From 370ad2f26a39804286fd9616f6a13e27c36fae0a Mon Sep 17 00:00:00 2001 From: Joshua Chi <112539+JoshuaChi@users.noreply.github.com> Date: Sat, 31 Oct 2026 22:35:02 +0800 Subject: [PATCH 01/19] refactor #400: Remove StateMachine::start, len and is_empty - start only set a flag that is already true when the state machine is built - len and is_empty were not called by the engine; tests now check entries with get - Update docs, sled example and MIGRATION_GUIDE --- MIGRATION_GUIDE.md | 2 +- .../leader_state_test/snapshot_worker_test.rs | 2 - d-engine-core/src/storage/state_machine.rs | 22 +-------- .../src/storage/state_machine_test.rs | 40 +++-------------- .../src/test_utils/mock/mock_raft_builder.rs | 2 - .../api/embedded_test/shutdown_order_test.rs | 6 --- d-engine-server/src/node/builder.rs | 3 -- d-engine-server/src/node/builder_test.rs | 14 ++++-- .../adaptors/file/file_state_machine.rs | 11 ----- .../adaptors/file/file_state_machine_test.rs | 10 ----- .../adaptors/rocksdb/rocksdb_state_machine.rs | 19 -------- .../rocksdb/rocksdb_state_machine_test.rs | 7 --- .../src/test_utils/mock/mock_node_builder.rs | 2 - ...oin_cluster_concurrent_joins_standalone.rs | 10 ++++- .../server_guide/customize-state-machine.md | 45 +++++++------------ .../sled-cluster/src/sled_state_machine.rs | 16 ++----- 16 files changed, 47 insertions(+), 164 deletions(-) diff --git a/MIGRATION_GUIDE.md b/MIGRATION_GUIDE.md index fbbac79d..334ad019 100644 --- a/MIGRATION_GUIDE.md +++ b/MIGRATION_GUIDE.md @@ -10,7 +10,7 @@ - Remove calls to `put_with_ttl` and the `ttl_secs` field of `Command::Insert`. Implement key expiry in the layer above d-engine. - Delete the `[raft.state_machine.lease]` section from your config files. -- Custom state machines: remove `lease_background_cleanup`. +- Custom state machines: remove `lease_background_cleanup`, `start`, `len` and `is_empty` from your `StateMachine` impl; the trait no longer declares them. `start` was called once when the node booted. Do that setup in your constructor (a state machine must be ready to serve as soon as it is built). If you need an entry count, keep `len` as a plain method on your own type. - Proto: `WriteCommand.Insert.ttl_secs` (field 3) is removed. Regenerate your client code. - The file state machine storage format changed. Start with a fresh data directory; old files are not supported. - RocksDB state machine: keys written with `put_with_ttl` under v0.2.x are kept forever after the upgrade. v0.3.0 neither reads the old `ttl_state` entry nor deletes expired keys. Before upgrading, either delete the keys you no longer want, or record their deadlines and have your own expiry layer delete them. Or start with a fresh data directory. diff --git a/d-engine-core/src/raft_role/leader_state_test/snapshot_worker_test.rs b/d-engine-core/src/raft_role/leader_state_test/snapshot_worker_test.rs index 1d2734dd..d585ff15 100644 --- a/d-engine-core/src/raft_role/leader_state_test/snapshot_worker_test.rs +++ b/d-engine-core/src/raft_role/leader_state_test/snapshot_worker_test.rs @@ -102,12 +102,10 @@ fn state_machine_with_snapshot() -> MockStateMachine { use bytes::Bytes; use d_engine_proto::common::LogId; let mut sm = MockStateMachine::new(); - sm.expect_start().returning(|| Ok(())); sm.expect_close().returning(|| Ok(())); sm.expect_is_running().returning(|| true); sm.expect_get().returning(|_| Ok(None)); sm.expect_apply_chunk().returning(|_| Ok(vec![])); - sm.expect_len().returning(|| 0); sm.expect_update_last_applied().returning(|_| ()); sm.expect_last_applied().return_const(LogId::default()); sm.expect_persist_last_applied().returning(|_| Ok(())); diff --git a/d-engine-core/src/storage/state_machine.rs b/d-engine-core/src/storage/state_machine.rs index 62178fe8..6f5309dc 100644 --- a/d-engine-core/src/storage/state_machine.rs +++ b/d-engine-core/src/storage/state_machine.rs @@ -79,21 +79,12 @@ impl ApplyResult { /// /// **CRITICAL**: Implementations MUST be thread-safe. /// -/// - Read methods (`get()`, `len()`) may be called concurrently +/// - Read methods (`get()`) may be called concurrently /// - Write methods should use internal synchronization /// - No assumptions about caller's threading model #[cfg_attr(any(test, feature = "__test_support"), automock)] #[async_trait] pub trait StateMachine: Send + Sync + 'static { - /// Starts the state machine service. - /// - /// This method: - /// 1. Flips internal state flags - /// 2. Loads persisted data from disk - /// - /// Called once during node startup. Performance is not critical. - async fn start(&self) -> Result<(), Error>; - /// Permanently closes the state machine: stops serving, persists what must survive a /// restart (e.g. data, metadata), flushes, and releases exclusive resources /// (e.g. a DB handle or its lock file). @@ -173,17 +164,6 @@ pub trait StateMachine: Send + Sync + 'static { chunk: &[ApplyEntry], ) -> Result, Error>; - /// Returns the number of entries in the state machine. - /// NOTE: This may be expensive for some implementations. - /// Sync operation but should be used cautiously. - fn len(&self) -> usize; - - /// Checks if the state machine is empty. - /// Sync operation that typically delegates to len(). - fn is_empty(&self) -> bool { - self.len() == 0 - } - /// Updates the last applied index in memory. /// Sync operation as it just updates atomic variables. fn update_last_applied( diff --git a/d-engine-core/src/storage/state_machine_test.rs b/d-engine-core/src/storage/state_machine_test.rs index 350ba27c..a69a73b5 100644 --- a/d-engine-core/src/storage/state_machine_test.rs +++ b/d-engine-core/src/storage/state_machine_test.rs @@ -36,7 +36,7 @@ pub trait StateMachineBuilder: Send + Sync { impl StateMachineTestSuite { /// Run all state machine tests pub async fn run_all_tests(builder: B) -> Result<(), Error> { - Self::test_start_close(builder.build().await?).await?; + Self::test_close_is_idempotent_and_stops_running(builder.build().await?).await?; Self::test_basic_kv_operations(builder.build().await?).await?; Self::test_apply_chunk_functionality(builder.build().await?).await?; Self::test_cas_operations(builder.build().await?).await?; @@ -80,14 +80,14 @@ impl StateMachineTestSuite { Ok(()) } - /// Test start/close lifecycle: a state machine is running after `start`, and not running + /// Test close lifecycle: a freshly built state machine is already running, and not running /// once `close`d. `close` is permanent and idempotent (#455), so there is no restart. - pub async fn test_start_close(state_machine: Arc) -> Result<(), Error> { - // Test default state + pub async fn test_close_is_idempotent_and_stops_running( + state_machine: Arc + ) -> Result<(), Error> { + // Running as soon as it is built: there is no separate start step assert!(state_machine.is_running()); - state_machine.start().await?; - assert!(state_machine.is_running()); state_machine.close()?; assert!(!state_machine.is_running()); state_machine.close()?; // idempotent @@ -129,8 +129,6 @@ impl StateMachineTestSuite { /// Test CAS operations pub async fn test_cas_operations(sm: Arc) -> Result<(), Error> { - sm.start().await?; - // Test 1: CAS on non-existent key (expected None) let entry1 = create_cas_entry(1, b"lock".to_vec().into(), None, b"owner1".to_vec().into()); sm.apply_chunk(&[entry1]).await?; @@ -165,8 +163,6 @@ impl StateMachineTestSuite { pub async fn test_batch_operations(sm: Arc) -> Result<(), Error> { use crate::BatchOp; - sm.start().await?; - let entry = create_batch_entry(1, vec![ BatchOp::Insert { key: b"k1".to_vec().into(), @@ -607,7 +603,6 @@ impl StateMachineTestSuite { pub async fn test_drop_flushes_data(builder: &B) -> Result<(), Error> { // Step 1: Create state machine, write data, call stop() let sm = builder.build().await?; - sm.start().await?; // Write test data let entries = vec![ @@ -640,7 +635,6 @@ impl StateMachineTestSuite { // Step 2: Reopen state machine from same path let sm2 = builder.build().await?; - sm2.start().await?; // Step 3: Verify data survived let value1 = sm2.get(b"drop_test_key1")?; @@ -678,7 +672,6 @@ impl StateMachineTestSuite { // Step 1: Write data - NO explicit stop() or flush() before drop { let sm = builder.build().await?; - sm.start().await?; let entries = vec![ create_insert_entry( @@ -716,7 +709,6 @@ impl StateMachineTestSuite { // Step 2: Reopen let sm2 = builder.build().await?; - sm2.start().await?; // Step 3: Critical assertion - last_applied must be persisted by Drop assert_eq!( @@ -756,7 +748,6 @@ impl StateMachineTestSuite { { let sm = builder.build().await?; - sm.start().await?; let entries = vec![ create_insert_entry( @@ -808,7 +799,6 @@ impl StateMachineTestSuite { tokio::time::sleep(Duration::from_millis(50)).await; let sm2 = builder.build().await?; - sm2.start().await?; assert_eq!( sm2.last_applied(), @@ -844,7 +834,6 @@ impl StateMachineTestSuite { // Step 1: Create, write, stop gracefully { let sm = builder.build().await?; - sm.start().await?; let entries = vec![ create_insert_entry( @@ -874,7 +863,6 @@ impl StateMachineTestSuite { // Step 2: Reopen and verify let sm2 = builder.build().await?; - sm2.start().await?; // Verify all data assert_eq!( @@ -913,7 +901,6 @@ impl StateMachineTestSuite { // Step 1: apply a batch with mixed Insert + Delete, then drop without stop() { let sm = builder.build().await?; - sm.start().await?; let entry = create_batch_entry(1, vec![ BatchOp::Insert { @@ -939,7 +926,6 @@ impl StateMachineTestSuite { // Step 2: reopen from same path — WAL must replay correctly let sm2 = builder.build().await?; - sm2.start().await?; assert_eq!( sm2.get(b"bk1")?, @@ -984,7 +970,6 @@ impl StateMachineTestSuite { ) -> Result<(), Error> { // Step 1: Write data and drop ungracefully (no stop()) let sm = builder.build().await?; - sm.start().await?; let entries = vec![ create_insert_entry( @@ -1015,7 +1000,6 @@ impl StateMachineTestSuite { // Step 2: Simulate restart - create new instance from same path let sm2 = builder.build().await?; - sm2.start().await?; // Step 3: Verify crash recovery - both data and metadata let value1 = sm2.get(b"crash_key1")?; @@ -1058,7 +1042,6 @@ impl StateMachineTestSuite { builder: &B ) -> Result<(), Error> { let sm = builder.build().await?; - sm.start().await?; sm.apply_chunk(&[ create_insert_entry( 1, @@ -1083,7 +1066,6 @@ impl StateMachineTestSuite { // `sm` stays alive until the end of the test: Drop must not be what saves the state. let sm2 = builder.build().await?; - sm2.start().await?; assert_eq!( sm2.last_applied().index, @@ -1107,7 +1089,6 @@ impl StateMachineTestSuite { /// down. A late read must not answer from state that is no longer maintained, and a late /// `apply_chunk` must not mutate a state machine that already persisted its final state. pub async fn test_operations_fail_after_close(sm: Arc) -> Result<(), Error> { - sm.start().await?; sm.apply_chunk(&[create_insert_entry( 1, Bytes::from_static(b"serving_key"), @@ -1761,11 +1742,6 @@ mod default_scan_prefix_tests { #[async_trait] impl StateMachine for MinimalSm { - async fn start(&self) -> Result<(), Error> { - self.running.store(true, Ordering::SeqCst); - Ok(()) - } - fn close(&self) -> Result<(), Error> { self.running.store(false, Ordering::SeqCst); Ok(()) @@ -1789,10 +1765,6 @@ mod default_scan_prefix_tests { Ok(vec![]) } - fn len(&self) -> usize { - 0 - } - fn update_last_applied( &self, _last_applied: LogId, diff --git a/d-engine-core/src/test_utils/mock/mock_raft_builder.rs b/d-engine-core/src/test_utils/mock/mock_raft_builder.rs index 715b1c0a..69509cf5 100644 --- a/d-engine-core/src/test_utils/mock/mock_raft_builder.rs +++ b/d-engine-core/src/test_utils/mock/mock_raft_builder.rs @@ -383,13 +383,11 @@ pub fn mock_replication_handler() -> MockReplicationCore { pub fn mock_state_machine() -> MockStateMachine { let mut mock = MockStateMachine::new(); - mock.expect_start().returning(|| Ok(())); mock.expect_close().returning(|| Ok(())); mock.expect_is_running().returning(|| true); mock.expect_get().returning(|_| Ok(None)); mock.expect_apply_chunk().returning(|_| Ok(vec![])); - mock.expect_len().returning(|| 0); mock.expect_update_last_applied().returning(|_| ()); mock.expect_last_applied().return_const(LogId::default()); diff --git a/d-engine-server/src/api/embedded_test/shutdown_order_test.rs b/d-engine-server/src/api/embedded_test/shutdown_order_test.rs index 154d981d..0f0e6ce5 100644 --- a/d-engine-server/src/api/embedded_test/shutdown_order_test.rs +++ b/d-engine-server/src/api/embedded_test/shutdown_order_test.rs @@ -60,9 +60,6 @@ impl StateMachine for SlowApplyProbe { } // Everything below delegates unchanged. - async fn start(&self) -> Result<(), Error> { - self.inner.start().await - } fn is_running(&self) -> bool { self.inner.is_running() } @@ -72,9 +69,6 @@ impl StateMachine for SlowApplyProbe { ) -> Result, Error> { self.inner.get(key_buffer) } - fn len(&self) -> usize { - self.inner.len() - } fn update_last_applied( &self, last_applied: LogId, diff --git a/d-engine-server/src/node/builder.rs b/d-engine-server/src/node/builder.rs index 208e0df3..6c77b364 100644 --- a/d-engine-server/src/node/builder.rs +++ b/d-engine-server/src/node/builder.rs @@ -309,9 +309,6 @@ where ) })?; - // Start state machine: flip flags and load persisted data - state_machine.start().await?; - // Handle storage engine initialization let storage_engine = self.storage_engine.take().ok_or_else(|| { SystemError::NodeStartFailed( diff --git a/d-engine-server/src/node/builder_test.rs b/d-engine-server/src/node/builder_test.rs index 51e0b236..59db795e 100644 --- a/d-engine-server/src/node/builder_test.rs +++ b/d-engine-server/src/node/builder_test.rs @@ -81,10 +81,16 @@ async fn test_set_raft_log_replaces_default() { builder.storage_engine.as_ref().unwrap().log_store().last_index(), expected_raft_log_ids[1] ); - assert_eq!( - builder.state_machine.as_ref().unwrap().len(), - expected_state_machine_ids.len() - ); + // Verify the builder kept the customized state machine: every pre-inserted entry is readable + let state_machine = builder.state_machine.as_ref().unwrap(); + for id in &expected_state_machine_ids { + let key = id.to_be_bytes(); + assert_eq!( + state_machine.get(&key).unwrap(), + Some(bytes::Bytes::from(key.to_vec())), + "pre-inserted entry {id} missing from the builder's state machine" + ); + } } #[tokio::test] diff --git a/d-engine-server/src/storage/adaptors/file/file_state_machine.rs b/d-engine-server/src/storage/adaptors/file/file_state_machine.rs index 873bbf80..1d9c32b2 100644 --- a/d-engine-server/src/storage/adaptors/file/file_state_machine.rs +++ b/d-engine-server/src/storage/adaptors/file/file_state_machine.rs @@ -859,13 +859,6 @@ impl Drop for FileStateMachine { #[async_trait] impl StateMachine for FileStateMachine { - async fn start(&self) -> Result<(), Error> { - self.running.store(true, Ordering::SeqCst); - - info!("File state machine started"); - Ok(()) - } - fn close(&self) -> Result<(), Error> { if !self.running.swap(false, Ordering::SeqCst) { return Ok(()); // already closed @@ -1086,10 +1079,6 @@ impl StateMachine for FileStateMachine { Ok(results) } - fn len(&self) -> usize { - self.data.read().len() - } - fn update_last_applied( &self, last_applied: LogId, diff --git a/d-engine-server/src/storage/adaptors/file/file_state_machine_test.rs b/d-engine-server/src/storage/adaptors/file/file_state_machine_test.rs index ff1336b9..04f17ac5 100644 --- a/d-engine-server/src/storage/adaptors/file/file_state_machine_test.rs +++ b/d-engine-server/src/storage/adaptors/file/file_state_machine_test.rs @@ -26,7 +26,6 @@ use crate::storage::adaptors::file::FileStateMachine; async fn test_file_sm_close_marks_not_running() { let dir = TempDir::new().unwrap(); let sm = FileStateMachine::new(dir.path().to_path_buf()).await.unwrap(); - sm.start().await.unwrap(); assert!(sm.is_running()); sm.close().unwrap(); @@ -43,7 +42,6 @@ async fn test_file_sm_close_marks_not_running() { async fn test_file_sm_reads_fail_after_close() { let dir = TempDir::new().unwrap(); let sm = FileStateMachine::new(dir.path().to_path_buf()).await.unwrap(); - sm.start().await.unwrap(); sm.apply_chunk(&[ApplyEntry { index: 1, term: 1, @@ -673,7 +671,6 @@ fn insert_at( async fn test_file_apply_snapshot_from_file_rejects_stale_snapshot() { let dir = TempDir::new().expect("Failed to create temp dir"); let sm = FileStateMachine::new(dir.path().join("sm")).await.expect("create sm"); - sm.start().await.expect("start"); let early: Vec = (1..=20) .map(|i| insert_at(format!("key_{i}").as_bytes(), b"early", i)) @@ -726,7 +723,6 @@ async fn test_file_apply_snapshot_from_file_rejects_stale_snapshot() { async fn test_file_apply_snapshot_from_file_ignores_duplicate_snapshot() { let dir = TempDir::new().expect("Failed to create temp dir"); let sm = FileStateMachine::new(dir.path().join("sm")).await.expect("create sm"); - sm.start().await.expect("start"); let entries: Vec = (1..=5).map(|i| insert_at(format!("key_{i}").as_bytes(), b"v", i)).collect(); @@ -764,7 +760,6 @@ async fn test_file_apply_snapshot_from_file_ignores_duplicate_snapshot() { async fn test_file_apply_snapshot_from_file_rejects_boundary_conflict() { let dir = TempDir::new().expect("Failed to create temp dir"); let sm = FileStateMachine::new(dir.path().join("sm")).await.expect("create sm"); - sm.start().await.expect("start"); let entries: Vec = (1..=5).map(|i| insert_at(format!("key_{i}").as_bytes(), b"v", i)).collect(); @@ -798,7 +793,6 @@ async fn test_file_apply_snapshot_from_file_rejects_boundary_conflict() { async fn test_file_apply_snapshot_from_file_reports_applied_for_newer_snapshot() { let dir = TempDir::new().expect("Failed to create temp dir"); let sm = FileStateMachine::new(dir.path().join("sm")).await.expect("create sm"); - sm.start().await.expect("start"); let entries: Vec = (1..=5).map(|i| insert_at(format!("key_{i}").as_bytes(), b"v", i)).collect(); @@ -838,7 +832,6 @@ async fn test_file_apply_snapshot_from_file_reports_applied_for_newer_snapshot() async fn test_file_apply_snapshot_from_file_rejects_missing_last_included() { let dir = TempDir::new().expect("Failed to create temp dir"); let sm = FileStateMachine::new(dir.path().join("sm")).await.expect("create sm"); - sm.start().await.expect("start"); let early_entries: Vec = (1..=5).map(|i| insert_at(format!("key_{i}").as_bytes(), b"v", i)).collect(); @@ -883,14 +876,12 @@ async fn test_replay_wal_skips_stale_entries_below_last_applied() { // Follower applies key_1..key_5 = "old" at indexes 1..=5. let sm = FileStateMachine::new(dir.path().join("sm")).await.expect("create follower"); - sm.start().await.expect("start follower"); let stale: Vec = (1..=5).map(|i| insert_at(format!("key_{i}").as_bytes(), b"old", i)).collect(); sm.apply_chunk(&stale).await.expect("apply pre-snapshot entries"); // Snapshot at index 10 overwrites the same keys with newer values. let gen_sm = FileStateMachine::new(dir.path().join("gen")).await.expect("create generator"); - gen_sm.start().await.expect("start generator"); let newer: Vec = (1..=10).map(|i| insert_at(format!("key_{i}").as_bytes(), b"new", i)).collect(); gen_sm.apply_chunk(&newer).await.expect("apply generator entries"); @@ -926,7 +917,6 @@ async fn test_replay_wal_skips_stale_entries_below_last_applied() { drop(sm); let recovered = FileStateMachine::new(dir.path().join("sm")).await.expect("reload after crash"); - recovered.start().await.expect("start recovered"); // The snapshot's newer values must survive; stale WAL must NOT clobber them. assert_eq!( diff --git a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs index f6806d15..670cd6d1 100644 --- a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs +++ b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs @@ -523,13 +523,6 @@ impl RocksDBStateMachine { #[async_trait] impl StateMachine for RocksDBStateMachine { - async fn start(&self) -> Result<(), Error> { - self.is_serving.store(true, Ordering::SeqCst); - - info!("RocksDB state machine started"); - Ok(()) - } - /// Sequence: stop serving → flush → save hard state → cancel background /// work → drop the DB handle (releasing the LOCK). A second call finds no DB and does nothing. fn close(&self) -> Result<(), Error> { @@ -716,18 +709,6 @@ impl StateMachine for RocksDBStateMachine { Ok(results) } - fn len(&self) -> usize { - let guard = self.db.load(); - let Some(db) = guard.as_deref() else { - return 0; - }; - let Some(cf) = db.cf_handle(STATE_MACHINE_CF) else { - return 0; - }; - // Note: This is an expensive operation because it iterates over all keys. - db.iterator_cf(&cf, IteratorMode::Start).count() - } - fn update_last_applied( &self, last_applied: LogId, diff --git a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine_test.rs b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine_test.rs index c73aedf5..4693b242 100644 --- a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine_test.rs +++ b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine_test.rs @@ -521,7 +521,6 @@ async fn test_get_multi_two_keys_still_uses_snapshot_isolation() { async fn test_close_prevents_reads() { let dir = TempDir::new().unwrap(); let (_storage, sm) = RocksDBUnifiedEngine::open(dir.path()).unwrap(); - sm.start().await.unwrap(); sm.apply_chunk(&[insert_at(b"k", b"v", 1)]).await.unwrap(); assert_eq!(sm.get(b"k").unwrap(), Some(Bytes::from_static(b"v"))); @@ -558,7 +557,6 @@ async fn test_drop_after_close_does_not_panic() { let dir = TempDir::new().unwrap(); { let (_storage, sm) = RocksDBUnifiedEngine::open(dir.path()).unwrap(); - sm.start().await.unwrap(); sm.apply_chunk(&[insert_at(b"k", b"v", 1)]).await.unwrap(); sm.close().unwrap(); // sm drops here — Drop must be a no-op @@ -815,7 +813,6 @@ async fn test_apply_snapshot_from_file_rejects_stale_snapshot() { let db_path = temp_dir.path().join("test_stale_snapshot"); let (_storage, state_machine) = RocksDBUnifiedEngine::open(&db_path).expect("Failed to open unified DB"); - state_machine.start().await.expect("Failed to start"); // 1. Apply entries 1..=20 — the state a snapshot generated "now" would capture. let early_entries: Vec = (1..=20) @@ -890,7 +887,6 @@ async fn test_apply_snapshot_from_file_ignores_duplicate_snapshot() { let db_path = temp_dir.path().join("test_duplicate_snapshot"); let (_storage, state_machine) = RocksDBUnifiedEngine::open(&db_path).expect("Failed to open unified DB"); - state_machine.start().await.expect("Failed to start"); let entries: Vec = (1..=5).map(|i| insert_at(format!("key_{i}").as_bytes(), b"v", i)).collect(); @@ -935,7 +931,6 @@ async fn test_apply_snapshot_from_file_rejects_boundary_conflict() { let db_path = temp_dir.path().join("test_boundary_conflict"); let (_storage, state_machine) = RocksDBUnifiedEngine::open(&db_path).expect("Failed to open unified DB"); - state_machine.start().await.expect("Failed to start"); let entries: Vec = (1..=5).map(|i| insert_at(format!("key_{i}").as_bytes(), b"v", i)).collect(); @@ -980,7 +975,6 @@ async fn test_apply_snapshot_from_file_reports_applied_for_newer_snapshot() { let db_path = temp_dir.path().join("test_applied_snapshot"); let (_storage, state_machine) = RocksDBUnifiedEngine::open(&db_path).expect("Failed to open unified DB"); - state_machine.start().await.expect("Failed to start"); let entries: Vec = (1..=5).map(|i| insert_at(format!("key_{i}").as_bytes(), b"v", i)).collect(); @@ -1025,7 +1019,6 @@ async fn test_apply_snapshot_from_file_rejects_missing_last_included() { let db_path = temp_dir.path().join("test_none_last_included"); let (_storage, state_machine) = RocksDBUnifiedEngine::open(&db_path).expect("Failed to open unified DB"); - state_machine.start().await.expect("Failed to start"); // Snapshot captured early, containing only key_1..key_5. let early_entries: Vec = diff --git a/d-engine-server/src/test_utils/mock/mock_node_builder.rs b/d-engine-server/src/test_utils/mock/mock_node_builder.rs index 71418506..e88ec394 100644 --- a/d-engine-server/src/test_utils/mock/mock_node_builder.rs +++ b/d-engine-server/src/test_utils/mock/mock_node_builder.rs @@ -625,13 +625,11 @@ pub(crate) fn mock_replication_handler() -> MockReplicationCore pub(crate) fn mock_state_machine() -> MockStateMachine { let mut mock = MockStateMachine::new(); - mock.expect_start().returning(|| Ok(())); mock.expect_close().returning(|| Ok(())); mock.expect_is_running().returning(|| true); mock.expect_get().returning(|_| Ok(None)); mock.expect_apply_chunk().returning(|_| Ok(vec![])); - mock.expect_len().returning(|| 0); mock.expect_update_last_applied().returning(|_| ()); mock.expect_last_applied().return_const(LogId::default()); diff --git a/d-engine-server/tests/cluster_membership/join_cluster_concurrent_joins_standalone.rs b/d-engine-server/tests/cluster_membership/join_cluster_concurrent_joins_standalone.rs index 1baa39e2..f4a28ef0 100644 --- a/d-engine-server/tests/cluster_membership/join_cluster_concurrent_joins_standalone.rs +++ b/d-engine-server/tests/cluster_membership/join_cluster_concurrent_joins_standalone.rs @@ -334,7 +334,15 @@ async fn test_join_cluster_scenario2() -> Result<(), ClientApiError> { let node1_sm = FileStateMachine::new(PathBuf::from(format!("{db_dir}/cs/1/state_machine"))) .await .expect("Failed to open node 1 state machine for verification"); - assert_eq!(node1_sm.len(), 11); + // Node 1 reopened from disk must hold the seeded entries plus entry 11. + for i in 1..=last_log_id { + let value = node1_sm.get(&safe_kv(i)).unwrap(); + assert_eq!(value, Some(Bytes::from(safe_kv(i).to_vec()))); + } + assert!( + node1_sm.get(&safe_kv(11)).unwrap().is_some(), + "entry 11 was not persisted on node 1" + ); // Verify leader is still node 3 assert_eq!(client_manager.list_leader_id().await.unwrap(), Some(3)); diff --git a/d-engine/src/docs/server_guide/customize-state-machine.md b/d-engine/src/docs/server_guide/customize-state-machine.md index 255b68a7..c04c0aa8 100644 --- a/d-engine/src/docs/server_guide/customize-state-machine.md +++ b/d-engine/src/docs/server_guide/customize-state-machine.md @@ -24,10 +24,6 @@ struct CustomStateMachine { #[async_trait] impl StateMachine for CustomStateMachine { - fn start(&self) -> Result<(), Error> { - // Initialize your state machine - Ok(()) - } // close() has a default no-op implementation. // Override it if your backend must persist something at shutdown (e.g. data, metadata) or @@ -98,10 +94,6 @@ impl StateMachine for CustomStateMachine { Ok(results) } - fn len(&self) -> usize { - // Return number of entries - self.backend.count() - } // Implement all required methods... } @@ -119,27 +111,24 @@ impl StateMachine for CustomStateMachine { ## 3. StateMachine API Reference -| Method | Purpose | Sync/Async | Criticality | -| ---------------------------------- | -------------------------------------------------------------------------------------------- | ---------- | ----------- | -| `start()` | Initialize state machine service | Sync | High | +| Method | Purpose | Sync/Async | Criticality | +| ---------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | ----------- | | `close()` | Final step at shutdown: persist what must survive a restart and release exclusive OS resources (e.g. DB lock file), even if persisting fails; returns the first error. Idempotent. Default `Ok(())`. | Sync | Medium | -| `is_running()` | Check service status | Sync | Medium | -| `get()` | Read value by key | Sync | High | -| `apply_chunk()` | Apply committed entries | Async | Critical | -| `len()` | Get entry count | Sync | Low | -| `is_empty()` | Check if empty | Sync | Low | -| `update_last_applied()` | Update applied index in memory | Sync | High | -| `last_applied()` | Get last applied index | Sync | High | -| `persist_last_applied()` | Persist applied index | Sync | High | -| `update_last_snapshot_metadata()` | Update snapshot metadata in memory | Sync | Medium | -| `snapshot_metadata()` | Get current snapshot metadata | Sync | Medium | -| `persist_last_snapshot_metadata()` | Persist snapshot metadata | Sync | Medium | -| `apply_snapshot_from_file()` | Replace state with snapshot | Async | Critical | -| `generate_snapshot_data()` | Create new snapshot | Async | High | -| `save_hard_state()` | Persist term/vote state | Sync | High | -| `flush()` | Sync writes to storage | Sync | High | -| `flush_async()` | Async flush | Async | High | -| `reset()` | Reset to initial state | Async | Medium | +| `is_running()` | Check service status | Sync | Medium | +| `get()` | Read value by key | Sync | High | +| `apply_chunk()` | Apply committed entries | Async | Critical | +| `update_last_applied()` | Update applied index in memory | Sync | High | +| `last_applied()` | Get last applied index | Sync | High | +| `persist_last_applied()` | Persist applied index | Sync | High | +| `update_last_snapshot_metadata()` | Update snapshot metadata in memory | Sync | Medium | +| `snapshot_metadata()` | Get current snapshot metadata | Sync | Medium | +| `persist_last_snapshot_metadata()` | Persist snapshot metadata | Sync | Medium | +| `apply_snapshot_from_file()` | Replace state with snapshot | Async | Critical | +| `generate_snapshot_data()` | Create new snapshot | Async | High | +| `save_hard_state()` | Persist term/vote state | Sync | High | +| `flush()` | Sync writes to storage | Sync | High | +| `flush_async()` | Async flush | Async | High | +| `reset()` | Reset to initial state | Async | Medium | ## 4. Testing Your Implementation diff --git a/examples/sled-cluster/src/sled_state_machine.rs b/examples/sled-cluster/src/sled_state_machine.rs index 96fa0710..f29a63e4 100644 --- a/examples/sled-cluster/src/sled_state_machine.rs +++ b/examples/sled-cluster/src/sled_state_machine.rs @@ -8,17 +8,17 @@ use bincode::config; use bytes::Bytes; use d_engine::BatchOp; use d_engine::{ - ApplyEntry, ApplyResult, Command, Result, ScanResult, SnapshotApplyResult, SnapshotError, - StateMachine, StorageError, common::LogId, server_storage::SnapshotMetadata, + common::LogId, server_storage::SnapshotMetadata, ApplyEntry, ApplyResult, Command, Result, + ScanResult, SnapshotApplyResult, SnapshotError, StateMachine, StorageError, }; use parking_lot::Mutex; use sled::Batch; use std::path::Path; use std::path::PathBuf; -use std::sync::Arc; use std::sync::atomic::AtomicBool; use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; +use std::sync::Arc; use std::time::Instant; use tracing::debug; use tracing::error; @@ -88,12 +88,6 @@ impl Drop for SledStateMachine { #[async_trait] impl StateMachine for SledStateMachine { - async fn start(&self) -> Result<()> { - debug!("start state machine"); - self.is_serving.store(true, Ordering::Release); - Ok(()) - } - fn close(&self) -> Result<()> { if self.db.load().is_none() { return Ok(()); @@ -525,10 +519,6 @@ impl StateMachine for SledStateMachine { }) } - fn len(&self) -> usize { - self.current_tree().map(|t| t.len()).unwrap_or(0) - } - async fn reset(&self) -> Result<()> { let db = self.db()?; for tree_name in &[ From 4a51f1aebfd6c2ce00ea2b560663f6c4d9b4c5d2 Mon Sep 17 00:00:00 2001 From: Joshua Chi <112539+JoshuaChi@users.noreply.github.com> Date: Sun, 1 Nov 2026 16:31:55 +0800 Subject: [PATCH 02/19] refactor #400: Remove StateMachine::is_running; report state machine read failures - is_running is gone: a state machine that cannot serve returns NotServing and the ReadActor falls back to the Raft path (ReadActorError::SmUnavailable) - A failed state machine read now returns an error code instead of an empty result: read_from_state_machine returns Result and reads with get_multi (one consistent view); clients get StorageIoError, DataCorruption, RetryRequired or Uncategorized, with no internal detail - EmbeddedReadHandle falls back only on NotServing; hard failures are returned - Add Error::is_not_serving and ErrorCode::for_read_failure as the single classification - Update docs, sled example and MIGRATION_GUIDE --- MIGRATION_GUIDE.md | 3 +- d-engine-core/src/client/client_api_error.rs | 2 +- d-engine-core/src/client/types.rs | 40 +++++ d-engine-core/src/client/types_test.rs | 69 ++++++++ d-engine-core/src/errors.rs | 12 ++ d-engine-core/src/errors_test.rs | 25 +++ .../src/raft_role/candidate_state_test.rs | 4 +- .../src/raft_role/follower_state_test.rs | 52 +++++- d-engine-core/src/raft_role/leader_state.rs | 48 ++---- .../leader_state_test/client_read_test.rs | 51 +++++- .../leader_state_test/event_handling_test.rs | 4 +- .../leader_state_test/replication_test.rs | 2 +- .../leader_state_test/snapshot_worker_test.rs | 1 - .../src/raft_role/learner_state_test.rs | 2 +- d-engine-core/src/raft_role/role_state.rs | 10 +- .../drain_based_batch_architecture_tests.rs | 4 +- .../raft_test/process_inbound_events_tests.rs | 155 ++++++++++++++++++ .../default_state_machine_handler.rs | 20 +-- .../default_state_machine_handler_test.rs | 69 ++++++++ .../src/state_machine_handler/mod.rs | 6 +- d-engine-core/src/storage/state_machine.rs | 4 - .../src/storage/state_machine_test.rs | 32 ++-- .../src/test_utils/mock/mock_raft_builder.rs | 3 +- .../src/api/embedded_read_handle.rs | 28 ++-- .../src/api/embedded_read_handle_test.rs | 76 +++++++-- .../api/embedded_test/shutdown_order_test.rs | 3 - .../src/api/standalone_read_handle.rs | 36 +++- .../src/api/standalone_read_handle_test.rs | 101 +++++++++--- .../grpc/grpc_raft_service_fast_path_test.rs | 33 ++-- d-engine-server/src/read_actor.rs | 29 ++-- d-engine-server/src/read_actor_test.rs | 26 +-- .../adaptors/file/file_state_machine.rs | 4 - .../adaptors/file/file_state_machine_test.rs | 15 +- .../adaptors/rocksdb/rocksdb_state_machine.rs | 4 - .../rocksdb/rocksdb_state_machine_test.rs | 6 +- .../src/test_utils/mock/mock_node_builder.rs | 3 +- .../server_guide/customize-state-machine.md | 12 +- .../sled-cluster/src/sled_state_machine.rs | 10 +- 38 files changed, 779 insertions(+), 225 deletions(-) diff --git a/MIGRATION_GUIDE.md b/MIGRATION_GUIDE.md index 334ad019..0bbf50f8 100644 --- a/MIGRATION_GUIDE.md +++ b/MIGRATION_GUIDE.md @@ -10,10 +10,11 @@ - Remove calls to `put_with_ttl` and the `ttl_secs` field of `Command::Insert`. Implement key expiry in the layer above d-engine. - Delete the `[raft.state_machine.lease]` section from your config files. -- Custom state machines: remove `lease_background_cleanup`, `start`, `len` and `is_empty` from your `StateMachine` impl; the trait no longer declares them. `start` was called once when the node booted. Do that setup in your constructor (a state machine must be ready to serve as soon as it is built). If you need an entry count, keep `len` as a plain method on your own type. +- Custom state machines: remove `lease_background_cleanup`, `start`, `is_running`, `len` and `is_empty` from your `StateMachine` impl; the trait no longer declares them. `start` was called once when the node booted. Do that setup in your constructor (a state machine must be ready to serve as soon as it is built). The engine no longer asks whether the state machine is running: a state machine that cannot serve a read, for example while it installs a snapshot or after `close()`, must return `StorageError::NotServing` from `get` and `get_multi`. If you need an entry count, keep `len` as a plain method on your own type. - Proto: `WriteCommand.Insert.ttl_secs` (field 3) is removed. Regenerate your client code. - The file state machine storage format changed. Start with a fresh data directory; old files are not supported. - RocksDB state machine: keys written with `put_with_ttl` under v0.2.x are kept forever after the upgrade. v0.3.0 neither reads the old `ttl_state` entry nor deletes expired keys. Before upgrading, either delete the keys you no longer want, or record their deadlines and have your own expiry layer delete them. Or start with a fresh data directory. +- A read that fails in the state machine now returns an error code instead of an empty result: `StorageIoError`, `DataCorruption` (the stored data may be damaged), `RetryRequired` (the state machine is replacing its state; retry) or `Uncategorized`. Before, such a read looked like "key not found". Handle these errors in your client. --- diff --git a/d-engine-core/src/client/client_api_error.rs b/d-engine-core/src/client/client_api_error.rs index 7352dfae..6fb47a78 100644 --- a/d-engine-core/src/client/client_api_error.rs +++ b/d-engine-core/src/client/client_api_error.rs @@ -254,7 +254,7 @@ impl From for ClientApiError { }, ErrorCode::DataCorruption => ClientApiError::Storage { code, - message: "Data corruption detected".to_string(), + message: "The stored data for this request may be damaged. Treat the result as unreliable and contact the administrator.".to_string(), }, ErrorCode::StorageIoError => ClientApiError::Storage { code, diff --git a/d-engine-core/src/client/types.rs b/d-engine-core/src/client/types.rs index e26cbac4..d62a608d 100644 --- a/d-engine-core/src/client/types.rs +++ b/d-engine-core/src/client/types.rs @@ -14,9 +14,14 @@ use bytes::Bytes; use serde::Deserialize; use serde::Serialize; +use tracing::error; +use tracing::warn; // ReadConsistencyPolicy already exists as a native type in crate::config; re-export it // here so all client-facing types live in one module. +use crate::Error; +use crate::StorageError; +use crate::SystemError; pub use crate::config::ReadConsistencyPolicy; // ─── Write request ──────────────────────────────────────────────────────────── @@ -173,6 +178,25 @@ impl TryFrom for ErrorCode { } } +impl ErrorCode { + /// The code a client sees when the state machine fails a read. + /// + /// `DataCorruption` is reported to the client, not escalated to a node shutdown: the read + /// failed and the stored data may be damaged, and the client application decides what to do. + pub fn for_read_failure(e: &Error) -> Self { + match e { + _ if e.is_not_serving() => Self::RetryRequired, + Error::System(SystemError::Storage(StorageError::DataCorruption { .. })) => { + Self::DataCorruption + } + Error::System(SystemError::Storage( + StorageError::IoError(_) | StorageError::DbError(_), + )) => Self::StorageIoError, + _ => Self::Uncategorized, + } + } +} + // ─── ClientResponse constructors ───────────────────────────────────────────── impl ClientResponse { @@ -266,4 +290,20 @@ impl ClientResponse { pub fn is_propose_failure(&self) -> bool { self.error == ErrorCode::ProposeFailed } + + /// Builds the response to a read. A failure is logged and answered with an error code that + /// carries no internal detail, never with an empty result. + pub fn from_read(result: Result, Error>) -> Self { + match result { + Ok(entries) => Self::read_results(entries), + Err(e) => { + if e.is_not_serving() { + warn!(error = ?e, "state machine cannot serve reads right now"); + } else { + error!(error = ?e, "state machine read failed"); + } + Self::client_error(ErrorCode::for_read_failure(&e)) + } + } + } } diff --git a/d-engine-core/src/client/types_test.rs b/d-engine-core/src/client/types_test.rs index 095f79f4..bebc4fb1 100644 --- a/d-engine-core/src/client/types_test.rs +++ b/d-engine-core/src/client/types_test.rs @@ -343,3 +343,72 @@ fn test_is_propose_failure_true() { fn test_is_propose_failure_false_for_success() { assert!(!ClientResponse::write_success().is_propose_failure()); } + +// ── State machine read failures ────────────────────────────────────────────── + +fn read_failure_cases() -> [(crate::Error, ErrorCode); 5] { + [ + // Transient: the state machine is replacing its state, a retry will succeed. + ( + crate::StorageError::NotServing("restoring".into()).into(), + ErrorCode::RetryRequired, + ), + ( + crate::StorageError::DbError("disk failure".into()).into(), + ErrorCode::StorageIoError, + ), + ( + crate::StorageError::IoError(std::io::Error::other("disk failure")).into(), + ErrorCode::StorageIoError, + ), + // The client is told the stored data may be damaged and decides what to do. + ( + crate::StorageError::DataCorruption { + location: "sst".into(), + } + .into(), + ErrorCode::DataCorruption, + ), + (crate::Error::Fatal("boom".into()), ErrorCode::Uncategorized), + ] +} + +/// Each failure class maps to one stable code, so clients can tell "retry", "storage problem", +/// "data may be damaged" and "internal bug" apart. +#[test] +fn test_for_read_failure_maps_each_failure_class_to_its_code() { + for (error, expected) in read_failure_cases() { + assert_eq!( + ErrorCode::for_read_failure(&error), + expected, + "for {error:?}" + ); + } +} + +/// A failed read is answered with an error code and no payload. It is never answered as a +/// successful empty read, which a client would take for "key not found". +#[test] +fn test_from_read_failure_is_an_error_response_without_result() { + for (error, expected) in read_failure_cases() { + let response = ClientResponse::from_read(Err(error)); + assert_eq!(response.error, expected); + assert!(response.result.is_none()); + } +} + +#[test] +fn test_from_read_success_carries_the_entries() { + let entries = vec![KvEntry { + key: Bytes::from_static(b"k"), + value: Bytes::from_static(b"v"), + }]; + + let response = ClientResponse::from_read(Ok(entries.clone())); + + assert_eq!(response.error, ErrorCode::Success); + assert_eq!( + response.result, + Some(ClientResponsePayload::Read(ReadResults { entries })) + ); +} diff --git a/d-engine-core/src/errors.rs b/d-engine-core/src/errors.rs index 65fbb142..7271a201 100644 --- a/d-engine-core/src/errors.rs +++ b/d-engine-core/src/errors.rs @@ -47,6 +47,18 @@ impl Error { )) ) } + + /// Returns true if the state machine refused the operation because it cannot serve right + /// now: it is replacing its state (snapshot install) or it has been closed. + /// + /// Transient from the caller's point of view: route the request through another path, or + /// ask the client to retry. Not an error in the data. + pub fn is_not_serving(&self) -> bool { + matches!( + self, + Error::System(SystemError::Storage(StorageError::NotServing(_))) + ) + } } #[derive(Debug, thiserror::Error)] diff --git a/d-engine-core/src/errors_test.rs b/d-engine-core/src/errors_test.rs index f4e50c4c..88f3c2a4 100644 --- a/d-engine-core/src/errors_test.rs +++ b/d-engine-core/src/errors_test.rs @@ -552,3 +552,28 @@ fn test_is_not_fatal_storage_log_storage() { let err: Error = StorageError::LogStorage("write failed".to_string()).into(); assert!(!err.is_fatal()); } + +/// `NotServing` is the state machine saying "not right now" (it is replacing its state or has +/// been closed). Callers route around it or ask the client to retry, so it must not be confused +/// with a real failure of the data. +#[test] +fn test_is_not_serving_is_true_only_for_not_serving() { + let not_serving: Error = StorageError::NotServing("restoring".to_string()).into(); + assert!(not_serving.is_not_serving()); + + let failures: [Error; 4] = [ + StorageError::DbError("disk failure".to_string()).into(), + StorageError::IoError(std::io::Error::other("disk failure")).into(), + StorageError::DataCorruption { + location: "sst".to_string(), + } + .into(), + Error::Fatal("boom".to_string()), + ]; + for error in failures { + assert!( + !error.is_not_serving(), + "{error:?} is a failure, not 'not serving'" + ); + } +} diff --git a/d-engine-core/src/raft_role/candidate_state_test.rs b/d-engine-core/src/raft_role/candidate_state_test.rs index 037b1288..202cebf3 100644 --- a/d-engine-core/src/raft_role/candidate_state_test.rs +++ b/d-engine-core/src/raft_role/candidate_state_test.rs @@ -1526,7 +1526,7 @@ mod handle_client_read_request { state_machine_handler .expect_read_from_state_machine() .times(1) - .returning(|_| Some(vec![])); + .returning(|_| Ok(vec![])); context.handlers.state_machine_handler = Arc::new(state_machine_handler); let mut state = CandidateState::::new(1, context.node_config.clone()); @@ -2176,7 +2176,6 @@ async fn candidate_with_unapplied_range( let temp_dir = tempfile::tempdir().unwrap(); let nc = node_config(temp_dir.path().to_str().unwrap()); let mut state_machine = crate::MockStateMachine::new(); - state_machine.expect_is_running().returning(|| true); state_machine.expect_last_applied().return_const(d_engine_proto::common::LogId { index: applied, term: 1, @@ -2512,7 +2511,6 @@ async fn test_candidate_tick_does_not_campaign_when_the_unapplied_range_cannot_b let temp_dir = tempfile::tempdir().unwrap(); let nc = node_config(temp_dir.path().to_str().unwrap()); let mut state_machine = crate::MockStateMachine::new(); - state_machine.expect_is_running().returning(|| true); state_machine .expect_last_applied() .return_const(d_engine_proto::common::LogId { index: 5, term: 1 }); diff --git a/d-engine-core/src/raft_role/follower_state_test.rs b/d-engine-core/src/raft_role/follower_state_test.rs index c7cb145f..04341068 100644 --- a/d-engine-core/src/raft_role/follower_state_test.rs +++ b/d-engine-core/src/raft_role/follower_state_test.rs @@ -1860,7 +1860,7 @@ async fn test_handle_client_read_request_eventual_consistency_succeeds() { state_machine_handler .expect_read_from_state_machine() .times(1) - .returning(|_| Some(vec![])); + .returning(|_| Ok(vec![])); context.handlers.state_machine_handler = Arc::new(state_machine_handler); let mut state = @@ -1886,6 +1886,56 @@ async fn test_handle_client_read_request_eventual_consistency_succeeds() { ); } +/// Business scenario: a follower's disk fails while a client reads from it with eventual +/// consistency. The client must get an error it can act on, not an empty "key not found" answer. +#[tokio::test] +async fn test_follower_eventual_read_reports_state_machine_failure_as_error() { + let cases: [(fn() -> Error, ErrorCode); 2] = [ + ( + || -> Error { crate::StorageError::DbError("disk failure".into()).into() }, + ErrorCode::StorageIoError, + ), + // Transient: the state machine is replacing its state, retrying will succeed. + ( + || -> Error { crate::StorageError::NotServing("restoring".into()).into() }, + ErrorCode::RetryRequired, + ), + ]; + + for (failure, expected) in cases { + let (_graceful_tx, graceful_rx) = watch::channel(()); + let (mut context, _temp_dir) = mock_raft_context_with_temp(graceful_rx, None); + + let mut state_machine_handler = MockStateMachineHandler::::new(); + state_machine_handler + .expect_read_from_state_machine() + .times(1) + .returning(move |_| Err(failure())); + context.handlers.state_machine_handler = Arc::new(state_machine_handler); + + let mut state = + FollowerState::::new(1, context.node_config.clone(), None, None); + let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); + let cmd = ClientCmd::Read( + ClientReadRequest { + client_id: 1, + consistency_policy: Some(ReadConsistencyPolicy::EventualConsistency), + keys: vec![bytes::Bytes::from_static(b"k")], + }, + resp_tx, + ); + + state.push_client_cmd(cmd, &context); + + let response = resp_rx.recv().await.unwrap().unwrap(); + assert_eq!( + response.error, expected, + "a failed read must be reported, not answered as an empty result" + ); + assert!(response.result.is_none()); + } +} + #[tokio::test] async fn test_handle_join_cluster_rejects_on_follower() { let (_graceful_tx, graceful_rx) = watch::channel(()); diff --git a/d-engine-core/src/raft_role/leader_state.rs b/d-engine-core/src/raft_role/leader_state.rs index 03af3ce1..cdbce695 100644 --- a/d-engine-core/src/raft_role/leader_state.rs +++ b/d-engine-core/src/raft_role/leader_state.rs @@ -4087,12 +4087,10 @@ impl LeaderState { ctx: &RaftContext, ) { for (req, sender) in read_batch { - let results = ctx - .handlers - .state_machine_handler - .read_from_state_machine(req.keys) - .unwrap_or_default(); - let _ = sender.send(Ok(ClientResponse::read_results(results))); + let response = ClientResponse::from_read( + ctx.handlers.state_machine_handler.read_from_state_machine(req.keys), + ); + let _ = sender.send(Ok(response)); } } @@ -4104,12 +4102,10 @@ impl LeaderState { ctx: &RaftContext, ) { while let Some(entry) = self.pending_lease_reads.pop_front() { - let results = ctx - .handlers - .state_machine_handler - .read_from_state_machine(entry.request.keys) - .unwrap_or_default(); - let _ = entry.sender.send(Ok(ClientResponse::read_results(results))); + let response = ClientResponse::from_read( + ctx.handlers.state_machine_handler.read_from_state_machine(entry.request.keys), + ); + let _ = entry.sender.send(Ok(response)); } } @@ -4208,12 +4204,9 @@ impl LeaderState { ) -> Result<()> { if self.is_lease_valid() { // Lease valid - serve immediately - let results = ctx - .handlers - .state_machine_handler - .read_from_state_machine(req.keys) - .unwrap_or_default(); - let response = ClientResponse::read_results(results); + let response = ClientResponse::from_read( + ctx.handlers.state_machine_handler.read_from_state_machine(req.keys), + ); let _ = sender.send(Ok(response)); } else { // Lease expired - need to confirm leadership before serving. @@ -4224,12 +4217,10 @@ impl LeaderState { ctx.node_config().raft.read_consistency.lease_duration_ms, ctx.state_machine().last_applied().index, ); - let results = ctx - .handlers - .state_machine_handler - .read_from_state_machine(req.keys) - .unwrap_or_default(); - let _ = sender.send(Ok(ClientResponse::read_results(results))); + let response = ClientResponse::from_read( + ctx.handlers.state_machine_handler.read_from_state_machine(req.keys), + ); + let _ = sender.send(Ok(response)); } else { // Multi-voter: queue the request and trigger a heartbeat. // drain_pending_lease_reads() is called from handle_append_result after @@ -4259,12 +4250,9 @@ impl LeaderState { ctx: &RaftContext, ) { // Eventual consistency: serve immediately without any verification - let results = ctx - .handlers - .state_machine_handler - .read_from_state_machine(req.keys) - .unwrap_or_default(); - let response = ClientResponse::read_results(results); + let response = ClientResponse::from_read( + ctx.handlers.state_machine_handler.read_from_state_machine(req.keys), + ); let _ = sender.send(Ok(response)); } diff --git a/d-engine-core/src/raft_role/leader_state_test/client_read_test.rs b/d-engine-core/src/raft_role/leader_state_test/client_read_test.rs index c412a256..00061199 100644 --- a/d-engine-core/src/raft_role/leader_state_test/client_read_test.rs +++ b/d-engine-core/src/raft_role/leader_state_test/client_read_test.rs @@ -511,9 +511,7 @@ async fn test_linearizable_read_quorum_success() { // read_from_state_machine is called by execute_pending_reads when ApplyCompleted fires. let mut state_machine_handler = MockStateMachineHandler::new(); state_machine_handler.expect_should_snapshot().returning(|_| false); - state_machine_handler - .expect_read_from_state_machine() - .returning(|_| Some(vec![])); + state_machine_handler.expect_read_from_state_machine().returning(|_| Ok(vec![])); let (_graceful_tx, graceful_rx) = watch::channel(()); @@ -889,9 +887,7 @@ async fn test_unspecified_policy_defaults_to_linearizable_read() { // read_from_state_machine called by execute_pending_reads when ApplyCompleted fires. let mut state_machine_handler = MockStateMachineHandler::new(); state_machine_handler.expect_should_snapshot().returning(|_| false); - state_machine_handler - .expect_read_from_state_machine() - .returning(|_| Some(vec![])); + state_machine_handler.expect_read_from_state_machine().returning(|_| Ok(vec![])); let (_graceful_tx, graceful_rx) = watch::channel(()); @@ -1209,7 +1205,6 @@ async fn test_lease_reuse_after_linearizable_read_refresh() { // Built from scratch: with the default mock, its `last_applied() == 0` expectation is // registered first and wins over a later one. let mut state_machine = crate::MockStateMachine::new(); - state_machine.expect_is_running().returning(|| true); state_machine .expect_last_applied() .return_const(d_engine_proto::common::LogId { index: 1, term: 1 }); @@ -2335,3 +2330,45 @@ async fn test_scan_cmd_leader_propagates_sm_error() { // failed" (the outer SystemError::Storage variant). Verify it's Internal. assert_eq!(err.code(), Code::Internal); } + +/// A lease read that fails in the state machine must reach the client as an error. The data +/// may be damaged, so the client gets the corruption code and decides what to do; it must not get +/// an empty result that looks like "key not found". +#[tokio::test] +async fn test_lease_read_reports_state_machine_failure_as_error() { + let mut state_machine_handler = MockStateMachineHandler::new(); + state_machine_handler.expect_read_from_state_machine().times(1).returning(|_| { + Err(crate::StorageError::DataCorruption { + location: "sst".into(), + } + .into()) + }); + let (_graceful_tx, graceful_rx) = watch::channel(()); + let context = MockBuilder::new(graceful_rx) + .with_db_path("/tmp/test_lease_read_reports_state_machine_failure_as_error") + .with_state_machine_handler(state_machine_handler) + .build_context(); + + let mut state = LeaderState::::new(1, context.node_config.clone()); + state.test_update_lease_timestamp(); // lease valid: served immediately + + let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); + state + .process_lease_read( + ClientReadRequest { + client_id: 1, + consistency_policy: Some(ReadConsistencyPolicy::LeaseRead), + keys: vec![safe_kv_bytes(1)], + }, + resp_tx, + &context, + &internal_event_tx, + ) + .await + .expect("answering the client is not a node failure"); + + let response = resp_rx.recv().await.unwrap().unwrap(); + assert_eq!(response.error, ErrorCode::DataCorruption); + assert!(response.result.is_none()); +} diff --git a/d-engine-core/src/raft_role/leader_state_test/event_handling_test.rs b/d-engine-core/src/raft_role/leader_state_test/event_handling_test.rs index 955a6e02..e555cce9 100644 --- a/d-engine-core/src/raft_role/leader_state_test/event_handling_test.rs +++ b/d-engine-core/src/raft_role/leader_state_test/event_handling_test.rs @@ -748,9 +748,7 @@ async fn test_handle_client_read_linearizable_success() { // Mock state machine handler: read_from_state_machine called when ApplyCompleted fires. let mut state_machine_handler = MockStateMachineHandler::new(); state_machine_handler.expect_should_snapshot().returning(|_| false); - state_machine_handler - .expect_read_from_state_machine() - .returning(|_| Some(vec![])); + state_machine_handler.expect_read_from_state_machine().returning(|_| Ok(vec![])); // Initializing Shutdown Signal let (_graceful_tx, graceful_rx) = watch::channel(()); diff --git a/d-engine-core/src/raft_role/leader_state_test/replication_test.rs b/d-engine-core/src/raft_role/leader_state_test/replication_test.rs index c74f4b0c..5877bdee 100644 --- a/d-engine-core/src/raft_role/leader_state_test/replication_test.rs +++ b/d-engine-core/src/raft_role/leader_state_test/replication_test.rs @@ -2677,7 +2677,7 @@ async fn five_voter_partition_context(path: &str) -> ProcessRaftRequestTestConte }); context.raft_context.storage.raft_log = Arc::new(raft_log); let mut sm_handler = crate::MockStateMachineHandler::new(); - sm_handler.expect_read_from_state_machine().returning(|_| Some(vec![])); + sm_handler.expect_read_from_state_machine().returning(|_| Ok(vec![])); context.raft_context.handlers.state_machine_handler = Arc::new(sm_handler); context diff --git a/d-engine-core/src/raft_role/leader_state_test/snapshot_worker_test.rs b/d-engine-core/src/raft_role/leader_state_test/snapshot_worker_test.rs index d585ff15..c8160518 100644 --- a/d-engine-core/src/raft_role/leader_state_test/snapshot_worker_test.rs +++ b/d-engine-core/src/raft_role/leader_state_test/snapshot_worker_test.rs @@ -103,7 +103,6 @@ fn state_machine_with_snapshot() -> MockStateMachine { use d_engine_proto::common::LogId; let mut sm = MockStateMachine::new(); sm.expect_close().returning(|| Ok(())); - sm.expect_is_running().returning(|| true); sm.expect_get().returning(|_| Ok(None)); sm.expect_apply_chunk().returning(|_| Ok(vec![])); sm.expect_update_last_applied().returning(|_| ()); diff --git a/d-engine-core/src/raft_role/learner_state_test.rs b/d-engine-core/src/raft_role/learner_state_test.rs index 06ed1e72..dcd497c5 100644 --- a/d-engine-core/src/raft_role/learner_state_test.rs +++ b/d-engine-core/src/raft_role/learner_state_test.rs @@ -1512,7 +1512,7 @@ async fn test_learner_serves_eventual_read_locally() { .times(1) .withf(|keys| keys.len() == 1 && keys[0] == "eventual_key") .returning(|_| { - Some(vec![KvEntry { + Ok(vec![KvEntry { key: bytes::Bytes::from("eventual_key"), value: bytes::Bytes::from("eventual_value"), }]) diff --git a/d-engine-core/src/raft_role/role_state.rs b/d-engine-core/src/raft_role/role_state.rs index da6ecda7..dd98757a 100644 --- a/d-engine-core/src/raft_role/role_state.rs +++ b/d-engine-core/src/raft_role/role_state.rs @@ -50,7 +50,6 @@ use crate::StateTransitionError; use crate::TypeConfig; use crate::client::ClientReadRequest; use crate::client::ClientResponse; -use crate::client::KvEntry; use crate::client::LeaderHint; use crate::event::ClientCmd; use crate::scoped_timer::ScopedTimer; @@ -520,12 +519,9 @@ pub(crate) trait RaftRoleState: Send + Sync + 'static { ctx: &RaftContext, ) { // Read directly from local state machine without any consistency checks - let results: Vec = ctx - .state_machine_handler() - .read_from_state_machine(req.keys) - .unwrap_or_default(); - - let response = ClientResponse::read_results(results); + let response = ClientResponse::from_read( + ctx.handlers.state_machine_handler.read_from_state_machine(req.keys), + ); let _ = sender.send(Ok(response)); } diff --git a/d-engine-core/src/raft_test/drain_based_batch_architecture_tests.rs b/d-engine-core/src/raft_test/drain_based_batch_architecture_tests.rs index d6d60cf4..54f841e5 100644 --- a/d-engine-core/src/raft_test/drain_based_batch_architecture_tests.rs +++ b/d-engine-core/src/raft_test/drain_based_batch_architecture_tests.rs @@ -380,7 +380,7 @@ async fn test_p0_follower_eventual_read_local_processing() { .times(1) .withf(|keys| keys.len() == 1 && keys[0] == "test_key") .returning(|_| { - Some(vec![d_engine_proto::client::ClientResult { + Ok(vec![d_engine_proto::client::ClientResult { key: bytes::Bytes::from("test_key"), value: bytes::Bytes::from("test_value"), }]) @@ -540,7 +540,7 @@ async fn test_p0_candidate_commands_handling() { .times(1) .withf(|keys| keys.len() == 1 && keys[0] == "eventual_key") .returning(|_| { - Some(vec![d_engine_proto::client::ClientResult { + Ok(vec![d_engine_proto::client::ClientResult { key: bytes::Bytes::from("eventual_key"), value: bytes::Bytes::from("eventual_value"), }]) diff --git a/d-engine-core/src/raft_test/process_inbound_events_tests.rs b/d-engine-core/src/raft_test/process_inbound_events_tests.rs index 1201983f..e2ad4692 100644 --- a/d-engine-core/src/raft_test/process_inbound_events_tests.rs +++ b/d-engine-core/src/raft_test/process_inbound_events_tests.rs @@ -341,3 +341,158 @@ async fn test_shutdown_drops_pending_events_without_responding() { "pending event must be dropped, not answered" ); } + +// ----------------------------------------------------------------------- +// reads that arrive while a snapshot installs +// ----------------------------------------------------------------------- + +/// Business scenario: a follower that fell far behind is installing the leader's snapshot. A +/// client sends it an eventual read; the read fell back from the ReadActor fast path to the Raft +/// loop, which is busy awaiting the install. +/// +/// Expected: the read waits behind the install. It is not answered from the half-installed state +/// and it does not fail. Once the install is done it is answered from the new state. +#[tokio::test] +async fn test_read_queued_during_snapshot_install_is_answered_from_new_state() { + use std::sync::Arc; + use std::sync::atomic::AtomicBool; + use std::sync::atomic::Ordering; + use std::time::Duration; + + use bytes::Bytes; + use d_engine_proto::server::storage::SnapshotChunk; + use tokio::sync::mpsc; + use tokio::sync::oneshot; + + use crate::ClientCmd; + use crate::MockStateMachineHandler; + use crate::PreparedSnapshot; + use crate::SnapshotApplyResult; + use crate::StateMachineCommand; + use crate::StateMachineCommandSender; + use crate::client::ClientReadRequest; + use crate::client::ClientResponsePayload; + use crate::client::ErrorCode; + use crate::client::KvEntry; + use crate::config::RaftNodeConfig; + use crate::config::ReadConsistencyPolicy; + + const INSTALL_TIME: Duration = Duration::from_millis(400); + + // The state machine holds "old" until the worker finishes installing, then "new". + let installed = Arc::new(AtomicBool::new(false)); + let mut smh = MockStateMachineHandler::new(); + smh.expect_update_pending().returning(|_| {}); + smh.expect_last_applied().returning(|| 0); + smh.expect_should_snapshot().returning(|_| false); + smh.expect_get_latest_snapshot_metadata().returning(|| None); + smh.expect_try_begin_local_snapshot_capture().returning(|| Ok(())); + smh.expect_end_local_snapshot_capture().returning(|| {}); + smh.expect_prepare_snapshot_stream().once().returning(|_, _, _, _| { + Ok(PreparedSnapshot { + metadata: Default::default(), + temp_dir: tempfile::tempdir().unwrap(), + }) + }); + let read_view = installed.clone(); + smh.expect_read_from_state_machine().returning(move |keys| { + let value = if read_view.load(Ordering::SeqCst) { + "new" + } else { + "old" + }; + Ok(keys + .into_iter() + .map(|key| KvEntry { + key, + value: Bytes::from(value), + }) + .collect()) + }); + + // The worker takes INSTALL_TIME to install, then switches the state machine to "new". + let (sm_tx, mut sm_rx) = mpsc::unbounded_channel(); + let (install_started_tx, install_started_rx) = oneshot::channel(); + let worker_state = installed.clone(); + tokio::spawn(async move { + if let Some(StateMachineCommand::InstallSnapshot { response, .. }) = sm_rx.recv().await { + let _ = install_started_tx.send(()); + tokio::time::sleep(INSTALL_TIME).await; + worker_state.store(true, Ordering::SeqCst); + let _ = response.send(Ok(SnapshotApplyResult::Applied { + last_included: Default::default(), + })); + } + }); + + let mut node_config = RaftNodeConfig::default(); + node_config.raft.read_consistency.default_policy = ReadConsistencyPolicy::EventualConsistency; + let (_graceful_tx, graceful_rx) = watch::channel(()); + let mut raft = MockBuilder::new(graceful_rx) + .with_node_config(node_config) + .with_state_machine_handler(smh) + .with_state_machine_commands(StateMachineCommandSender::new(sm_tx)) + .turn_on_election(false) + .build_raft(); + let event_tx = raft.event_sender(); + let cmd_tx = raft.cmd_sender(); + let _loop = tokio::spawn(async move { raft.run().await }); + + // The leader starts a snapshot transfer; the loop now awaits the worker. + let (chunk_tx, chunk_rx) = mpsc::channel(8); + chunk_tx + .send(SnapshotChunk { + leader_term: 1, + leader_id: 1, + ..Default::default() + }) + .await + .unwrap(); + drop(chunk_tx); + let (install_resp_tx, _install_resp_rx) = MaybeCloneOneshot::new(); + event_tx + .send(InboundEvent::InstallSnapshotChunk( + chunk_rx, + install_resp_tx, + )) + .await + .unwrap(); + install_started_rx.await.expect("the install must reach the worker"); + + // A read arrives while the install is running. + let (read_tx, mut read_rx) = MaybeCloneOneshot::new(); + cmd_tx + .send(ClientCmd::Read( + ClientReadRequest { + client_id: 1, + consistency_policy: None, // server default: eventual + keys: vec![Bytes::from_static(b"k")], + }, + read_tx, + )) + .await + .unwrap(); + + assert!( + tokio::time::timeout(INSTALL_TIME / 4, read_rx.recv()).await.is_err(), + "the read must wait behind the install, not be answered from the old state" + ); + + let response = tokio::time::timeout(Duration::from_secs(5), read_rx.recv()) + .await + .expect("the queued read must be answered once the install is done") + .expect("the response channel must stay open") + .expect("the read must not fail"); + assert_eq!(response.error, ErrorCode::Success); + match response.result { + Some(ClientResponsePayload::Read(results)) => { + assert_eq!(results.entries.len(), 1); + assert_eq!( + results.entries[0].value, + Bytes::from_static(b"new"), + "the read must see the state the snapshot installed" + ); + } + other => panic!("expected read results, got: {other:?}"), + } +} diff --git a/d-engine-core/src/state_machine_handler/default_state_machine_handler.rs b/d-engine-core/src/state_machine_handler/default_state_machine_handler.rs index 6f53e05b..4f8dc9a1 100644 --- a/d-engine-core/src/state_machine_handler/default_state_machine_handler.rs +++ b/d-engine-core/src/state_machine_handler/default_state_machine_handler.rs @@ -189,19 +189,13 @@ where fn read_from_state_machine( &self, keys: Vec, - ) -> Option> { - let mut result = Vec::new(); - for key in keys { - if let Ok(Some(value)) = self.state_machine.get(&key) { - result.push(KvEntry { key, value }); - } - } - - if !result.is_empty() { - Some(result) - } else { - None - } + ) -> Result> { + let values = self.state_machine.get_multi(&keys)?; + Ok(keys + .into_iter() + .zip(values) + .filter_map(|(key, value)| value.map(|value| KvEntry { key, value })) + .collect()) } async fn prepare_snapshot_stream( diff --git a/d-engine-core/src/state_machine_handler/default_state_machine_handler_test.rs b/d-engine-core/src/state_machine_handler/default_state_machine_handler_test.rs index 210b2e15..c35276fe 100644 --- a/d-engine-core/src/state_machine_handler/default_state_machine_handler_test.rs +++ b/d-engine-core/src/state_machine_handler/default_state_machine_handler_test.rs @@ -2853,3 +2853,72 @@ mod wait_applied_tests { ); } } + +// ── read_from_state_machine ────────────────────────────────────────────────── + +fn handler_over( + state_machine: MockStateMachine, + temp_dir: &Path, +) -> DefaultStateMachineHandler { + DefaultStateMachineHandler::new( + 1, + 0, + Arc::new(state_machine), + temp_dir.to_path_buf(), + snapshot_config(temp_dir.to_path_buf()), + MockSnapshotPolicy::new(), + ) +} + +/// A state machine failure must come back as an error. Turning it into "no keys found" would +/// make a failing disk look like an empty database. +#[test] +fn test_read_from_state_machine_returns_the_state_machine_error() { + let temp_dir = tempdir().unwrap(); + let mut state_machine = MockStateMachine::new(); + state_machine + .expect_get_multi() + .returning(|_| Err(crate::StorageError::DbError("disk failure".into()).into())); + let handler = handler_over(state_machine, temp_dir.path()); + + let err = handler + .read_from_state_machine(vec![bytes::Bytes::from_static(b"k")]) + .expect_err("a failing state machine must not look like an empty one"); + + assert!(format!("{err:?}").contains("disk failure"), "got: {err:?}"); +} + +/// Keys that do not exist are left out; the others keep the order they were asked in. All keys +/// are read with one `get_multi` call, so they come from one consistent view. +#[test] +fn test_read_from_state_machine_skips_missing_keys_and_reads_one_view() { + let temp_dir = tempdir().unwrap(); + let mut state_machine = MockStateMachine::new(); + state_machine.expect_get_multi().times(1).returning(|_| { + Ok(vec![ + Some(bytes::Bytes::from_static(b"a")), + None, + Some(bytes::Bytes::from_static(b"c")), + ]) + }); + let handler = handler_over(state_machine, temp_dir.path()); + + let entries = handler + .read_from_state_machine(vec![ + bytes::Bytes::from_static(b"k1"), + bytes::Bytes::from_static(b"k2"), + bytes::Bytes::from_static(b"k3"), + ]) + .unwrap(); + + let keys: Vec<_> = entries.iter().map(|e| e.key.clone()).collect(); + let values: Vec<_> = entries.iter().map(|e| e.value.clone()).collect(); + assert_eq!(keys, vec![ + bytes::Bytes::from_static(b"k1"), + bytes::Bytes::from_static(b"k3") + ]); + assert_eq!(values, vec![ + bytes::Bytes::from_static(b"a"), + bytes::Bytes::from_static(b"c") + ]); +} diff --git a/d-engine-core/src/state_machine_handler/mod.rs b/d-engine-core/src/state_machine_handler/mod.rs index 94bfc5c8..5be35566 100644 --- a/d-engine-core/src/state_machine_handler/mod.rs +++ b/d-engine-core/src/state_machine_handler/mod.rs @@ -100,12 +100,12 @@ where timeout: std::time::Duration, ) -> Result<()>; - /// Reads values from the state machine for given keys - /// Returns None if any key doesn't exist + /// Reads `keys` from one consistent view of the state machine. Keys that do not exist are left + /// out of the result. A state machine failure is returned. fn read_from_state_machine( &self, keys: Vec, - ) -> Option>; + ) -> Result>; /// Receives the remaining chunks of a snapshot stream, ACKs each one, and assembles /// + decompresses them into a `PreparedSnapshot`. diff --git a/d-engine-core/src/storage/state_machine.rs b/d-engine-core/src/storage/state_machine.rs index 6f5309dc..52099b96 100644 --- a/d-engine-core/src/storage/state_machine.rs +++ b/d-engine-core/src/storage/state_machine.rs @@ -103,10 +103,6 @@ pub trait StateMachine: Send + Sync + 'static { Ok(()) } - /// Checks if the state machine is currently running. - /// Sync operation as it just checks an atomic boolean. - fn is_running(&self) -> bool; - /// Retrieves a value by key from the state machine. /// Sync operation as it accesses in-memory data structures. fn get( diff --git a/d-engine-core/src/storage/state_machine_test.rs b/d-engine-core/src/storage/state_machine_test.rs index a69a73b5..d11b94b8 100644 --- a/d-engine-core/src/storage/state_machine_test.rs +++ b/d-engine-core/src/storage/state_machine_test.rs @@ -36,7 +36,7 @@ pub trait StateMachineBuilder: Send + Sync { impl StateMachineTestSuite { /// Run all state machine tests pub async fn run_all_tests(builder: B) -> Result<(), Error> { - Self::test_close_is_idempotent_and_stops_running(builder.build().await?).await?; + Self::test_close_is_idempotent_and_stops_serving(builder.build().await?).await?; Self::test_basic_kv_operations(builder.build().await?).await?; Self::test_apply_chunk_functionality(builder.build().await?).await?; Self::test_cas_operations(builder.build().await?).await?; @@ -80,18 +80,24 @@ impl StateMachineTestSuite { Ok(()) } - /// Test close lifecycle: a freshly built state machine is already running, and not running - /// once `close`d. `close` is permanent and idempotent (#455), so there is no restart. - pub async fn test_close_is_idempotent_and_stops_running( + /// Test close lifecycle: a freshly built state machine already serves, and answers + /// `NotServing` once `close`d. `close` is permanent and idempotent (#455), so there is no + /// restart. + pub async fn test_close_is_idempotent_and_stops_serving( state_machine: Arc ) -> Result<(), Error> { - // Running as soon as it is built: there is no separate start step - assert!(state_machine.is_running()); + let is_closed = |sm: &Arc| { + sm.get(b"lifecycle-probe") + .is_err_and(|err| format!("{err:?}").contains("NotServing")) + }; + + // Serving as soon as it is built: there is no separate start step + assert!(!is_closed(&state_machine)); state_machine.close()?; - assert!(!state_machine.is_running()); + assert!(is_closed(&state_machine)); state_machine.close()?; // idempotent - assert!(!state_machine.is_running()); + assert!(is_closed(&state_machine)); Ok(()) } @@ -1167,9 +1173,6 @@ impl StateMachineTestSuite { // Note: snapshot_metadata may or may not be None depending on previous tests // The test focuses on verifying reset() clears everything, not initial state - // Store running state for verification - let was_running = state_machine.is_running(); - // Perform reset state_machine.reset().await?; @@ -1182,8 +1185,7 @@ impl StateMachineTestSuite { assert_eq!(state_machine.last_applied(), LogId { index: 0, term: 0 }); assert!(state_machine.snapshot_metadata().is_none()); - // Verify operational state is maintained - assert_eq!(state_machine.is_running(), was_running); + // Verify operational state is maintained: reads above succeeded, so it still serves // Test that we can add new data after reset let new_entries = vec![ @@ -1747,10 +1749,6 @@ mod default_scan_prefix_tests { Ok(()) } - fn is_running(&self) -> bool { - self.running.load(Ordering::SeqCst) - } - fn get( &self, _key_buffer: &[u8], diff --git a/d-engine-core/src/test_utils/mock/mock_raft_builder.rs b/d-engine-core/src/test_utils/mock/mock_raft_builder.rs index 69509cf5..42ed1566 100644 --- a/d-engine-core/src/test_utils/mock/mock_raft_builder.rs +++ b/d-engine-core/src/test_utils/mock/mock_raft_builder.rs @@ -384,7 +384,6 @@ pub fn mock_state_machine() -> MockStateMachine { let mut mock = MockStateMachine::new(); mock.expect_close().returning(|| Ok(())); - mock.expect_is_running().returning(|| true); mock.expect_get().returning(|_| Ok(None)); mock.expect_apply_chunk().returning(|_| Ok(vec![])); @@ -416,7 +415,7 @@ pub fn mock_state_machine_handler() -> MockStateMachineHandler { state_machine_handler.expect_update_pending().returning(|_| {}); // Commit delivery (also run on shutdown) reads the applied position every iteration. state_machine_handler.expect_last_applied().returning(|| 0); - state_machine_handler.expect_read_from_state_machine().returning(|_| None); + state_machine_handler.expect_read_from_state_machine().returning(|_| Ok(vec![])); state_machine_handler.expect_should_snapshot().returning(|_| false); state_machine_handler.expect_get_latest_snapshot_metadata().returning(|| None); // handle_create_snapshot (#436 Task #21) always acquires/releases this guard diff --git a/d-engine-server/src/api/embedded_read_handle.rs b/d-engine-server/src/api/embedded_read_handle.rs index 110fe190..65f5e36c 100644 --- a/d-engine-server/src/api/embedded_read_handle.rs +++ b/d-engine-server/src/api/embedded_read_handle.rs @@ -8,7 +8,7 @@ //! //! ```text //! Eventual / LeaseRead (valid) → sm.get_multi() (direct, zero channel) -//! Eventual / LeaseRead (error) → cmd_tx fallback (SM stopped or error) +//! Eventual / LeaseRead (error) → cmd_tx fallback (SM unavailable or error) //! LeaseRead (invalid lease) → cmd_tx fallback //! LinearizableRead → cmd_tx always (Raft readIndex) //! ``` @@ -42,6 +42,7 @@ use super::standalone_read_handle::extract_read_payload; use super::standalone_read_handle::map_error_response; use super::standalone_read_handle::server_error; use super::standalone_read_handle::timeout_error; +use crate::api::standalone_read_handle::read_failure; // ── EmbeddedReadHandle ──────────────────────────────────────────────────────── @@ -107,20 +108,21 @@ impl EmbeddedReadHandle { client_id: u32, timeout: Duration, ) -> ClientApiResult>> { + // Only `NotServing` (the state machine is replacing its state or is closed) falls through + // to cmd_tx. A hard failure would fail again on the Raft path, so it is reported now. match consistency { - ReadConsistencyPolicy::EventualConsistency => { - if let Ok(values) = self.sm.get_multi(keys) { - return Ok(values); - } - // SM stopped or hard error → fall through to cmd_tx - } + ReadConsistencyPolicy::EventualConsistency => match self.sm.get_multi(keys) { + Ok(values) => return Ok(values), + Err(e) if e.is_not_serving() => {} + Err(e) => return Err(read_failure(&e)), + }, ReadConsistencyPolicy::LeaseRead => { - if self.lease.is_valid(now_ms()) - && let Ok(values) = self.sm.get_multi(keys) - { - return Ok(values); - - // SM stopped or hard error → fall through + if self.lease.is_valid(now_ms()) { + match self.sm.get_multi(keys) { + Ok(values) => return Ok(values), + Err(e) if e.is_not_serving() => {} + Err(e) => return Err(read_failure(&e)), + } } // invalid lease → fall through to cmd_tx } diff --git a/d-engine-server/src/api/embedded_read_handle_test.rs b/d-engine-server/src/api/embedded_read_handle_test.rs index 44cdf3cb..89dd8950 100644 --- a/d-engine-server/src/api/embedded_read_handle_test.rs +++ b/d-engine-server/src/api/embedded_read_handle_test.rs @@ -9,9 +9,13 @@ mod embedded_read_handle_tests { use std::time::Duration; use bytes::Bytes; + use d_engine_core::ClientApiError; + use d_engine_core::Error; use d_engine_core::MockStateMachine; use d_engine_core::MockTypeConfig; use d_engine_core::ReadLease; + use d_engine_core::StorageError; + use d_engine_core::client::ErrorCode; use d_engine_core::config::ReadConsistencyPolicy; use d_engine_core::now_ms; use tokio::sync::mpsc; @@ -39,8 +43,8 @@ mod embedded_read_handle_tests { sm } - fn sm_stopped() -> MockStateMachine { - // SM stopped: get_multi returns NotServing error + fn sm_unavailable() -> MockStateMachine { + // SM unavailable: get_multi returns NotServing error let mut sm = MockStateMachine::new(); sm.expect_get_multi().returning(|_| { Err(d_engine_core::StorageError::NotServing("state machine stopped".into()).into()) @@ -61,6 +65,58 @@ mod embedded_read_handle_tests { EmbeddedReadHandle::new(Arc::new(sm), lease, cmd_tx) } + fn sm_failing_with(make_error: fn() -> Error) -> MockStateMachine { + let mut sm = MockStateMachine::new(); + sm.expect_get_multi().returning(move |_| Err(make_error())); + sm + } + + // ── Hard failures are reported, never retried through cmd_tx ───────────── + + /// A storage failure is returned to the caller with its storage code. cmd_rx is dropped: a + /// fallback would produce a Network error instead, so a Storage error proves there was none. + #[tokio::test] + async fn test_hard_failure_is_reported_without_falling_back() { + let cases: [(fn() -> Error, ErrorCode); 2] = [ + ( + || StorageError::DbError("secret detail".into()).into(), + ErrorCode::StorageIoError, + ), + ( + // The client is told the data may be damaged; the decision is theirs. + || { + StorageError::DataCorruption { + location: "secret detail".into(), + } + .into() + }, + ErrorCode::DataCorruption, + ), + ]; + + for consistency in [ + ReadConsistencyPolicy::EventualConsistency, + ReadConsistencyPolicy::LeaseRead, + ] { + for (make_error, expected) in &cases { + let handle = make_handle(sm_failing_with(*make_error), valid_lease(), true); + let result = + handle.get(b"k", consistency.clone(), 1, Duration::from_millis(100)).await; + + match result { + Err(ClientApiError::Storage { code, message }) => { + assert_eq!(code, *expected); + assert!( + !message.contains("secret detail"), + "detail leaked: {message}" + ); + } + other => panic!("expected a Storage error for {expected:?}, got: {other:?}"), + } + } + } + } + // ── Eventual fast path ──────────────────────────────────────────────────── /// Eventual read goes directly to SM; cmd_rx dropped proves cmd_tx was not used. @@ -78,11 +134,11 @@ mod embedded_read_handle_tests { assert_eq!(result.unwrap(), Some(Bytes::from("v"))); } - /// Eventual read when SM is stopped falls back to cmd_tx. + /// Eventual read when the SM is unavailable falls back to cmd_tx. /// cmd_rx dropped → fallback returns error (proves fallback was taken, not panic). #[tokio::test] - async fn test_eventual_sm_stopped_falls_back_to_cmd_tx() { - let handle = make_handle(sm_stopped(), valid_lease(), true); + async fn test_eventual_sm_unavailable_falls_back_to_cmd_tx() { + let handle = make_handle(sm_unavailable(), valid_lease(), true); let result = handle .get( b"k", @@ -91,7 +147,7 @@ mod embedded_read_handle_tests { Duration::from_millis(100), ) .await; - assert!(result.is_err(), "stopped SM must fall back to cmd_tx"); + assert!(result.is_err(), "unavailable SM must fall back to cmd_tx"); } // ── LeaseRead fast path ─────────────────────────────────────────────────── @@ -130,10 +186,10 @@ mod embedded_read_handle_tests { assert!(result.is_err(), "invalid lease must fall back to cmd_tx"); } - /// LeaseRead with valid lease but stopped SM falls back to cmd_tx. + /// LeaseRead with valid lease but unavailable SM falls back to cmd_tx. #[tokio::test] - async fn test_lease_read_sm_stopped_falls_back_to_cmd_tx() { - let handle = make_handle(sm_stopped(), valid_lease(), true); + async fn test_lease_read_sm_unavailable_falls_back_to_cmd_tx() { + let handle = make_handle(sm_unavailable(), valid_lease(), true); let result = handle .get( b"k", @@ -142,7 +198,7 @@ mod embedded_read_handle_tests { Duration::from_millis(100), ) .await; - assert!(result.is_err(), "stopped SM must fall back to cmd_tx"); + assert!(result.is_err(), "unavailable SM must fall back to cmd_tx"); } // ── LinearizableRead ───────────────────────────────────────────────────── diff --git a/d-engine-server/src/api/embedded_test/shutdown_order_test.rs b/d-engine-server/src/api/embedded_test/shutdown_order_test.rs index 0f0e6ce5..8ff99674 100644 --- a/d-engine-server/src/api/embedded_test/shutdown_order_test.rs +++ b/d-engine-server/src/api/embedded_test/shutdown_order_test.rs @@ -60,9 +60,6 @@ impl StateMachine for SlowApplyProbe { } // Everything below delegates unchanged. - fn is_running(&self) -> bool { - self.inner.is_running() - } fn get( &self, key_buffer: &[u8], diff --git a/d-engine-server/src/api/standalone_read_handle.rs b/d-engine-server/src/api/standalone_read_handle.rs index 80b66ec5..0e41d44a 100644 --- a/d-engine-server/src/api/standalone_read_handle.rs +++ b/d-engine-server/src/api/standalone_read_handle.rs @@ -9,6 +9,7 @@ use std::time::Duration; use bytes::Bytes; +use d_engine_core::Error; use d_engine_core::MaybeCloneOneshot; use d_engine_core::RaftOneshot; use d_engine_core::client::ClientApiError; @@ -21,6 +22,7 @@ use d_engine_core::client::ReadResults; use d_engine_core::config::ReadConsistencyPolicy; use tokio::sync::mpsc; use tokio::sync::oneshot; +use tracing::error; use crate::read_actor::ReadActorError; use crate::read_actor::ReadCmd; @@ -119,6 +121,34 @@ pub(crate) fn extract_read_payload( }), } } +/// Turns a state machine read failure into a client error without internal detail. +/// The original error goes to the log and to a counter, so operators still see the cause. +pub(crate) fn read_failure(e: &Error) -> ClientApiError { + let code = ErrorCode::for_read_failure(e); + // The failure class a dashboard groups by. + let kind = match code { + ErrorCode::StorageIoError => "storage_io", + ErrorCode::DataCorruption => "data_corruption", + ErrorCode::RetryRequired => "not_serving", + _ => "other", + }; + error!(error = ?e, kind, "state machine read failed"); + metrics::counter!("server.read.errors", "kind" => kind).increment(1); + + match code { + // Fixed, friendly messages from the error code, e.g. "Storage I/O error" + ErrorCode::StorageIoError | ErrorCode::DataCorruption | ErrorCode::RetryRequired => { + ClientApiError::from(code) + } + _ => ClientApiError::Business { + code, + message: + "Read failed due to an internal error. Please retry or contact the administrator." + .to_string(), + required_action: None, + }, + } +} // ── StandaloneReadHandle ────────────────────────────────────────────────────── @@ -200,9 +230,9 @@ impl StandaloneReadHandle { { match tokio::time::timeout(timeout, reply_rx).await { Ok(Ok(Ok(values))) => return Ok(values), - Ok(Ok(Err(ReadActorError::SmError(e)))) => return Err(server_error(e)), - // LeaseInvalid / SmStopped → fall through to cmd_tx - Ok(Ok(Err(ReadActorError::LeaseInvalid | ReadActorError::SmStopped))) => {} + Ok(Ok(Err(ReadActorError::SmError(e)))) => return Err(read_failure(&e)), + // LeaseInvalid / SmUnavailable → fall through to cmd_tx + Ok(Ok(Err(ReadActorError::LeaseInvalid | ReadActorError::SmUnavailable))) => {} // ReadActor exited without replying → fall through Ok(Err(_)) => {} // ReadActor stalled beyond timeout → fall through to cmd_tx diff --git a/d-engine-server/src/api/standalone_read_handle_test.rs b/d-engine-server/src/api/standalone_read_handle_test.rs index a0335e2b..04e175b6 100644 --- a/d-engine-server/src/api/standalone_read_handle_test.rs +++ b/d-engine-server/src/api/standalone_read_handle_test.rs @@ -1,7 +1,7 @@ /// Unit tests for StandaloneReadHandle routing logic. /// /// StandaloneReadHandle is the single source of truth for read routing: -/// Eventual/LeaseRead → ReadActor fast path; fallback to cmd_tx on LeaseInvalid/SmStopped +/// Eventual/LeaseRead → ReadActor fast path; fallback to cmd_tx on LeaseInvalid/SmUnavailable /// LinearizableRead → cmd_tx always /// SmError → returned directly, NO cmd_tx fallback /// @@ -20,7 +20,9 @@ mod read_handle_tests { use d_engine_core::Error; use d_engine_core::MockStateMachine; use d_engine_core::ReadLease; + use d_engine_core::StorageError; use d_engine_core::client::ClientResponse; + use d_engine_core::client::ErrorCode; use d_engine_core::client::KvEntry; use d_engine_core::config::ReadConsistencyPolicy; use d_engine_core::now_ms; @@ -52,7 +54,6 @@ mod read_handle_tests { fn sm_running_with(value: Bytes) -> MockStateMachine { let mut sm = MockStateMachine::new(); - sm.expect_is_running().returning(|| true); sm.expect_get_multi() .returning(move |keys| Ok(keys.iter().map(|_| Some(value.clone())).collect())); sm @@ -60,20 +61,19 @@ mod read_handle_tests { fn sm_running_empty() -> MockStateMachine { let mut sm = MockStateMachine::new(); - sm.expect_is_running().returning(|| true); sm.expect_get_multi().returning(|keys| Ok(keys.iter().map(|_| None).collect())); sm } - fn sm_stopped() -> MockStateMachine { + fn sm_unavailable() -> MockStateMachine { let mut sm = MockStateMachine::new(); - sm.expect_is_running().returning(|| false); + sm.expect_get_multi() + .returning(|_| Err(d_engine_core::StorageError::NotServing("stopped".into()).into())); sm } fn sm_with_error() -> MockStateMachine { let mut sm = MockStateMachine::new(); - sm.expect_is_running().returning(|| true); sm.expect_get_multi().returning(|_| Err(Error::Fatal("disk failure".into()))); sm } @@ -130,15 +130,15 @@ mod read_handle_tests { handle.await.unwrap(); } - /// Eventual, SM stopped → SmStopped → fallback to cmd_tx. + /// Eventual, SM unavailable → SmUnavailable → fallback to cmd_tx. /// cmd_rx dropped: error is Network (channel-closed), proving cmd_tx fallback triggered. #[tokio::test] - async fn test_eventual_sm_stopped_falls_back_to_cmd_tx() { - let (rh, handle) = make_handle(sm_stopped(), valid_lease(), true).await; + async fn test_eventual_sm_unavailable_falls_back_to_cmd_tx() { + let (rh, handle) = make_handle(sm_unavailable(), valid_lease(), true).await; let result = rh.get(b"k", ReadConsistencyPolicy::EventualConsistency, 1, TIMEOUT).await; assert!( matches!(result, Err(ClientApiError::Network { .. })), - "SmStopped must fall back to cmd_tx (Network error), got: {:?}", + "SmUnavailable must fall back to cmd_tx (Network error), got: {:?}", result ); drop(rh); @@ -192,14 +192,14 @@ mod read_handle_tests { handle.await.unwrap(); } - /// LeaseRead, SM stopped (even with valid lease) → LeaseInvalid → fallback. + /// LeaseRead, SM unavailable (even with valid lease) → SmUnavailable → fallback to cmd_tx. #[tokio::test] - async fn test_lease_read_sm_stopped_falls_back_to_cmd_tx() { - let (rh, handle) = make_handle(sm_stopped(), valid_lease(), true).await; + async fn test_lease_read_sm_unavailable_falls_back_to_cmd_tx() { + let (rh, handle) = make_handle(sm_unavailable(), valid_lease(), true).await; let result = rh.get(b"k", ReadConsistencyPolicy::LeaseRead, 1, TIMEOUT).await; assert!( matches!(result, Err(ClientApiError::Network { .. })), - "SM stopped must fall back to cmd_tx, got: {:?}", + "SM unavailable must fall back to cmd_tx, got: {:?}", result ); drop(rh); @@ -224,6 +224,72 @@ mod read_handle_tests { handle.await.unwrap(); } + /// The engine's own error text is internal detail: the client gets a fixed, friendly + /// message and a stable code; the original error only goes to the log. + #[tokio::test] + async fn test_sm_error_message_hides_engine_detail() { + let (rh, handle) = make_handle(sm_with_error(), valid_lease(), true).await; + let result = rh.get(b"k", ReadConsistencyPolicy::EventualConsistency, 1, TIMEOUT).await; + match result { + Err(ClientApiError::Business { code, message, .. }) => { + assert_eq!(code, ErrorCode::Uncategorized); + assert!( + !message.contains("disk failure"), + "engine detail leaked to the client: {message}" + ); + } + other => panic!("expected a Business error, got: {other:?}"), + } + drop(rh); + handle.await.unwrap(); + } + + /// Storage failures keep a storage error code, so a client can tell a storage problem from + /// an internal bug, and the message stays free of engine detail. + #[tokio::test] + async fn test_sm_error_storage_failures_keep_their_storage_code() { + let cases: [(fn() -> Error, ErrorCode); 3] = [ + ( + || StorageError::DbError("rocksdb: secret detail".into()).into(), + ErrorCode::StorageIoError, + ), + ( + || StorageError::IoError(std::io::Error::other("secret detail")).into(), + ErrorCode::StorageIoError, + ), + ( + || { + StorageError::DataCorruption { + location: "secret detail".into(), + } + .into() + }, + ErrorCode::DataCorruption, + ), + ]; + + for (make_error, expected) in cases { + let mut sm = MockStateMachine::new(); + sm.expect_get_multi().returning(move |_| Err(make_error())); + let (rh, handle) = make_handle(sm, valid_lease(), true).await; + + let result = rh.get(b"k", ReadConsistencyPolicy::EventualConsistency, 1, TIMEOUT).await; + + match result { + Err(ClientApiError::Storage { code, message }) => { + assert_eq!(code, expected); + assert!( + !message.contains("secret detail"), + "detail leaked: {message}" + ); + } + other => panic!("expected a Storage error for {expected:?}, got: {other:?}"), + } + drop(rh); + handle.await.unwrap(); + } + } + // ── Linearizable — always cmd_tx ───────────────────────────────────────── /// LinearizableRead must never touch ReadActor — always routed to cmd_tx. @@ -231,8 +297,7 @@ mod read_handle_tests { #[tokio::test] async fn test_linearizable_always_uses_cmd_tx() { // SM has no get() expectation — mockall panics if get() is called (implicit safety net). - let mut sm = MockStateMachine::new(); - sm.expect_is_running().returning(|| true); + let sm = MockStateMachine::new(); let (rh, handle) = make_handle(sm, valid_lease(), true).await; let result = rh.get(b"k", ReadConsistencyPolicy::LinearizableRead, 1, TIMEOUT).await; @@ -451,7 +516,6 @@ mod read_handle_tests { fn sm_running_with_multi(vals: Vec<(&'static [u8], &'static [u8])>) -> MockStateMachine { let mut sm = MockStateMachine::new(); - sm.expect_is_running().returning(|| true); sm.expect_get_multi().returning(move |keys| { Ok(keys .iter() @@ -557,8 +621,7 @@ mod read_handle_tests { /// cmd_rx dropped: Network error proves cmd_tx was used. #[tokio::test] async fn test_get_batch_linearizable_always_uses_cmd_tx() { - let mut sm = MockStateMachine::new(); - sm.expect_is_running().returning(|| true); + let sm = MockStateMachine::new(); // no expect_get() — mockall panics if get() is called let (rh, handle) = make_handle(sm, valid_lease(), true).await; diff --git a/d-engine-server/src/network/grpc/grpc_raft_service_fast_path_test.rs b/d-engine-server/src/network/grpc/grpc_raft_service_fast_path_test.rs index 73b5581f..9e1d86a1 100644 --- a/d-engine-server/src/network/grpc/grpc_raft_service_fast_path_test.rs +++ b/d-engine-server/src/network/grpc/grpc_raft_service_fast_path_test.rs @@ -1,8 +1,8 @@ /// Fast-path tests for `handle_client_read` via `ReadHandle`. /// /// Routing rules under test: -/// Eventual/LeaseRead (any key count) → ReadActor fast path; fallback on LeaseInvalid/SmStopped -/// LinearizableRead → cmd_tx always +/// Eventual/LeaseRead (any key count) → ReadActor fast path; fallback on +/// LeaseInvalid/SmUnavailable LinearizableRead → cmd_tx always /// SmError → error returned directly (no cmd_tx fallback) /// /// Proof technique (identical to read_handle_test.rs): @@ -48,21 +48,20 @@ mod grpc_fast_path_tests { fn sm_running_with(value: Bytes) -> MockStateMachine { let mut sm = MockStateMachine::new(); - sm.expect_is_running().returning(|| true); sm.expect_get_multi() .returning(move |keys| Ok(keys.iter().map(|_| Some(value.clone())).collect())); sm } - fn sm_stopped() -> MockStateMachine { + fn sm_unavailable() -> MockStateMachine { let mut sm = MockStateMachine::new(); - sm.expect_is_running().returning(|| false); + sm.expect_get_multi() + .returning(|_| Err(d_engine_core::StorageError::NotServing("stopped".into()).into())); sm } fn sm_with_error() -> MockStateMachine { let mut sm = MockStateMachine::new(); - sm.expect_is_running().returning(|| true); sm.expect_get_multi() .returning(|_| Err(d_engine_core::Error::Fatal("disk failure".into()))); sm @@ -144,7 +143,6 @@ mod grpc_fast_path_tests { #[tokio::test] async fn test_eventual_single_key_fast_path_returns_empty_for_missing_key() { let mut sm = MockStateMachine::new(); - sm.expect_is_running().returning(|| true); sm.expect_get_multi().returning(|keys| Ok(keys.iter().map(|_| None).collect())); let (node, handle) = make_node(sm, valid_lease(), true).await; @@ -167,11 +165,11 @@ mod grpc_fast_path_tests { handle.await.unwrap(); } - /// Eventual, SM stopped → SmStopped → fallback to cmd_tx. + /// Eventual, SM unavailable → SmUnavailable → fallback to cmd_tx. /// cmd_rx dropped: error proves fallback was triggered. #[tokio::test] - async fn test_eventual_sm_stopped_falls_back_to_cmd_tx() { - let (node, handle) = make_node(sm_stopped(), valid_lease(), true).await; + async fn test_eventual_sm_unavailable_falls_back_to_cmd_tx() { + let (node, handle) = make_node(sm_unavailable(), valid_lease(), true).await; let result = node .handle_client_read(Request::new(req(b"k", ProtoPolicy::EventualConsistency))) @@ -179,7 +177,7 @@ mod grpc_fast_path_tests { assert!( result.is_err(), - "SM stopped must fall back to closed cmd_tx" + "SM unavailable must fall back to closed cmd_tx" ); drop(node); handle.await.unwrap(); @@ -216,16 +214,16 @@ mod grpc_fast_path_tests { handle.await.unwrap(); } - /// LeaseRead, SM stopped (even valid lease) → LeaseInvalid → fallback. + /// LeaseRead, SM unavailable (even valid lease) → SmUnavailable → fallback to cmd_tx. #[tokio::test] - async fn test_lease_read_sm_stopped_falls_back_to_cmd_tx() { - let (node, handle) = make_node(sm_stopped(), valid_lease(), true).await; + async fn test_lease_read_sm_unavailable_falls_back_to_cmd_tx() { + let (node, handle) = make_node(sm_unavailable(), valid_lease(), true).await; let result = node.handle_client_read(Request::new(req(b"k", ProtoPolicy::LeaseRead))).await; assert!( result.is_err(), - "SM stopped must fall back to closed cmd_tx" + "SM unavailable must fall back to closed cmd_tx" ); drop(node); handle.await.unwrap(); @@ -260,8 +258,7 @@ mod grpc_fast_path_tests { /// cmd_rx dropped: error proves cmd_tx was used (not fast path). #[tokio::test] async fn test_linearizable_always_uses_cmd_tx() { - let mut sm = MockStateMachine::new(); - sm.expect_is_running().returning(|| true); + let sm = MockStateMachine::new(); // no expect_get_multi() — any get_multi() call panics let (node, handle) = make_node(sm, valid_lease(), true).await; @@ -282,7 +279,6 @@ mod grpc_fast_path_tests { #[tokio::test] async fn test_multi_key_eventual_fast_path_returns_values() { let mut sm = MockStateMachine::new(); - sm.expect_is_running().returning(|| true); sm.expect_get_multi() .returning(|keys| Ok(keys.iter().map(|_| Some(Bytes::from_static(b"val"))).collect())); @@ -309,7 +305,6 @@ mod grpc_fast_path_tests { #[tokio::test] async fn test_multi_key_lease_read_fast_path_returns_values() { let mut sm = MockStateMachine::new(); - sm.expect_is_running().returning(|| true); sm.expect_get_multi() .returning(|keys| Ok(keys.iter().map(|_| Some(Bytes::from_static(b"val"))).collect())); diff --git a/d-engine-server/src/read_actor.rs b/d-engine-server/src/read_actor.rs index 7bc520eb..909b6b75 100644 --- a/d-engine-server/src/read_actor.rs +++ b/d-engine-server/src/read_actor.rs @@ -21,6 +21,7 @@ use std::sync::Arc; use bytes::Bytes; +use d_engine_core::Error; use d_engine_core::ReadLease; use d_engine_core::StateMachine; use d_engine_core::config::ReadConsistencyPolicy; @@ -49,16 +50,18 @@ pub(crate) struct ReadCmd { /// Error returned by the ReadActor to the caller. /// -/// `LeaseInvalid` and `SmStopped` are recoverable: fall back to `cmd_tx`. +/// `LeaseInvalid` and `SmUnavailable` are recoverable: fall back to `cmd_tx`. /// `SmError` is a hard failure; do NOT retry via cmd_tx. #[derive(Debug)] pub(crate) enum ReadActorError { /// Lease has been revoked or expired. Retry via cmd_tx. LeaseInvalid, - /// State machine is not running (node is stopping). Retry via cmd_tx. - SmStopped, - /// SM returned a hard error. - SmError(String), + /// The state machine cannot serve reads right now: it is installing a snapshot or has been + /// closed. Retry via cmd_tx, which waits behind the install or reports the shutdown. + SmUnavailable, + /// The state machine failed to read. Keeps the original error for logging and + /// classification; it must not reach the client as is. + SmError(Error), } // ── run_read_actor ───────────────────────────────────────────────────────────── @@ -113,13 +116,9 @@ where SM: StateMachine, { match cmd.consistency { - ReadConsistencyPolicy::EventualConsistency => { - if !sm.is_running() { - return Err(ReadActorError::SmStopped); - } - } + ReadConsistencyPolicy::EventualConsistency => {} ReadConsistencyPolicy::LeaseRead => { - if !lease.is_valid(now_ms()) || !sm.is_running() { + if !lease.is_valid(now_ms()) { return Err(ReadActorError::LeaseInvalid); } } @@ -127,5 +126,11 @@ where _ => return Err(ReadActorError::LeaseInvalid), } - sm.get_multi(&cmd.keys).map_err(|e| ReadActorError::SmError(e.to_string())) + sm.get_multi(&cmd.keys).map_err(|e| { + if e.is_not_serving() { + ReadActorError::SmUnavailable + } else { + ReadActorError::SmError(e) + } + }) } diff --git a/d-engine-server/src/read_actor_test.rs b/d-engine-server/src/read_actor_test.rs index 8bc3202d..90944557 100644 --- a/d-engine-server/src/read_actor_test.rs +++ b/d-engine-server/src/read_actor_test.rs @@ -14,14 +14,13 @@ use crate::read_actor::ReadCmd; use crate::read_actor::run_read_actor; fn make_sm_running() -> MockStateMachine { - let mut sm = MockStateMachine::new(); - sm.expect_is_running().returning(|| true); - sm + MockStateMachine::new() } -fn make_sm_stopped() -> MockStateMachine { +fn make_sm_unavailable() -> MockStateMachine { let mut sm = MockStateMachine::new(); - sm.expect_is_running().returning(|| false); + sm.expect_get_multi() + .returning(|_| Err(d_engine_core::StorageError::NotServing("stopped".into()).into())); sm } @@ -146,10 +145,10 @@ async fn test_read_actor_eventual_multi_key_returns_ordered_values() { handle.await.unwrap(); } -/// Eventual read fails when SM is stopped +/// Eventual read answers SmUnavailable when the SM cannot serve (NotServing) #[tokio::test] -async fn test_read_actor_eventual_sm_stopped_returns_error() { - let sm = make_sm_stopped(); +async fn test_read_actor_eventual_sm_unavailable_returns_error() { + let sm = make_sm_unavailable(); let (tx, rx) = mpsc::channel(8); let handle = tokio::spawn(run_read_actor(rx, valid_lease(), Arc::new(sm), 64)); @@ -157,7 +156,7 @@ async fn test_read_actor_eventual_sm_stopped_returns_error() { drop(tx); let err = reply_rx.await.unwrap().unwrap_err(); - assert!(matches!(err, ReadActorError::SmStopped)); + assert!(matches!(err, ReadActorError::SmUnavailable)); handle.await.unwrap(); } @@ -211,10 +210,11 @@ async fn test_read_actor_lease_expired_returns_lease_invalid() { handle.await.unwrap(); } -/// LeaseRead returns LeaseInvalid when SM is stopped (even if lease is valid) +/// LeaseRead falls back (SmUnavailable, not a hard SmError) when the SM answers NotServing, even if +/// the lease is valid #[tokio::test] -async fn test_read_actor_lease_valid_but_sm_stopped_returns_error() { - let sm = make_sm_stopped(); +async fn test_read_actor_lease_valid_but_sm_unavailable_returns_error() { + let sm = make_sm_unavailable(); let (tx, rx) = mpsc::channel(8); let handle = tokio::spawn(run_read_actor(rx, valid_lease(), Arc::new(sm), 64)); @@ -222,7 +222,7 @@ async fn test_read_actor_lease_valid_but_sm_stopped_returns_error() { drop(tx); let err = reply_rx.await.unwrap().unwrap_err(); - assert!(matches!(err, ReadActorError::LeaseInvalid)); + assert!(matches!(err, ReadActorError::SmUnavailable)); handle.await.unwrap(); } diff --git a/d-engine-server/src/storage/adaptors/file/file_state_machine.rs b/d-engine-server/src/storage/adaptors/file/file_state_machine.rs index 1d9c32b2..072df885 100644 --- a/d-engine-server/src/storage/adaptors/file/file_state_machine.rs +++ b/d-engine-server/src/storage/adaptors/file/file_state_machine.rs @@ -869,10 +869,6 @@ impl StateMachine for FileStateMachine { self.save_hard_state() } - fn is_running(&self) -> bool { - self.running.load(Ordering::SeqCst) - } - fn get( &self, key_buffer: &[u8], diff --git a/d-engine-server/src/storage/adaptors/file/file_state_machine_test.rs b/d-engine-server/src/storage/adaptors/file/file_state_machine_test.rs index 04f17ac5..1dfb2897 100644 --- a/d-engine-server/src/storage/adaptors/file/file_state_machine_test.rs +++ b/d-engine-server/src/storage/adaptors/file/file_state_machine_test.rs @@ -19,18 +19,22 @@ use crate::storage::adaptors::file::FileStateMachine; // ── close() tests ───────────────────────────────────────────────────────────── -/// close() must mark the SM as not running. +/// close() must stop the SM from serving: reads answer NotServing afterwards. /// FileStateMachine has no exclusive OS lock, so close() is a lifecycle /// signal and flushes state to disk. #[tokio::test] -async fn test_file_sm_close_marks_not_running() { +async fn test_file_sm_close_stops_serving() { let dir = TempDir::new().unwrap(); let sm = FileStateMachine::new(dir.path().to_path_buf()).await.unwrap(); - assert!(sm.is_running()); + assert!( + sm.get(b"k").is_ok(), + "a freshly built state machine serves reads" + ); sm.close().unwrap(); - assert!(!sm.is_running(), "close must mark SM not running"); + let err = sm.get(b"k").expect_err("close must stop serving"); + assert!(format!("{err:?}").contains("NotServing"), "got: {err:?}"); } /// Business scenario: the node shuts down while the developer still holds a clone of the state @@ -80,7 +84,8 @@ async fn test_file_sm_close_twice_is_safe() { let sm = FileStateMachine::new(dir.path().to_path_buf()).await.unwrap(); sm.close().unwrap(); sm.close().unwrap(); - assert!(!sm.is_running()); + let err = sm.get(b"k").expect_err("still closed after the second close"); + assert!(format!("{err:?}").contains("NotServing"), "got: {err:?}"); } /// Builder for FileStateMachine test instances diff --git a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs index 670cd6d1..6169b7ee 100644 --- a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs +++ b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs @@ -552,10 +552,6 @@ impl StateMachine for RocksDBStateMachine { first_err.map_or(Ok(()), Err) } - fn is_running(&self) -> bool { - self.is_serving.load(Ordering::SeqCst) - } - fn get( &self, key_buffer: &[u8], diff --git a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine_test.rs b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine_test.rs index 4693b242..f21f663f 100644 --- a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine_test.rs +++ b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine_test.rs @@ -516,7 +516,7 @@ async fn test_get_multi_two_keys_still_uses_snapshot_isolation() { // immediately, without waiting for Arc to be the sole owner (the developer may keep // their own clone). -/// close() marks SM not-running and makes reads return a NotServing error, never an empty result. +/// close() makes reads return a NotServing error, never an empty result. #[tokio::test] async fn test_close_prevents_reads() { let dir = TempDir::new().unwrap(); @@ -526,7 +526,6 @@ async fn test_close_prevents_reads() { sm.close().unwrap(); - assert!(!sm.is_running(), "is_running must be false after close"); let err = sm.get(b"k").expect_err("get must fail after close"); assert!( format!("{err:?}").contains("NotServing"), @@ -546,7 +545,8 @@ fn test_close_twice_is_safe() { let (_storage, sm) = RocksDBUnifiedEngine::open(dir.path()).unwrap(); sm.close().unwrap(); sm.close().unwrap(); - assert!(!sm.is_running()); + let err = sm.get(b"k").expect_err("still closed after the second close"); + assert!(format!("{err:?}").contains("NotServing"), "got: {err:?}"); } /// Drop after close() must not panic or double-flush. diff --git a/d-engine-server/src/test_utils/mock/mock_node_builder.rs b/d-engine-server/src/test_utils/mock/mock_node_builder.rs index e88ec394..bb519628 100644 --- a/d-engine-server/src/test_utils/mock/mock_node_builder.rs +++ b/d-engine-server/src/test_utils/mock/mock_node_builder.rs @@ -626,7 +626,6 @@ pub(crate) fn mock_state_machine() -> MockStateMachine { let mut mock = MockStateMachine::new(); mock.expect_close().returning(|| Ok(())); - mock.expect_is_running().returning(|| true); mock.expect_get().returning(|_| Ok(None)); mock.expect_apply_chunk().returning(|_| Ok(vec![])); @@ -659,7 +658,7 @@ pub(crate) fn mock_state_machine_handler() -> MockStateMachineHandler bool { - // Return running status - true - } - fn get(&self, key_buffer: &[u8]) -> Result>, Error> { // Retrieve value by key self.backend.get(key_buffer) @@ -114,7 +109,6 @@ impl StateMachine for CustomStateMachine { | Method | Purpose | Sync/Async | Criticality | | ---------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | ----------- | | `close()` | Final step at shutdown: persist what must survive a restart and release exclusive OS resources (e.g. DB lock file), even if persisting fails; returns the first error. Idempotent. Default `Ok(())`. | Sync | Medium | -| `is_running()` | Check service status | Sync | Medium | | `get()` | Read value by key | Sync | High | | `apply_chunk()` | Apply committed entries | Async | Critical | | `update_last_applied()` | Update applied index in memory | Sync | High | @@ -157,14 +151,10 @@ async fn test_custom_state_machine() -> Result<(), Error> { let builder = TestStateMachineBuilder::new(); let sm = builder.build().await?; - // Test basic functionality - sm.start()?; - assert!(sm.is_running()); - // Test apply_chunk with sample entries let entries = vec![create_test_entry(1, 1)]; sm.apply_chunk(&entries).await?; - assert_eq!(sm.len(), 1); + assert_eq!(sm.last_applied().index, 1); // Test snapshot operations let temp_dir = TempDir::new()?; diff --git a/examples/sled-cluster/src/sled_state_machine.rs b/examples/sled-cluster/src/sled_state_machine.rs index f29a63e4..1497f96c 100644 --- a/examples/sled-cluster/src/sled_state_machine.rs +++ b/examples/sled-cluster/src/sled_state_machine.rs @@ -8,17 +8,17 @@ use bincode::config; use bytes::Bytes; use d_engine::BatchOp; use d_engine::{ - common::LogId, server_storage::SnapshotMetadata, ApplyEntry, ApplyResult, Command, Result, - ScanResult, SnapshotApplyResult, SnapshotError, StateMachine, StorageError, + ApplyEntry, ApplyResult, Command, Result, ScanResult, SnapshotApplyResult, SnapshotError, + StateMachine, StorageError, common::LogId, server_storage::SnapshotMetadata, }; use parking_lot::Mutex; use sled::Batch; use std::path::Path; use std::path::PathBuf; +use std::sync::Arc; use std::sync::atomic::AtomicBool; use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; -use std::sync::Arc; use std::time::Instant; use tracing::debug; use tracing::error; @@ -100,10 +100,6 @@ impl StateMachine for SledStateMachine { saved } - fn is_running(&self) -> bool { - self.is_serving.load(Ordering::Acquire) - } - fn update_last_applied( &self, last_applied: LogId, From 24aa6ebec5599e8d22939067046ffa4738e29537 Mon Sep 17 00:00:00 2001 From: Joshua Chi <112539+JoshuaChi@users.noreply.github.com> Date: Sun, 1 Nov 2026 18:32:04 +0800 Subject: [PATCH 03/19] refactor #400: Take the state machine by value; route embedded reads through the ReadActor The engine now owns the state machine. start_custom, start_node and run_custom (embedded and standalone) take `SM` instead of `Arc`, and the Arc is created inside NodeBuilder. Callers can no longer keep a second reference to the instance the engine uses, so the engine controls when it is dropped. Embedded reads used to hold their own `Arc` (EmbeddedReadHandle), which kept the state machine alive after stop(). They now go through the ReadActor like the standalone path, so one ReadHandle serves both modes. EmbeddedReadHandle is removed. EmbeddedEngine and EmbeddedClient no longer carry type parameters; the start functions are generic over the storage engine and state machine instead. Remove the forwarding start_node_with_lock helpers (and their allow(dead_code)). Tests: - Add test_stop_drops_state_machine_before_returning: stop() must return only after the state machine is dropped. - Add SharedStateMachine, a test wrapper for tests that must reach the instance the engine uses (read it directly, or simulate a crash). - Migrate Arc call sites in unit and integration tests; update docs, examples and benches. --- CHANGELOG.md | 2 + MIGRATION_GUIDE.md | 19 ++ benches/embedded-bench/src/main.rs | 10 +- d-engine-server/benches/watch_overhead.rs | 1 - d-engine-server/src/api/embedded.rs | 106 +++---- d-engine-server/src/api/embedded_client.rs | 31 +-- .../embedded_client_test.rs | 18 +- .../src/api/embedded_read_handle.rs | 182 ------------ .../src/api/embedded_read_handle_test.rs | 258 ------------------ .../src/api/embedded_test/embedded_test.rs | 27 +- .../api/embedded_test/shutdown_order_test.rs | 86 +++++- d-engine-server/src/api/mod.rs | 20 +- ...andalone_read_handle.rs => read_handle.rs} | 12 +- ...ead_handle_test.rs => read_handle_test.rs} | 30 +- d-engine-server/src/api/standalone.rs | 47 +--- d-engine-server/src/api/standalone_test.rs | 7 +- .../grpc/grpc_raft_service_fast_path_test.rs | 6 +- d-engine-server/src/node/builder.rs | 6 +- d-engine-server/src/node/builder_test.rs | 4 +- d-engine-server/src/node/mod.rs | 2 +- d-engine-server/src/node/node_test.rs | 4 +- .../src/test_utils/mock/mock_node_builder.rs | 4 +- d-engine-server/src/test_utils/mod.rs | 4 + .../bank_transfer_invariant_embedded.rs | 2 +- .../distributed_lock_embedded.rs | 2 +- .../leader_failover_cas_embedded.rs | 2 +- .../snapshot_recovery_embedded.rs | 4 +- .../scale_single_to_three_node_embedded.rs | 14 +- ...oin_cluster_concurrent_joins_standalone.rs | 12 +- .../join_cluster_single_join_standalone.rs | 19 +- .../cluster_state_consensus_embedded.rs | 18 +- d-engine-server/tests/common/mod.rs | 14 +- .../tests/common/shared_state_machine.rs | 120 ++++++++ .../linearizable_read_consistency_embedded.rs | 2 +- .../leader_failover_embedded.rs | 10 +- .../sm_metadata_atomicity_embedded.rs | 3 +- ...sm_wal_disabled_crash_recovery_embedded.rs | 9 +- .../learner_readonly_sync_embedded.rs | 6 +- ...k_listener_released_after_stop_embedded.rs | 2 +- ...napshot_concurrent_replication_embedded.rs | 4 +- .../snapshot_concurrent_writes_embedded.rs | 2 +- ...shot_correctness_after_install_embedded.rs | 12 +- .../snapshot_follower_generation_embedded.rs | 2 +- .../snapshot_generation_standalone.rs | 14 +- .../snapshot_interrupted_transfer_embedded.rs | 4 +- ..._leader_change_during_transfer_embedded.rs | 4 +- ..._transfer_does_not_block_apply_embedded.rs | 5 +- .../watch_events_embedded.rs | 6 +- .../watch_events_grpc_standalone.rs | 2 +- .../watch_membership_embedded.rs | 2 +- .../watch_performance_gate_embedded.rs | 2 +- d-engine/src/docs/quick-start-5min.md | 8 +- .../server_guide/customize-state-machine.md | 2 +- .../server_guide/customize-storage-engine.md | 2 +- examples/raft-engine-log/src/main.rs | 5 +- examples/single-node-expansion/src/main.rs | 4 +- examples/sled-cluster/src/main.rs | 3 +- examples/three-nodes-standalone/src/main.rs | 4 +- 58 files changed, 445 insertions(+), 767 deletions(-) delete mode 100644 d-engine-server/src/api/embedded_read_handle.rs delete mode 100644 d-engine-server/src/api/embedded_read_handle_test.rs rename d-engine-server/src/api/{standalone_read_handle.rs => read_handle.rs} (96%) rename d-engine-server/src/api/{standalone_read_handle_test.rs => read_handle_test.rs} (96%) create mode 100644 d-engine-server/tests/common/shared_state_machine.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 8fbb698b..361e5bce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -116,6 +116,8 @@ All notable changes to this project will be documented in this file. "MemFirst"` or `"DiskFirst"` are silently ignored, not an error — remove the field, `flush_policy` is the only persistence knob now. +- **⚠️ The engine takes the state machine by value** (#400): `EmbeddedEngine::start_custom`/`start_node`, `StandaloneEngine::run_custom`/`start_node` take `SM` instead of `Arc`; the `Arc` is created inside the engine. Nothing outside the engine holds the state machine, so `stop()` returns only after it is dropped and its resources are released. Embedded reads now go through the ReadActor like standalone reads (`EmbeddedReadHandle` is removed). `EmbeddedEngine` and `EmbeddedClient` no longer have type parameters. See [MIGRATION_GUIDE.md](./MIGRATION_GUIDE.md). + ### Removed - **TTL and per-key lease** (#464): d-engine now does consensus and replication only. Removed `put_with_ttl`, the `Lease` trait, `TtlLease`, the `[raft.state_machine.lease]` config section and `StateMachine::lease_background_cleanup`. `Command::Insert` no longer has `ttl_secs`. In the proto, `WriteCommand.Insert.ttl_secs` (field 3) is removed and the number is reserved. Key expiry is now the job of the layer above d-engine. The leader read lease (`LeaseRead`) is a different feature and is unchanged. diff --git a/MIGRATION_GUIDE.md b/MIGRATION_GUIDE.md index 0bbf50f8..5ad028c9 100644 --- a/MIGRATION_GUIDE.md +++ b/MIGRATION_GUIDE.md @@ -16,6 +16,25 @@ - RocksDB state machine: keys written with `put_with_ttl` under v0.2.x are kept forever after the upgrade. v0.3.0 neither reads the old `ttl_state` entry nor deletes expired keys. Before upgrading, either delete the keys you no longer want, or record their deadlines and have your own expiry layer delete them. Or start with a fresh data directory. - A read that fails in the state machine now returns an error code instead of an empty result: `StorageIoError`, `DataCorruption` (the stored data may be damaged), `RetryRequired` (the state machine is replacing its state; retry) or `Uncategorized`. Before, such a read looked like "key not found". Handle these errors in your client. +## Upgrading to v0.3.0: The Engine Owns the State Machine + +`start_custom`, `start_node` and `run_custom` (embedded and standalone) now take the state machine by value. The engine creates the `Arc` itself, so by the time `stop()` returns it has dropped the state machine and released whatever it held (a database handle, a lock file). + +```rust +// before +let sm = Arc::new(MyStateMachine::new()?); +EmbeddedEngine::::start_custom(dir, storage, sm, None).await?; + +// after +let sm = MyStateMachine::new()?; +EmbeddedEngine::start_custom(dir, storage, sm, None).await?; +``` + +- **`EmbeddedEngine` and `EmbeddedClient` no longer have type parameters.** Write `EmbeddedEngine` and `EmbeddedClient`. `DefaultEmbeddedEngine` and `DefaultEmbeddedClient` still exist and are now plain aliases of them. +- **If you kept a clone of your `Arc` to read local data directly** (for example `DLmdb` reading its own LMDB), you can no longer clone the instance the engine uses. Share a handle to your storage instead: put the handles (LMDB `Env`, sled `Db`, `Arc`) behind an `Arc` inside your state machine type, derive `Clone` on it, give the engine one clone and keep the other. +- **Embedded reads (`get_eventual`, lease reads) are answered by the ReadActor**, as they already were in standalone mode, instead of reading the state machine on the caller's thread. Expect one extra channel hop per read. +- When `stop()` returns, the engine has dropped the state machine. If your state machine releases its resources in `Drop` (a database handle, a lock file), they are free at that point and the same data directory can be reopened. + --- ## For v0.2.4 Users: Wire Protocol Changes in v0.3.0 diff --git a/benches/embedded-bench/src/main.rs b/benches/embedded-bench/src/main.rs index 3fc81923..22a67ba6 100644 --- a/benches/embedded-bench/src/main.rs +++ b/benches/embedded-bench/src/main.rs @@ -2,11 +2,7 @@ // `raft-engine` feature replaces only the Raft log store (the state machine, // config file and data layout are unchanged), so the two builds differ by the // log store alone and their results stay comparable. -#[cfg(not(feature = "raft-engine"))] -type BenchEngine = d_engine::DefaultEmbeddedEngine; -#[cfg(feature = "raft-engine")] -type BenchEngine = - d_engine::EmbeddedEngine; +type BenchEngine = d_engine::EmbeddedEngine; #[cfg(not(feature = "raft-engine"))] async fn start_engine( @@ -28,9 +24,7 @@ async fn start_engine( format!("{data_dir}/storage"), &options, )?); - let sm = Arc::new(d_engine::RocksDBStateMachine::new(format!( - "{data_dir}/state_machine" - ))?); + let sm = d_engine::RocksDBStateMachine::new(format!("{data_dir}/state_machine"))?; BenchEngine::start_custom(data_dir, storage, sm, Some(config_path)).await } use d_engine::protocol::ReadConsistencyPolicy; diff --git a/d-engine-server/benches/watch_overhead.rs b/d-engine-server/benches/watch_overhead.rs index 8ff45ef2..e72f69e2 100644 --- a/d-engine-server/benches/watch_overhead.rs +++ b/d-engine-server/benches/watch_overhead.rs @@ -101,7 +101,6 @@ watcher_buffer_size = 100 let (storage, state_machine) = RocksDBUnifiedEngine::open(&db_open_path)?; let storage = Arc::new(storage); - let state_machine = Arc::new(state_machine); let engine = DefaultEmbeddedEngine::start_custom( &db_path, diff --git a/d-engine-server/src/api/embedded.rs b/d-engine-server/src/api/embedded.rs index 03c37a8e..bdde2e91 100644 --- a/d-engine-server/src/api/embedded.rs +++ b/d-engine-server/src/api/embedded.rs @@ -115,7 +115,6 @@ use std::fmt::Debug; use std::sync::Arc; use std::time::Duration; -use d_engine_core::TypeConfig; use tokio::sync::Mutex; use tokio::sync::watch; use tokio::task::JoinHandle; @@ -123,7 +122,6 @@ use tracing::error; use tracing::info; use super::embedded_client::EmbeddedClient; -use super::embedded_read_handle::EmbeddedReadHandle; use crate::Result; #[cfg(feature = "rocksdb")] use crate::RocksDBStateMachine; @@ -134,12 +132,11 @@ use crate::RocksDBUnifiedEngine; use crate::StateMachine; use crate::StorageEngine; use crate::node::NodeBuilder; -use crate::node::RaftTypeConfig; -struct Inner { +struct Inner { node_handle: Mutex>>>, shutdown_tx: watch::Sender<()>, - client: Arc>, + client: Arc, leader_elected_rx: watch::Receiver>, membership_rx: watch::Receiver, is_stopped: Mutex, @@ -170,11 +167,11 @@ struct Inner { /// /// engine.stop().await?; /// ``` -pub struct EmbeddedEngine { - inner: Arc>, +pub struct EmbeddedEngine { + inner: Arc, } -impl Clone for EmbeddedEngine { +impl Clone for EmbeddedEngine { fn clone(&self) -> Self { Self { inner: Arc::clone(&self.inner), @@ -185,7 +182,7 @@ impl Clone for EmbeddedEngine { // ─── RocksDB default start ─────────────────────────────────────────────────── #[cfg(feature = "rocksdb")] -impl EmbeddedEngine> { +impl EmbeddedEngine { /// Start engine with an explicit data directory. /// /// The directory is created automatically if it does not exist. @@ -221,11 +218,11 @@ impl EmbeddedEngine> { (storage, sm) }; - Self::start_node_with_lock( + Self::start_node_inner( data_dir.as_path().to_path_buf(), - data_dir_lock, + Some(data_dir_lock), Arc::new(storage), - Arc::new(sm), + sm, config, ) .await @@ -271,15 +268,15 @@ impl EmbeddedEngine> { (storage, sm) }; - // start_node_with_lock directly, not start_custom — config is + // start_node_inner directly, not start_custom — config is // already loaded/validated here, and the lock is already held; // start_custom would re-parse config_path again for no benefit and // can't accept a pre-acquired lock anyway. - Self::start_node_with_lock( + Self::start_node_inner( data_dir.as_path().to_path_buf(), - data_dir_lock, + Some(data_dir_lock), Arc::new(storage), - Arc::new(sm), + sm, config, ) .await @@ -288,11 +285,7 @@ impl EmbeddedEngine> { // ─── Custom SE/SM: start_custom + start_node ───────────────────────────────── -impl EmbeddedEngine> -where - SE: StorageEngine + Debug + 'static, - SM: StateMachine + Debug + 'static, -{ +impl EmbeddedEngine { /// Starts the engine with a custom storage engine and state machine. /// /// `data_dir` is required and explicit — `config_path` is only for @@ -312,15 +305,19 @@ where /// # Example /// ```ignore /// let storage = Arc::new(MyCustomStorage::new()?); - /// let sm = Arc::new(MyCustomStateMachine::new()?); + /// let sm = MyCustomStateMachine::new()?; /// let engine = EmbeddedEngine::start_custom("./data/my-node", storage, sm, None).await?; /// ``` - pub async fn start_custom( + pub async fn start_custom( data_dir: impl AsRef, storage_engine: Arc, - state_machine: Arc, + state_machine: SM, config_path: Option<&str>, - ) -> Result { + ) -> Result + where + SE: StorageEngine + Debug + 'static, + SM: StateMachine + Debug + 'static, + { let node_config = if let Some(path) = config_path { d_engine_core::RaftNodeConfig::default().with_override_config(path)? } else { @@ -336,12 +333,16 @@ where /// For library wrappers that build their own config layer and /// translate to `RaftNodeConfig` in Rust — no config file required. /// Same lock scope as [`start_custom`](Self::start_custom) — see its doc. - pub async fn start_node( + pub async fn start_node( data_dir: impl AsRef, storage_engine: Arc, - state_machine: Arc, + state_machine: SM, node_config: d_engine_core::RaftNodeConfig, - ) -> Result { + ) -> Result + where + SE: StorageEngine + Debug + 'static, + SM: StateMachine + Debug + 'static, + { Self::start_node_inner( data_dir.as_ref().to_path_buf(), None, @@ -352,44 +353,22 @@ where .await } - /// Same as `start_node`, but for callers (`start`/`start_with`) that - /// already opened storage and therefore must have acquired the lock - /// *before* that — see `NodeBuilder::data_dir_lock`'s doc comment. - #[allow(dead_code)] - async fn start_node_with_lock( - data_dir: std::path::PathBuf, - data_dir_lock: crate::node::DataDirLock, - storage_engine: Arc, - state_machine: Arc, - node_config: d_engine_core::RaftNodeConfig, - ) -> Result { - Self::start_node_inner( - data_dir, - Some(data_dir_lock), - storage_engine, - state_machine, - node_config, - ) - .await - } - - async fn start_node_inner( + async fn start_node_inner( data_dir: std::path::PathBuf, data_dir_lock: Option, storage_engine: Arc, - state_machine: Arc, + state_machine: SM, node_config: d_engine_core::RaftNodeConfig, - ) -> Result { + ) -> Result + where + SE: StorageEngine + Debug + 'static, + SM: StateMachine + Debug + 'static, + { info!("Starting embedded d-engine"); d_engine_core::init_clock(); let (shutdown_tx, shutdown_rx) = watch::channel(()); - // Clone SM before NodeBuilder consumes it — EmbeddedReadHandle holds this - // for the direct-SM fast path. LOCK release is decoupled from Arc counting - // (via close_db()), so holding multiple Arc clones is safe. - let sm_for_client = Arc::clone(&state_machine); - let mut builder = NodeBuilder::init(data_dir, node_config.validate()?, shutdown_rx) .storage_engine(storage_engine) .state_machine(state_machine); @@ -406,8 +385,9 @@ where let leader_elected_rx = node.leader_change_notifier(); let membership_rx = node.membership_change_notifier(); - let read_handle = - EmbeddedReadHandle::new(sm_for_client, node.read_lease(), node.cmd_tx.clone()); + // Reads go through the ReadActor, the only holder of the state machine besides the + // Raft loop and the apply worker, so the engine controls when it is dropped. + let read_handle = node.read_handle.clone(); let client = { let base = EmbeddedClient::new_internal( @@ -456,9 +436,9 @@ where } } -// ─── General API (all TypeConfig) ──────────────────────────────────────────── +// ─── General API ──────────────────────────────────────────── -impl EmbeddedEngine { +impl EmbeddedEngine { /// Wait until the cluster is ready to serve requests. /// /// Blocks until a leader has been elected **and** its no-op entry is committed @@ -661,7 +641,7 @@ impl EmbeddedEngine { /// let client = engine.client(); /// client.put(b"key", b"value").await?; /// ``` - pub fn client(&self) -> Arc> { + pub fn client(&self) -> Arc { Arc::clone(&self.inner.client) } @@ -732,7 +712,7 @@ impl EmbeddedEngine { } } -impl Drop for EmbeddedEngine { +impl Drop for EmbeddedEngine { fn drop(&mut self) { // Warn if stop() was not called if let Ok(handle) = self.inner.node_handle.try_lock() diff --git a/d-engine-server/src/api/embedded_client.rs b/d-engine-server/src/api/embedded_client.rs index 5164cea6..b1982b03 100644 --- a/d-engine-server/src/api/embedded_client.rs +++ b/d-engine-server/src/api/embedded_client.rs @@ -24,7 +24,6 @@ use d_engine_core::BatchOp; use d_engine_core::InboundEvent; use d_engine_core::MaybeCloneOneshot; use d_engine_core::RaftOneshot; -use d_engine_core::TypeConfig; use d_engine_core::client::ClientApi; use d_engine_core::client::ClientApiError; use d_engine_core::client::ClientApiResult; @@ -37,30 +36,30 @@ use d_engine_core::types::ScanResults; use d_engine_core::watch::WatchRegistry; use tokio::sync::mpsc; -use super::embedded_read_handle::EmbeddedReadHandle; -pub(crate) use super::standalone_read_handle::channel_closed_error; -pub(crate) use super::standalone_read_handle::map_error_response; -pub(crate) use super::standalone_read_handle::server_error; -pub(crate) use super::standalone_read_handle::timeout_error; +pub(crate) use super::read_handle::channel_closed_error; +pub(crate) use super::read_handle::map_error_response; +pub(crate) use super::read_handle::server_error; +pub(crate) use super::read_handle::timeout_error; +use crate::api::ReadHandle; use crate::api::types::WriteOperation; use crate::proto_convert; -/// Zero-overhead KV client for embedded mode. Obtained via `EmbeddedEngine::client()`. +/// KV client for embedded mode. Obtained via `EmbeddedEngine::client()`. /// -/// `T` is the [`TypeConfig`] that carries the concrete `StateMachine` type, enabling -/// direct monomorphized calls to `T::SM::get()` on the fast path — no vtable dispatch. +/// Eventual and lease reads are answered by the ReadActor, the only reader of the state machine +/// besides the Raft loop; linearizable reads and all writes go through Raft. /// /// For standalone/gRPC mode use `GrpcClient` instead. Both implement `ClientApi`. -pub struct EmbeddedClient { +pub struct EmbeddedClient { event_tx: mpsc::Sender, - read_handle: EmbeddedReadHandle, + read_handle: ReadHandle, client_id: u32, timeout: Duration, #[cfg(feature = "watch")] watch_registry: Option>, } -impl Clone for EmbeddedClient { +impl Clone for EmbeddedClient { fn clone(&self) -> Self { Self { event_tx: self.event_tx.clone(), @@ -73,11 +72,11 @@ impl Clone for EmbeddedClient { } } -impl EmbeddedClient { +impl EmbeddedClient { /// Internal constructor (used by EmbeddedEngine). pub(crate) fn new_internal( event_tx: mpsc::Sender, - read_handle: EmbeddedReadHandle, + read_handle: ReadHandle, client_id: u32, timeout: Duration, ) -> Self { @@ -561,7 +560,7 @@ impl EmbeddedClient { } } -impl std::fmt::Debug for EmbeddedClient { +impl std::fmt::Debug for EmbeddedClient { fn fmt( &self, f: &mut std::fmt::Formatter<'_>, @@ -575,7 +574,7 @@ impl std::fmt::Debug for EmbeddedClient { // Implement ClientApi trait #[async_trait::async_trait] -impl ClientApi for EmbeddedClient { +impl ClientApi for EmbeddedClient { async fn put( &self, key: impl AsRef<[u8]> + Send, diff --git a/d-engine-server/src/api/embedded_client_test/embedded_client_test.rs b/d-engine-server/src/api/embedded_client_test/embedded_client_test.rs index aef20722..c187b17b 100644 --- a/d-engine-server/src/api/embedded_client_test/embedded_client_test.rs +++ b/d-engine-server/src/api/embedded_client_test/embedded_client_test.rs @@ -391,31 +391,23 @@ max_batch_size = 1 // also the only way to force a NotLeader response, since a real single-node // engine (used by `integration_tests` above) is always its own leader. mod scan_prefix_error_handling { - use std::sync::Arc; use std::time::Duration; use d_engine_core::ClientCmd; - use d_engine_core::MockStateMachine; - use d_engine_core::MockTypeConfig; - use d_engine_core::ReadLease; use d_engine_core::client::ClientApiError; use d_engine_core::client::ClientResponse; use d_engine_core::client::ErrorCode; use d_engine_core::client::LeaderHint; use tokio::sync::mpsc; + use crate::api::ReadHandle; use crate::api::embedded_client::EmbeddedClient; - use crate::api::embedded_read_handle::EmbeddedReadHandle; - /// Build a client wired to a manually-controlled cmd channel. sm/lease are - /// unused by scan_prefix (it only goes through cmd_tx), so bare stand-ins suffice. - fn make_client(cmd_tx: mpsc::Sender) -> EmbeddedClient { + /// Build a client wired to a manually-controlled cmd channel. scan_prefix only goes through + /// cmd_tx, so the read handle has no ReadActor. + fn make_client(cmd_tx: mpsc::Sender) -> EmbeddedClient { let (event_tx, _event_rx) = mpsc::channel(1); - let read_handle = EmbeddedReadHandle::new( - Arc::new(MockStateMachine::new()), - Arc::new(ReadLease::new()), - cmd_tx, - ); + let read_handle = ReadHandle::new(None, cmd_tx); EmbeddedClient::new_internal(event_tx, read_handle, 1, Duration::from_millis(200)) } diff --git a/d-engine-server/src/api/embedded_read_handle.rs b/d-engine-server/src/api/embedded_read_handle.rs deleted file mode 100644 index 65f5e36c..00000000 --- a/d-engine-server/src/api/embedded_read_handle.rs +++ /dev/null @@ -1,182 +0,0 @@ -//! EmbeddedReadHandle — zero-channel read path for embedded mode. -//! -//! Replaces `ReadHandle` (which routes through the ReadActor channel) for -//! `EmbeddedClient`. Eventual and LeaseRead call `SM::get_multi()` directly — -//! no oneshot allocation, no task context switch, no channel round-trip. -//! -//! # Routing -//! -//! ```text -//! Eventual / LeaseRead (valid) → sm.get_multi() (direct, zero channel) -//! Eventual / LeaseRead (error) → cmd_tx fallback (SM unavailable or error) -//! LeaseRead (invalid lease) → cmd_tx fallback -//! LinearizableRead → cmd_tx always (Raft readIndex) -//! ``` -//! -//! # Shutdown safety -//! -//! After `EmbeddedEngine::stop()` calls `sm.close_db()`, `sm.get_multi()` returns -//! `NotServing`. The handle falls through to `cmd_tx` which is also closed at -//! that point, so callers receive a `channel_closed_error` — the same behaviour -//! as any other shutdown scenario. - -use std::marker::PhantomData; -use std::sync::Arc; -use std::time::Duration; - -use bytes::Bytes; -use d_engine_core::MaybeCloneOneshot; -use d_engine_core::RaftOneshot; -use d_engine_core::ReadLease; -use d_engine_core::StateMachine; -use d_engine_core::TypeConfig; -use d_engine_core::client::ClientApiResult; -use d_engine_core::client::ClientReadRequest; -use d_engine_core::client::ErrorCode; -use d_engine_core::config::ReadConsistencyPolicy; -use d_engine_core::now_ms; -use tokio::sync::mpsc; - -use super::standalone_read_handle::channel_closed_error; -use super::standalone_read_handle::extract_read_payload; -use super::standalone_read_handle::map_error_response; -use super::standalone_read_handle::server_error; -use super::standalone_read_handle::timeout_error; -use crate::api::standalone_read_handle::read_failure; - -// ── EmbeddedReadHandle ──────────────────────────────────────────────────────── - -/// Direct-SM read handle for embedded mode. -/// -/// Cloneable — `EmbeddedClient` and its clones each hold their own instance. -/// Cloning is cheap: all fields are `Arc` or `mpsc::Sender` (reference-counted). -pub(crate) struct EmbeddedReadHandle { - sm: Arc, - lease: Arc, - pub(crate) cmd_tx: mpsc::Sender, - _phantom: PhantomData T>, -} - -impl Clone for EmbeddedReadHandle { - fn clone(&self) -> Self { - Self { - sm: Arc::clone(&self.sm), - lease: Arc::clone(&self.lease), - cmd_tx: self.cmd_tx.clone(), - _phantom: PhantomData, - } - } -} - -impl EmbeddedReadHandle { - pub(crate) fn new( - sm: Arc, - lease: Arc, - cmd_tx: mpsc::Sender, - ) -> Self { - Self { - sm, - lease, - cmd_tx, - _phantom: PhantomData, - } - } - - /// Single-key read. Convenience wrapper around [`Self::get_batch`]. - pub async fn get( - &self, - key: &[u8], - consistency: ReadConsistencyPolicy, - client_id: u32, - timeout: Duration, - ) -> ClientApiResult> { - let key_bytes = Bytes::copy_from_slice(key); - let mut results = self.get_batch(&[key_bytes], consistency, client_id, timeout).await?; - Ok(results.pop().flatten()) - } - - /// Multi-key read with consistency routing. - /// - /// For `Eventual` and `LeaseRead` (when the lease is valid), all keys are - /// served from a single `SM::get_multi()` snapshot — no Raft round-trip. - /// On any SM error, or when the lease is invalid, the request falls through - /// to `cmd_tx` (Raft readIndex protocol). - pub async fn get_batch( - &self, - keys: &[Bytes], - consistency: ReadConsistencyPolicy, - client_id: u32, - timeout: Duration, - ) -> ClientApiResult>> { - // Only `NotServing` (the state machine is replacing its state or is closed) falls through - // to cmd_tx. A hard failure would fail again on the Raft path, so it is reported now. - match consistency { - ReadConsistencyPolicy::EventualConsistency => match self.sm.get_multi(keys) { - Ok(values) => return Ok(values), - Err(e) if e.is_not_serving() => {} - Err(e) => return Err(read_failure(&e)), - }, - ReadConsistencyPolicy::LeaseRead => { - if self.lease.is_valid(now_ms()) { - match self.sm.get_multi(keys) { - Ok(values) => return Ok(values), - Err(e) if e.is_not_serving() => {} - Err(e) => return Err(read_failure(&e)), - } - } - // invalid lease → fall through to cmd_tx - } - ReadConsistencyPolicy::LinearizableRead => { - // always cmd_tx — Raft readIndex protocol - } - } - - self.cmd_tx_path(keys, consistency, client_id, timeout).await - } - - // ── cmd_tx fallback ─────────────────────────────────────────────────────── - - async fn cmd_tx_path( - &self, - keys: &[Bytes], - consistency: ReadConsistencyPolicy, - client_id: u32, - timeout: Duration, - ) -> ClientApiResult>> { - let request = ClientReadRequest { - client_id, - keys: keys.to_vec(), - consistency_policy: Some(consistency), - }; - let (resp_tx, resp_rx) = MaybeCloneOneshot::new(); - self.cmd_tx - .send(d_engine_core::ClientCmd::Read(request, resp_tx)) - .await - .map_err(|_| channel_closed_error())?; - - let result = tokio::time::timeout(timeout, resp_rx) - .await - .map_err(|_| timeout_error(timeout))? - .map_err(|_| channel_closed_error())?; - - let response = - result.map_err(|status| server_error(format!("RPC error: {}", status.message())))?; - - if response.error != ErrorCode::Success { - return Err(map_error_response( - response.error, - response.leader_hint, - response.retry_after_ms, - )); - } - - let read_results = extract_read_payload(response.result)?; - let entry_map: std::collections::HashMap = - read_results.entries.into_iter().map(|e| (e.key, e.value)).collect(); - Ok(keys.iter().map(|k| entry_map.get(k).cloned()).collect()) - } -} - -#[cfg(test)] -#[path = "embedded_read_handle_test.rs"] -mod tests; diff --git a/d-engine-server/src/api/embedded_read_handle_test.rs b/d-engine-server/src/api/embedded_read_handle_test.rs deleted file mode 100644 index 89dd8950..00000000 --- a/d-engine-server/src/api/embedded_read_handle_test.rs +++ /dev/null @@ -1,258 +0,0 @@ -/// Tests for EmbeddedReadHandle — the zero-channel direct-SM read path. -/// -/// Proof technique (same as embedded_client_fast_path_test): -/// cmd_rx is dropped so any fallback to cmd_tx returns a channel-closed error. -/// Success proves the SM was called directly; error proves fallback was triggered. -#[cfg(test)] -mod embedded_read_handle_tests { - use std::sync::Arc; - use std::time::Duration; - - use bytes::Bytes; - use d_engine_core::ClientApiError; - use d_engine_core::Error; - use d_engine_core::MockStateMachine; - use d_engine_core::MockTypeConfig; - use d_engine_core::ReadLease; - use d_engine_core::StorageError; - use d_engine_core::client::ErrorCode; - use d_engine_core::config::ReadConsistencyPolicy; - use d_engine_core::now_ms; - use tokio::sync::mpsc; - - use super::super::EmbeddedReadHandle; - - // ── Helpers ─────────────────────────────────────────────────────────────── - - fn valid_lease() -> Arc { - let lease = Arc::new(ReadLease::new()); - lease.renew(1, now_ms() + 60_000); - lease - } - - fn invalid_lease() -> Arc { - let lease = Arc::new(ReadLease::new()); - lease.revoke(); - lease - } - - fn sm_with_value(value: Bytes) -> MockStateMachine { - let mut sm = MockStateMachine::new(); - sm.expect_get_multi() - .returning(move |keys| Ok(keys.iter().map(|_| Some(value.clone())).collect())); - sm - } - - fn sm_unavailable() -> MockStateMachine { - // SM unavailable: get_multi returns NotServing error - let mut sm = MockStateMachine::new(); - sm.expect_get_multi().returning(|_| { - Err(d_engine_core::StorageError::NotServing("state machine stopped".into()).into()) - }); - sm - } - - /// Build a handle. `drop_cmd_rx=true` closes the cmd channel so any fallback errors. - fn make_handle( - sm: MockStateMachine, - lease: Arc, - drop_cmd_rx: bool, - ) -> EmbeddedReadHandle { - let (cmd_tx, cmd_rx) = mpsc::channel(1); - if drop_cmd_rx { - drop(cmd_rx); - } - EmbeddedReadHandle::new(Arc::new(sm), lease, cmd_tx) - } - - fn sm_failing_with(make_error: fn() -> Error) -> MockStateMachine { - let mut sm = MockStateMachine::new(); - sm.expect_get_multi().returning(move |_| Err(make_error())); - sm - } - - // ── Hard failures are reported, never retried through cmd_tx ───────────── - - /// A storage failure is returned to the caller with its storage code. cmd_rx is dropped: a - /// fallback would produce a Network error instead, so a Storage error proves there was none. - #[tokio::test] - async fn test_hard_failure_is_reported_without_falling_back() { - let cases: [(fn() -> Error, ErrorCode); 2] = [ - ( - || StorageError::DbError("secret detail".into()).into(), - ErrorCode::StorageIoError, - ), - ( - // The client is told the data may be damaged; the decision is theirs. - || { - StorageError::DataCorruption { - location: "secret detail".into(), - } - .into() - }, - ErrorCode::DataCorruption, - ), - ]; - - for consistency in [ - ReadConsistencyPolicy::EventualConsistency, - ReadConsistencyPolicy::LeaseRead, - ] { - for (make_error, expected) in &cases { - let handle = make_handle(sm_failing_with(*make_error), valid_lease(), true); - let result = - handle.get(b"k", consistency.clone(), 1, Duration::from_millis(100)).await; - - match result { - Err(ClientApiError::Storage { code, message }) => { - assert_eq!(code, *expected); - assert!( - !message.contains("secret detail"), - "detail leaked: {message}" - ); - } - other => panic!("expected a Storage error for {expected:?}, got: {other:?}"), - } - } - } - } - - // ── Eventual fast path ──────────────────────────────────────────────────── - - /// Eventual read goes directly to SM; cmd_rx dropped proves cmd_tx was not used. - #[tokio::test] - async fn test_eventual_read_bypasses_cmd_tx() { - let handle = make_handle(sm_with_value(Bytes::from("v")), valid_lease(), true); - let result = handle - .get( - b"k", - ReadConsistencyPolicy::EventualConsistency, - 1, - Duration::from_millis(100), - ) - .await; - assert_eq!(result.unwrap(), Some(Bytes::from("v"))); - } - - /// Eventual read when the SM is unavailable falls back to cmd_tx. - /// cmd_rx dropped → fallback returns error (proves fallback was taken, not panic). - #[tokio::test] - async fn test_eventual_sm_unavailable_falls_back_to_cmd_tx() { - let handle = make_handle(sm_unavailable(), valid_lease(), true); - let result = handle - .get( - b"k", - ReadConsistencyPolicy::EventualConsistency, - 1, - Duration::from_millis(100), - ) - .await; - assert!(result.is_err(), "unavailable SM must fall back to cmd_tx"); - } - - // ── LeaseRead fast path ─────────────────────────────────────────────────── - - /// LeaseRead with valid lease goes directly to SM. - #[tokio::test] - async fn test_lease_read_valid_lease_bypasses_cmd_tx() { - let handle = make_handle(sm_with_value(Bytes::from("lease_val")), valid_lease(), true); - let result = handle - .get( - b"k", - ReadConsistencyPolicy::LeaseRead, - 1, - Duration::from_millis(100), - ) - .await; - assert_eq!(result.unwrap(), Some(Bytes::from("lease_val"))); - } - - /// LeaseRead with invalid (revoked) lease falls back to cmd_tx. - #[tokio::test] - async fn test_lease_read_invalid_lease_falls_back_to_cmd_tx() { - let mut sm = MockStateMachine::new(); - // get_multi must NOT be called — lease is invalid - sm.expect_get_multi().never(); - - let handle = make_handle(sm, invalid_lease(), true); - let result = handle - .get( - b"k", - ReadConsistencyPolicy::LeaseRead, - 1, - Duration::from_millis(100), - ) - .await; - assert!(result.is_err(), "invalid lease must fall back to cmd_tx"); - } - - /// LeaseRead with valid lease but unavailable SM falls back to cmd_tx. - #[tokio::test] - async fn test_lease_read_sm_unavailable_falls_back_to_cmd_tx() { - let handle = make_handle(sm_unavailable(), valid_lease(), true); - let result = handle - .get( - b"k", - ReadConsistencyPolicy::LeaseRead, - 1, - Duration::from_millis(100), - ) - .await; - assert!(result.is_err(), "unavailable SM must fall back to cmd_tx"); - } - - // ── LinearizableRead ───────────────────────────────────────────────────── - - /// Linearizable always uses cmd_tx. cmd_rx dropped → error proves cmd_tx was used. - #[tokio::test] - async fn test_linearizable_always_uses_cmd_tx() { - let mut sm = MockStateMachine::new(); - sm.expect_get_multi().never(); // must not be called - - let handle = make_handle(sm, valid_lease(), true); - let result = handle - .get( - b"k", - ReadConsistencyPolicy::LinearizableRead, - 1, - Duration::from_millis(100), - ) - .await; - assert!( - result.is_err(), - "linearizable must always go through cmd_tx" - ); - } - - // ── get_batch multi-key ─────────────────────────────────────────────────── - - /// get_batch returns values for all keys in order. - #[tokio::test] - async fn test_get_batch_returns_all_keys_in_order() { - let mut sm = MockStateMachine::new(); - sm.expect_get_multi().returning(|keys| { - Ok(keys - .iter() - .enumerate() - .map(|(i, _)| Some(Bytes::from(format!("v{i}")))) - .collect()) - }); - - let handle = make_handle(sm, valid_lease(), true); - let keys = vec![Bytes::from("k0"), Bytes::from("k1"), Bytes::from("k2")]; - let result = handle - .get_batch( - &keys, - ReadConsistencyPolicy::EventualConsistency, - 1, - Duration::from_millis(100), - ) - .await - .unwrap(); - - assert_eq!(result.len(), 3); - assert_eq!(result[0], Some(Bytes::from("v0"))); - assert_eq!(result[1], Some(Bytes::from("v1"))); - assert_eq!(result[2], Some(Bytes::from("v2"))); - } -} diff --git a/d-engine-server/src/api/embedded_test/embedded_test.rs b/d-engine-server/src/api/embedded_test/embedded_test.rs index 8874b62a..898087fb 100644 --- a/d-engine-server/src/api/embedded_test/embedded_test.rs +++ b/d-engine-server/src/api/embedded_test/embedded_test.rs @@ -8,14 +8,12 @@ mod embedded_engine_tests { use crate::api::EmbeddedEngine; use crate::storage::FileStateMachine; use crate::storage::FileStorageEngine; + use crate::test_utils::SharedStateMachine; - type TestEngine = EmbeddedEngine; + type TestEngine = EmbeddedEngine; - async fn create_test_storage_and_sm() -> ( - Arc, - Arc, - tempfile::TempDir, - ) { + async fn create_test_storage_and_sm() + -> (Arc, FileStateMachine, tempfile::TempDir) { // Clear CONFIG_PATH to avoid external config interference unsafe { std::env::remove_var("CONFIG_PATH"); @@ -30,8 +28,7 @@ mod embedded_engine_tests { let storage = Arc::new(FileStorageEngine::new(storage_path).expect("Failed to create storage")); - let sm = - Arc::new(FileStateMachine::new(sm_path).await.expect("Failed to create state machine")); + let sm = FileStateMachine::new(sm_path).await.expect("Failed to create state machine"); (storage, sm, temp_dir) } @@ -592,7 +589,10 @@ listen_addr = "127.0.0.1:0" #[serial(start_node)] async fn test_start_node_uses_passed_se_and_sm_instances() { let (storage, sm, temp_dir) = create_test_storage_and_sm().await; - let sm_for_direct_read = Arc::clone(&sm); + // The engine takes the state machine by value; the shared wrapper lets this test keep + // a reference to the very instance the engine writes to. + let sm_for_direct_read = Arc::new(sm); + let sm = SharedStateMachine::new(Arc::clone(&sm_for_direct_read)); let config = make_config(1); let engine = @@ -735,9 +735,8 @@ listen_addr = "127.0.0.1:0" let storage = Arc::new( FileStorageEngine::new(storage_path).expect("Failed to create storage"), ); - let sm = Arc::new( - FileStateMachine::new(sm_path).await.expect("Failed to create state machine"), - ); + let sm = + FileStateMachine::new(sm_path).await.expect("Failed to create state machine"); (storage, sm) // temp_dir dropped here — underlying paths are now invalid @@ -793,9 +792,7 @@ listen_addr = "127.0.0.1:0" let storage = Arc::new(FileStorageEngine::new(storage_path).expect("Failed to create storage")); - let sm = Arc::new( - FileStateMachine::new(sm_path).await.expect("Failed to create state machine"), - ); + let sm = FileStateMachine::new(sm_path).await.expect("Failed to create state machine"); println!("Created storage and SM, TempDir kept alive"); diff --git a/d-engine-server/src/api/embedded_test/shutdown_order_test.rs b/d-engine-server/src/api/embedded_test/shutdown_order_test.rs index 8ff99674..463ebe01 100644 --- a/d-engine-server/src/api/embedded_test/shutdown_order_test.rs +++ b/d-engine-server/src/api/embedded_test/shutdown_order_test.rs @@ -32,10 +32,25 @@ use crate::storage::FileStorageEngine; struct SlowApplyProbe { inner: FileStateMachine, apply_delay: Duration, + flags: Arc, +} + +/// What the probe observed. Shared with the test through an `Arc`, because the engine takes the +/// state machine by value and the test can no longer hold the state machine itself. +#[derive(Debug, Default)] +struct ProbeFlags { apply_started: AtomicBool, storage_closed: AtomicBool, /// Entries whose apply started or finished after `close_storage()` had been called. applied_after_close: AtomicUsize, + /// Set when the probe (the state machine) is dropped. + dropped: AtomicBool, +} + +impl Drop for SlowApplyProbe { + fn drop(&mut self) { + self.flags.dropped.store(true, Ordering::SeqCst); + } } #[async_trait] @@ -44,18 +59,18 @@ impl StateMachine for SlowApplyProbe { &self, chunk: &[ApplyEntry], ) -> Result, Error> { - self.apply_started.store(true, Ordering::SeqCst); - let closed_at_start = self.storage_closed.load(Ordering::SeqCst); + self.flags.apply_started.store(true, Ordering::SeqCst); + let closed_at_start = self.flags.storage_closed.load(Ordering::SeqCst); tokio::time::sleep(self.apply_delay).await; let result = self.inner.apply_chunk(chunk).await; - if closed_at_start || self.storage_closed.load(Ordering::SeqCst) { - self.applied_after_close.fetch_add(chunk.len(), Ordering::SeqCst); + if closed_at_start || self.flags.storage_closed.load(Ordering::SeqCst) { + self.flags.applied_after_close.fetch_add(chunk.len(), Ordering::SeqCst); } result } fn close(&self) -> Result<(), Error> { - self.storage_closed.store(true, Ordering::SeqCst); + self.flags.storage_closed.store(true, Ordering::SeqCst); self.inner.close() } @@ -145,15 +160,14 @@ async fn test_stop_closes_state_machine_storage_only_after_in_flight_applies_fin let temp_dir = tempfile::tempdir().expect("temp dir"); let storage = Arc::new(FileStorageEngine::new(temp_dir.path().join("storage")).expect("storage")); - let probe = Arc::new(SlowApplyProbe { + let flags = Arc::new(ProbeFlags::default()); + let probe = SlowApplyProbe { inner: FileStateMachine::new(temp_dir.path().join("sm")).await.expect("sm"), apply_delay: Duration::from_millis(300), - apply_started: AtomicBool::new(false), - storage_closed: AtomicBool::new(false), - applied_after_close: AtomicUsize::new(0), - }); + flags: Arc::clone(&flags), + }; - let engine = EmbeddedEngine::start_custom(temp_dir.path(), storage, probe.clone(), None) + let engine = EmbeddedEngine::start_custom(temp_dir.path(), storage, probe, None) .await .expect("start engine"); engine.wait_ready(Duration::from_secs(5)).await.expect("leader"); @@ -167,14 +181,60 @@ async fn test_stop_closes_state_machine_storage_only_after_in_flight_applies_fin }); } // Wait until the state machine is busy applying, then stop the engine. - while !probe.apply_started.load(Ordering::SeqCst) { + while !flags.apply_started.load(Ordering::SeqCst) { tokio::time::sleep(Duration::from_millis(5)).await; } engine.stop().await.expect("stop"); assert_eq!( - probe.applied_after_close.load(Ordering::SeqCst), + flags.applied_after_close.load(Ordering::SeqCst), 0, "stop() closed the state machine's storage while committed entries were still being applied" ); } + +/// Business scenario: an operator stops a node and immediately reopens the same data directory +/// (rolling restart, or a test harness reusing a path). The state machine owns an exclusive +/// resource (for RocksDB: the DB handle and its LOCK file) that is only released when the state +/// machine is dropped. +/// +/// Purpose: the engine takes the state machine by value and owns every reference to it, so by the +/// time `stop()` returns the state machine must already be dropped. If any internal holder (read +/// path, apply worker, Raft core) outlives `stop()`, the exclusive resource stays held and the +/// reopen fails. +#[tokio::test] +async fn test_stop_drops_state_machine_before_returning() { + unsafe { + std::env::remove_var("CONFIG_PATH"); + } + let temp_dir = tempfile::tempdir().expect("temp dir"); + let storage = + Arc::new(FileStorageEngine::new(temp_dir.path().join("storage")).expect("storage")); + let flags = Arc::new(ProbeFlags::default()); + let probe = SlowApplyProbe { + inner: FileStateMachine::new(temp_dir.path().join("sm")).await.expect("sm"), + apply_delay: Duration::ZERO, + flags: Arc::clone(&flags), + }; + + let engine = EmbeddedEngine::start_custom(temp_dir.path(), storage, probe, None) + .await + .expect("start engine"); + engine.wait_ready(Duration::from_secs(5)).await.expect("leader"); + + // Exercise both the write path and the read path, so every internal holder has been used. + let client = engine.client(); + client.put("key", "value").await.expect("put"); + let _ = client.get_eventual("key").await.expect("get"); + assert!( + !flags.dropped.load(Ordering::SeqCst), + "state machine must stay alive while the engine is running" + ); + + engine.stop().await.expect("stop"); + + assert!( + flags.dropped.load(Ordering::SeqCst), + "stop() returned while the state machine was still referenced by an internal holder" + ); +} diff --git a/d-engine-server/src/api/mod.rs b/d-engine-server/src/api/mod.rs index 5a42028b..97babd28 100644 --- a/d-engine-server/src/api/mod.rs +++ b/d-engine-server/src/api/mod.rs @@ -2,27 +2,19 @@ mod embedded; mod embedded_client; -mod embedded_read_handle; +mod read_handle; mod standalone; -mod standalone_read_handle; pub(crate) mod types; +pub use embedded::EmbeddedEngine; +pub use embedded_client::EmbeddedClient; +pub(crate) use read_handle::ReadHandle; pub use standalone::StandaloneEngine; -pub(crate) use standalone_read_handle::StandaloneReadHandle; - -/// Embedded engine generic over any `(SE, SM)` pair. -pub type EmbeddedEngine = embedded::EmbeddedEngine>; - -/// Embedded client generic over any `(SE, SM)` pair. -pub type EmbeddedClient = - embedded_client::EmbeddedClient>; /// Zero-param embedded engine alias using the default RocksDB backend. #[cfg(feature = "rocksdb")] -pub type DefaultEmbeddedEngine = - EmbeddedEngine; +pub type DefaultEmbeddedEngine = EmbeddedEngine; /// Zero-param embedded client alias using the default RocksDB backend. #[cfg(feature = "rocksdb")] -pub type DefaultEmbeddedClient = - EmbeddedClient; +pub type DefaultEmbeddedClient = EmbeddedClient; diff --git a/d-engine-server/src/api/standalone_read_handle.rs b/d-engine-server/src/api/read_handle.rs similarity index 96% rename from d-engine-server/src/api/standalone_read_handle.rs rename to d-engine-server/src/api/read_handle.rs index 0e41d44a..7148b6ea 100644 --- a/d-engine-server/src/api/standalone_read_handle.rs +++ b/d-engine-server/src/api/read_handle.rs @@ -1,4 +1,4 @@ -//! StandaloneReadHandle — read routing for standalone (gRPC) mode. +//! ReadHandle — read routing for standalone (gRPC) mode. //! //! Routes reads through the ReadActor channel for Eventual/LeaseRead fast path, //! falling back to `cmd_tx` (Raft readIndex) for LinearizableRead or on error. @@ -150,22 +150,22 @@ pub(crate) fn read_failure(e: &Error) -> ClientApiError { } } -// ── StandaloneReadHandle ────────────────────────────────────────────────────── +// ── ReadHandle ────────────────────────────────────────────────────── /// Routes read requests to the ReadActor fast path or `cmd_tx` for standalone gRPC mode. /// /// Cloneable — the gRPC `Node` handler holds a clone per request. /// The underlying channels are shared (`mpsc::Sender` is cheaply cloneable). #[derive(Clone)] -pub(crate) struct StandaloneReadHandle { +pub(crate) struct ReadHandle { /// ReadActor channel. `None` if node was started without a ReadActor. pub(crate) read_tx: Option>, /// Raft command channel for the cmd_tx fallback path. pub(crate) cmd_tx: mpsc::Sender, } -impl StandaloneReadHandle { - /// Create a `StandaloneReadHandle` wiring the fast-path sender and the Raft cmd channel. +impl ReadHandle { + /// Create a `ReadHandle` wiring the fast-path sender and the Raft cmd channel. pub(crate) fn new( read_tx: Option>, cmd_tx: mpsc::Sender, @@ -278,5 +278,5 @@ impl StandaloneReadHandle { } #[cfg(test)] -#[path = "standalone_read_handle_test.rs"] +#[path = "read_handle_test.rs"] mod tests; diff --git a/d-engine-server/src/api/standalone_read_handle_test.rs b/d-engine-server/src/api/read_handle_test.rs similarity index 96% rename from d-engine-server/src/api/standalone_read_handle_test.rs rename to d-engine-server/src/api/read_handle_test.rs index 04e175b6..6017937b 100644 --- a/d-engine-server/src/api/standalone_read_handle_test.rs +++ b/d-engine-server/src/api/read_handle_test.rs @@ -1,6 +1,6 @@ -/// Unit tests for StandaloneReadHandle routing logic. +/// Unit tests for ReadHandle routing logic. /// -/// StandaloneReadHandle is the single source of truth for read routing: +/// ReadHandle is the single source of truth for read routing: /// Eventual/LeaseRead → ReadActor fast path; fallback to cmd_tx on LeaseInvalid/SmUnavailable /// LinearizableRead → cmd_tx always /// SmError → returned directly, NO cmd_tx fallback @@ -29,7 +29,7 @@ mod read_handle_tests { use tokio::sync::mpsc; use tokio::task::JoinHandle; - use super::super::StandaloneReadHandle; + use super::super::ReadHandle; use crate::read_actor::run_read_actor; // ── Helpers ─────────────────────────────────────────────────────────────── @@ -78,7 +78,7 @@ mod read_handle_tests { sm } - /// Build a StandaloneReadHandle with a live ReadActor. + /// Build a ReadHandle with a live ReadActor. /// /// `drop_cmd_rx=true` closes the cmd_tx receiver — any cmd_tx fallback /// returns a channel-closed Network error (proof the fast path was NOT taken). @@ -86,14 +86,14 @@ mod read_handle_tests { sm: MockStateMachine, lease: Arc, drop_cmd_rx: bool, - ) -> (StandaloneReadHandle, JoinHandle<()>) { + ) -> (ReadHandle, JoinHandle<()>) { let (read_tx, read_rx) = mpsc::channel(8); let (cmd_tx, cmd_rx) = mpsc::channel(1); if drop_cmd_rx { drop(cmd_rx); } let handle = tokio::spawn(run_read_actor(read_rx, lease, Arc::new(sm), 64)); - let rh = StandaloneReadHandle::new(Some(read_tx), cmd_tx); + let rh = ReadHandle::new(Some(read_tx), cmd_tx); (rh, handle) } @@ -312,12 +312,12 @@ mod read_handle_tests { // ── No read_tx — always cmd_tx ──────────────────────────────────────────── - /// StandaloneReadHandle without read_tx (None) always falls through to cmd_tx for any policy. + /// ReadHandle without read_tx (None) always falls through to cmd_tx for any policy. #[tokio::test] async fn test_no_read_tx_all_policies_use_cmd_tx() { let (cmd_tx, cmd_rx) = mpsc::channel::(1); drop(cmd_rx); - let rh = StandaloneReadHandle::new(None, cmd_tx); + let rh = ReadHandle::new(None, cmd_tx); for policy in [ ReadConsistencyPolicy::EventualConsistency, @@ -344,7 +344,7 @@ mod read_handle_tests { drop(read_rx); // ReadActor is gone drop(cmd_rx); // cmd_tx also closed — proves fallback was attempted - let rh = StandaloneReadHandle::new(Some(read_tx), cmd_tx); + let rh = ReadHandle::new(Some(read_tx), cmd_tx); let result = rh.get(b"k", ReadConsistencyPolicy::EventualConsistency, 1, TIMEOUT).await; assert!( matches!(result, Err(ClientApiError::Network { .. })), @@ -355,19 +355,19 @@ mod read_handle_tests { // ── cmd_tx path: alignment helper ──────────────────────────────────────── - /// Builds a StandaloneReadHandle wired to a one-shot mock Raft responder. + /// Builds a ReadHandle wired to a one-shot mock Raft responder. /// /// The responder returns the given sparse `entries` exactly as /// `read_from_state_machine` would — caller is responsible for verifying /// that `get_batch` re-aligns them to the input key positions. - async fn make_cmd_tx_handle(entries: Vec) -> StandaloneReadHandle { + async fn make_cmd_tx_handle(entries: Vec) -> ReadHandle { let (cmd_tx, mut cmd_rx) = mpsc::channel(1); tokio::spawn(async move { if let Some(ClientCmd::Read(_, resp_tx)) = cmd_rx.recv().await { let _ = resp_tx.send(Ok(ClientResponse::read_results(entries))); } }); - StandaloneReadHandle::new(None, cmd_tx) + ReadHandle::new(None, cmd_tx) } // ── cmd_tx path: HashMap re-alignment correctness ───────────────────────── @@ -646,7 +646,7 @@ mod read_handle_tests { let (cmd_tx, cmd_rx) = mpsc::channel::(1); drop(cmd_rx); - let rh = StandaloneReadHandle::new(Some(read_tx), cmd_tx); + let rh = ReadHandle::new(Some(read_tx), cmd_tx); let short_timeout = Duration::from_millis(50); let start = tokio::time::Instant::now(); @@ -674,8 +674,8 @@ mod read_handle_tests { // ── Error helper tests ──────────────────────────────────────────────────────── // -// These functions live in standalone_read_handle.rs and are shared by both -// StandaloneReadHandle and EmbeddedReadHandle (via re-export). Testing them +// These functions live in read_handle.rs and are shared by both +// ReadHandle and EmbeddedReadHandle (via re-export). Testing them // here keeps the coverage co-located with the implementation. #[cfg(test)] diff --git a/d-engine-server/src/api/standalone.rs b/d-engine-server/src/api/standalone.rs index 7ad01719..6d462b27 100644 --- a/d-engine-server/src/api/standalone.rs +++ b/d-engine-server/src/api/standalone.rs @@ -58,11 +58,11 @@ impl StandaloneEngine { (storage, sm) }; - Self::start_node_with_lock( + Self::start_node_inner( data_dir.as_path().to_path_buf(), - data_dir_lock, + Some(data_dir_lock), Arc::new(storage), - Arc::new(sm), + sm, shutdown_rx, config, ) @@ -107,13 +107,13 @@ impl StandaloneEngine { (storage, sm) }; - // start_node_with_lock directly — config already loaded/validated, + // start_node_inner directly — config already loaded/validated, // lock already held. - Self::start_node_with_lock( + Self::start_node_inner( data_dir.as_path().to_path_buf(), - data_dir_lock, + Some(data_dir_lock), Arc::new(storage), - Arc::new(sm), + sm, shutdown_rx, config, ) @@ -141,7 +141,7 @@ impl StandaloneEngine { pub async fn run_custom( data_dir: impl AsRef, storage_engine: Arc, - state_machine: Arc, + state_machine: SM, shutdown_rx: watch::Receiver<()>, config_path: Option>, ) -> Result<()> @@ -168,7 +168,7 @@ impl StandaloneEngine { pub async fn start_node( data_dir: impl AsRef, storage_engine: Arc, - state_machine: Arc, + state_machine: SM, shutdown_rx: watch::Receiver<()>, config: d_engine_core::RaftNodeConfig, ) -> Result<()> @@ -187,38 +187,11 @@ impl StandaloneEngine { .await } - /// Same as `start_node`, but for callers (`run`/`run_with`) that already - /// opened storage and therefore must have acquired the lock *before* - /// that — see `NodeBuilder::data_dir_lock`'s doc comment. - #[allow(dead_code)] - async fn start_node_with_lock( - data_dir: std::path::PathBuf, - data_dir_lock: crate::node::DataDirLock, - storage_engine: Arc, - state_machine: Arc, - shutdown_rx: watch::Receiver<()>, - config: d_engine_core::RaftNodeConfig, - ) -> Result<()> - where - SE: StorageEngine + std::fmt::Debug + 'static, - SM: StateMachine + std::fmt::Debug + 'static, - { - Self::start_node_inner( - data_dir, - Some(data_dir_lock), - storage_engine, - state_machine, - shutdown_rx, - config, - ) - .await - } - async fn start_node_inner( data_dir: std::path::PathBuf, data_dir_lock: Option, storage_engine: Arc, - state_machine: Arc, + state_machine: SM, shutdown_rx: watch::Receiver<()>, config: d_engine_core::RaftNodeConfig, ) -> Result<()> diff --git a/d-engine-server/src/api/standalone_test.rs b/d-engine-server/src/api/standalone_test.rs index 4fdb9df3..9b619ecc 100644 --- a/d-engine-server/src/api/standalone_test.rs +++ b/d-engine-server/src/api/standalone_test.rs @@ -279,7 +279,7 @@ mod start_node_tests { /// Create a RocksDB storage engine and state machine. fn create_rocksdb_se_and_sm( temp_dir: &tempfile::TempDir - ) -> (Arc, Arc) { + ) -> (Arc, RocksDBStateMachine) { let storage_path = temp_dir.path().join("storage"); let sm_path = temp_dir.path().join("sm"); std::fs::create_dir_all(&storage_path).unwrap(); @@ -288,7 +288,6 @@ mod start_node_tests { let storage = Arc::new(RocksDBStorageEngine::new(&storage_path).expect("create storage engine")); let sm = RocksDBStateMachine::new(&sm_path).expect("create state machine"); - let sm = Arc::new(sm); (storage, sm) } @@ -378,7 +377,6 @@ mod start_node_tests { let storage = Arc::new(RocksDBStorageEngine::new(&storage_path).expect("create storage")); let sm = RocksDBStateMachine::new(&sm_path).expect("create sm"); - let sm = Arc::new(sm); let config = make_config(1); @@ -423,7 +421,7 @@ mod run_custom_tests { /// Create a RocksDB storage engine and state machine. fn create_rocksdb_se_and_sm( temp_dir: &tempfile::TempDir - ) -> (Arc, Arc) { + ) -> (Arc, RocksDBStateMachine) { let storage_path = temp_dir.path().join("storage"); let sm_path = temp_dir.path().join("sm"); std::fs::create_dir_all(&storage_path).unwrap(); @@ -432,7 +430,6 @@ mod run_custom_tests { let storage = Arc::new(RocksDBStorageEngine::new(&storage_path).expect("create storage engine")); let sm = RocksDBStateMachine::new(&sm_path).expect("create state machine"); - let sm = Arc::new(sm); (storage, sm) } diff --git a/d-engine-server/src/network/grpc/grpc_raft_service_fast_path_test.rs b/d-engine-server/src/network/grpc/grpc_raft_service_fast_path_test.rs index 9e1d86a1..71d3eb26 100644 --- a/d-engine-server/src/network/grpc/grpc_raft_service_fast_path_test.rs +++ b/d-engine-server/src/network/grpc/grpc_raft_service_fast_path_test.rs @@ -28,7 +28,7 @@ mod grpc_fast_path_tests { use tonic::Request; use crate::Node; - use crate::api::StandaloneReadHandle; + use crate::api::ReadHandle; use crate::read_actor::run_read_actor; use crate::test_utils::mock_node; @@ -82,7 +82,7 @@ mod grpc_fast_path_tests { drop(cmd_rx); } let handle = tokio::spawn(run_read_actor(read_rx, lease, Arc::new(sm), 64)); - let rh = StandaloneReadHandle::new(Some(read_tx), cmd_tx); + let rh = ReadHandle::new(Some(read_tx), cmd_tx); let (_, graceful_rx) = tokio::sync::watch::channel(()); let mut node = mock_node("/tmp/grpc_fast_path_test", graceful_rx, None); @@ -336,7 +336,7 @@ mod grpc_fast_path_tests { let (_, graceful_rx) = tokio::sync::watch::channel(()); let mut node = mock_node("/tmp/grpc_fast_path_test_no_rt", graceful_rx, None); - node.read_handle = StandaloneReadHandle::new(None, cmd_tx); + node.read_handle = ReadHandle::new(None, cmd_tx); node.ready.store(true, Ordering::SeqCst); for policy in [ diff --git a/d-engine-server/src/node/builder.rs b/d-engine-server/src/node/builder.rs index 6c77b364..44ac3bcc 100644 --- a/d-engine-server/src/node/builder.rs +++ b/d-engine-server/src/node/builder.rs @@ -224,9 +224,9 @@ where /// Sets a custom state machine implementation pub fn state_machine( mut self, - state_machine: Arc, + state_machine: SM, ) -> Self { - self.state_machine = Some(state_machine); + self.state_machine = Some(Arc::new(state_machine)); self } @@ -609,7 +609,7 @@ where state_machine, max_drain, )); - let read_handle = crate::api::StandaloneReadHandle::new(Some(read_tx), cmd_tx.clone()); + let read_handle = crate::api::ReadHandle::new(Some(read_tx), cmd_tx.clone()); let node = Node::> { node_id, diff --git a/d-engine-server/src/node/builder_test.rs b/d-engine-server/src/node/builder_test.rs index 59db795e..de1558a6 100644 --- a/d-engine-server/src/node/builder_test.rs +++ b/d-engine-server/src/node/builder_test.rs @@ -63,7 +63,7 @@ async fn test_set_raft_log_replaces_default() { insert_raft_log(&buffered_raft_log, expected_raft_log_ids.clone(), 1).await; let mock_state_machine = - Arc::new(FileStateMachine::new(temp_dir.path().join("state_machine")).await.unwrap()); + FileStateMachine::new(temp_dir.path().join("state_machine")).await.unwrap(); let expected_state_machine_ids = vec![1, 2, 3]; insert_state_machine(&mock_state_machine, expected_state_machine_ids.clone(), 1).await; @@ -108,7 +108,7 @@ async fn test_build_creates_node() { temp_dir.path().to_str().unwrap(), shutdown_rx, ) - .state_machine(Arc::new(file_sm)) + .state_machine(file_sm) .storage_engine(Arc::new(MockStorageEngine::new())) .build() .await diff --git a/d-engine-server/src/node/mod.rs b/d-engine-server/src/node/mod.rs index d8dd0d46..a9603cab 100644 --- a/d-engine-server/src/node/mod.rs +++ b/d-engine-server/src/node/mod.rs @@ -133,7 +133,7 @@ where pub(crate) _data_dir_lock: DataDirLock, /// Fast-path read routing. Always set; `read_tx = None` means no ReadActor. - pub(crate) read_handle: crate::api::StandaloneReadHandle, + pub(crate) read_handle: crate::api::ReadHandle, } impl Debug for Node diff --git a/d-engine-server/src/node/node_test.rs b/d-engine-server/src/node/node_test.rs index f873f250..b26f9c1b 100644 --- a/d-engine-server/src/node/node_test.rs +++ b/d-engine-server/src/node/node_test.rs @@ -1085,7 +1085,7 @@ mod sm_worker_lifecycle_tests { let storage = Arc::new(FileStorageEngine::new(temp_dir.path().join("storage")).expect("storage")); - let sm = Arc::new(FileStateMachine::new(temp_dir.path().join("sm")).await.expect("sm")); + let sm = FileStateMachine::new(temp_dir.path().join("sm")).await.expect("sm"); let node = NodeBuilder::::from_node_config( temp_dir.path(), config, @@ -1135,7 +1135,7 @@ mod sm_worker_lifecycle_tests { let storage = Arc::new(FileStorageEngine::new(temp_dir.path().join("storage")).expect("storage")); - let sm = Arc::new(FileStateMachine::new(temp_dir.path().join("sm")).await.expect("sm")); + let sm = FileStateMachine::new(temp_dir.path().join("sm")).await.expect("sm"); let node = NodeBuilder::::from_node_config( temp_dir.path(), config, diff --git a/d-engine-server/src/test_utils/mock/mock_node_builder.rs b/d-engine-server/src/test_utils/mock/mock_node_builder.rs index bb519628..1f647700 100644 --- a/d-engine-server/src/test_utils/mock/mock_node_builder.rs +++ b/d-engine-server/src/test_utils/mock/mock_node_builder.rs @@ -351,7 +351,7 @@ impl MockBuilder { raft_core: Arc::new(Mutex::new(raft)), membership, event_tx, - read_handle: crate::api::StandaloneReadHandle::new(None, cmd_tx.clone()), + read_handle: crate::api::ReadHandle::new(None, cmd_tx.clone()), cmd_tx, ready: AtomicBool::new(false), rpc_ready_tx, @@ -409,7 +409,7 @@ impl MockBuilder { raft_core: Arc::new(Mutex::new(raft)), membership, event_tx, - read_handle: crate::api::StandaloneReadHandle::new(None, cmd_tx.clone()), + read_handle: crate::api::ReadHandle::new(None, cmd_tx.clone()), cmd_tx, ready: AtomicBool::new(false), rpc_ready_tx, diff --git a/d-engine-server/src/test_utils/mod.rs b/d-engine-server/src/test_utils/mod.rs index f27a01d1..f93d8f4b 100644 --- a/d-engine-server/src/test_utils/mod.rs +++ b/d-engine-server/src/test_utils/mod.rs @@ -23,6 +23,10 @@ mod integration; mod mock; +// One source of truth for the shared-state-machine wrapper used by both unit and integration tests. +#[path = "../../tests/common/shared_state_machine.rs"] +mod shared_state_machine; pub use integration::*; pub use mock::*; +pub use shared_state_machine::SharedStateMachine; diff --git a/d-engine-server/tests/cas_operations/bank_transfer_invariant_embedded.rs b/d-engine-server/tests/cas_operations/bank_transfer_invariant_embedded.rs index 02e5a3a1..0b6eaad8 100644 --- a/d-engine-server/tests/cas_operations/bank_transfer_invariant_embedded.rs +++ b/d-engine-server/tests/cas_operations/bank_transfer_invariant_embedded.rs @@ -83,7 +83,7 @@ async fn test_concurrent_transfers_preserve_bank_invariant() let engine = DefaultEmbeddedEngine::start_custom( &node_db_root, Arc::new(storage), - Arc::new(state_machine), + state_machine, Some(&config_path), ) .await?; diff --git a/d-engine-server/tests/cas_operations/distributed_lock_embedded.rs b/d-engine-server/tests/cas_operations/distributed_lock_embedded.rs index ddcc04c6..a702559c 100644 --- a/d-engine-server/tests/cas_operations/distributed_lock_embedded.rs +++ b/d-engine-server/tests/cas_operations/distributed_lock_embedded.rs @@ -58,7 +58,7 @@ async fn test_distributed_lock_embedded() -> Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), ClientApiError> { Some(last_log_id.saturating_sub(node_config.raft.snapshot.retained_log_entries)); // Start the node with its specific state machine and storage engine - // Create fresh Arc for state machine to ensure single ownership - let state_machine = Arc::new( - prepare_state_machine(node_id as u32, &format!("{db_dir}/cs/{node_id}")).await, - ); + let state_machine = + prepare_state_machine(node_id as u32, &format!("{db_dir}/cs/{node_id}")).await; let (graceful_tx, node_handle) = start_node( format!("{db_dir}/cs/{node_id}"), node_config, @@ -223,8 +220,7 @@ async fn test_join_cluster_scenario2() -> Result<(), ClientApiError> { node_config.raft.snapshot.cleanup_retain_count = 2; node_config.raft.snapshot.chunk_size = 100; - // Create fresh Arc for node 4 state machine to ensure single ownership - let state_machine_4 = Arc::new(prepare_state_machine(4, &format!("{db_dir}/cs/4")).await); + let state_machine_4 = prepare_state_machine(4, &format!("{db_dir}/cs/4")).await; let (graceful_tx4, node_n4) = start_node( format!("{db_dir}/cs/4"), node_config, @@ -267,7 +263,7 @@ async fn test_join_cluster_scenario2() -> Result<(), ClientApiError> { node_config.raft.snapshot.chunk_size = 100; // Create fresh Arc for node 5 state machine to ensure single ownership - let state_machine_5 = Arc::new(prepare_state_machine(5, &format!("{db_dir}/cs/5")).await); + let state_machine_5 = prepare_state_machine(5, &format!("{db_dir}/cs/5")).await; let (graceful_tx5, node_n5) = start_node( format!("{db_dir}/cs/5"), node_config, diff --git a/d-engine-server/tests/cluster_membership/join_cluster_single_join_standalone.rs b/d-engine-server/tests/cluster_membership/join_cluster_single_join_standalone.rs index d0834bf5..87bc7ede 100644 --- a/d-engine-server/tests/cluster_membership/join_cluster_single_join_standalone.rs +++ b/d-engine-server/tests/cluster_membership/join_cluster_single_join_standalone.rs @@ -3,7 +3,6 @@ //! This test verifies that a new node can successfully join an existing Raft cluster, //! synchronize its state via snapshot, and become a fully functional member. -use std::sync::Arc; use std::time::Duration; use bytes::Bytes; @@ -102,14 +101,12 @@ async fn test_join_cluster_scenario1() -> Result<(), ClientApiError> { snapshot_last_included_id = Some(last_log_id.saturating_sub(node_config.raft.snapshot.retained_log_entries)); - // Create state machine and storage engine for this node (Arc refcount = 1) - let state_machine = Arc::new( - prepare_state_machine( - node_id as u32, - &format!("{JOIN_CLUSTER_CASE1_DATA_DIR}/cs/{node_id}"), - ) - .await, - ); + // Create state machine and storage engine for this node + let state_machine = prepare_state_machine( + node_id as u32, + &format!("{JOIN_CLUSTER_CASE1_DATA_DIR}/cs/{node_id}"), + ) + .await; let storage_engine = match i { 0 => storage_engine_1.clone(), 1 => storage_engine_2.clone(), @@ -174,9 +171,9 @@ async fn test_join_cluster_scenario1() -> Result<(), ClientApiError> { node_config.raft.snapshot.cleanup_retain_count = 2; node_config.raft.snapshot.chunk_size = 100; - // Create state machine and storage engine for node 4 (Arc refcount = 1) + // Create state machine and storage engine for node 4 let node4_state_machine = - Arc::new(prepare_state_machine(4, &format!("{JOIN_CLUSTER_CASE1_DATA_DIR}/cs/4")).await); + prepare_state_machine(4, &format!("{JOIN_CLUSTER_CASE1_DATA_DIR}/cs/4")).await; let node4_storage_engine = prepare_storage_engine(4, &format!("{JOIN_CLUSTER_CASE1_DATA_DIR}/cs/4"), 0); diff --git a/d-engine-server/tests/cluster_state_and_metadata/cluster_state_consensus_embedded.rs b/d-engine-server/tests/cluster_state_and_metadata/cluster_state_consensus_embedded.rs index d80af582..2b816ce7 100644 --- a/d-engine-server/tests/cluster_state_and_metadata/cluster_state_consensus_embedded.rs +++ b/d-engine-server/tests/cluster_state_and_metadata/cluster_state_consensus_embedded.rs @@ -106,7 +106,7 @@ listen_address = '127.0.0.1:{}' let engine1 = DefaultEmbeddedEngine::start_custom( &node1_data_dir, Arc::new(storage1), - Arc::new(sm1), + sm1, Some(node1_config_path), ) .await?; @@ -118,7 +118,7 @@ listen_address = '127.0.0.1:{}' let engine2 = DefaultEmbeddedEngine::start_custom( &node2_data_dir, Arc::new(storage2), - Arc::new(sm2), + sm2, Some(node2_config_path), ) .await?; @@ -130,7 +130,7 @@ listen_address = '127.0.0.1:{}' let engine3 = DefaultEmbeddedEngine::start_custom( &node3_data_dir, Arc::new(storage3), - Arc::new(sm3), + sm3, Some(node3_config_path), ) .await?; @@ -254,7 +254,7 @@ general_raft_timeout_duration_in_ms = 3000 let engine1 = DefaultEmbeddedEngine::start_custom( &node1_data_dir, Arc::new(storage1), - Arc::new(sm1), + sm1, Some(node1_config_path), ) .await?; @@ -266,7 +266,7 @@ general_raft_timeout_duration_in_ms = 3000 let engine2 = DefaultEmbeddedEngine::start_custom( &node2_data_dir, Arc::new(storage2), - Arc::new(sm2), + sm2, Some(node2_config_path), ) .await?; @@ -278,7 +278,7 @@ general_raft_timeout_duration_in_ms = 3000 let engine3 = DefaultEmbeddedEngine::start_custom( &node3_data_dir, Arc::new(storage3), - Arc::new(sm3), + sm3, Some(node3_config_path), ) .await?; @@ -434,7 +434,7 @@ general_raft_timeout_duration_in_ms = 3000 let engine1 = DefaultEmbeddedEngine::start_custom( &node1_data_dir, Arc::new(storage1), - Arc::new(sm1), + sm1, Some(node1_config_path), ) .await?; @@ -446,7 +446,7 @@ general_raft_timeout_duration_in_ms = 3000 let engine2 = DefaultEmbeddedEngine::start_custom( &node2_data_dir, Arc::new(storage2), - Arc::new(sm2), + sm2, Some(node2_config_path), ) .await?; @@ -458,7 +458,7 @@ general_raft_timeout_duration_in_ms = 3000 let engine3 = DefaultEmbeddedEngine::start_custom( &node3_data_dir, Arc::new(storage3), - Arc::new(sm3), + sm3, Some(node3_config_path), ) .await?; diff --git a/d-engine-server/tests/common/mod.rs b/d-engine-server/tests/common/mod.rs index 1a0398f4..db03ecef 100644 --- a/d-engine-server/tests/common/mod.rs +++ b/d-engine-server/tests/common/mod.rs @@ -47,6 +47,10 @@ use tracing::error; use crate::client_manager::ClientManager; +mod shared_state_machine; + +pub use shared_state_machine::SharedStateMachine; + pub const WAIT_FOR_NODE_READY_IN_SEC: u64 = 6; // we are testing linearizable read from Leader directly, the latency should less than 1ms ideally @@ -282,7 +286,7 @@ pub async fn start_cluster( pub async fn start_node( data_dir: impl AsRef, config: RaftNodeConfig, - state_machine: Option>>>, + state_machine: Option>>, storage_engine: Option>>>, ) -> std::result::Result< ( @@ -310,11 +314,9 @@ pub async fn start_node( Some(sm) => sm, None => { let state_machine_path = data_dir.join("state_machine"); - Arc::new( - FileStateMachine::new(state_machine_path).await.map_err(|e| { - std::io::Error::other(format!("failed to create file state machine: {e}")) - })?, - ) + FileStateMachine::new(state_machine_path).await.map_err(|e| { + std::io::Error::other(format!("failed to create file state machine: {e}")) + })? } }; diff --git a/d-engine-server/tests/common/shared_state_machine.rs b/d-engine-server/tests/common/shared_state_machine.rs new file mode 100644 index 00000000..03c8c4b5 --- /dev/null +++ b/d-engine-server/tests/common/shared_state_machine.rs @@ -0,0 +1,120 @@ +//! A state machine that forwards every call to an inner state machine shared through an `Arc`. +//! +//! The engine takes its state machine by value, so a test can no longer keep a second reference +//! to the very instance the engine uses. Tests that must reach that instance while the engine +//! runs (to read it directly, or to simulate a crash by closing its storage) wrap the shared +//! `Arc` in this type, hand the wrapper to the engine and keep their own `Arc`. +//! +//! This file is also compiled into the crate's unit tests, so it only names `d_engine_core` and +//! `d_engine_proto` paths. + +use std::fmt::Debug; +use std::sync::Arc; + +use async_trait::async_trait; +use bytes::Bytes; +use d_engine_core::ApplyEntry; +use d_engine_core::ApplyResult; +use d_engine_core::Error; +use d_engine_core::ScanResult; +use d_engine_core::SnapshotApplyResult; +use d_engine_core::StateMachine; +use d_engine_proto::common::LogId; +use d_engine_proto::server::storage::SnapshotMetadata; + +/// Forwards the whole [`StateMachine`] trait to a shared inner state machine. +#[derive(Debug)] +pub struct SharedStateMachine(Arc); + +impl SharedStateMachine { + pub fn new(inner: Arc) -> Self { + Self(inner) + } +} + +#[async_trait] +impl StateMachine for SharedStateMachine { + fn close(&self) -> Result<(), Error> { + self.0.close() + } + fn get( + &self, + key_buffer: &[u8], + ) -> Result, Error> { + self.0.get(key_buffer) + } + fn get_multi( + &self, + keys: &[Bytes], + ) -> Result>, Error> { + self.0.get_multi(keys) + } + async fn apply_chunk( + &self, + chunk: &[ApplyEntry], + ) -> Result, Error> { + self.0.apply_chunk(chunk).await + } + fn update_last_applied( + &self, + last_applied: LogId, + ) { + self.0.update_last_applied(last_applied) + } + fn last_applied(&self) -> LogId { + self.0.last_applied() + } + fn persist_last_applied( + &self, + last_applied: LogId, + ) -> Result<(), Error> { + self.0.persist_last_applied(last_applied) + } + fn update_last_snapshot_metadata( + &self, + snapshot_metadata: &SnapshotMetadata, + ) -> Result<(), Error> { + self.0.update_last_snapshot_metadata(snapshot_metadata) + } + fn snapshot_metadata(&self) -> Option { + self.0.snapshot_metadata() + } + fn persist_last_snapshot_metadata( + &self, + snapshot_metadata: &SnapshotMetadata, + ) -> Result<(), Error> { + self.0.persist_last_snapshot_metadata(snapshot_metadata) + } + async fn apply_snapshot_from_file( + &self, + metadata: &SnapshotMetadata, + snapshot_path: std::path::PathBuf, + ) -> Result { + self.0.apply_snapshot_from_file(metadata, snapshot_path).await + } + async fn generate_snapshot_data( + &self, + new_snapshot_dir: std::path::PathBuf, + last_included: LogId, + ) -> Result { + self.0.generate_snapshot_data(new_snapshot_dir, last_included).await + } + fn save_hard_state(&self) -> Result<(), Error> { + self.0.save_hard_state() + } + fn flush(&self) -> Result<(), Error> { + self.0.flush() + } + async fn flush_async(&self) -> Result<(), Error> { + self.0.flush_async().await + } + async fn reset(&self) -> Result<(), Error> { + self.0.reset().await + } + fn scan_prefix( + &self, + prefix: &[u8], + ) -> Result { + self.0.scan_prefix(prefix) + } +} diff --git a/d-engine-server/tests/consistent_reads/linearizable_read_consistency_embedded.rs b/d-engine-server/tests/consistent_reads/linearizable_read_consistency_embedded.rs index f970356f..3d145d05 100644 --- a/d-engine-server/tests/consistent_reads/linearizable_read_consistency_embedded.rs +++ b/d-engine-server/tests/consistent_reads/linearizable_read_consistency_embedded.rs @@ -333,7 +333,7 @@ async fn test_read_index_fixed_with_concurrent_writes_multi_node() let engine = DefaultEmbeddedEngine::start_custom( &db_path, Arc::new(storage), - Arc::new(state_machine), + state_machine, Some(&config_path), ) .await?; diff --git a/d-engine-server/tests/failover_and_recovery/leader_failover_embedded.rs b/d-engine-server/tests/failover_and_recovery/leader_failover_embedded.rs index 8a32b1cc..c8ead465 100644 --- a/d-engine-server/tests/failover_and_recovery/leader_failover_embedded.rs +++ b/d-engine-server/tests/failover_and_recovery/leader_failover_embedded.rs @@ -63,7 +63,7 @@ async fn test_embedded_leader_failover() -> Result<(), Box Result<(), Box> { let engine = DefaultEmbeddedEngine::start_custom( &node_db_root, Arc::new(storage), - Arc::new(state_machine), + state_machine, Some(&configs[i].1), ) .await?; @@ -333,7 +333,7 @@ async fn test_embedded_node_rejoin() -> Result<(), Box> { let restarted_engine = DefaultEmbeddedEngine::start_custom( &node_db_root, Arc::new(storage), - Arc::new(state_machine), + state_machine, Some(&killed_config.1), ) .await?; @@ -408,7 +408,7 @@ async fn test_minority_failure_blocks_writes() -> Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), ClientApiError> { ) .await; - // Create fresh Arc for state machine to ensure single ownership - let state_machine = Arc::new( - prepare_state_machine( - node_id as u32, - &format!("{SNAPSHOT_CASE1_DATA_DIR}/cs/{node_id}"), - ) - .await, - ); + let state_machine = prepare_state_machine( + node_id as u32, + &format!("{SNAPSHOT_CASE1_DATA_DIR}/cs/{node_id}"), + ) + .await; let raft_log = match i { 0 => Some(r1.clone()), diff --git a/d-engine-server/tests/snapshot_and_recovery/snapshot_interrupted_transfer_embedded.rs b/d-engine-server/tests/snapshot_and_recovery/snapshot_interrupted_transfer_embedded.rs index f2c6eeef..48d1bf45 100644 --- a/d-engine-server/tests/snapshot_and_recovery/snapshot_interrupted_transfer_embedded.rs +++ b/d-engine-server/tests/snapshot_and_recovery/snapshot_interrupted_transfer_embedded.rs @@ -100,7 +100,7 @@ retained_log_entries = {RETAINED_LOGS} let engine = DefaultEmbeddedEngine::start_custom( &db_path, Arc::new(storage), - Arc::new(sm), + sm, Some(config_path.to_str().unwrap()), ) .await?; @@ -177,7 +177,7 @@ retained_log_entries = {RETAINED_LOGS} let learner_engine = DefaultEmbeddedEngine::start_custom( &learner_db_path, Arc::new(learner_storage), - Arc::new(learner_sm), + learner_sm, Some(learner_config_path.to_str().unwrap()), ) .await?; diff --git a/d-engine-server/tests/snapshot_and_recovery/snapshot_leader_change_during_transfer_embedded.rs b/d-engine-server/tests/snapshot_and_recovery/snapshot_leader_change_during_transfer_embedded.rs index b0206a88..85901ce4 100644 --- a/d-engine-server/tests/snapshot_and_recovery/snapshot_leader_change_during_transfer_embedded.rs +++ b/d-engine-server/tests/snapshot_and_recovery/snapshot_leader_change_during_transfer_embedded.rs @@ -100,7 +100,7 @@ retained_log_entries = {RETAINED_LOGS} let engine = DefaultEmbeddedEngine::start_custom( &db_path, Arc::new(storage), - Arc::new(sm), + sm, Some(config_path.to_str().unwrap()), ) .await?; @@ -169,7 +169,7 @@ retained_log_entries = {RETAINED_LOGS} let learner_engine = DefaultEmbeddedEngine::start_custom( &learner_db_path, Arc::new(learner_storage), - Arc::new(learner_sm), + learner_sm, Some(learner_config_path.to_str().unwrap()), ) .await?; diff --git a/d-engine-server/tests/snapshot_and_recovery/snapshot_transfer_does_not_block_apply_embedded.rs b/d-engine-server/tests/snapshot_and_recovery/snapshot_transfer_does_not_block_apply_embedded.rs index 37a2d295..831c3448 100644 --- a/d-engine-server/tests/snapshot_and_recovery/snapshot_transfer_does_not_block_apply_embedded.rs +++ b/d-engine-server/tests/snapshot_and_recovery/snapshot_transfer_does_not_block_apply_embedded.rs @@ -21,6 +21,7 @@ use d_engine_server::install_snapshot_transfer_gate; use serial_test::serial; use tracing::info; +use crate::common::SharedStateMachine; use crate::common::get_available_ports; /// Deterministic, incompressible payload so the snapshot does not shrink under gzip — @@ -166,7 +167,7 @@ push_queue_size = 1 let engine = DefaultEmbeddedEngine::start_custom( &db_path, Arc::new(storage), - sm, + SharedStateMachine::new(sm), Some(config_path.to_str().unwrap()), ) .await?; @@ -295,7 +296,7 @@ push_queue_size = 1 let learner_engine = DefaultEmbeddedEngine::start_custom( &learner_db_path, Arc::new(learner_storage), - Arc::::clone(&learner_sm), + SharedStateMachine::new(Arc::::clone(&learner_sm)), Some(learner_config_path.to_str().unwrap()), ) .await?; diff --git a/d-engine-server/tests/watch_and_subscriptions/watch_events_embedded.rs b/d-engine-server/tests/watch_and_subscriptions/watch_events_embedded.rs index 8bb6089a..793a428f 100644 --- a/d-engine-server/tests/watch_and_subscriptions/watch_events_embedded.rs +++ b/d-engine-server/tests/watch_and_subscriptions/watch_events_embedded.rs @@ -39,7 +39,7 @@ watcher_buffer_size = 10 let engine = DefaultEmbeddedEngine::start_custom( &db_path, Arc::new(storage), - Arc::new(state_machine), + state_machine, Some(config_path.to_str().unwrap()), ) .await?; @@ -122,7 +122,7 @@ listen_address = "127.0.0.1:{port}" let engine = DefaultEmbeddedEngine::start_custom( &db_path, Arc::new(storage), - Arc::new(state_machine), + state_machine, Some(config_path.to_str().unwrap()), ) .await?; @@ -365,7 +365,7 @@ watcher_buffer_size = 10 let engine = DefaultEmbeddedEngine::start_custom( &db_path, Arc::new(storage), - Arc::new(state_machine), + state_machine, Some(config_path.to_str().unwrap()), ) .await?; diff --git a/d-engine-server/tests/watch_and_subscriptions/watch_events_grpc_standalone.rs b/d-engine-server/tests/watch_and_subscriptions/watch_events_grpc_standalone.rs index 1e884b18..21277b28 100644 --- a/d-engine-server/tests/watch_and_subscriptions/watch_events_grpc_standalone.rs +++ b/d-engine-server/tests/watch_and_subscriptions/watch_events_grpc_standalone.rs @@ -73,7 +73,7 @@ async fn setup_standalone_cluster() tokio::fs::create_dir_all(&sm_path).await?; let storage = Arc::new(FileStorageEngine::new(storage_path)?); - let state_machine = Arc::new(FileStateMachine::new(sm_path).await?); + let state_machine = FileStateMachine::new(sm_path).await?; // Start node let (graceful_tx, node_handle) = diff --git a/d-engine-server/tests/watch_and_subscriptions/watch_membership_embedded.rs b/d-engine-server/tests/watch_and_subscriptions/watch_membership_embedded.rs index bbd454a0..7a5d4233 100644 --- a/d-engine-server/tests/watch_and_subscriptions/watch_membership_embedded.rs +++ b/d-engine-server/tests/watch_and_subscriptions/watch_membership_embedded.rs @@ -192,7 +192,7 @@ async fn start_engine( Ok(DefaultEmbeddedEngine::start_custom( &node_data_dir, Arc::new(storage), - Arc::new(sm), + sm, Some(config_path), ) .await?) diff --git a/d-engine-server/tests/watch_and_subscriptions/watch_performance_gate_embedded.rs b/d-engine-server/tests/watch_and_subscriptions/watch_performance_gate_embedded.rs index 092f3032..8daf7c9c 100644 --- a/d-engine-server/tests/watch_and_subscriptions/watch_performance_gate_embedded.rs +++ b/d-engine-server/tests/watch_and_subscriptions/watch_performance_gate_embedded.rs @@ -50,7 +50,7 @@ watcher_buffer_size = 100 let engine = DefaultEmbeddedEngine::start_custom( temp_dir.path(), Arc::new(storage), - Arc::new(state_machine), + state_machine, Some(config_path.to_str().unwrap()), ) .await diff --git a/d-engine/src/docs/quick-start-5min.md b/d-engine/src/docs/quick-start-5min.md index c4ee9ab4..cf6ae7a0 100644 --- a/d-engine/src/docs/quick-start-5min.md +++ b/d-engine/src/docs/quick-start-5min.md @@ -209,18 +209,18 @@ DefaultEmbeddedEngine::start_with( ) -> Result // Start with programmatic config — no config file needed -EmbeddedEngine::::start_node( +EmbeddedEngine::start_node( data_dir: impl AsRef, storage: Arc, - state_machine: Arc, + state_machine: impl StateMachine, config: RaftNodeConfig ) -> Result // Advanced (custom storage + state machine with optional config file) -EmbeddedEngine::::start_custom( +EmbeddedEngine::start_custom( data_dir: impl AsRef, storage: Arc, - state_machine: Arc, + state_machine: impl StateMachine, config_path: Option<&str> ) -> Result diff --git a/d-engine/src/docs/server_guide/customize-state-machine.md b/d-engine/src/docs/server_guide/customize-state-machine.md index 03db8657..a207a22e 100644 --- a/d-engine/src/docs/server_guide/customize-state-machine.md +++ b/d-engine/src/docs/server_guide/customize-state-machine.md @@ -213,7 +213,7 @@ Run performance tests locally: `cargo test -- --ignored` use d_engine::{EmbeddedEngine, FileStorageEngine}; let storage_engine = Arc::new(FileStorageEngine::new("./data/storage").await?); -let state_machine = Arc::new(CustomStateMachine::new().await?); +let state_machine = CustomStateMachine::new().await?; let engine = EmbeddedEngine::start_custom( "./data", // data_dir diff --git a/d-engine/src/docs/server_guide/customize-storage-engine.md b/d-engine/src/docs/server_guide/customize-storage-engine.md index c337141e..dafa26b4 100644 --- a/d-engine/src/docs/server_guide/customize-storage-engine.md +++ b/d-engine/src/docs/server_guide/customize-storage-engine.md @@ -179,7 +179,7 @@ async fn test_performance() -> Result<(), Error> { use d_engine::{EmbeddedEngine, FileStateMachine}; let storage_engine = Arc::new(CustomStorageEngine::new().await?); -let state_machine = Arc::new(FileStateMachine::new("./data/state_machine").await?); +let state_machine = FileStateMachine::new("./data/state_machine").await?; let engine = EmbeddedEngine::start_custom( "./data", // data_dir diff --git a/examples/raft-engine-log/src/main.rs b/examples/raft-engine-log/src/main.rs index 1896dadf..a4e036c2 100644 --- a/examples/raft-engine-log/src/main.rs +++ b/examples/raft-engine-log/src/main.rs @@ -14,7 +14,7 @@ use raft_engine_log::{RaftEngineOptions, RaftEngineStorage}; use serde::{Deserialize, Serialize}; use tokio::sync::watch; -type Engine = EmbeddedEngine; +type Engine = EmbeddedEngine; #[derive(Debug, Deserialize)] struct PutRequest { @@ -67,8 +67,7 @@ async fn main() { RaftEngineStorage::open_with(format!("{}/log", cli.data_dir), &options) .expect("open log"), ); - let sm = - Arc::new(RocksDBStateMachine::new(format!("{}/state", cli.data_dir)).expect("open state")); + let sm = RocksDBStateMachine::new(format!("{}/state", cli.data_dir)).expect("open state"); let engine = Arc::new( Engine::start_custom(&cli.data_dir, storage, sm, Some(cli.config_path.as_str())) diff --git a/examples/single-node-expansion/src/main.rs b/examples/single-node-expansion/src/main.rs index 9f56a3db..c32faf9b 100644 --- a/examples/single-node-expansion/src/main.rs +++ b/examples/single-node-expansion/src/main.rs @@ -95,7 +95,7 @@ async fn start_dengine_server( ) { // Option 1: RAW FILE // let storage_engine = Arc::new(FileStorageEngine::new(db_path.join("storage_engine")).unwrap()); - // let state_machine = Arc::new(FileStateMachine::new(db_path.join("state_machine")).await.unwrap()); + // let state_machine = FileStateMachine::new(db_path.join("state_machine")).await.unwrap(); // Option 2: ROCKSDB (unified engine — single DB instance, 4 column families) let (storage_engine, state_machine) = RocksDBUnifiedEngine::open(db_path.join("db")).unwrap(); @@ -104,7 +104,7 @@ async fn start_dengine_server( if let Err(e) = StandaloneEngine::run_custom( &db_path, Arc::new(storage_engine), - Arc::new(state_machine), + state_machine, graceful_rx, None::<&str>, ) diff --git a/examples/sled-cluster/src/main.rs b/examples/sled-cluster/src/main.rs index 66d9aab6..4ade5a40 100644 --- a/examples/sled-cluster/src/main.rs +++ b/examples/sled-cluster/src/main.rs @@ -67,8 +67,7 @@ async fn start_dengine_server( ) { let storage_engine = Arc::new(SledStorageEngine::new(db_path.join("storage_engine"), node_id).unwrap()); - let state_machine = - Arc::new(SledStateMachine::new(db_path.join("state_machine"), node_id).unwrap()); + let state_machine = SledStateMachine::new(db_path.join("state_machine"), node_id).unwrap(); // Start Node — blocks until shutdown. if let Err(e) = StandaloneEngine::run_custom( diff --git a/examples/three-nodes-standalone/src/main.rs b/examples/three-nodes-standalone/src/main.rs index 322dfcd5..7c3ea564 100644 --- a/examples/three-nodes-standalone/src/main.rs +++ b/examples/three-nodes-standalone/src/main.rs @@ -141,9 +141,7 @@ async fn run_engine( format!("{data_dir}/storage"), &options, )?); - let sm = Arc::new(d_engine::RocksDBStateMachine::new(format!( - "{data_dir}/state_machine" - ))?); + let sm = d_engine::RocksDBStateMachine::new(format!("{data_dir}/state_machine"))?; StandaloneEngine::run_custom(data_dir, storage, sm, graceful_rx, Some(config_path)).await } From 907cebbd70e20e339479c484323fa797dcec6b17 Mon Sep 17 00:00:00 2001 From: Joshua Chi <112539+JoshuaChi@users.noreply.github.com> Date: Sun, 1 Nov 2026 19:13:09 +0800 Subject: [PATCH 04/19] refactor #400: Make snapshot checksums an engine responsibility generate_snapshot_data now returns Result<()>. The engine computes the SHA-256 of the exported snapshot directory when it captures a snapshot, and verifies it before calling apply_snapshot_from_file, so a state machine neither computes nor checks a checksum. The algorithm is unchanged. The file and RocksDB state machines drop their own checksum code. The sled example follows, and no longer needs sha2. Tests: - Add engine-level tests: capture records the checksum of the exported directory; a mismatching checksum is rejected without calling the state machine; a matching one installs exactly once. - Remove test_snapshot_checksum_validation. It required a state machine to return and enforce a checksum, which is no longer its job. - test_build_local_snapshot_removes_partial_archive_on_compression_failure builds its staging directory by hand: a real capture now reads every exported file to checksum it, so an unreadable file would fail the capture before compression. The property it asserts is unchanged. Docs: MIGRATION_GUIDE and CHANGELOG describe the new contract. --- CHANGELOG.md | 2 + MIGRATION_GUIDE.md | 1 + .../leader_state_test/snapshot_worker_test.rs | 4 +- .../default_state_machine_handler_test.rs | 176 +++++++++++++++--- .../src/state_machine_handler/writer.rs | 30 ++- d-engine-core/src/storage/state_machine.rs | 15 +- .../src/storage/state_machine_test.rs | 84 +-------- .../src/test_utils/mock/mock_raft_builder.rs | 4 +- .../api/embedded_test/shutdown_order_test.rs | 2 +- .../adaptors/file/file_state_machine.rs | 16 +- .../adaptors/file/file_state_machine_test.rs | 5 +- .../adaptors/rocksdb/rocksdb_state_machine.rs | 17 +- .../src/test_utils/mock/mock_node_builder.rs | 4 +- .../tests/common/shared_state_machine.rs | 2 +- examples/sled-cluster/Cargo.toml | 1 - examples/sled-cluster/src/lib.rs | 33 ---- .../sled-cluster/src/sled_state_machine.rs | 34 +--- .../state_machine/sled_state_machine_test.rs | 5 +- 18 files changed, 208 insertions(+), 227 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 361e5bce..8d7da7d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -116,6 +116,8 @@ All notable changes to this project will be documented in this file. "MemFirst"` or `"DiskFirst"` are silently ignored, not an error — remove the field, `flush_policy` is the only persistence knob now. +- **⚠️ `StateMachine::generate_snapshot_data` returns `Result<()>`; the engine owns snapshot checksums** (#400): the engine computes the SHA-256 of the exported snapshot directory and verifies it before calling `apply_snapshot_from_file`, which now only ever receives a directory that matches the recorded checksum. A state machine neither returns nor checks a checksum. The checksum algorithm is unchanged. See [MIGRATION_GUIDE.md](./MIGRATION_GUIDE.md). + - **⚠️ The engine takes the state machine by value** (#400): `EmbeddedEngine::start_custom`/`start_node`, `StandaloneEngine::run_custom`/`start_node` take `SM` instead of `Arc`; the `Arc` is created inside the engine. Nothing outside the engine holds the state machine, so `stop()` returns only after it is dropped and its resources are released. Embedded reads now go through the ReadActor like standalone reads (`EmbeddedReadHandle` is removed). `EmbeddedEngine` and `EmbeddedClient` no longer have type parameters. See [MIGRATION_GUIDE.md](./MIGRATION_GUIDE.md). ### Removed diff --git a/MIGRATION_GUIDE.md b/MIGRATION_GUIDE.md index 5ad028c9..00ac72c5 100644 --- a/MIGRATION_GUIDE.md +++ b/MIGRATION_GUIDE.md @@ -33,6 +33,7 @@ EmbeddedEngine::start_custom(dir, storage, sm, None).await?; - **`EmbeddedEngine` and `EmbeddedClient` no longer have type parameters.** Write `EmbeddedEngine` and `EmbeddedClient`. `DefaultEmbeddedEngine` and `DefaultEmbeddedClient` still exist and are now plain aliases of them. - **If you kept a clone of your `Arc` to read local data directly** (for example `DLmdb` reading its own LMDB), you can no longer clone the instance the engine uses. Share a handle to your storage instead: put the handles (LMDB `Env`, sled `Db`, `Arc`) behind an `Arc` inside your state machine type, derive `Clone` on it, give the engine one clone and keep the other. - **Embedded reads (`get_eventual`, lease reads) are answered by the ReadActor**, as they already were in standalone mode, instead of reading the state machine on the caller's thread. Expect one extra channel hop per read. +- **Snapshot checksums belong to the engine.** `generate_snapshot_data` now returns `Result<()>`: write the snapshot into the directory you are given and return. The engine computes the checksum of that directory itself, and verifies it before it calls `apply_snapshot_from_file`, so you no longer compute or check a checksum. Delete that code (and any hashing dependency you added only for it). - When `stop()` returns, the engine has dropped the state machine. If your state machine releases its resources in `Drop` (a database handle, a lock file), they are free at that point and the same data directory can be reopened. --- diff --git a/d-engine-core/src/raft_role/leader_state_test/snapshot_worker_test.rs b/d-engine-core/src/raft_role/leader_state_test/snapshot_worker_test.rs index c8160518..4ffea524 100644 --- a/d-engine-core/src/raft_role/leader_state_test/snapshot_worker_test.rs +++ b/d-engine-core/src/raft_role/leader_state_test/snapshot_worker_test.rs @@ -99,7 +99,6 @@ fn stub_snapshot_metadata() -> SnapshotMetadata { } fn state_machine_with_snapshot() -> MockStateMachine { - use bytes::Bytes; use d_engine_proto::common::LogId; let mut sm = MockStateMachine::new(); sm.expect_close().returning(|| Ok(())); @@ -117,8 +116,7 @@ fn state_machine_with_snapshot() -> MockStateMachine { last_included: LogId::default(), }) }); - sm.expect_generate_snapshot_data() - .returning(|_, _| Ok(Bytes::copy_from_slice(&[0u8; 32]))); + sm.expect_generate_snapshot_data().returning(|_, _| Ok(())); sm.expect_save_hard_state().returning(|| Ok(())); sm.expect_flush().returning(|| Ok(())); sm diff --git a/d-engine-core/src/state_machine_handler/default_state_machine_handler_test.rs b/d-engine-core/src/state_machine_handler/default_state_machine_handler_test.rs index c35276fe..77efa365 100644 --- a/d-engine-core/src/state_machine_handler/default_state_machine_handler_test.rs +++ b/d-engine-core/src/state_machine_handler/default_state_machine_handler_test.rs @@ -909,7 +909,7 @@ mod create_snapshot_tests { path.ends_with("temp-5-1") && last_included.index == 5 && last_included.term == 1 }) - .returning(|_, _| Ok(Bytes::from(vec![0; 32]))); + .returning(|_, _| Ok(())); sm.expect_persist_last_snapshot_metadata().returning(|_| Ok(())); let mut config = snapshot_config(temp_path.to_path_buf()); @@ -979,7 +979,7 @@ mod create_snapshot_tests { .returning(|path, _| { fs::create_dir_all(path.clone()).unwrap(); fs::create_dir(path.join("state_machine")).unwrap(); - Ok(Bytes::from(vec![0; 32])) + Ok(()) }); // The publish must come after generate_snapshot_data (capture), i.e. during // build_local_snapshot once the archive exists — never during capture. @@ -1037,7 +1037,7 @@ mod create_snapshot_tests { fs::create_dir_all(path.clone()).unwrap(); let db_path = path.join("state_machine"); fs::create_dir(&db_path).unwrap(); - Ok(Bytes::copy_from_slice(&[0; 32])) + Ok(()) } else { Err(SnapshotError::OperationFailed("Concurrency failure".into()).into()) } @@ -1120,7 +1120,7 @@ mod create_snapshot_tests { debug!(?path, "expect_generate_snapshot_data"); std::fs::create_dir_all(path).expect("Failed to create directory"); - Ok(Bytes::copy_from_slice(&[0; 32])) + Ok(()) }); sm.expect_persist_last_snapshot_metadata().returning(|_| Ok(())); let snapshot_dir = temp_path.to_path_buf(); @@ -1202,7 +1202,7 @@ mod create_snapshot_tests { sm.expect_last_applied().returning(|| LogId { term: 1, index: 5 }); sm.expect_generate_snapshot_data().returning(|path, _| { fs::create_dir_all(path).unwrap(); - Ok(Bytes::from(vec![0; 32])) + Ok(()) }); sm.expect_persist_last_snapshot_metadata().returning(|_| Ok(())); @@ -1303,7 +1303,7 @@ mod create_snapshot_tests { }) .returning(|path, _| { fs::create_dir_all(path).unwrap(); - Ok(Bytes::from(vec![0; 32])) + Ok(()) }); sm.expect_persist_last_snapshot_metadata().returning(|_| Ok(())); @@ -1380,7 +1380,7 @@ mod create_snapshot_tests { // New code: last_included = last_applied = { index: 100, term: 2 } — truthful last_included.index == 100 && last_included.term == 2 }) - .returning(|_, _| Ok(Bytes::from(vec![0; 32]))); + .returning(|_, _| Ok(())); sm.expect_persist_last_snapshot_metadata().returning(|_| Ok(())); let mut config = snapshot_config(temp_path.to_path_buf()); @@ -1440,7 +1440,7 @@ mod create_snapshot_tests { .withf(|_, last_included| last_included.index == 0 && last_included.term == 0) .returning(|path, _| { fs::create_dir_all(path).unwrap(); - Ok(Bytes::from(vec![0; 32])) + Ok(()) }); sm.expect_persist_last_snapshot_metadata().returning(|_| Ok(())); @@ -1518,7 +1518,7 @@ mod create_snapshot_tests { return Err(SnapshotError::OperationFailed("mkdir: File exists".into()).into()); } fs::create_dir_all(&path).unwrap(); - Ok(Bytes::from(vec![0; 32])) + Ok(()) }); sm.expect_persist_last_snapshot_metadata().returning(|_| Ok(())); @@ -1970,7 +1970,7 @@ async fn test_snapshot_compression() { std::fs::write(&file1, "This is a test file").unwrap(); std::fs::write(&file2, vec![0u8; 1024]).unwrap(); - Ok(Bytes::from(vec![0; 32])) + Ok(()) }); sm.expect_persist_last_snapshot_metadata().returning(|_| Ok(())); @@ -2046,22 +2046,9 @@ async fn test_build_local_snapshot_removes_partial_archive_on_compression_failur let mut sm = MockStateMachine::new(); sm.expect_last_applied().returning(|| LogId { index: 10, term: 2 }); - sm.expect_generate_snapshot_data().returning(|path, _| { - fs::create_dir_all(&path).unwrap(); - fs::write(path.join("readable.txt"), "fine").unwrap(); - - // Unreadable file: tar_builder.append_dir_all must fail trying to - // read its content, after compress_directory has already created - // final_path via File::create. - let unreadable = path.join("unreadable.bin"); - fs::write(&unreadable, vec![0u8; 16]).unwrap(); - fs::set_permissions(&unreadable, std::fs::Permissions::from_mode(0o000)).unwrap(); - - Ok(Bytes::from(vec![0; 32])) - }); let config = snapshot_config(snapshot_dir.clone()); - let (handler, writer) = new_reader_writer_pair::( + let (handler, _writer) = new_reader_writer_pair::( 1, 0, Arc::new(sm), @@ -2072,10 +2059,26 @@ async fn test_build_local_snapshot_removes_partial_archive_on_compression_failur Arc::new(AtomicUsize::new(0)), ); - handler.try_begin_local_snapshot_capture().unwrap(); - let captured = writer.capture_local_snapshot().await.unwrap(); - let captured_temp_dir = captured.temp_dir.path().to_path_buf(); + // The staging directory a capture would have produced. It is built by hand because a real + // capture now reads every exported file to checksum it, so an unreadable file would fail the + // capture before compression, and this test is about compression failing. + let captured_temp_dir = snapshot_dir.join("staging"); + fs::create_dir_all(&captured_temp_dir).unwrap(); + fs::write(captured_temp_dir.join("readable.txt"), "fine").unwrap(); + // Unreadable file: tar_builder.append_dir_all must fail trying to read its content, after + // compress_directory has already created final_path via File::create. + let unreadable = captured_temp_dir.join("unreadable.bin"); + fs::write(&unreadable, vec![0u8; 16]).unwrap(); + fs::set_permissions(&unreadable, std::fs::Permissions::from_mode(0o000)).unwrap(); + let captured = crate::CapturedLocalSnapshot { + metadata: SnapshotMetadata { + last_included: Some(LogId { index: 10, term: 2 }), + checksum: Bytes::new(), + }, + temp_dir: crate::OwnedSnapshotDir::from_existing(captured_temp_dir.clone()).unwrap(), + }; + handler.try_begin_local_snapshot_capture().unwrap(); let result = handler.build_local_snapshot(captured).await; handler.end_local_snapshot_capture(); @@ -2922,3 +2925,122 @@ fn test_read_from_state_machine_skips_missing_keys_and_reads_one_view() { bytes::Bytes::from_static(b"c") ]); } + +fn writer_with_mock_state_machine( + sm: MockStateMachine, + root: &Path, +) -> ( + DefaultStateMachineHandler, + DefaultStateMachineWriter, +) { + new_reader_writer_pair::( + 1, + 0, + Arc::new(sm), + root.to_path_buf(), + snapshot_config(root.to_path_buf()), + MockSnapshotPolicy::new(), + None, + Arc::new(AtomicUsize::new(0)), + ) +} + +/// A leader exports a snapshot for a lagging follower and records the checksum the follower will +/// verify against. The state machine only writes files; it never returns a checksum. +/// +/// Purpose: the framework must record the SHA-256 of the exported directory itself. If it did not, +/// followers would verify against a value nobody produced. +#[tokio::test] +#[traced_test] +async fn test_capture_local_snapshot_records_checksum_of_exported_directory() { + let temp_dir = tempfile::tempdir().unwrap(); + let mut sm = MockStateMachine::new(); + sm.expect_last_applied().returning(|| LogId { index: 10, term: 2 }); + sm.expect_generate_snapshot_data().returning(|path, _| { + fs::create_dir_all(path.clone()).unwrap(); + fs::write(path.join("a.bin"), b"alpha").unwrap(); + fs::write(path.join("b.bin"), b"beta").unwrap(); + Ok(()) + }); + let (_handler, writer) = writer_with_mock_state_machine(sm, temp_dir.path()); + + let captured = writer.capture_local_snapshot().await.unwrap(); + + let expected = crate::file_io::compute_checksum_from_folder_path(captured.temp_dir.path()) + .await + .unwrap(); + assert_eq!( + captured.metadata.checksum.as_ref(), + expected.as_ref(), + "recorded checksum must be the checksum of the exported directory" + ); + assert!( + captured.metadata.checksum.iter().any(|&b| b != 0), + "recorded checksum must not be an all-zero placeholder" + ); +} + +/// A follower receives a snapshot whose files do not match the checksum the leader recorded +/// (corrupted or tampered in transit, after the per-chunk crc32 checks passed). +/// +/// Purpose: the framework rejects the install and the state machine is never called, so the +/// follower keeps its current state instead of replacing it with bad data. +#[tokio::test] +#[traced_test] +async fn test_install_prepared_snapshot_checksum_mismatch_rejects_without_calling_state_machine() { + let temp_dir = tempfile::tempdir().unwrap(); + let received = temp_dir.path().join("received"); + fs::create_dir_all(&received).unwrap(); + fs::write(received.join("data.bin"), b"content").unwrap(); + + let mut sm = MockStateMachine::new(); + sm.expect_apply_snapshot_from_file().times(0); + let (_handler, writer) = writer_with_mock_state_machine(sm, temp_dir.path()); + + let metadata = SnapshotMetadata { + last_included: Some(LogId { index: 5, term: 1 }), + checksum: Bytes::from(vec![0; 32]), + }; + let err = writer + .install_prepared_snapshot(metadata, received) + .await + .expect_err("a snapshot with a mismatched checksum must be rejected"); + + assert!( + format!("{err:?}").contains("ChecksumMismatch"), + "expected ChecksumMismatch, got: {err:?}" + ); +} + +/// The same follower receives an intact snapshot. +/// +/// Purpose: when the checksum matches, the framework hands the directory to the state machine +/// exactly once and returns its result, so the verification does not block valid installs. +#[tokio::test] +#[traced_test] +async fn test_install_prepared_snapshot_matching_checksum_installs_once() { + let temp_dir = tempfile::tempdir().unwrap(); + let received = temp_dir.path().join("received"); + fs::create_dir_all(&received).unwrap(); + fs::write(received.join("data.bin"), b"content").unwrap(); + let checksum = crate::file_io::compute_checksum_from_folder_path(&received).await.unwrap(); + + let mut sm = MockStateMachine::new(); + sm.expect_apply_snapshot_from_file().times(1).returning(|metadata, _| { + Ok(crate::SnapshotApplyResult::Applied { + last_included: metadata.last_included.unwrap(), + }) + }); + let (_handler, writer) = writer_with_mock_state_machine(sm, temp_dir.path()); + + let metadata = SnapshotMetadata { + last_included: Some(LogId { index: 5, term: 1 }), + checksum: Bytes::copy_from_slice(&checksum), + }; + let result = writer.install_prepared_snapshot(metadata, received).await.unwrap(); + + assert!( + matches!(result, crate::SnapshotApplyResult::Applied { .. }), + "expected Applied, got: {result:?}" + ); +} diff --git a/d-engine-core/src/state_machine_handler/writer.rs b/d-engine-core/src/state_machine_handler/writer.rs index ab6217db..c5cff413 100644 --- a/d-engine-core/src/state_machine_handler/writer.rs +++ b/d-engine-core/src/state_machine_handler/writer.rs @@ -2,8 +2,10 @@ use std::path::PathBuf; use std::sync::Arc; use async_trait::async_trait; +use bytes::Bytes; use d_engine_proto::common::Entry; use d_engine_proto::server::storage::SnapshotMetadata; +use tracing::error; use tracing::trace; use super::StateMachineWriterOps; @@ -19,6 +21,7 @@ use crate::TypeConfig; use crate::alias::SMOF; use crate::convert::classify_error; use crate::decode_entries; +use crate::file_io::compute_checksum_from_folder_path; use crate::scoped_timer::ScopedTimer; use crate::state_machine_handler::applied_state::AppliedState; @@ -176,6 +179,17 @@ impl StateMachineWriterOps for DefaultStateMachineWriter { metadata: SnapshotMetadata, dir: PathBuf, ) -> Result { + // The framework verifies the snapshot's end-to-end checksum, so a state machine only + // ever receives a directory whose content matches what the leader exported. + let computed = compute_checksum_from_folder_path(&dir).await?; + if metadata.checksum.as_ref() != computed.as_ref() { + error!( + "Snapshot checksum mismatch! Computed: {:?}, Expected: {:?}", + computed, metadata.checksum + ); + return Err(SnapshotError::ChecksumMismatch.into()); + } + let result = self.state_machine.apply_snapshot_from_file(&metadata, dir).await?; if let SnapshotApplyResult::Applied { last_included } = &result { self.applied.advance(last_included.index); @@ -198,16 +212,20 @@ impl StateMachineWriterOps for DefaultStateMachineWriter { .into()); } - // generate_snapshot_data commits SnapshotMetadata as the state machine's - // current snapshot internally (each StateMachine adapter does this as part - // of the same call) — by the time this returns, the boundary is durable, - // not just captured in memory. - let checksum = match self + // generate_snapshot_data only exports the state; it does not publish snapshot + // metadata. The metadata is published later, once the archive is durably built. + if let Err(e) = self .state_machine .generate_snapshot_data(temp_path.clone(), last_included) .await { - Ok(bytes) => bytes, + let _ = tokio::fs::remove_dir_all(&temp_path).await; + return Err(e); + } + + // The framework, not the state machine, checksums the exported directory. + let checksum = match compute_checksum_from_folder_path(&temp_path).await { + Ok(sum) => Bytes::copy_from_slice(&sum), Err(e) => { let _ = tokio::fs::remove_dir_all(&temp_path).await; return Err(e); diff --git a/d-engine-core/src/storage/state_machine.rs b/d-engine-core/src/storage/state_machine.rs index 52099b96..4dea4b79 100644 --- a/d-engine-core/src/storage/state_machine.rs +++ b/d-engine-core/src/storage/state_machine.rs @@ -220,11 +220,11 @@ pub trait StateMachine: Send + Sync + 'static { /// Generates a snapshot of the state machine's current key-value entries /// up to the specified `last_included_index`. /// - /// Exports the current state (as of `last_included`) to `new_snapshot_dir` and - /// returns its checksum. Does NOT publish snapshot metadata — the caller publishes - /// it (via `persist_last_snapshot_metadata`) only after the final archive is - /// durably built, so a crash mid-snapshot never leaves metadata pointing at a - /// missing archive. + /// Exports the current state (as of `last_included`) to `new_snapshot_dir`. The framework + /// computes the checksum of that directory itself; the state machine does not. Does NOT + /// publish snapshot metadata — the caller publishes it (via + /// `persist_last_snapshot_metadata`) only after the final archive is durably built, so a + /// crash mid-snapshot never leaves metadata pointing at a missing archive. /// /// # Concurrency contract (#436) /// May run concurrently with `apply_chunk` (never with `apply_snapshot_from_file` — the two @@ -237,14 +237,11 @@ pub trait StateMachine: Send + Sync + 'static { /// * `new_snapshot_dir` - Temporary path to store the snapshot data. /// * `last_included_index` - Last log index included in the snapshot. /// * `last_included_term` - Last log term included in the snapshot. - /// - /// # Returns - /// * if success, checksum will be returned async fn generate_snapshot_data( &self, new_snapshot_dir: std::path::PathBuf, last_included: LogId, - ) -> Result; + ) -> Result<(), Error>; /// Saves the hard state of the state machine. /// Sync operation as it typically just delegates to other persistence methods. diff --git a/d-engine-core/src/storage/state_machine_test.rs b/d-engine-core/src/storage/state_machine_test.rs index d11b94b8..84358726 100644 --- a/d-engine-core/src/storage/state_machine_test.rs +++ b/d-engine-core/src/storage/state_machine_test.rs @@ -43,7 +43,6 @@ impl StateMachineTestSuite { Self::test_batch_operations(builder.build().await?).await?; Self::test_last_applied_detection(builder.build().await?).await?; Self::test_snapshot_operations(builder.build().await?).await?; - Self::test_snapshot_checksum_validation(builder.build().await?).await?; Self::test_snapshot_apply_rejects_missing_boundary(builder.build().await?).await?; Self::test_persistence(builder.build().await?).await?; @@ -305,9 +304,10 @@ impl StateMachineTestSuite { // Generate snapshot let last_included = LogId { index: 3, term: 1 }; - let checksum = state_machine + state_machine .generate_snapshot_data(snapshot_dir.clone(), last_included) .await?; + let checksum = crate::file_io::compute_checksum_from_folder_path(&snapshot_dir).await?; // (#436) generate_snapshot_data no longer publishes snapshot metadata — that // moved to build_local_snapshot (the handler), after the archive is durable. @@ -376,76 +376,6 @@ impl StateMachineTestSuite { Ok(()) } - /// Snapshot checksum must be both computed and enforced. - /// - /// `generate_snapshot_data` must return a real checksum of the exported data (never an - /// all-zero placeholder), and `apply_snapshot_from_file` must reject a snapshot whose - /// checksum does not match — so a corrupted or tampered snapshot is never installed - /// silently. The per-chunk crc32 only proves "bytes received == bytes sent"; this - /// end-to-end checksum proves "bytes received == bytes the leader recorded". - /// - /// # Scenario - /// - Apply entries, then generate a snapshot at the applied boundary. - /// - Expected: the returned checksum has at least one non-zero byte (a real checksum). - /// - Reset, then install the same snapshot with a WRONG checksum. - /// - Expected: the install is rejected. - pub async fn test_snapshot_checksum_validation( - state_machine: Arc - ) -> Result<(), Error> { - let entries = vec![ - create_insert_entry( - 1, - Bytes::from(b"key1".to_vec()), - Bytes::from(b"value1".to_vec()), - ), - create_insert_entry( - 2, - Bytes::from(b"key2".to_vec()), - Bytes::from(b"value2".to_vec()), - ), - create_insert_entry( - 3, - Bytes::from(b"key3".to_vec()), - Bytes::from(b"value3".to_vec()), - ), - ]; - state_machine.apply_chunk(&entries).await?; - - let temp_dir = TempDir::new()?; - let snapshot_dir = temp_dir.path().join("snapshot"); - let last_included = LogId { index: 3, term: 1 }; - - let checksum = state_machine - .generate_snapshot_data(snapshot_dir.clone(), last_included) - .await?; - - // A real checksum is never all-zero; a placeholder (e.g. [0; 32]) signals the - // adapter is not computing a checksum at all. - assert!( - checksum.iter().any(|&b| b != 0), - "generate_snapshot_data returned an all-zero checksum — the adapter must compute \ - a real checksum of the snapshot content, not a placeholder" - ); - - // Reset so the snapshot is strictly newer than current state, then install it with a - // deliberately wrong checksum. The install must be rejected, not silently accepted. - state_machine.reset().await?; - - let wrong_metadata = SnapshotMetadata { - last_included: Some(last_included), - checksum: Bytes::from(vec![0; 32]), - }; - let result = state_machine.apply_snapshot_from_file(&wrong_metadata, snapshot_dir).await; - - assert!( - result.is_err(), - "apply_snapshot_from_file accepted a snapshot with a mismatched checksum — the \ - adapter must reject it" - ); - - Ok(()) - } - /// `apply_snapshot_from_file` must reject `last_included: None` outright — a missing /// boundary can't be classified stale/duplicate/conflict/eligible, so silently /// defaulting it (`unwrap_or_default()`) turns malformed metadata into a fake no-op. @@ -776,7 +706,8 @@ impl StateMachineTestSuite { let temp_dir = TempDir::new()?; let snapshot_dir = temp_dir.path().join("snapshot"); - let checksum = sm.generate_snapshot_data(snapshot_dir.clone(), last_included).await?; + sm.generate_snapshot_data(snapshot_dir.clone(), last_included).await?; + let checksum = crate::file_io::compute_checksum_from_folder_path(&snapshot_dir).await?; // Fresh node receiving this snapshot for the first time (current == default), // so the install is strictly "Greater" — eligible, not a stale/duplicate no-op. @@ -1610,7 +1541,8 @@ impl StateMachineTestSuite { }; let (export_result, apply_result) = tokio::join!(export_fut, apply_fut); - let checksum = export_result?; + export_result?; + let checksum = crate::file_io::compute_checksum_from_folder_path(&snapshot_dir).await?; apply_result?; next_index += 2; @@ -1812,8 +1744,8 @@ mod default_scan_prefix_tests { &self, _new_snapshot_dir: std::path::PathBuf, _last_included: LogId, - ) -> Result { - Ok(Bytes::new()) + ) -> Result<(), Error> { + Ok(()) } fn save_hard_state(&self) -> Result<(), Error> { diff --git a/d-engine-core/src/test_utils/mock/mock_raft_builder.rs b/d-engine-core/src/test_utils/mock/mock_raft_builder.rs index 42ed1566..5b2dbe76 100644 --- a/d-engine-core/src/test_utils/mock/mock_raft_builder.rs +++ b/d-engine-core/src/test_utils/mock/mock_raft_builder.rs @@ -1,7 +1,6 @@ use std::path::Path; use std::sync::Arc; -use bytes::Bytes; use d_engine_proto::common::LogId; use d_engine_proto::server::cluster::ClusterMembership; use tokio::sync::mpsc; @@ -401,8 +400,7 @@ pub fn mock_state_machine() -> MockStateMachine { last_included: LogId::default(), }) }); - mock.expect_generate_snapshot_data() - .returning(|_, _| Ok(Bytes::copy_from_slice(&[0u8; 32]))); + mock.expect_generate_snapshot_data().returning(|_, _| Ok(())); mock.expect_save_hard_state().returning(|| Ok(())); mock.expect_flush().returning(|| Ok(())); diff --git a/d-engine-server/src/api/embedded_test/shutdown_order_test.rs b/d-engine-server/src/api/embedded_test/shutdown_order_test.rs index 463ebe01..f4ec1a22 100644 --- a/d-engine-server/src/api/embedded_test/shutdown_order_test.rs +++ b/d-engine-server/src/api/embedded_test/shutdown_order_test.rs @@ -122,7 +122,7 @@ impl StateMachine for SlowApplyProbe { &self, new_snapshot_dir: std::path::PathBuf, last_included: LogId, - ) -> Result { + ) -> Result<(), Error> { self.inner.generate_snapshot_data(new_snapshot_dir, last_included).await } fn save_hard_state(&self) -> Result<(), Error> { diff --git a/d-engine-server/src/storage/adaptors/file/file_state_machine.rs b/d-engine-server/src/storage/adaptors/file/file_state_machine.rs index 072df885..53bba332 100644 --- a/d-engine-server/src/storage/adaptors/file/file_state_machine.rs +++ b/d-engine-server/src/storage/adaptors/file/file_state_machine.rs @@ -72,7 +72,6 @@ use d_engine_core::SnapshotApplyResult; use d_engine_core::SnapshotError; use d_engine_core::StateMachine; use d_engine_core::StorageError; -use d_engine_core::file_io::compute_checksum_from_folder_path; use d_engine_proto::common::LogId; use d_engine_proto::server::storage::SnapshotMetadata; use parking_lot::RwLock; @@ -1150,16 +1149,6 @@ impl StateMachine for FileStateMachine { std::cmp::Ordering::Greater => {} } - // Verify the snapshot's end-to-end checksum before installing (#436). - let computed_checksum = compute_checksum_from_folder_path(&snapshot_dir).await?; - if metadata.checksum.as_ref() != computed_checksum.as_ref() { - error!( - "Snapshot checksum mismatch! Computed: {:?}, Expected: {:?}", - computed_checksum, metadata.checksum - ); - return Err(SnapshotError::ChecksumMismatch.into()); - } - info!("Applying snapshot from file: {:?}", snapshot_dir); // Read from the snapshot.bin file inside the directory @@ -1277,7 +1266,7 @@ impl StateMachine for FileStateMachine { &self, new_snapshot_dir: std::path::PathBuf, _last_included: LogId, - ) -> Result { + ) -> Result<(), Error> { // Create snapshot directory fs::create_dir_all(&new_snapshot_dir).await?; @@ -1309,8 +1298,7 @@ impl StateMachine for FileStateMachine { info!("Snapshot generated at {:?}", snapshot_path); - let checksum = compute_checksum_from_folder_path(&new_snapshot_dir).await?; - Ok(Bytes::copy_from_slice(&checksum)) + Ok(()) } fn save_hard_state(&self) -> Result<(), Error> { diff --git a/d-engine-server/src/storage/adaptors/file/file_state_machine_test.rs b/d-engine-server/src/storage/adaptors/file/file_state_machine_test.rs index 1dfb2897..0431a9d7 100644 --- a/d-engine-server/src/storage/adaptors/file/file_state_machine_test.rs +++ b/d-engine-server/src/storage/adaptors/file/file_state_machine_test.rs @@ -1012,10 +1012,13 @@ async fn test_file_sm_install_empty_snapshot_clears_existing_data() { let source = FileStateMachine::new(dir.path().join("source")).await.unwrap(); let snapshot_dir = dir.path().join("snapshot"); - let checksum = source + source .generate_snapshot_data(snapshot_dir.clone(), LogId { term: 1, index: 5 }) .await .unwrap(); + let checksum = d_engine_core::file_io::compute_checksum_from_folder_path(&snapshot_dir) + .await + .unwrap(); let receiver = FileStateMachine::new(dir.path().join("receiver")).await.unwrap(); receiver.apply_chunk(&[insert_at(b"stale", b"v", 1)]).await.unwrap(); diff --git a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs index 6169b7ee..9076ff54 100644 --- a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs +++ b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs @@ -16,7 +16,6 @@ use d_engine_core::SnapshotApplyResult; use d_engine_core::SnapshotError; use d_engine_core::StateMachine; use d_engine_core::StorageError; -use d_engine_core::file_io::compute_checksum_from_folder_path; use d_engine_proto::common::LogId; use d_engine_proto::server::storage::SnapshotMetadata; use parking_lot::RwLock; @@ -789,16 +788,6 @@ impl StateMachine for RocksDBStateMachine { } } - // Verify the snapshot's end-to-end checksum before installing (#436). - let computed_checksum = compute_checksum_from_folder_path(&snapshot_dir).await?; - if metadata.checksum.as_ref() != computed_checksum.as_ref() { - error!( - "Snapshot checksum mismatch! Computed: {:?}, Expected: {:?}", - computed_checksum, metadata.checksum - ); - return Err(SnapshotError::ChecksumMismatch.into()); - } - info!("Applying snapshot from: {:?}", snapshot_dir); // Stop serving — prevents SM reads/writes during restore @@ -826,7 +815,7 @@ impl StateMachine for RocksDBStateMachine { &self, new_snapshot_dir: std::path::PathBuf, _last_included: LogId, - ) -> Result { + ) -> Result<(), Error> { // Export only SM CFs: compact SST-only export, no Raft log data. // export_column_family() creates only the final subdirectory (e.g. "sm"), so the parent // (new_snapshot_dir) must already exist before calling it. @@ -875,10 +864,8 @@ impl StateMachine for RocksDBStateMachine { .await .map_err(Self::map_snapshot_join_error)??; - let checksum = compute_checksum_from_folder_path(&new_snapshot_dir).await?; - info!("Snapshot generated at {:?}", new_snapshot_dir); - Ok(Bytes::copy_from_slice(&checksum)) + Ok(()) } fn save_hard_state(&self) -> Result<(), Error> { diff --git a/d-engine-server/src/test_utils/mock/mock_node_builder.rs b/d-engine-server/src/test_utils/mock/mock_node_builder.rs index 1f647700..7e85bd1f 100644 --- a/d-engine-server/src/test_utils/mock/mock_node_builder.rs +++ b/d-engine-server/src/test_utils/mock/mock_node_builder.rs @@ -4,7 +4,6 @@ use std::path::Path; use std::sync::Arc; use std::sync::atomic::AtomicBool; -use bytes::Bytes; use d_engine_core::ElectionConfig; use d_engine_core::InboundEvent; use d_engine_core::InternalEvent; @@ -643,8 +642,7 @@ pub(crate) fn mock_state_machine() -> MockStateMachine { last_included: LogId::default(), }) }); - mock.expect_generate_snapshot_data() - .returning(|_, _| Ok(Bytes::copy_from_slice(&[0u8; 32]))); + mock.expect_generate_snapshot_data().returning(|_, _| Ok(())); mock.expect_save_hard_state().returning(|| Ok(())); mock.expect_flush().returning(|| Ok(())); diff --git a/d-engine-server/tests/common/shared_state_machine.rs b/d-engine-server/tests/common/shared_state_machine.rs index 03c8c4b5..4573e162 100644 --- a/d-engine-server/tests/common/shared_state_machine.rs +++ b/d-engine-server/tests/common/shared_state_machine.rs @@ -96,7 +96,7 @@ impl StateMachine for SharedStateMachine { &self, new_snapshot_dir: std::path::PathBuf, last_included: LogId, - ) -> Result { + ) -> Result<(), Error> { self.0.generate_snapshot_data(new_snapshot_dir, last_included).await } fn save_hard_state(&self) -> Result<(), Error> { diff --git a/examples/sled-cluster/Cargo.toml b/examples/sled-cluster/Cargo.toml index 34f58d06..9318b18f 100644 --- a/examples/sled-cluster/Cargo.toml +++ b/examples/sled-cluster/Cargo.toml @@ -24,7 +24,6 @@ bincode = { version = "2.0", features = ["serde"] } arc-swap = "1.7" parking_lot = "0.12" anyhow = "1.0" -sha2 = "0.10" uuid = { version = "1.4", features = ["v4"] } [dev-dependencies] diff --git a/examples/sled-cluster/src/lib.rs b/examples/sled-cluster/src/lib.rs index 881c5509..c46cbf20 100644 --- a/examples/sled-cluster/src/lib.rs +++ b/examples/sled-cluster/src/lib.rs @@ -5,40 +5,7 @@ pub(crate) use converters::{safe_kv, safe_vk_ivec}; pub use sled_state_machine::*; pub use sled_storage_engine::*; -use sha2::Digest; -use sha2::Sha256; -use std::path::Path; -use tokio::fs; - #[cfg(test)] mod sled_engine_test; #[cfg(test)] mod state_machine; - -pub(crate) async fn compute_checksum_from_folder_path( - folder_path: &Path -) -> Result<[u8; 32], std::io::Error> { - let mut hasher = Sha256::new(); - let mut entries = fs::read_dir(folder_path).await?; - - // Collect files while preserving DirEntry information - let mut files = Vec::new(); - while let Some(entry) = entries.next_entry().await? { - if entry.file_type().await?.is_file() { - files.push(entry); - } - } - - // Critical sorting: Filenames compared as OsString but sorted by simple byte order. - // This matches lexical order for valid UTF-8 names, and provides consistent ordering - // for non-UTF names across platforms. - files.sort_by_key(|entry| entry.file_name()); - - // Process files in deterministic sorted order - for entry in files { - let data = fs::read(entry.path()).await?; - hasher.update(data); - } - - Ok(hasher.finalize().into()) -} diff --git a/examples/sled-cluster/src/sled_state_machine.rs b/examples/sled-cluster/src/sled_state_machine.rs index 1497f96c..24a3ca20 100644 --- a/examples/sled-cluster/src/sled_state_machine.rs +++ b/examples/sled-cluster/src/sled_state_machine.rs @@ -1,6 +1,5 @@ //! It works as KV storage for client business CRUDs. -use crate::compute_checksum_from_folder_path; use crate::{safe_kv, safe_vk_ivec}; use arc_swap::ArcSwapOption; use async_trait::async_trait; @@ -330,7 +329,7 @@ impl StateMachine for SledStateMachine { &self, new_snapshot_dir: PathBuf, _last_included: LogId, - ) -> Result { + ) -> Result<()> { // No internal locking needed: `StateMachineWorker` is the sole caller of // generate_snapshot_data/apply_snapshot_from_file/apply_chunk, and it processes // them one at a time from a single queue (#436) — concurrent snapshot @@ -370,10 +369,7 @@ impl StateMachine for SledStateMachine { // Make sure flush into disk new_db.flush().map_err(|e| StorageError::DbError(e.to_string()))?; - // Calculate the checksum after generating snapshot data - let checksum = compute_checksum_from_folder_path(&new_snapshot_dir).await?; - - Ok(Bytes::copy_from_slice(&checksum)) + Ok(()) } async fn apply_snapshot_from_file( @@ -417,32 +413,6 @@ impl StateMachine for SledStateMachine { } } - debug!( - ?decompressed_snapshot_path, - "Validating checksum before install" - ); - // CRITICAL SECURITY STEP: prevents tampered or corrupted snapshots from being applied. - let computed_checksum = - compute_checksum_from_folder_path(&decompressed_snapshot_path).await?; - - if metadata.checksum.as_ref() != computed_checksum.as_ref() { - error!( - "Snapshot checksum mismatch! Computed: {:?}, Expected: {:?}", - computed_checksum, metadata.checksum - ); - - metrics::counter!( - "snapshot.checksum_failures", - &[ - ("node_id", self.node_id.to_string()), - ("snapshot_index", new_last_included.index.to_string()), - ] - ) - .increment(1); - - return Err(SnapshotError::ChecksumMismatch.into()); - } - // `decompressed_snapshot_path` is a temporary directory owned by the caller // (see `PreparedSnapshot`), not this state machine's own `self.path`. Opening a // sled::Db directly there and swapping `self.db` to point at it — the previous diff --git a/examples/sled-cluster/src/state_machine/sled_state_machine_test.rs b/examples/sled-cluster/src/state_machine/sled_state_machine_test.rs index 52ddae2b..b5ea4ad5 100644 --- a/examples/sled-cluster/src/state_machine/sled_state_machine_test.rs +++ b/examples/sled-cluster/src/state_machine/sled_state_machine_test.rs @@ -169,14 +169,15 @@ async fn test_snapshot_install_content_survives_restart() { let last_included = LogId { index: 3, term: 1 }; let export_dir = TempDir::new().unwrap(); - let checksum = source_sm + source_sm .generate_snapshot_data(export_dir.path().to_path_buf(), last_included) .await .unwrap(); + // The framework checksums and verifies snapshots; the state machine ignores this field. let metadata = d_engine::server_storage::SnapshotMetadata { last_included: Some(last_included), - checksum: Bytes::from(checksum.to_vec()), + checksum: Bytes::new(), }; // Node B: fresh, empty, behind the purge boundary — installs the snapshot. From b74bc59bebaf9da98467a99e67aad808ae102e46 Mon Sep 17 00:00:00 2001 From: Joshua Chi <112539+JoshuaChi@users.noreply.github.com> Date: Sun, 1 Nov 2026 20:44:49 +0800 Subject: [PATCH 05/19] fix #400: Make snapshot checksums cover every file of the snapshot directory --- .../default_state_machine_handler_test.rs | 38 ++++++ d-engine-core/src/utils/file_io.rs | 69 ++++++++-- d-engine-core/src/utils/file_io_test.rs | 127 ++++++++++++++++++ 3 files changed, 224 insertions(+), 10 deletions(-) diff --git a/d-engine-core/src/state_machine_handler/default_state_machine_handler_test.rs b/d-engine-core/src/state_machine_handler/default_state_machine_handler_test.rs index 77efa365..bfe1bfff 100644 --- a/d-engine-core/src/state_machine_handler/default_state_machine_handler_test.rs +++ b/d-engine-core/src/state_machine_handler/default_state_machine_handler_test.rs @@ -3044,3 +3044,41 @@ async fn test_install_prepared_snapshot_matching_checksum_installs_once() { "expected Applied, got: {result:?}" ); } + +/// A follower receives a RocksDB-style snapshot (data files in a sub-directory) in which one data +/// file was corrupted after the leader recorded the checksum. +/// +/// Purpose: the install must be rejected and the state machine never called. If the checksum only +/// covered top-level files, this corruption would be installed silently. +#[tokio::test] +#[traced_test] +async fn test_install_prepared_snapshot_nested_file_corruption_rejects_without_calling_state_machine() + { + let temp_dir = tempfile::tempdir().unwrap(); + let received = temp_dir.path().join("received"); + fs::create_dir_all(received.join("sm")).unwrap(); + fs::write(received.join("sm_metadata.bin"), b"metadata").unwrap(); + fs::write(received.join("sm").join("000008.sst"), b"table-data").unwrap(); + let recorded = crate::file_io::compute_checksum_from_folder_path(&received).await.unwrap(); + + // Corrupted in transit, after the checksum was recorded. + fs::write(received.join("sm").join("000008.sst"), b"table-dat!").unwrap(); + + let mut sm = MockStateMachine::new(); + sm.expect_apply_snapshot_from_file().times(0); + let (_handler, writer) = writer_with_mock_state_machine(sm, temp_dir.path()); + + let metadata = SnapshotMetadata { + last_included: Some(LogId { index: 5, term: 1 }), + checksum: Bytes::copy_from_slice(&recorded), + }; + let err = writer + .install_prepared_snapshot(metadata, received) + .await + .expect_err("a snapshot with a corrupted nested file must be rejected"); + + assert!( + format!("{err:?}").contains("ChecksumMismatch"), + "expected ChecksumMismatch, got: {err:?}" + ); +} diff --git a/d-engine-core/src/utils/file_io.rs b/d-engine-core/src/utils/file_io.rs index 1cd17060..c682a59c 100644 --- a/d-engine-core/src/utils/file_io.rs +++ b/d-engine-core/src/utils/file_io.rs @@ -255,20 +255,69 @@ pub async fn compute_checksum_from_file_path(file_path: &Path) -> Result<[u8; 32 Ok(hasher.finalize().into()) } -/// SHA-256 over every file in a directory, in deterministic (sorted) filename order. +/// Domain separator mixed into every snapshot checksum, so it can never equal a hash of the +/// same bytes computed for another purpose. +const SNAPSHOT_CHECKSUM_DOMAIN: &[u8] = b"d-engine/snapshot-checksum\0"; + +/// SHA-256 of a snapshot directory: every regular file at any depth, in sorted relative-path +/// order. Each file contributes its relative path and its length ahead of its bytes, so renaming +/// a file, moving it to another directory, or moving a byte across a file boundary changes the +/// checksum. Symbolic links and other special entries are rejected rather than skipped. pub async fn compute_checksum_from_folder_path(folder_path: &Path) -> Result<[u8; 32]> { - let mut hasher = Sha256::new(); - let mut entries = tokio::fs::read_dir(folder_path).await.map_err(StorageError::IoError)?; + let root = folder_path.to_path_buf(); + tokio::task::spawn_blocking(move || checksum_of_tree(&root)) + .await + .map_err(|e| StorageError::IoError(std::io::Error::other(e.to_string())))? + .map_err(|e| StorageError::IoError(e).into()) +} + +fn checksum_of_tree(root: &Path) -> std::io::Result<[u8; 32]> { let mut files = Vec::new(); - while let Some(entry) = entries.next_entry().await.map_err(StorageError::IoError)? { - if entry.file_type().await.map_err(StorageError::IoError)?.is_file() { - files.push(entry); + let mut pending = vec![root.to_path_buf()]; + while let Some(dir) = pending.pop() { + for entry in std::fs::read_dir(&dir)? { + let entry = entry?; + let file_type = entry.file_type()?; + if file_type.is_dir() { + pending.push(entry.path()); + } else if file_type.is_file() { + let path = entry.path(); + let relative = + path.strip_prefix(root).map_err(std::io::Error::other)?.to_path_buf(); + files.push((relative, path)); + } else { + return Err(std::io::Error::other(format!( + "unsupported entry in snapshot directory: {}", + entry.path().display() + ))); + } } } - files.sort_by_key(|entry| entry.file_name()); - for entry in files { - let data = tokio::fs::read(entry.path()).await.map_err(StorageError::IoError)?; - hasher.update(data); + files.sort(); + + let mut hasher = Sha256::new(); + hasher.update(SNAPSHOT_CHECKSUM_DOMAIN); + for (relative, path) in files { + // Components joined with '/', so the same tree hashes the same on every platform. + let mut name = Vec::new(); + for (index, component) in relative.components().enumerate() { + if index > 0 { + name.push(b'/'); + } + name.extend_from_slice(component.as_os_str().as_encoded_bytes()); + } + let file = std::fs::File::open(&path)?; + let length = file.metadata()?.len(); + hasher.update((name.len() as u64).to_be_bytes()); + hasher.update(&name); + hasher.update(length.to_be_bytes()); + let copied = std::io::copy(&mut std::io::BufReader::new(file), &mut hasher)?; + if copied != length { + return Err(std::io::Error::other(format!( + "{} changed while it was being checksummed", + path.display() + ))); + } } Ok(hasher.finalize().into()) } diff --git a/d-engine-core/src/utils/file_io_test.rs b/d-engine-core/src/utils/file_io_test.rs index b7e8a84a..f2278a97 100644 --- a/d-engine-core/src/utils/file_io_test.rs +++ b/d-engine-core/src/utils/file_io_test.rs @@ -538,3 +538,130 @@ mod compute_checksum_from_file_path_tests { ); } } + +// ── compute_checksum_from_folder_path ──────────────────────────────────────── +// +// The snapshot checksum is what a follower checks a received snapshot against. These tests pin +// what it must cover: every file at any depth, the boundaries between files, and file names. + +fn write_tree( + root: &std::path::Path, + files: &[(&str, &[u8])], +) { + for (relative, content) in files { + let path = root.join(relative); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(path, content).unwrap(); + } +} + +/// RocksDB exports its data files into sub-directories (`sm/000008.sst`). A byte flipped in one +/// of them is exactly the corruption the checksum exists to catch. +#[tokio::test] +async fn test_folder_checksum_changes_when_a_nested_file_is_modified() { + let dir = tempdir().unwrap(); + write_tree(dir.path(), &[ + ("meta.bin", b"meta"), + ("sm/000008.sst", b"table-data"), + ]); + let before = file_io::compute_checksum_from_folder_path(dir.path()).await.unwrap(); + + std::fs::write(dir.path().join("sm/000008.sst"), b"table-dat!").unwrap(); + let after = file_io::compute_checksum_from_folder_path(dir.path()).await.unwrap(); + + assert_ne!( + before, after, + "a modified file in a sub-directory must change the checksum" + ); +} + +/// A snapshot that arrives with an extra file somewhere below the root is not the snapshot the +/// leader exported. +#[tokio::test] +async fn test_folder_checksum_changes_when_a_nested_file_is_added() { + let dir = tempdir().unwrap(); + write_tree(dir.path(), &[ + ("meta.bin", b"meta"), + ("sm/000008.sst", b"table-data"), + ]); + let before = file_io::compute_checksum_from_folder_path(dir.path()).await.unwrap(); + + write_tree(dir.path(), &[("sm/000009.sst", b"extra")]); + let after = file_io::compute_checksum_from_folder_path(dir.path()).await.unwrap(); + + assert_ne!( + before, after, + "an added file in a sub-directory must change the checksum" + ); +} + +/// Where one file ends and the next begins is part of the content. Hashing the bytes of the files +/// back to back lets `a="xy", b="z"` and `a="x", b="yz"` collide. +#[tokio::test] +async fn test_folder_checksum_distinguishes_where_one_file_ends_and_the_next_begins() { + let first = tempdir().unwrap(); + write_tree(first.path(), &[("a.bin", b"xy"), ("b.bin", b"z")]); + let second = tempdir().unwrap(); + write_tree(second.path(), &[("a.bin", b"x"), ("b.bin", b"yz")]); + + let first_sum = file_io::compute_checksum_from_folder_path(first.path()).await.unwrap(); + let second_sum = file_io::compute_checksum_from_folder_path(second.path()).await.unwrap(); + + assert_ne!( + first_sum, second_sum, + "moving a byte across a file boundary must change the checksum" + ); +} + +/// Two snapshots holding the same bytes under different file names are different snapshots (a +/// state machine reads its files by name). +#[tokio::test] +async fn test_folder_checksum_changes_when_a_file_is_renamed() { + let first = tempdir().unwrap(); + write_tree(first.path(), &[("a.bin", b"same")]); + let second = tempdir().unwrap(); + write_tree(second.path(), &[("b.bin", b"same")]); + + let first_sum = file_io::compute_checksum_from_folder_path(first.path()).await.unwrap(); + let second_sum = file_io::compute_checksum_from_folder_path(second.path()).await.unwrap(); + + assert_ne!( + first_sum, second_sum, + "the file name must be part of the checksum" + ); +} + +/// The leader and a follower build the same directory in different orders; they must agree. +#[tokio::test] +async fn test_folder_checksum_is_independent_of_creation_order() { + let first = tempdir().unwrap(); + write_tree(first.path(), &[ + ("a.bin", b"1"), + ("sm/b.sst", b"2"), + ("sm/c.sst", b"3"), + ]); + let second = tempdir().unwrap(); + write_tree(second.path(), &[ + ("sm/c.sst", b"3"), + ("sm/b.sst", b"2"), + ("a.bin", b"1"), + ]); + + let first_sum = file_io::compute_checksum_from_folder_path(first.path()).await.unwrap(); + let second_sum = file_io::compute_checksum_from_folder_path(second.path()).await.unwrap(); + + assert_eq!(first_sum, second_sum); +} + +/// Unchanged behaviour the rewrite must keep: a top-level change is still detected. +#[tokio::test] +async fn test_folder_checksum_changes_when_a_top_level_file_is_modified() { + let dir = tempdir().unwrap(); + write_tree(dir.path(), &[("meta.bin", b"meta")]); + let before = file_io::compute_checksum_from_folder_path(dir.path()).await.unwrap(); + + std::fs::write(dir.path().join("meta.bin"), b"Meta").unwrap(); + let after = file_io::compute_checksum_from_folder_path(dir.path()).await.unwrap(); + + assert_ne!(before, after); +} From 42dedd0c23e8fc399be0f889f0f92cd9d4a307a5 Mon Sep 17 00:00:00 2001 From: Joshua Chi <112539+JoshuaChi@users.noreply.github.com> Date: Sun, 1 Nov 2026 20:44:59 +0800 Subject: [PATCH 06/19] refactor #400: Trim StateMachine to what the engine calls; persist snapshot metadata Remove update_last_applied, persist_last_applied, update_last_snapshot_metadata, save_hard_state, flush and flush_async from the trait. The engine never called them; the built-in implementations keep them as private helpers. Document the durability contract on the trait and check it in the conformance suite: snapshot metadata passed to persist_last_snapshot_metadata must survive a reopen. That test replaces test_persistence, which asserted nothing. It found that the file state machine and the sled example lost the metadata on restart; both are fixed. Docs, CHANGELOG and MIGRATION_GUIDE follow. --- CHANGELOG.md | 8 +- MIGRATION_GUIDE.md | 3 + .../leader_state_test/client_read_test.rs | 2 - .../leader_state_test/snapshot_worker_test.rs | 5 - d-engine-core/src/storage/state_machine.rs | 56 +++---- .../src/storage/state_machine_test.rs | 95 ++++------- .../src/test_utils/mock/mock_raft_builder.rs | 6 - .../api/embedded_test/shutdown_order_test.rs | 27 ---- d-engine-server/src/node/node_test.rs | 2 +- .../adaptors/file/file_state_machine.rs | 152 ++++++++++++------ .../adaptors/rocksdb/rocksdb_state_machine.rs | 108 ++++++------- .../rocksdb/rocksdb_state_machine_test.rs | 4 +- .../src/test_utils/mock/mock_node_builder.rs | 6 - .../tests/common/shared_state_machine.rs | 27 ---- .../sm_metadata_atomicity_embedded.rs | 4 +- .../server_guide/customize-state-machine.md | 17 +- .../sled-cluster/src/sled_state_machine.rs | 149 +++++++++-------- .../state_machine/sled_state_machine_test.rs | 5 +- 18 files changed, 301 insertions(+), 375 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d7da7d3..4447fea3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,10 @@ All notable changes to this project will be documented in this file. - **File state machine lost empty values on WAL replay** (#464): `replay_wal` treated a zero value length as "no value", so an `Insert` with an empty value was dropped after a restart. It now restores the empty value. +- **File state machine forgot its snapshot metadata on restart** (#400): `persist_last_snapshot_metadata` only updated memory, and `metadata.bin` stores only `last_applied`. The metadata is now written to `snapshot_metadata.bin` (temporary file, `fsync`, rename) and loaded when the state machine opens; a file that cannot be decoded fails the open instead of being ignored. + +- **Snapshot checksum ignored files in sub-directories and file boundaries** (#400): see the snapshot checksum entry under Changed. + ### Changed - **⚠️ Wire protocol is not compatible with v0.2.4**: `ClientResponse.metadata` moved from field 4 to 5 (field 4 is now `scan_data`); `HandleClientScan` returns `ClientResponse` (the `ScanResponse` message is now `ScanResults`); the `ChunkStatus` values are renumbered with `CHUNK_STATUS_UNSPECIFIED = 0`; `WriteCommand.Insert.ttl_secs` (field 3) is reserved; new RPC `PreVote`. Do not run v0.2.4 and v0.3.0 nodes or clients together. See `MIGRATION_GUIDE.md`. @@ -116,7 +120,9 @@ All notable changes to this project will be documented in this file. "MemFirst"` or `"DiskFirst"` are silently ignored, not an error — remove the field, `flush_policy` is the only persistence knob now. -- **⚠️ `StateMachine::generate_snapshot_data` returns `Result<()>`; the engine owns snapshot checksums** (#400): the engine computes the SHA-256 of the exported snapshot directory and verifies it before calling `apply_snapshot_from_file`, which now only ever receives a directory that matches the recorded checksum. A state machine neither returns nor checks a checksum. The checksum algorithm is unchanged. See [MIGRATION_GUIDE.md](./MIGRATION_GUIDE.md). +- **⚠️ `StateMachine` no longer declares `update_last_applied`, `persist_last_applied`, `update_last_snapshot_metadata`, `save_hard_state`, `flush` and `flush_async`** (#400): the engine never called them. Keep any you still need as methods on your own type. Your state machine must restore `last_applied` and its snapshot metadata when it is opened; the `StateMachine` docs now state this as a durability contract, and `StateMachineTestSuite` checks it. See [MIGRATION_GUIDE.md](./MIGRATION_GUIDE.md). + +- **⚠️ `StateMachine::generate_snapshot_data` returns `Result<()>`; the engine owns snapshot checksums** (#400): the engine computes the SHA-256 of the exported snapshot directory and verifies it before calling `apply_snapshot_from_file`, which now only ever receives a directory that matches the recorded checksum. A state machine neither returns nor checks a checksum. The checksum now covers every file at any depth together with its relative path and length; before, it covered only top-level files, so the data files RocksDB exports into sub-directories were not protected. Snapshot archives created before the upgrade fail verification, so remove old snapshot archives when upgrading. See [MIGRATION_GUIDE.md](./MIGRATION_GUIDE.md). - **⚠️ The engine takes the state machine by value** (#400): `EmbeddedEngine::start_custom`/`start_node`, `StandaloneEngine::run_custom`/`start_node` take `SM` instead of `Arc`; the `Arc` is created inside the engine. Nothing outside the engine holds the state machine, so `stop()` returns only after it is dropped and its resources are released. Embedded reads now go through the ReadActor like standalone reads (`EmbeddedReadHandle` is removed). `EmbeddedEngine` and `EmbeddedClient` no longer have type parameters. See [MIGRATION_GUIDE.md](./MIGRATION_GUIDE.md). diff --git a/MIGRATION_GUIDE.md b/MIGRATION_GUIDE.md index 00ac72c5..f512d575 100644 --- a/MIGRATION_GUIDE.md +++ b/MIGRATION_GUIDE.md @@ -34,6 +34,9 @@ EmbeddedEngine::start_custom(dir, storage, sm, None).await?; - **If you kept a clone of your `Arc` to read local data directly** (for example `DLmdb` reading its own LMDB), you can no longer clone the instance the engine uses. Share a handle to your storage instead: put the handles (LMDB `Env`, sled `Db`, `Arc`) behind an `Arc` inside your state machine type, derive `Clone` on it, give the engine one clone and keep the other. - **Embedded reads (`get_eventual`, lease reads) are answered by the ReadActor**, as they already were in standalone mode, instead of reading the state machine on the caller's thread. Expect one extra channel hop per read. - **Snapshot checksums belong to the engine.** `generate_snapshot_data` now returns `Result<()>`: write the snapshot into the directory you are given and return. The engine computes the checksum of that directory itself, and verifies it before it calls `apply_snapshot_from_file`, so you no longer compute or check a checksum. Delete that code (and any hashing dependency you added only for it). +- **Six methods left the trait:** `update_last_applied`, `persist_last_applied`, `update_last_snapshot_metadata`, `save_hard_state`, `flush` and `flush_async`. The engine never called them, so deleting them from your `impl StateMachine` block changes no behaviour. Keep any your own code still uses as plain methods on your type. +- **Restore your state when you open it.** The engine does not do it for you: after reopening, `last_applied()` must return the last entry whose effects are durable, and `snapshot_metadata()` must return the value most recently passed to `persist_last_snapshot_metadata()` (which must therefore update memory and write to disk). `StateMachineTestSuite` now checks the snapshot metadata case. +- **The snapshot directory must contain only regular files and directories.** The engine rejects symbolic links and other special entries instead of skipping them. - When `stop()` returns, the engine has dropped the state machine. If your state machine releases its resources in `Drop` (a database handle, a lock file), they are free at that point and the same data directory can be reopened. --- diff --git a/d-engine-core/src/raft_role/leader_state_test/client_read_test.rs b/d-engine-core/src/raft_role/leader_state_test/client_read_test.rs index 00061199..2ce3a622 100644 --- a/d-engine-core/src/raft_role/leader_state_test/client_read_test.rs +++ b/d-engine-core/src/raft_role/leader_state_test/client_read_test.rs @@ -1208,8 +1208,6 @@ async fn test_lease_reuse_after_linearizable_read_refresh() { state_machine .expect_last_applied() .return_const(d_engine_proto::common::LogId { index: 1, term: 1 }); - state_machine.expect_update_last_applied().returning(|_| ()); - state_machine.expect_persist_last_applied().returning(|_| Ok(())); let ctx = MockBuilder::new(shutdown_rx) .with_db_path("/tmp/test_lease_reuse") diff --git a/d-engine-core/src/raft_role/leader_state_test/snapshot_worker_test.rs b/d-engine-core/src/raft_role/leader_state_test/snapshot_worker_test.rs index 4ffea524..d92d235d 100644 --- a/d-engine-core/src/raft_role/leader_state_test/snapshot_worker_test.rs +++ b/d-engine-core/src/raft_role/leader_state_test/snapshot_worker_test.rs @@ -104,10 +104,7 @@ fn state_machine_with_snapshot() -> MockStateMachine { sm.expect_close().returning(|| Ok(())); sm.expect_get().returning(|_| Ok(None)); sm.expect_apply_chunk().returning(|_| Ok(vec![])); - sm.expect_update_last_applied().returning(|_| ()); sm.expect_last_applied().return_const(LogId::default()); - sm.expect_persist_last_applied().returning(|_| Ok(())); - sm.expect_update_last_snapshot_metadata().returning(|_| Ok(())); // Return a valid snapshot so Phase 6 fires the Snapshot task to the worker. sm.expect_snapshot_metadata().returning(|| Some(stub_snapshot_metadata())); sm.expect_persist_last_snapshot_metadata().returning(|_| Ok(())); @@ -117,8 +114,6 @@ fn state_machine_with_snapshot() -> MockStateMachine { }) }); sm.expect_generate_snapshot_data().returning(|_, _| Ok(())); - sm.expect_save_hard_state().returning(|| Ok(())); - sm.expect_flush().returning(|| Ok(())); sm } diff --git a/d-engine-core/src/storage/state_machine.rs b/d-engine-core/src/storage/state_machine.rs index 4dea4b79..0a989003 100644 --- a/d-engine-core/src/storage/state_machine.rs +++ b/d-engine-core/src/storage/state_machine.rs @@ -82,6 +82,22 @@ impl ApplyResult { /// - Read methods (`get()`) may be called concurrently /// - Write methods should use internal synchronization /// - No assumptions about caller's threading model +/// +/// # Durability contract +/// +/// The engine never asks a state machine to restore itself; it only calls the methods below. An +/// implementation therefore has to restore its own state whenever it is opened: +/// +/// - After reopening the same storage, [`last_applied`](Self::last_applied) returns the position of +/// the last entry whose effects are durable. Persist it together with the data it describes (in +/// one transaction or write batch where the storage allows), so a restart never replays an entry +/// that was already applied. +/// - After reopening, [`snapshot_metadata`](Self::snapshot_metadata) returns the value most +/// recently passed to [`persist_last_snapshot_metadata`](Self::persist_last_snapshot_metadata). +/// The engine publishes snapshot metadata only through that method, so it must update what +/// `snapshot_metadata()` returns and write it durably. +/// +/// `StateMachineTestSuite` checks these rules; run it against your implementation. #[cfg_attr(any(test, feature = "__test_support"), automock)] #[async_trait] pub trait StateMachine: Send + Sync + 'static { @@ -90,9 +106,10 @@ pub trait StateMachine: Send + Sync + 'static { /// (e.g. a DB handle or its lock file). /// /// Called once by the state machine worker as its last act, after everything queued for - /// it has been applied. The developer may keep their own `Arc` to the state machine, so - /// this must not rely on `Drop`. Must be synchronous and idempotent; afterwards every - /// operation returns `NotServing`. + /// it has been applied. The engine owns the state machine, but a state machine that shares + /// its storage handle with the developer (a cheap `Clone`) is not dropped when the engine + /// stops, so this must not rely on `Drop`. Must be synchronous and idempotent; afterwards + /// every operation returns `NotServing`. /// /// When it returns, reopening the same storage restores every applied entry and a /// `last_applied` that is not older than the one before the call, and every exclusive @@ -160,31 +177,10 @@ pub trait StateMachine: Send + Sync + 'static { chunk: &[ApplyEntry], ) -> Result, Error>; - /// Updates the last applied index in memory. - /// Sync operation as it just updates atomic variables. - fn update_last_applied( - &self, - last_applied: LogId, - ); - /// Gets the last applied log index and term. /// Sync operation as it reads from atomic variables. fn last_applied(&self) -> LogId; - /// Persists the last applied index to durable storage. - /// Should be async as it involves disk I/O. - fn persist_last_applied( - &self, - last_applied: LogId, - ) -> Result<(), Error>; - - /// Updates snapshot metadata in memory. - /// Sync operation as it updates in-memory structures. - fn update_last_snapshot_metadata( - &self, - snapshot_metadata: &SnapshotMetadata, - ) -> Result<(), Error>; - /// Retrieves the current snapshot metadata. /// Sync operation as it reads from in-memory structures. fn snapshot_metadata(&self) -> Option; @@ -243,18 +239,6 @@ pub trait StateMachine: Send + Sync + 'static { last_included: LogId, ) -> Result<(), Error>; - /// Saves the hard state of the state machine. - /// Sync operation as it typically just delegates to other persistence methods. - fn save_hard_state(&self) -> Result<(), Error>; - - /// Flushes any pending writes to durable storage. - /// Sync operation that may block but provides a synchronous interface. - fn flush(&self) -> Result<(), Error>; - - /// Flushes any pending writes to durable storage. - /// Async operation as it involves disk I/O. - async fn flush_async(&self) -> Result<(), Error>; - /// Resets the state machine to its initial state. /// Async operation as it may involve cleaning up files and data. async fn reset(&self) -> Result<(), Error>; diff --git a/d-engine-core/src/storage/state_machine_test.rs b/d-engine-core/src/storage/state_machine_test.rs index 84358726..7d70dd01 100644 --- a/d-engine-core/src/storage/state_machine_test.rs +++ b/d-engine-core/src/storage/state_machine_test.rs @@ -44,7 +44,7 @@ impl StateMachineTestSuite { Self::test_last_applied_detection(builder.build().await?).await?; Self::test_snapshot_operations(builder.build().await?).await?; Self::test_snapshot_apply_rejects_missing_boundary(builder.build().await?).await?; - Self::test_persistence(builder.build().await?).await?; + Self::test_snapshot_metadata_survives_reopen(&builder).await?; Self::test_drop_flushes_data(&builder).await?; Self::test_drop_persists_last_applied(&builder).await?; @@ -402,36 +402,43 @@ impl StateMachineTestSuite { Ok(()) } - /// Test data persistence - pub async fn test_persistence(state_machine: Arc) -> Result<(), Error> { - // Add test data - let entries = vec![ - create_insert_entry( - 1, - Bytes::from(b"key1".to_vec()), - Bytes::from(b"value1".to_vec()), - ), - create_insert_entry( - 2, - Bytes::from(b"key2".to_vec()), - Bytes::from(b"value2".to_vec()), - ), - ]; - state_machine.apply_chunk(&entries).await?; - - // Update last applied - let last_applied = LogId { index: 2, term: 1 }; - state_machine.persist_last_applied(last_applied)?; + /// Snapshot metadata published through `persist_last_snapshot_metadata` must survive a + /// restart. + /// + /// # Scenario + /// A node builds a snapshot and the engine publishes its metadata, once the archive is + /// durably built, by calling `persist_last_snapshot_metadata`. The node then restarts. + /// + /// # Expected + /// The reopened state machine reports exactly that metadata from `snapshot_metadata()`. + /// Otherwise the node forgets which snapshot it holds and can no longer tell followers + /// where its snapshot ends. + pub async fn test_snapshot_metadata_survives_reopen( + builder: &B + ) -> Result<(), Error> { + let sm = builder.build().await?; + sm.apply_chunk(&[ + create_insert_entry(1, Bytes::from_static(b"k1"), Bytes::from_static(b"v1")), + create_insert_entry(2, Bytes::from_static(b"k2"), Bytes::from_static(b"v2")), + create_insert_entry(3, Bytes::from_static(b"k3"), Bytes::from_static(b"v3")), + ]) + .await?; - // Update snapshot metadata - let snapshot_metadata = SnapshotMetadata { - last_included: Some(last_applied), - checksum: Bytes::from(vec![0; 32]), + let metadata = SnapshotMetadata { + last_included: Some(LogId { index: 3, term: 1 }), + checksum: Bytes::from(vec![7; 32]), }; - state_machine.persist_last_snapshot_metadata(&snapshot_metadata)?; + sm.persist_last_snapshot_metadata(&metadata)?; + sm.close()?; + // `sm` stays alive until the end of the test: Drop must not be what saves the state. - // Flush to ensure persistence - state_machine.flush()?; + let reopened = builder.build().await?; + assert_eq!( + reopened.snapshot_metadata(), + Some(metadata), + "snapshot metadata published with persist_last_snapshot_metadata must survive a \ + restart" + ); Ok(()) } @@ -1695,30 +1702,10 @@ mod default_scan_prefix_tests { Ok(vec![]) } - fn update_last_applied( - &self, - _last_applied: LogId, - ) { - } - fn last_applied(&self) -> LogId { LogId::default() } - fn persist_last_applied( - &self, - _last_applied: LogId, - ) -> Result<(), Error> { - Ok(()) - } - - fn update_last_snapshot_metadata( - &self, - _snapshot_metadata: &SnapshotMetadata, - ) -> Result<(), Error> { - Ok(()) - } - fn snapshot_metadata(&self) -> Option { None } @@ -1748,18 +1735,6 @@ mod default_scan_prefix_tests { Ok(()) } - fn save_hard_state(&self) -> Result<(), Error> { - Ok(()) - } - - fn flush(&self) -> Result<(), Error> { - Ok(()) - } - - async fn flush_async(&self) -> Result<(), Error> { - Ok(()) - } - async fn reset(&self) -> Result<(), Error> { Ok(()) } diff --git a/d-engine-core/src/test_utils/mock/mock_raft_builder.rs b/d-engine-core/src/test_utils/mock/mock_raft_builder.rs index 5b2dbe76..d7ab4631 100644 --- a/d-engine-core/src/test_utils/mock/mock_raft_builder.rs +++ b/d-engine-core/src/test_utils/mock/mock_raft_builder.rs @@ -387,11 +387,8 @@ pub fn mock_state_machine() -> MockStateMachine { mock.expect_get().returning(|_| Ok(None)); mock.expect_apply_chunk().returning(|_| Ok(vec![])); - mock.expect_update_last_applied().returning(|_| ()); mock.expect_last_applied().return_const(LogId::default()); - mock.expect_persist_last_applied().returning(|_| Ok(())); - mock.expect_update_last_snapshot_metadata().returning(|_| Ok(())); mock.expect_snapshot_metadata().returning(|| None); mock.expect_persist_last_snapshot_metadata().returning(|_| Ok(())); @@ -402,9 +399,6 @@ pub fn mock_state_machine() -> MockStateMachine { }); mock.expect_generate_snapshot_data().returning(|_, _| Ok(())); - mock.expect_save_hard_state().returning(|| Ok(())); - mock.expect_flush().returning(|| Ok(())); - mock } diff --git a/d-engine-server/src/api/embedded_test/shutdown_order_test.rs b/d-engine-server/src/api/embedded_test/shutdown_order_test.rs index f4ec1a22..9ffab31f 100644 --- a/d-engine-server/src/api/embedded_test/shutdown_order_test.rs +++ b/d-engine-server/src/api/embedded_test/shutdown_order_test.rs @@ -81,27 +81,9 @@ impl StateMachine for SlowApplyProbe { ) -> Result, Error> { self.inner.get(key_buffer) } - fn update_last_applied( - &self, - last_applied: LogId, - ) { - self.inner.update_last_applied(last_applied) - } fn last_applied(&self) -> LogId { self.inner.last_applied() } - fn persist_last_applied( - &self, - last_applied: LogId, - ) -> Result<(), Error> { - self.inner.persist_last_applied(last_applied) - } - fn update_last_snapshot_metadata( - &self, - snapshot_metadata: &SnapshotMetadata, - ) -> Result<(), Error> { - self.inner.update_last_snapshot_metadata(snapshot_metadata) - } fn snapshot_metadata(&self) -> Option { self.inner.snapshot_metadata() } @@ -125,15 +107,6 @@ impl StateMachine for SlowApplyProbe { ) -> Result<(), Error> { self.inner.generate_snapshot_data(new_snapshot_dir, last_included).await } - fn save_hard_state(&self) -> Result<(), Error> { - self.inner.save_hard_state() - } - fn flush(&self) -> Result<(), Error> { - self.inner.flush() - } - async fn flush_async(&self) -> Result<(), Error> { - self.inner.flush_async().await - } async fn reset(&self) -> Result<(), Error> { self.inner.reset().await } diff --git a/d-engine-server/src/node/node_test.rs b/d-engine-server/src/node/node_test.rs index b26f9c1b..a86760ba 100644 --- a/d-engine-server/src/node/node_test.rs +++ b/d-engine-server/src/node/node_test.rs @@ -1175,7 +1175,7 @@ mod sm_worker_lifecycle_tests { /// fails at its first step, before anything else was started. /// /// Purpose: the unconditional tail still runs and hands the state machine back closed even - /// though the worker never ran, because the developer may keep their own `Arc` to it. + /// though the worker never ran, because a handle the developer shares may keep it alive. #[test] fn test_run_failing_to_start_the_worker_still_closes_the_state_machine() { let closed = Arc::new(std::sync::atomic::AtomicUsize::new(0)); diff --git a/d-engine-server/src/storage/adaptors/file/file_state_machine.rs b/d-engine-server/src/storage/adaptors/file/file_state_machine.rs index 53bba332..683c634c 100644 --- a/d-engine-server/src/storage/adaptors/file/file_state_machine.rs +++ b/d-engine-server/src/storage/adaptors/file/file_state_machine.rs @@ -75,6 +75,7 @@ use d_engine_core::StorageError; use d_engine_proto::common::LogId; use d_engine_proto::server::storage::SnapshotMetadata; use parking_lot::RwLock; +use prost::Message; use tokio::fs; use tokio::fs::File; use tokio::fs::OpenOptions; @@ -86,6 +87,9 @@ use tracing::error; use tracing::info; use tracing::warn; +/// File holding the snapshot metadata published by `persist_last_snapshot_metadata`. +const SNAPSHOT_METADATA_FILE: &str = "snapshot_metadata.bin"; + type FileStateMachineDataType = RwLock>; /// WAL operation codes for fixed-size encoding @@ -266,6 +270,9 @@ impl FileStateMachine { // Load last applied index and term from metadata file self.load_metadata().await?; + // Load the snapshot metadata published by the last persist_last_snapshot_metadata + self.load_snapshot_metadata().await?; + // Load key-value data from data file self.load_data().await?; @@ -276,6 +283,40 @@ impl FileStateMachine { Ok(()) } + /// Loads the snapshot metadata file, if one was ever persisted. + /// + /// A file that exists but cannot be decoded is an error: silently starting without it would + /// make the node forget which snapshot it holds. + async fn load_snapshot_metadata(&self) -> Result<(), Error> { + let path = self.data_dir.join(SNAPSHOT_METADATA_FILE); + if !path.exists() { + return Ok(()); + } + let bytes = fs::read(&path).await?; + let metadata = SnapshotMetadata::decode(bytes.as_slice()).map_err(|e| { + StorageError::StateMachineError(format!("corrupt {SNAPSHOT_METADATA_FILE}: {e}")) + })?; + *self.last_snapshot_metadata.write() = Some(metadata); + Ok(()) + } + + /// Writes the snapshot metadata durably: to a temporary file first, then renamed over the + /// real one, so a crash leaves either the old file or the new one, never a partial file. + fn persist_snapshot_metadata( + &self, + metadata: &SnapshotMetadata, + ) -> Result<(), Error> { + let path = self.data_dir.join(SNAPSHOT_METADATA_FILE); + let temp_path = self.data_dir.join(format!("{SNAPSHOT_METADATA_FILE}.tmp")); + { + let mut file = std::fs::File::create(&temp_path)?; + file.write_all(&metadata.encode_to_vec())?; + file.sync_all()?; + } + std::fs::rename(&temp_path, &path)?; + std::fs::File::open(&self.data_dir)?.sync_all()?; + Ok(()) + } /// Loads metadata from disk async fn load_metadata(&self) -> Result<(), Error> { let metadata_path = self.data_dir.join("metadata.bin"); @@ -619,6 +660,49 @@ impl FileStateMachine { Ok(()) } + pub(crate) fn update_last_applied( + &self, + last_applied: LogId, + ) { + self.last_applied_index.store(last_applied.index, Ordering::SeqCst); + self.last_applied_term.store(last_applied.term, Ordering::SeqCst); + } + + pub(crate) fn persist_last_applied( + &self, + last_applied: LogId, + ) -> Result<(), Error> { + self.update_last_applied(last_applied); + self.persist_metadata() + } + + pub(crate) fn update_last_snapshot_metadata( + &self, + snapshot_metadata: &SnapshotMetadata, + ) -> Result<(), Error> { + *self.last_snapshot_metadata.write() = Some(snapshot_metadata.clone()); + Ok(()) + } + + pub(crate) fn save_hard_state(&self) -> Result<(), Error> { + let last_applied = self.last_applied(); + self.persist_last_applied(last_applied)?; + + if let Some(last_snapshot_metadata) = self.snapshot_metadata() { + self.persist_last_snapshot_metadata(&last_snapshot_metadata)?; + } + + self.flush()?; + Ok(()) + } + + pub(crate) fn flush(&self) -> Result<(), Error> { + self.persist_data()?; + self.persist_metadata()?; + // self.clear_wal()?; + Ok(()) + } + /// Persists metadata to disk fn persist_metadata(&self) -> Result<(), Error> { let metadata_path = self.data_dir.join("metadata.bin"); @@ -785,11 +869,20 @@ impl FileStateMachine { file.write_all(&0u64.to_be_bytes()).await?; file.flush().await?; + + // The snapshot metadata is part of the state being reset + match fs::remove_file(self.data_dir.join(SNAPSHOT_METADATA_FILE)).await { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => return Err(e.into()), + } + Ok(()) } - /// Reads are served only while running. After `close()` the developer may still hold a clone - /// of the state machine, and it must not answer from state that is no longer maintained. + /// Reads are served only while running. After `close()` the state machine may still be + /// reachable through a handle the developer shares, and it must not answer from state that + /// is no longer maintained. fn ensure_serving(&self) -> Result<(), Error> { if self.running.load(Ordering::Acquire) { Ok(()) @@ -862,8 +955,8 @@ impl StateMachine for FileStateMachine { if !self.running.swap(false, Ordering::SeqCst) { return Ok(()); // already closed } - // The developer may keep an `Arc` to this state machine, so `Drop` may never run: - // persist everything a restart needs here. + // `Drop` does not run when the engine stops if the developer still holds a shared + // handle: persist everything a restart needs here. // No exclusive OS lock file to release. self.save_hard_state() } @@ -1074,14 +1167,6 @@ impl StateMachine for FileStateMachine { Ok(results) } - fn update_last_applied( - &self, - last_applied: LogId, - ) { - self.last_applied_index.store(last_applied.index, Ordering::SeqCst); - self.last_applied_term.store(last_applied.term, Ordering::SeqCst); - } - fn last_applied(&self) -> LogId { LogId { index: self.last_applied_index.load(Ordering::SeqCst), @@ -1089,22 +1174,6 @@ impl StateMachine for FileStateMachine { } } - fn persist_last_applied( - &self, - last_applied: LogId, - ) -> Result<(), Error> { - self.update_last_applied(last_applied); - self.persist_metadata() - } - - fn update_last_snapshot_metadata( - &self, - snapshot_metadata: &SnapshotMetadata, - ) -> Result<(), Error> { - *self.last_snapshot_metadata.write() = Some(snapshot_metadata.clone()); - Ok(()) - } - fn snapshot_metadata(&self) -> Option { self.last_snapshot_metadata.read().clone() } @@ -1113,7 +1182,8 @@ impl StateMachine for FileStateMachine { &self, snapshot_metadata: &SnapshotMetadata, ) -> Result<(), Error> { - self.update_last_snapshot_metadata(snapshot_metadata) + self.update_last_snapshot_metadata(snapshot_metadata)?; + self.persist_snapshot_metadata(snapshot_metadata) } async fn apply_snapshot_from_file( @@ -1301,30 +1371,6 @@ impl StateMachine for FileStateMachine { Ok(()) } - fn save_hard_state(&self) -> Result<(), Error> { - let last_applied = self.last_applied(); - self.persist_last_applied(last_applied)?; - - if let Some(last_snapshot_metadata) = self.snapshot_metadata() { - self.persist_last_snapshot_metadata(&last_snapshot_metadata)?; - } - - self.flush()?; - Ok(()) - } - - fn flush(&self) -> Result<(), Error> { - self.persist_data()?; - self.persist_metadata()?; - // self.clear_wal()?; - Ok(()) - } - - async fn flush_async(&self) -> Result<(), Error> { - // Force checkpoint on shutdown to ensure WAL entries are not lost. - self.checkpoint().await - } - async fn reset(&self) -> Result<(), Error> { self.reset().await } diff --git a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs index 9076ff54..dee6339a 100644 --- a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs +++ b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs @@ -496,6 +496,53 @@ impl RocksDBStateMachine { Ok(ScanResult { entries, revision }) } + pub(crate) fn update_last_applied( + &self, + last_applied: LogId, + ) { + self.last_applied_index.store(last_applied.index, Ordering::SeqCst); + self.last_applied_term.store(last_applied.term, Ordering::SeqCst); + } + + pub(crate) fn update_last_snapshot_metadata( + &self, + snapshot_metadata: &SnapshotMetadata, + ) -> Result<(), Error> { + *self.last_snapshot_metadata.write() = Some(snapshot_metadata.clone()); + Ok(()) + } + + pub(crate) fn save_hard_state(&self) -> Result<(), Error> { + // last_applied_index/term are now persisted atomically with each + // apply_chunk() batch (see apply_chunk()) — no longer written here. + self.persist_snapshot_metadata() + } + + /// Flushes the WAL and both state machine column families to disk. + pub fn flush(&self) -> Result<(), Error> { + self.with_db(|db| { + db.flush_wal(true).map_err(|e| StorageError::DbError(e.to_string()))?; + + // db.flush() with no CF argument only flushes RocksDB's implicit + // "default" column family, which this state machine never writes to — + // it is a silent no-op for STATE_MACHINE_CF and STATE_MACHINE_META_CF. + // Both must be flushed explicitly, matching the precedent already used + // in create_snapshot()'s pre-export flush. + let cf_sm = db + .cf_handle(STATE_MACHINE_CF) + .ok_or_else(|| StorageError::DbError("State machine CF not found".to_string()))?; + let cf_sm_meta = db.cf_handle(STATE_MACHINE_META_CF).ok_or_else(|| { + StorageError::DbError("State machine meta CF not found".to_string()) + })?; + let flush_opts = rocksdb::FlushOptions::default(); + db.flush_cfs_opt(&[&cf_sm, &cf_sm_meta], &flush_opts) + .map_err(|e| StorageError::DbError(e.to_string()))?; + Ok(()) + }) + // last_applied is already durable as of apply_chunk() — no separate + // persist_state_machine_metadata() call needed here anymore. + } + /// Test-only: quiesce RocksDB's background compaction/flush WITHOUT flushing /// MemTable or saving hard state — simulates what a /// real `kill -9` leaves behind on disk (no "dying gasp" persistence a real @@ -704,14 +751,6 @@ impl StateMachine for RocksDBStateMachine { Ok(results) } - fn update_last_applied( - &self, - last_applied: LogId, - ) { - self.last_applied_index.store(last_applied.index, Ordering::SeqCst); - self.last_applied_term.store(last_applied.term, Ordering::SeqCst); - } - fn last_applied(&self) -> LogId { LogId { index: self.last_applied_index.load(Ordering::SeqCst), @@ -719,22 +758,6 @@ impl StateMachine for RocksDBStateMachine { } } - fn persist_last_applied( - &self, - last_applied: LogId, - ) -> Result<(), Error> { - self.update_last_applied(last_applied); - self.persist_state_machine_metadata() - } - - fn update_last_snapshot_metadata( - &self, - snapshot_metadata: &SnapshotMetadata, - ) -> Result<(), Error> { - *self.last_snapshot_metadata.write() = Some(snapshot_metadata.clone()); - Ok(()) - } - fn snapshot_metadata(&self) -> Option { self.last_snapshot_metadata.read().clone() } @@ -868,40 +891,6 @@ impl StateMachine for RocksDBStateMachine { Ok(()) } - fn save_hard_state(&self) -> Result<(), Error> { - // last_applied_index/term are now persisted atomically with each - // apply_chunk() batch (see apply_chunk()) — no longer written here. - self.persist_snapshot_metadata() - } - - fn flush(&self) -> Result<(), Error> { - self.with_db(|db| { - db.flush_wal(true).map_err(|e| StorageError::DbError(e.to_string()))?; - - // db.flush() with no CF argument only flushes RocksDB's implicit - // "default" column family, which this state machine never writes to — - // it is a silent no-op for STATE_MACHINE_CF and STATE_MACHINE_META_CF. - // Both must be flushed explicitly, matching the precedent already used - // in create_snapshot()'s pre-export flush. - let cf_sm = db - .cf_handle(STATE_MACHINE_CF) - .ok_or_else(|| StorageError::DbError("State machine CF not found".to_string()))?; - let cf_sm_meta = db.cf_handle(STATE_MACHINE_META_CF).ok_or_else(|| { - StorageError::DbError("State machine meta CF not found".to_string()) - })?; - let flush_opts = rocksdb::FlushOptions::default(); - db.flush_cfs_opt(&[&cf_sm, &cf_sm_meta], &flush_opts) - .map_err(|e| StorageError::DbError(e.to_string()))?; - Ok(()) - }) - // last_applied is already durable as of apply_chunk() — no separate - // persist_state_machine_metadata() call needed here anymore. - } - - async fn flush_async(&self) -> Result<(), Error> { - self.flush() - } - async fn reset(&self) -> Result<(), Error> { self.with_db(|db| { let cf = db.cf_handle(STATE_MACHINE_CF).ok_or_else(|| { @@ -978,7 +967,8 @@ impl StateMachine for RocksDBStateMachine { } impl Drop for RocksDBStateMachine { fn drop(&mut self) { - // Safety net for an owner that never called `close()`; a no-op if it already did. + // Safety net when the state machine is dropped without `close()` having been called; + // a no-op if it already was. if let Err(e) = StateMachine::close(self) { error!("RocksDB state machine: close on drop failed: {e:?}"); } diff --git a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine_test.rs b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine_test.rs index f21f663f..6af5eaf0 100644 --- a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine_test.rs +++ b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine_test.rs @@ -513,8 +513,8 @@ async fn test_get_multi_two_keys_still_uses_snapshot_isolation() { // ── close_db() tests ────────────────────────────────────────────────────────── // // close() is a permanent, one-way shutdown that releases the RocksDB LOCK file -// immediately, without waiting for Arc to be the sole owner (the developer may keep -// their own clone). +// immediately, without waiting for every reference to the state machine to be dropped (a +// handle the developer shares may outlive the engine). /// close() makes reads return a NotServing error, never an empty result. #[tokio::test] diff --git a/d-engine-server/src/test_utils/mock/mock_node_builder.rs b/d-engine-server/src/test_utils/mock/mock_node_builder.rs index 7e85bd1f..4f4290f1 100644 --- a/d-engine-server/src/test_utils/mock/mock_node_builder.rs +++ b/d-engine-server/src/test_utils/mock/mock_node_builder.rs @@ -629,11 +629,8 @@ pub(crate) fn mock_state_machine() -> MockStateMachine { mock.expect_get().returning(|_| Ok(None)); mock.expect_apply_chunk().returning(|_| Ok(vec![])); - mock.expect_update_last_applied().returning(|_| ()); mock.expect_last_applied().return_const(LogId::default()); - mock.expect_persist_last_applied().returning(|_| Ok(())); - mock.expect_update_last_snapshot_metadata().returning(|_| Ok(())); mock.expect_snapshot_metadata().returning(|| None); mock.expect_persist_last_snapshot_metadata().returning(|_| Ok(())); @@ -644,9 +641,6 @@ pub(crate) fn mock_state_machine() -> MockStateMachine { }); mock.expect_generate_snapshot_data().returning(|_, _| Ok(())); - mock.expect_save_hard_state().returning(|| Ok(())); - mock.expect_flush().returning(|| Ok(())); - mock } diff --git a/d-engine-server/tests/common/shared_state_machine.rs b/d-engine-server/tests/common/shared_state_machine.rs index 4573e162..03239fb5 100644 --- a/d-engine-server/tests/common/shared_state_machine.rs +++ b/d-engine-server/tests/common/shared_state_machine.rs @@ -55,27 +55,9 @@ impl StateMachine for SharedStateMachine { ) -> Result, Error> { self.0.apply_chunk(chunk).await } - fn update_last_applied( - &self, - last_applied: LogId, - ) { - self.0.update_last_applied(last_applied) - } fn last_applied(&self) -> LogId { self.0.last_applied() } - fn persist_last_applied( - &self, - last_applied: LogId, - ) -> Result<(), Error> { - self.0.persist_last_applied(last_applied) - } - fn update_last_snapshot_metadata( - &self, - snapshot_metadata: &SnapshotMetadata, - ) -> Result<(), Error> { - self.0.update_last_snapshot_metadata(snapshot_metadata) - } fn snapshot_metadata(&self) -> Option { self.0.snapshot_metadata() } @@ -99,15 +81,6 @@ impl StateMachine for SharedStateMachine { ) -> Result<(), Error> { self.0.generate_snapshot_data(new_snapshot_dir, last_included).await } - fn save_hard_state(&self) -> Result<(), Error> { - self.0.save_hard_state() - } - fn flush(&self) -> Result<(), Error> { - self.0.flush() - } - async fn flush_async(&self) -> Result<(), Error> { - self.0.flush_async().await - } async fn reset(&self) -> Result<(), Error> { self.0.reset().await } diff --git a/d-engine-server/tests/failover_and_recovery/sm_metadata_atomicity_embedded.rs b/d-engine-server/tests/failover_and_recovery/sm_metadata_atomicity_embedded.rs index d95ceff6..2a0a751c 100644 --- a/d-engine-server/tests/failover_and_recovery/sm_metadata_atomicity_embedded.rs +++ b/d-engine-server/tests/failover_and_recovery/sm_metadata_atomicity_embedded.rs @@ -127,7 +127,7 @@ async fn test_flush_then_crash_preserves_data_and_last_applied_together() ) .await?; } - // Guarantees the SM has applied every write above before flush_async() runs. + // Guarantees the SM has applied every write above before flush() runs. let last = engine .client() .get_linearizable(format!("key-{:02}", N - 1).into_bytes()) @@ -137,7 +137,7 @@ async fn test_flush_then_crash_preserves_data_and_last_applied_together() Some(format!("val-{:02}", N - 1).as_bytes()) ); - sm_arc.flush_async().await?; + sm_arc.flush()?; let last_applied_after_flush = sm_arc.last_applied().index; // True crash: no graceful close, no further flush. diff --git a/d-engine/src/docs/server_guide/customize-state-machine.md b/d-engine/src/docs/server_guide/customize-state-machine.md index a207a22e..58e95f46 100644 --- a/d-engine/src/docs/server_guide/customize-state-machine.md +++ b/d-engine/src/docs/server_guide/customize-state-machine.md @@ -32,7 +32,8 @@ impl StateMachine for CustomStateMachine { // drains cleanly, but shutdown continues without that guarantee if // `shutdown_apply_timeout_ms` expires. It must be synchronous and idempotent, release its // resources even if persisting fails, and return the first error. - // You may keep your own Arc to the state machine, so do not rely on Drop alone. + // If you share your storage handle with the application (a cloneable state machine), the + // state machine is not dropped when the engine stops, so do not rely on Drop alone. // // fn close(&self) -> Result<(), Error> { // let flushed = self.backend.flush(); @@ -100,7 +101,8 @@ impl StateMachine for CustomStateMachine { - **Atomic Operations**: Ensure `apply_chunk()` either fully applies or fails the entire batch - **Idempotency**: Handle duplicate entries safely—Raft may resend committed entries - **Snapshot Isolation**: `apply_snapshot_from_file()` must atomically replace state -- **Checksum Validation**: Mandatory for snapshot integrity—validate before application +- **Snapshot Checksums**: The engine computes the checksum of the directory your `generate_snapshot_data()` writes, and verifies it before it calls `apply_snapshot_from_file()`. You neither compute nor check one +- **Restore On Open**: The engine never asks you to restore state. When your state machine is opened, `last_applied()` must return the last entry whose effects are durable (persist it together with the data it describes), and `snapshot_metadata()` must return the value most recently passed to `persist_last_snapshot_metadata()` - **Concurrency Control**: Use appropriate locking for state mutations - **Resource Cleanup**: Implement `Drop` to ensure proper flush on shutdown @@ -111,17 +113,11 @@ impl StateMachine for CustomStateMachine { | `close()` | Final step at shutdown: persist what must survive a restart and release exclusive OS resources (e.g. DB lock file), even if persisting fails; returns the first error. Idempotent. Default `Ok(())`. | Sync | Medium | | `get()` | Read value by key | Sync | High | | `apply_chunk()` | Apply committed entries | Async | Critical | -| `update_last_applied()` | Update applied index in memory | Sync | High | | `last_applied()` | Get last applied index | Sync | High | -| `persist_last_applied()` | Persist applied index | Sync | High | -| `update_last_snapshot_metadata()` | Update snapshot metadata in memory | Sync | Medium | | `snapshot_metadata()` | Get current snapshot metadata | Sync | Medium | | `persist_last_snapshot_metadata()` | Persist snapshot metadata | Sync | Medium | | `apply_snapshot_from_file()` | Replace state with snapshot | Async | Critical | -| `generate_snapshot_data()` | Create new snapshot | Async | High | -| `save_hard_state()` | Persist term/vote state | Sync | High | -| `flush()` | Sync writes to storage | Sync | High | -| `flush_async()` | Async flush | Async | High | +| `generate_snapshot_data()` | Export a snapshot into the given directory (the engine checksums it) | Async | High | | `reset()` | Reset to initial state | Async | Medium | ## 4. Testing Your Implementation @@ -158,8 +154,7 @@ async fn test_custom_state_machine() -> Result<(), Error> { // Test snapshot operations let temp_dir = TempDir::new()?; - let checksum = sm.generate_snapshot_data(temp_dir.path().to_path_buf(), LogId::new(1, 1)).await?; - assert!(!checksum.is_zero()); + sm.generate_snapshot_data(temp_dir.path().to_path_buf(), LogId::new(1, 1)).await?; sm.close()?; Ok(()) diff --git a/examples/sled-cluster/src/sled_state_machine.rs b/examples/sled-cluster/src/sled_state_machine.rs index 24a3ca20..5a6f928d 100644 --- a/examples/sled-cluster/src/sled_state_machine.rs +++ b/examples/sled-cluster/src/sled_state_machine.rs @@ -99,31 +99,6 @@ impl StateMachine for SledStateMachine { saved } - fn update_last_applied( - &self, - last_applied: LogId, - ) { - debug!(%self.node_id, ?last_applied, "update_last_applied"); - self.last_applied_index.store(last_applied.index, Ordering::SeqCst); - self.last_applied_term.store(last_applied.term, Ordering::SeqCst); - } - - fn update_last_snapshot_metadata( - &self, - snapshot_metadata: &SnapshotMetadata, - ) -> Result<()> { - debug!(%self.node_id, ?snapshot_metadata, "update_last_snapshot_metadata"); - let last_included = snapshot_metadata.last_included.unwrap(); - self.last_included_index.store(last_included.index, Ordering::SeqCst); - self.last_included_term.store(last_included.term, Ordering::SeqCst); - let checksum = snapshot_metadata - .checksum_array() - .map_err(|e| SnapshotError::OperationFailed(format!("{e:?}")))?; - *self.last_snapshot_checksum.lock() = Some(checksum); - - Ok(()) - } - fn last_applied(&self) -> LogId { LogId { index: self.last_applied_index.load(Ordering::SeqCst), @@ -261,47 +236,14 @@ impl StateMachine for SledStateMachine { Ok(results) } - fn save_hard_state(&self) -> Result<()> { - let last_applied = self.last_applied(); - self.persist_last_applied(last_applied)?; - - if let Some(last_snapshot_metadata) = self.snapshot_metadata() { - self.persist_last_snapshot_metadata(&last_snapshot_metadata)?; - } - - self.flush()?; - Ok(()) - } - - fn persist_last_applied( - &self, - last_applied: LogId, - ) -> Result<()> { - debug!(%self.node_id, ?last_applied, "persist_last_applied"); - let db = self.db()?; - let tree = db - .open_tree(STATE_MACHINE_META_NAMESPACE) - .map_err(|e| StorageError::DbError(e.to_string()))?; - tree.insert( - STATE_MACHINE_META_KEY_LAST_APPLIED_INDEX, - safe_kv(last_applied.index), - ) - .map_err(|e| StorageError::DbError(e.to_string()))?; - tree.insert( - STATE_MACHINE_META_KEY_LAST_APPLIED_TERM, - safe_kv(last_applied.term), - ) - .map_err(|e| StorageError::DbError(e.to_string()))?; - - tree.flush().map_err(|e| StorageError::DbError(e.to_string()))?; - Ok(()) - } - fn persist_last_snapshot_metadata( &self, last_snapshot_metadata: &SnapshotMetadata, ) -> Result<()> { debug!(%self.node_id, ?last_snapshot_metadata, "persist_last_snapshot_metadata"); + // The engine only ever calls this method, so it must also update the in-memory copy that + // `snapshot_metadata()` reads and that `close()` writes back. + self.update_last_snapshot_metadata(last_snapshot_metadata)?; let db = self.db()?; let tree = db .open_tree(STATE_SNAPSHOT_METADATA_TREE) @@ -311,20 +253,6 @@ impl StateMachine for SledStateMachine { Ok(()) } - fn flush(&self) -> Result<()> { - let db = self.db()?; - let bytes = db.flush().map_err(|e| StorageError::DbError(e.to_string()))?; - info!( - "Successfully flushed State Machine, bytes flushed: {}", - bytes - ); - Ok(()) - } - - async fn flush_async(&self) -> Result<()> { - self.flush() - } - async fn generate_snapshot_data( &self, new_snapshot_dir: PathBuf, @@ -677,6 +605,77 @@ impl SledStateMachine { tree.flush().map_err(|e| StorageError::DbError(e.to_string()))?; Ok(()) } + + pub(crate) fn update_last_applied( + &self, + last_applied: LogId, + ) { + debug!(%self.node_id, ?last_applied, "update_last_applied"); + self.last_applied_index.store(last_applied.index, Ordering::SeqCst); + self.last_applied_term.store(last_applied.term, Ordering::SeqCst); + } + + pub(crate) fn persist_last_applied( + &self, + last_applied: LogId, + ) -> Result<()> { + debug!(%self.node_id, ?last_applied, "persist_last_applied"); + let db = self.db()?; + let tree = db + .open_tree(STATE_MACHINE_META_NAMESPACE) + .map_err(|e| StorageError::DbError(e.to_string()))?; + tree.insert( + STATE_MACHINE_META_KEY_LAST_APPLIED_INDEX, + safe_kv(last_applied.index), + ) + .map_err(|e| StorageError::DbError(e.to_string()))?; + tree.insert( + STATE_MACHINE_META_KEY_LAST_APPLIED_TERM, + safe_kv(last_applied.term), + ) + .map_err(|e| StorageError::DbError(e.to_string()))?; + + tree.flush().map_err(|e| StorageError::DbError(e.to_string()))?; + Ok(()) + } + + pub(crate) fn update_last_snapshot_metadata( + &self, + snapshot_metadata: &SnapshotMetadata, + ) -> Result<()> { + debug!(%self.node_id, ?snapshot_metadata, "update_last_snapshot_metadata"); + let last_included = snapshot_metadata.last_included.unwrap(); + self.last_included_index.store(last_included.index, Ordering::SeqCst); + self.last_included_term.store(last_included.term, Ordering::SeqCst); + let checksum = snapshot_metadata + .checksum_array() + .map_err(|e| SnapshotError::OperationFailed(format!("{e:?}")))?; + *self.last_snapshot_checksum.lock() = Some(checksum); + + Ok(()) + } + + pub(crate) fn save_hard_state(&self) -> Result<()> { + let last_applied = self.last_applied(); + self.persist_last_applied(last_applied)?; + + if let Some(last_snapshot_metadata) = self.snapshot_metadata() { + self.persist_last_snapshot_metadata(&last_snapshot_metadata)?; + } + + self.flush()?; + Ok(()) + } + + pub(crate) fn flush(&self) -> Result<()> { + let db = self.db()?; + let bytes = db.flush().map_err(|e| StorageError::DbError(e.to_string()))?; + info!( + "Successfully flushed State Machine, bytes flushed: {}", + bytes + ); + Ok(()) + } } /// Recursively copies `src` into `dst`, creating `dst` if needed. Fallback for diff --git a/examples/sled-cluster/src/state_machine/sled_state_machine_test.rs b/examples/sled-cluster/src/state_machine/sled_state_machine_test.rs index b5ea4ad5..a9b43643 100644 --- a/examples/sled-cluster/src/state_machine/sled_state_machine_test.rs +++ b/examples/sled-cluster/src/state_machine/sled_state_machine_test.rs @@ -174,10 +174,11 @@ async fn test_snapshot_install_content_survives_restart() { .await .unwrap(); - // The framework checksums and verifies snapshots; the state machine ignores this field. + // The engine verifies the checksum before it calls the state machine, which only stores it, + // so any 32-byte value will do here. let metadata = d_engine::server_storage::SnapshotMetadata { last_included: Some(last_included), - checksum: Bytes::new(), + checksum: Bytes::from(vec![0u8; 32]), }; // Node B: fresh, empty, behind the purge boundary — installs the snapshot. From 0a589bd730b98a4098d7aedd92aca4ad358bf7e3 Mon Sep 17 00:00:00 2001 From: Joshua Chi <112539+JoshuaChi@users.noreply.github.com> Date: Sun, 1 Nov 2026 22:26:04 +0800 Subject: [PATCH 07/19] refactor #400: The engine decides which snapshots are installed apply_snapshot_from_file returns Result<()> and is called only for a snapshot newer than last_applied(). The engine compares the snapshot's boundary with last_applied() first: older is ignored, the same index and term is ignored, the same index with a different term is a boundary conflict error, and a missing boundary is rejected. Afterwards it checks that last_applied() equals the snapshot's boundary and reports an error if not. Before, every state machine had to implement this decision, and one that did not (d-lmdb) would install a stale snapshot and roll its data back. The boundary is decided before the checksum, so a snapshot that is ignored anyway is not read and hashed. Tests: add engine-level tests for each case, including the state machine that does not move to the boundary. Remove the state-machine-level stale, duplicate, conflict and missing-boundary tests (file and RocksDB adaptors) and the suite's missing-boundary test: the decision no longer belongs to the state machine. Mocks that return a value for last_applied now model a node that moves to the boundary. Docs, CHANGELOG and MIGRATION_GUIDE follow. --- CHANGELOG.md | 2 + MIGRATION_GUIDE.md | 54 +--- .../leader_state_test/snapshot_worker_test.rs | 7 +- .../default_state_machine_handler_test.rs | 261 +++++++++++++++++- .../src/state_machine_handler/writer.rs | 54 +++- d-engine-core/src/storage/state_machine.rs | 45 ++- .../src/storage/state_machine_test.rs | 43 +-- .../src/test_utils/mock/mock_raft_builder.rs | 7 +- .../api/embedded_test/shutdown_order_test.rs | 3 +- d-engine-server/src/lib.rs | 15 +- .../adaptors/file/file_state_machine.rs | 31 +-- .../adaptors/file/file_state_machine_test.rs | 186 +------------ .../adaptors/rocksdb/rocksdb_state_machine.rs | 39 +-- .../rocksdb/rocksdb_state_machine_test.rs | 218 +-------------- .../src/test_utils/mock/mock_node_builder.rs | 7 +- .../tests/common/shared_state_machine.rs | 3 +- .../server_guide/customize-state-machine.md | 1 + .../sled-cluster/src/sled_state_machine.rs | 67 ++--- .../state_machine/sled_state_machine_test.rs | 32 ++- 19 files changed, 399 insertions(+), 676 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4447fea3..4cc6b121 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -120,6 +120,8 @@ All notable changes to this project will be documented in this file. "MemFirst"` or `"DiskFirst"` are silently ignored, not an error — remove the field, `flush_policy` is the only persistence knob now. +- **⚠️ `StateMachine::apply_snapshot_from_file` returns `Result<()>`; the engine decides which snapshots are installed** (#400): the engine compares the snapshot's boundary with `last_applied()` before it calls the state machine, so stale, duplicate and conflicting snapshots never reach it, and it checks afterwards that `last_applied()` equals the snapshot's boundary, reporting an error if not. Before, every state machine had to implement this decision itself, and one that did not (d-lmdb) would install a stale snapshot and roll its data back. See [MIGRATION_GUIDE.md](./MIGRATION_GUIDE.md). + - **⚠️ `StateMachine` no longer declares `update_last_applied`, `persist_last_applied`, `update_last_snapshot_metadata`, `save_hard_state`, `flush` and `flush_async`** (#400): the engine never called them. Keep any you still need as methods on your own type. Your state machine must restore `last_applied` and its snapshot metadata when it is opened; the `StateMachine` docs now state this as a durability contract, and `StateMachineTestSuite` checks it. See [MIGRATION_GUIDE.md](./MIGRATION_GUIDE.md). - **⚠️ `StateMachine::generate_snapshot_data` returns `Result<()>`; the engine owns snapshot checksums** (#400): the engine computes the SHA-256 of the exported snapshot directory and verifies it before calling `apply_snapshot_from_file`, which now only ever receives a directory that matches the recorded checksum. A state machine neither returns nor checks a checksum. The checksum now covers every file at any depth together with its relative path and length; before, it covered only top-level files, so the data files RocksDB exports into sub-directories were not protected. Snapshot archives created before the upgrade fail verification, so remove old snapshot archives when upgrading. See [MIGRATION_GUIDE.md](./MIGRATION_GUIDE.md). diff --git a/MIGRATION_GUIDE.md b/MIGRATION_GUIDE.md index f512d575..c8ae810b 100644 --- a/MIGRATION_GUIDE.md +++ b/MIGRATION_GUIDE.md @@ -34,6 +34,7 @@ EmbeddedEngine::start_custom(dir, storage, sm, None).await?; - **If you kept a clone of your `Arc` to read local data directly** (for example `DLmdb` reading its own LMDB), you can no longer clone the instance the engine uses. Share a handle to your storage instead: put the handles (LMDB `Env`, sled `Db`, `Arc`) behind an `Arc` inside your state machine type, derive `Clone` on it, give the engine one clone and keep the other. - **Embedded reads (`get_eventual`, lease reads) are answered by the ReadActor**, as they already were in standalone mode, instead of reading the state machine on the caller's thread. Expect one extra channel hop per read. - **Snapshot checksums belong to the engine.** `generate_snapshot_data` now returns `Result<()>`: write the snapshot into the directory you are given and return. The engine computes the checksum of that directory itself, and verifies it before it calls `apply_snapshot_from_file`, so you no longer compute or check a checksum. Delete that code (and any hashing dependency you added only for it). +- **`apply_snapshot_from_file` returns `Result<()>` again, and you no longer classify snapshots.** The engine compares the snapshot's boundary with `last_applied()` first and calls you only for a newer snapshot, so delete your stale / duplicate / conflict handling and return `Ok(())` after installing. After your call the engine checks that `last_applied()` equals `metadata.last_included` and reports an error if it does not, so make sure you set it (durably) as part of the install. - **Six methods left the trait:** `update_last_applied`, `persist_last_applied`, `update_last_snapshot_metadata`, `save_hard_state`, `flush` and `flush_async`. The engine never called them, so deleting them from your `impl StateMachine` block changes no behaviour. Keep any your own code still uses as plain methods on your type. - **Restore your state when you open it.** The engine does not do it for you: after reopening, `last_applied()` must return the last entry whose effects are durable, and `snapshot_metadata()` must return the value most recently passed to `persist_last_snapshot_metadata()` (which must therefore update memory and write to disk). `StateMachineTestSuite` now checks the snapshot metadata case. - **The snapshot directory must contain only regular files and directories.** The engine rejects symbolic links and other special entries instead of skipping them. @@ -612,58 +613,11 @@ Compilation will fail with "method not found in trait" if you forget to remove i --- -## For v0.2.4 Users: `StateMachine::apply_snapshot_from_file` Return Type Changed in v0.3.0 (#436) +## For v0.2.4 Users: `StateMachine::apply_snapshot_from_file` Now Receives Only Snapshots To Install (#436, #400) -### What Changed - -`apply_snapshot_from_file` now returns `Result` instead of `Result<()>`, so -the caller knows whether the install actually ran or was a no-op instead of every outcome looking -identical. - -```rust -pub enum SnapshotApplyResult { - Applied { last_included: LogId }, // install actually ran - IgnoredStale { current: LogId }, // incoming.index < current — no-op, not an error - IgnoredDuplicate { current: LogId }, // incoming == current, same term — no-op, not an error -} -``` - -### Impact - -⚠️ **Breaking for custom `StateMachine` implementations.** - -```rust -// Old (v0.2.4) -async fn apply_snapshot_from_file( - &self, - metadata: &SnapshotMetadata, - path: PathBuf, -) -> Result<()> { - // ... install logic ... - Ok(()) -} - -// New (v0.3.0) -async fn apply_snapshot_from_file( - &self, - metadata: &SnapshotMetadata, - path: PathBuf, -) -> Result { - // ... same install logic ... - Ok(SnapshotApplyResult::Applied { last_included: new_last_included }) -} -``` - -Return `IgnoredStale` for a snapshot strictly older than what's already applied, and -`IgnoredDuplicate` only when the index **and** term both match what's already applied — that's -Raft's own idempotency check on the install call, not a failure. An equal index with a -**different** term is not a duplicate: it means two different leadership terms produced -conflicting content at the same boundary, which the public error contract classifies as fatal — -return `Err(SnapshotError::BoundaryConflict { .. })` for that case. Reserve `Err` generally for a -genuine failure (checksum mismatch, I/O error, corrupted data, or this boundary conflict). +The signature is unchanged: it still returns `Result<()>`. What changed is who decides. -Built-in implementations (`FileStateMachine`, `RocksDBStateMachine`) are already updated. -Compilation fails with "incompatible type for trait" until a custom implementation is updated. +In v0.2.4 every state machine had to decide whether a received snapshot should be installed. In v0.3.0 the engine decides before it calls you, so stale, duplicate and conflicting snapshots (older than `last_applied()`, equal to it, or the same index with a different term) never reach your code. If your implementation compared the snapshot's boundary with its own position, delete that code. After your call the engine checks that `last_applied()` equals `metadata.last_included`, and reports an error if it does not. --- diff --git a/d-engine-core/src/raft_role/leader_state_test/snapshot_worker_test.rs b/d-engine-core/src/raft_role/leader_state_test/snapshot_worker_test.rs index d92d235d..2825b942 100644 --- a/d-engine-core/src/raft_role/leader_state_test/snapshot_worker_test.rs +++ b/d-engine-core/src/raft_role/leader_state_test/snapshot_worker_test.rs @@ -51,7 +51,6 @@ use crate::MockRaftLog; use crate::MockStateMachine; use crate::MockTransport; use crate::RaftRequestWithSignal; -use crate::SnapshotApplyResult; use crate::event::InternalEvent; use crate::maybe_clone_oneshot::MaybeCloneOneshot; use crate::maybe_clone_oneshot::RaftOneshot; @@ -108,11 +107,7 @@ fn state_machine_with_snapshot() -> MockStateMachine { // Return a valid snapshot so Phase 6 fires the Snapshot task to the worker. sm.expect_snapshot_metadata().returning(|| Some(stub_snapshot_metadata())); sm.expect_persist_last_snapshot_metadata().returning(|_| Ok(())); - sm.expect_apply_snapshot_from_file().returning(|_, _| { - Ok(SnapshotApplyResult::Applied { - last_included: LogId::default(), - }) - }); + sm.expect_apply_snapshot_from_file().returning(|_, _| Ok(())); sm.expect_generate_snapshot_data().returning(|_, _| Ok(())); sm } diff --git a/d-engine-core/src/state_machine_handler/default_state_machine_handler_test.rs b/d-engine-core/src/state_machine_handler/default_state_machine_handler_test.rs index bfe1bfff..73bed761 100644 --- a/d-engine-core/src/state_machine_handler/default_state_machine_handler_test.rs +++ b/d-engine-core/src/state_machine_handler/default_state_machine_handler_test.rs @@ -2994,6 +2994,8 @@ async fn test_install_prepared_snapshot_checksum_mismatch_rejects_without_callin fs::write(received.join("data.bin"), b"content").unwrap(); let mut sm = MockStateMachine::new(); + // A node that has applied nothing yet, receiving a snapshot that ends at log 5. + sm.expect_last_applied().return_const(LogId::default()); sm.expect_apply_snapshot_from_file().times(0); let (_handler, writer) = writer_with_mock_state_machine(sm, temp_dir.path()); @@ -3025,11 +3027,18 @@ async fn test_install_prepared_snapshot_matching_checksum_installs_once() { fs::write(received.join("data.bin"), b"content").unwrap(); let checksum = crate::file_io::compute_checksum_from_folder_path(&received).await.unwrap(); + // A node that has applied nothing yet; installing the snapshot moves it to log 5. + let position = Arc::new(std::sync::atomic::AtomicU64::new(0)); let mut sm = MockStateMachine::new(); - sm.expect_apply_snapshot_from_file().times(1).returning(|metadata, _| { - Ok(crate::SnapshotApplyResult::Applied { - last_included: metadata.last_included.unwrap(), - }) + let reported = position.clone(); + sm.expect_last_applied().returning(move || LogId { + index: reported.load(std::sync::atomic::Ordering::SeqCst), + term: 1, + }); + let installer = position.clone(); + sm.expect_apply_snapshot_from_file().times(1).returning(move |_, _| { + installer.store(5, std::sync::atomic::Ordering::SeqCst); + Ok(()) }); let (_handler, writer) = writer_with_mock_state_machine(sm, temp_dir.path()); @@ -3065,6 +3074,8 @@ async fn test_install_prepared_snapshot_nested_file_corruption_rejects_without_c fs::write(received.join("sm").join("000008.sst"), b"table-dat!").unwrap(); let mut sm = MockStateMachine::new(); + // A node that has applied nothing yet, receiving a snapshot that ends at log 5. + sm.expect_last_applied().return_const(LogId::default()); sm.expect_apply_snapshot_from_file().times(0); let (_handler, writer) = writer_with_mock_state_machine(sm, temp_dir.path()); @@ -3082,3 +3093,245 @@ async fn test_install_prepared_snapshot_nested_file_corruption_rejects_without_c "expected ChecksumMismatch, got: {err:?}" ); } + +// ── install_prepared_snapshot: boundary decision and result verification ───────────────────── +// +// The engine, not the state machine, decides whether a received snapshot is installed, and it +// checks afterwards that the state machine really moved to the snapshot's boundary. + +/// A snapshot directory with real content, and the metadata (boundary + real checksum) a leader +/// would have recorded for it. +async fn received_snapshot( + root: &Path, + last_included: Option, +) -> (PathBuf, SnapshotMetadata) { + let dir = root.join("received"); + fs::create_dir_all(&dir).unwrap(); + fs::write(dir.join("data.bin"), b"content").unwrap(); + let checksum = crate::file_io::compute_checksum_from_folder_path(&dir).await.unwrap(); + (dir, SnapshotMetadata { + last_included, + checksum: Bytes::copy_from_slice(&checksum), + }) +} + +/// Node B has applied up to log 100 and receives a delayed snapshot that ends at log 50. +/// +/// Purpose: installing it would roll B back to 50 and lose data. The engine must ignore it and +/// never call the state machine. +#[tokio::test] +#[traced_test] +async fn test_install_prepared_snapshot_older_than_applied_is_ignored_without_calling_state_machine() + { + let temp_dir = tempfile::tempdir().unwrap(); + let (dir, metadata) = + received_snapshot(temp_dir.path(), Some(LogId { index: 50, term: 3 })).await; + let mut sm = MockStateMachine::new(); + sm.expect_last_applied().return_const(LogId { + index: 100, + term: 3, + }); + sm.expect_apply_snapshot_from_file().times(0); + let (_handler, writer) = writer_with_mock_state_machine(sm, temp_dir.path()); + + let result = writer.install_prepared_snapshot(metadata, dir).await.unwrap(); + + assert_eq!(result, crate::SnapshotApplyResult::IgnoredStale { + current: LogId { + index: 100, + term: 3 + } + }); +} + +/// B receives the snapshot it already has (same index, same term). +/// +/// Purpose: nothing to install; the state machine is not called. +#[tokio::test] +#[traced_test] +async fn test_install_prepared_snapshot_same_boundary_is_ignored_without_calling_state_machine() { + let temp_dir = tempfile::tempdir().unwrap(); + let (dir, metadata) = received_snapshot( + temp_dir.path(), + Some(LogId { + index: 100, + term: 3, + }), + ) + .await; + let mut sm = MockStateMachine::new(); + sm.expect_last_applied().return_const(LogId { + index: 100, + term: 3, + }); + sm.expect_apply_snapshot_from_file().times(0); + let (_handler, writer) = writer_with_mock_state_machine(sm, temp_dir.path()); + + let result = writer.install_prepared_snapshot(metadata, dir).await.unwrap(); + + assert_eq!(result, crate::SnapshotApplyResult::IgnoredDuplicate { + current: LogId { + index: 100, + term: 3 + } + }); +} + +/// B is at log 100 of term 3; the snapshot says log 100 belongs to term 4. +/// +/// Purpose: two histories disagree about the same log index. That is a serious error, not +/// something to ignore or to overwrite silently. The state machine is not called. +#[tokio::test] +#[traced_test] +async fn test_install_prepared_snapshot_same_index_different_term_is_a_conflict_error() { + let temp_dir = tempfile::tempdir().unwrap(); + let (dir, metadata) = received_snapshot( + temp_dir.path(), + Some(LogId { + index: 100, + term: 4, + }), + ) + .await; + let mut sm = MockStateMachine::new(); + sm.expect_last_applied().return_const(LogId { + index: 100, + term: 3, + }); + sm.expect_apply_snapshot_from_file().times(0); + let (_handler, writer) = writer_with_mock_state_machine(sm, temp_dir.path()); + + let err = writer + .install_prepared_snapshot(metadata, dir) + .await + .expect_err("a conflicting boundary must be an error"); + + assert!( + format!("{err:?}").contains("BoundaryConflict"), + "got: {err:?}" + ); +} + +/// Metadata without a boundary cannot be classified as older, equal or newer. +/// +/// Purpose: reject it outright instead of guessing, and never call the state machine. +#[tokio::test] +#[traced_test] +async fn test_install_prepared_snapshot_without_boundary_is_rejected() { + let temp_dir = tempfile::tempdir().unwrap(); + let (dir, metadata) = received_snapshot(temp_dir.path(), None).await; + let mut sm = MockStateMachine::new(); + sm.expect_last_applied().return_const(LogId { + index: 100, + term: 3, + }); + sm.expect_apply_snapshot_from_file().times(0); + let (_handler, writer) = writer_with_mock_state_machine(sm, temp_dir.path()); + + let result = writer.install_prepared_snapshot(metadata, dir).await; + + assert!( + result.is_err(), + "a snapshot without a boundary must be rejected" + ); +} + +/// A snapshot that is older than B's state also has a wrong checksum. +/// +/// Purpose: the boundary is decided first. A snapshot that would be ignored anyway is not read +/// and hashed (it can be large), and the answer is "ignored", not "corrupt". +#[tokio::test] +#[traced_test] +async fn test_install_prepared_snapshot_stale_is_ignored_before_its_checksum_is_checked() { + let temp_dir = tempfile::tempdir().unwrap(); + let (dir, mut metadata) = + received_snapshot(temp_dir.path(), Some(LogId { index: 50, term: 3 })).await; + metadata.checksum = Bytes::from(vec![0; 32]); + let mut sm = MockStateMachine::new(); + sm.expect_last_applied().return_const(LogId { + index: 100, + term: 3, + }); + sm.expect_apply_snapshot_from_file().times(0); + let (_handler, writer) = writer_with_mock_state_machine(sm, temp_dir.path()); + + let result = writer.install_prepared_snapshot(metadata, dir).await.unwrap(); + + assert!( + matches!(result, crate::SnapshotApplyResult::IgnoredStale { .. }), + "got: {result:?}" + ); +} + +/// B at log 100 receives a newer snapshot (log 500) and the state machine installs it correctly, +/// including moving its position to 500. +/// +/// Purpose: the normal path still works: installed exactly once, reported as applied. +#[tokio::test] +#[traced_test] +async fn test_install_prepared_snapshot_newer_is_installed_once() { + let temp_dir = tempfile::tempdir().unwrap(); + let (dir, metadata) = received_snapshot( + temp_dir.path(), + Some(LogId { + index: 500, + term: 3, + }), + ) + .await; + let position = Arc::new(std::sync::atomic::AtomicU64::new(100)); + let mut sm = MockStateMachine::new(); + let reported = position.clone(); + sm.expect_last_applied().returning(move || LogId { + index: reported.load(std::sync::atomic::Ordering::SeqCst), + term: 3, + }); + let installer = position.clone(); + sm.expect_apply_snapshot_from_file().times(1).returning(move |_, _| { + installer.store(500, std::sync::atomic::Ordering::SeqCst); + Ok(()) + }); + let (_handler, writer) = writer_with_mock_state_machine(sm, temp_dir.path()); + + let result = writer.install_prepared_snapshot(metadata, dir).await.unwrap(); + + assert_eq!(result, crate::SnapshotApplyResult::Applied { + last_included: LogId { + index: 500, + term: 3 + } + }); +} + +/// The state machine installs the data but forgets to move its position: it still reports log 100 +/// after being given a snapshot that ends at 500. +/// +/// Purpose: the engine must notice and fail loudly. Otherwise it would treat B as still at 100 +/// and replay logs 101..500 on top of state that already contains them. +#[tokio::test] +#[traced_test] +async fn test_install_prepared_snapshot_state_machine_not_advanced_is_an_error() { + let temp_dir = tempfile::tempdir().unwrap(); + let (dir, metadata) = received_snapshot( + temp_dir.path(), + Some(LogId { + index: 500, + term: 3, + }), + ) + .await; + let mut sm = MockStateMachine::new(); + sm.expect_last_applied().return_const(LogId { + index: 100, + term: 3, + }); + sm.expect_apply_snapshot_from_file().times(1).returning(|_, _| Ok(())); + let (_handler, writer) = writer_with_mock_state_machine(sm, temp_dir.path()); + + let result = writer.install_prepared_snapshot(metadata, dir).await; + + assert!( + result.is_err(), + "a state machine that did not move to the snapshot's boundary must be reported" + ); +} diff --git a/d-engine-core/src/state_machine_handler/writer.rs b/d-engine-core/src/state_machine_handler/writer.rs index c5cff413..3f4cf259 100644 --- a/d-engine-core/src/state_machine_handler/writer.rs +++ b/d-engine-core/src/state_machine_handler/writer.rs @@ -6,6 +6,7 @@ use bytes::Bytes; use d_engine_proto::common::Entry; use d_engine_proto::server::storage::SnapshotMetadata; use tracing::error; +use tracing::info; use tracing::trace; use super::StateMachineWriterOps; @@ -179,8 +180,36 @@ impl StateMachineWriterOps for DefaultStateMachineWriter { metadata: SnapshotMetadata, dir: PathBuf, ) -> Result { - // The framework verifies the snapshot's end-to-end checksum, so a state machine only - // ever receives a directory whose content matches what the leader exported. + // The engine, not the state machine, decides whether a snapshot is installed. A snapshot + // without a boundary cannot be classified at all, so it is rejected rather than guessed. + let incoming = metadata.last_included.ok_or_else(|| { + SnapshotError::OperationFailed("Missing last_included in snapshot metadata".into()) + })?; + let current = self.state_machine.last_applied(); + match incoming.index.cmp(¤t.index) { + std::cmp::Ordering::Less => { + info!(?incoming, ?current, "Ignoring stale snapshot"); + return Ok(SnapshotApplyResult::IgnoredStale { current }); + } + std::cmp::Ordering::Equal if incoming.term == current.term => { + info!(?incoming, ?current, "Ignoring duplicate snapshot"); + return Ok(SnapshotApplyResult::IgnoredDuplicate { current }); + } + // The same log index with a different term means the two histories disagree. That + // is a real inconsistency: it must be neither ignored nor installed over. + std::cmp::Ordering::Equal => { + return Err(SnapshotError::BoundaryConflict { + index: current.index, + local_term: current.term, + incoming_term: incoming.term, + } + .into()); + } + std::cmp::Ordering::Greater => {} + } + + // Only a snapshot that will be installed is read and hashed. The checksum guarantees a + // state machine only ever receives a directory that matches what the leader exported. let computed = compute_checksum_from_folder_path(&dir).await?; if metadata.checksum.as_ref() != computed.as_ref() { error!( @@ -190,11 +219,24 @@ impl StateMachineWriterOps for DefaultStateMachineWriter { return Err(SnapshotError::ChecksumMismatch.into()); } - let result = self.state_machine.apply_snapshot_from_file(&metadata, dir).await?; - if let SnapshotApplyResult::Applied { last_included } = &result { - self.applied.advance(last_included.index); + self.state_machine.apply_snapshot_from_file(&metadata, dir).await?; + + // The state machine must now be at the snapshot's boundary. If it is not, the engine + // would believe the node is still behind and replay entries that the installed state + // already contains, so this is reported instead of carried on from. + let reached = self.state_machine.last_applied(); + if reached != incoming { + return Err(SnapshotError::OperationFailed(format!( + "state machine reports last_applied {reached:?} after installing a snapshot that \ + ends at {incoming:?}" + )) + .into()); } - Ok(result) + + self.applied.advance(incoming.index); + Ok(SnapshotApplyResult::Applied { + last_included: incoming, + }) } async fn capture_local_snapshot(&self) -> Result { let last_included = self.state_machine.last_applied(); diff --git a/d-engine-core/src/storage/state_machine.rs b/d-engine-core/src/storage/state_machine.rs index 0a989003..c94ca922 100644 --- a/d-engine-core/src/storage/state_machine.rs +++ b/d-engine-core/src/storage/state_machine.rs @@ -121,7 +121,10 @@ pub trait StateMachine: Send + Sync + 'static { } /// Retrieves a value by key from the state machine. - /// Sync operation as it accesses in-memory data structures. + /// + /// Called concurrently from the read path, so it must be thread-safe and cheap. Return + /// `StorageError::NotServing` while the state machine cannot answer (for example during a + /// snapshot install, or after `close()`): the engine then routes the read through Raft. fn get( &self, key_buffer: &[u8], @@ -129,12 +132,8 @@ pub trait StateMachine: Send + Sync + 'static { /// Retrieves multiple values by key from the state machine in a single call. /// - /// # Why this belongs in the protocol layer - /// - /// `get()` is already part of the protocol interface because the ReadActor hot path - /// requires direct SM access. `get_multi` is its natural batch extension — the same - /// reasoning applies. An Iterator's `fold` has a default impl; `Vec` overrides it for - /// performance. Users don't need to know which path executes. + /// The engine reads through this method, so a request with several keys is answered in one + /// call, from one consistent state. /// /// # Read coherence requirement /// @@ -185,33 +184,30 @@ pub trait StateMachine: Send + Sync + 'static { /// Sync operation as it reads from in-memory structures. fn snapshot_metadata(&self) -> Option; - /// Persists snapshot metadata to durable storage. - /// Should be async as it involves disk I/O. + /// Persists snapshot metadata durably and makes it visible to `snapshot_metadata()`. + /// + /// The engine calls this once per snapshot it builds, after the archive is durably built. fn persist_last_snapshot_metadata( &self, snapshot_metadata: &SnapshotMetadata, ) -> Result<(), Error>; - /// Applies a snapshot received from the Raft leader to the local state machine + /// Installs a snapshot received from the Raft leader. /// - /// # Critical Security and Integrity Measures - /// 1. Checksum Validation: Verifies snapshot integrity before application - /// 2. Version Validation: Ensures snapshot is newer than current state - /// 3. Atomic Application: Uses locking to prevent concurrent modifications - /// 4. File Validation: Confirms compressed format before decompression + /// The engine decides whether to install: this is called only for a snapshot that is newer + /// than `last_applied()`. `snapshot_path` is a directory the engine has already decompressed + /// the archive into and whose checksum it has verified against `metadata`. /// - /// # Workflow - /// 1. Validate snapshot metadata and version - /// 2. Verify compressed file format - /// 3. Decompress to temporary directory - /// 4. Validate checklsum - /// 5. Atomically replace current state with the decompressed data - /// 6. Update Raft metadata and indexes + /// The implementation must: + /// 1. Replace its state atomically. A concurrent reader sees the old state, the new state, or + /// `NotServing`, never a mix. + /// 2. Set `last_applied()` to `metadata.last_included` and make that durable. The engine checks + /// this after the call and reports an error if it did not happen. async fn apply_snapshot_from_file( &self, metadata: &SnapshotMetadata, snapshot_path: std::path::PathBuf, - ) -> Result; + ) -> Result<(), Error>; /// Generates a snapshot of the state machine's current key-value entries /// up to the specified `last_included_index`. @@ -231,8 +227,7 @@ pub trait StateMachine: Send + Sync + 'static { /// /// # Arguments /// * `new_snapshot_dir` - Temporary path to store the snapshot data. - /// * `last_included_index` - Last log index included in the snapshot. - /// * `last_included_term` - Last log term included in the snapshot. + /// * `last_included` - Index and term of the last log entry the snapshot includes. async fn generate_snapshot_data( &self, new_snapshot_dir: std::path::PathBuf, diff --git a/d-engine-core/src/storage/state_machine_test.rs b/d-engine-core/src/storage/state_machine_test.rs index 7d70dd01..ee30ea79 100644 --- a/d-engine-core/src/storage/state_machine_test.rs +++ b/d-engine-core/src/storage/state_machine_test.rs @@ -12,7 +12,6 @@ use tempfile::TempDir; use crate::ApplyEntry; use crate::Command; use crate::Error; -use crate::SnapshotApplyResult; use crate::storage::StateMachine; /// Test suite for StateMachine implementations @@ -43,7 +42,6 @@ impl StateMachineTestSuite { Self::test_batch_operations(builder.build().await?).await?; Self::test_last_applied_detection(builder.build().await?).await?; Self::test_snapshot_operations(builder.build().await?).await?; - Self::test_snapshot_apply_rejects_missing_boundary(builder.build().await?).await?; Self::test_snapshot_metadata_survives_reopen(&builder).await?; Self::test_drop_flushes_data(&builder).await?; @@ -376,32 +374,6 @@ impl StateMachineTestSuite { Ok(()) } - /// `apply_snapshot_from_file` must reject `last_included: None` outright — a missing - /// boundary can't be classified stale/duplicate/conflict/eligible, so silently - /// defaulting it (`unwrap_or_default()`) turns malformed metadata into a fake no-op. - pub async fn test_snapshot_apply_rejects_missing_boundary( - state_machine: Arc - ) -> Result<(), Error> { - let metadata = SnapshotMetadata { - last_included: None, - checksum: Bytes::from(vec![0; 32]), - }; - let temp_dir = TempDir::new()?; - - let result = state_machine - .apply_snapshot_from_file(&metadata, temp_dir.path().to_path_buf()) - .await; - - assert!( - result.is_err(), - "apply_snapshot_from_file accepted SnapshotMetadata{{last_included: None}} — a \ - missing boundary can't be classified as stale/duplicate/conflict/eligible, it \ - must be rejected, not silently substituted with a default LogId" - ); - - Ok(()) - } - /// Snapshot metadata published through `persist_last_snapshot_metadata` must survive a /// restart. /// @@ -725,11 +697,7 @@ impl StateMachineTestSuite { last_included: Some(last_included), checksum: Bytes::from(checksum.to_vec()), }; - let result = sm.apply_snapshot_from_file(&metadata, snapshot_dir).await?; - assert!( - matches!(result, SnapshotApplyResult::Applied { .. }), - "expected Applied, got: {result:?}" - ); + sm.apply_snapshot_from_file(&metadata, snapshot_dir).await?; assert_eq!( sm.last_applied(), last_included, @@ -1673,7 +1641,6 @@ mod default_scan_prefix_tests { use crate::ApplyEntry; use crate::Error; - use crate::SnapshotApplyResult; use crate::storage::state_machine::ApplyResult; use crate::storage::state_machine::StateMachine; @@ -1719,12 +1686,10 @@ mod default_scan_prefix_tests { async fn apply_snapshot_from_file( &self, - metadata: &SnapshotMetadata, + _metadata: &SnapshotMetadata, _snapshot_path: std::path::PathBuf, - ) -> Result { - Ok(SnapshotApplyResult::Applied { - last_included: metadata.last_included.unwrap_or_default(), - }) + ) -> Result<(), Error> { + Ok(()) } async fn generate_snapshot_data( diff --git a/d-engine-core/src/test_utils/mock/mock_raft_builder.rs b/d-engine-core/src/test_utils/mock/mock_raft_builder.rs index d7ab4631..28ff3c9b 100644 --- a/d-engine-core/src/test_utils/mock/mock_raft_builder.rs +++ b/d-engine-core/src/test_utils/mock/mock_raft_builder.rs @@ -28,7 +28,6 @@ use crate::RaftNodeConfig; use crate::RaftRole; use crate::RaftStorageHandles; use crate::SignalParams; -use crate::SnapshotApplyResult; use crate::StateMachine; use crate::follower_state::FollowerState; @@ -392,11 +391,7 @@ pub fn mock_state_machine() -> MockStateMachine { mock.expect_snapshot_metadata().returning(|| None); mock.expect_persist_last_snapshot_metadata().returning(|_| Ok(())); - mock.expect_apply_snapshot_from_file().returning(|_, _| { - Ok(SnapshotApplyResult::Applied { - last_included: LogId::default(), - }) - }); + mock.expect_apply_snapshot_from_file().returning(|_, _| Ok(())); mock.expect_generate_snapshot_data().returning(|_, _| Ok(())); mock diff --git a/d-engine-server/src/api/embedded_test/shutdown_order_test.rs b/d-engine-server/src/api/embedded_test/shutdown_order_test.rs index 9ffab31f..87b38c2f 100644 --- a/d-engine-server/src/api/embedded_test/shutdown_order_test.rs +++ b/d-engine-server/src/api/embedded_test/shutdown_order_test.rs @@ -17,7 +17,6 @@ use d_engine_core::ApplyEntry; use d_engine_core::ApplyResult; use d_engine_core::Error; use d_engine_core::ScanResult; -use d_engine_core::SnapshotApplyResult; use d_engine_core::StateMachine; use d_engine_proto::common::LogId; use d_engine_proto::server::storage::SnapshotMetadata; @@ -97,7 +96,7 @@ impl StateMachine for SlowApplyProbe { &self, metadata: &SnapshotMetadata, snapshot_path: std::path::PathBuf, - ) -> Result { + ) -> Result<(), Error> { self.inner.apply_snapshot_from_file(metadata, snapshot_path).await } async fn generate_snapshot_data( diff --git a/d-engine-server/src/lib.rs b/d-engine-server/src/lib.rs index 4ade9797..7f1a1984 100644 --- a/d-engine-server/src/lib.rs +++ b/d-engine-server/src/lib.rs @@ -166,18 +166,17 @@ pub use d_engine_core::Result; pub use d_engine_core::ScanResult; /// Client-facing prefix scan result, returned by `EmbeddedClient::scan_prefix()`. pub use d_engine_core::ScanResults; -/// Outcome of installing a snapshot via [`StateMachine::apply_snapshot_from_file`]. +/// Outcome the engine reports for a received snapshot. /// /// Distinguishes "the install actually ran" (`Applied`) from two no-op cases that are not /// errors: the incoming snapshot is behind what's already applied (`IgnoredStale`), or is an -/// idempotent retry of the currently-applied boundary (`IgnoredDuplicate`). A genuine problem -/// (checksum mismatch, corrupted data, conflicting boundary) should be returned as `Err`, not -/// encoded as a variant here. +/// idempotent retry of the currently-applied boundary (`IgnoredDuplicate`). The engine decides +/// this itself before it calls [`StateMachine::apply_snapshot_from_file`], which only ever +/// receives a snapshot to install and returns `Result<()>`. pub use d_engine_core::SnapshotApplyResult; -/// Snapshot-specific error type. Construct these in -/// [`StateMachine::apply_snapshot_from_file`] (e.g. `SnapshotError::ChecksumMismatch`) to -/// report a genuine install failure — the no-op cases are already covered by -/// [`SnapshotApplyResult`], not this type. +/// Snapshot-specific error type. Return one from [`StateMachine::apply_snapshot_from_file`] to +/// report a genuine install failure; stale, duplicate and conflicting snapshots never reach +/// that method, because the engine rejects them first. pub use d_engine_core::SnapshotError; /// Storage trait for implementing custom storage backends /// diff --git a/d-engine-server/src/storage/adaptors/file/file_state_machine.rs b/d-engine-server/src/storage/adaptors/file/file_state_machine.rs index 683c634c..228f0e15 100644 --- a/d-engine-server/src/storage/adaptors/file/file_state_machine.rs +++ b/d-engine-server/src/storage/adaptors/file/file_state_machine.rs @@ -68,7 +68,6 @@ use d_engine_core::ApplyResult; use d_engine_core::Command; use d_engine_core::Error; use d_engine_core::ScanResult; -use d_engine_core::SnapshotApplyResult; use d_engine_core::SnapshotError; use d_engine_core::StateMachine; use d_engine_core::StorageError; @@ -1190,35 +1189,11 @@ impl StateMachine for FileStateMachine { &self, metadata: &SnapshotMetadata, snapshot_dir: std::path::PathBuf, - ) -> Result { - // (#436) — every StateMachine backend must honor this contract, not just RocksDB. - // A missing boundary can't be classified — reject rather than fall through - // to an unconditional install. + ) -> Result<(), Error> { let incoming = metadata.last_included.ok_or_else(|| { SnapshotError::OperationFailed("Missing last_included in snapshot metadata".into()) })?; - let current = self.last_applied(); - match incoming.index.cmp(¤t.index) { - std::cmp::Ordering::Less => { - info!(?incoming, ?current, "Ignoring stale snapshot"); - return Ok(SnapshotApplyResult::IgnoredStale { current }); - } - std::cmp::Ordering::Equal if incoming.term == current.term => { - info!(?incoming, ?current, "Ignoring duplicate snapshot"); - return Ok(SnapshotApplyResult::IgnoredDuplicate { current }); - } - std::cmp::Ordering::Equal => { - return Err(SnapshotError::BoundaryConflict { - index: current.index, - local_term: current.term, - incoming_term: incoming.term, - } - .into()); - } - std::cmp::Ordering::Greater => {} - } - info!("Applying snapshot from file: {:?}", snapshot_dir); // Read from the snapshot.bin file inside the directory @@ -1327,9 +1302,7 @@ impl StateMachine for FileStateMachine { info!("Snapshot applied successfully"); - Ok(SnapshotApplyResult::Applied { - last_included: incoming, - }) + Ok(()) } async fn generate_snapshot_data( diff --git a/d-engine-server/src/storage/adaptors/file/file_state_machine_test.rs b/d-engine-server/src/storage/adaptors/file/file_state_machine_test.rs index 0431a9d7..22cc144e 100644 --- a/d-engine-server/src/storage/adaptors/file/file_state_machine_test.rs +++ b/d-engine-server/src/storage/adaptors/file/file_state_machine_test.rs @@ -5,7 +5,6 @@ use bytes::Bytes; use d_engine_core::ApplyEntry; use d_engine_core::Command; use d_engine_core::Error; -use d_engine_core::SnapshotApplyResult; use d_engine_core::StateMachine; use d_engine_core::file_io::compute_checksum_from_folder_path; use d_engine_core::storage::state_machine_test::StateMachineBuilder; @@ -670,128 +669,6 @@ fn insert_at( } } -/// A stale snapshot (`last_included` behind current `last_applied`) must not be -/// installed — doing so rolls back already-applied data. -#[tokio::test] -async fn test_file_apply_snapshot_from_file_rejects_stale_snapshot() { - let dir = TempDir::new().expect("Failed to create temp dir"); - let sm = FileStateMachine::new(dir.path().join("sm")).await.expect("create sm"); - - let early: Vec = (1..=20) - .map(|i| insert_at(format!("key_{i}").as_bytes(), b"early", i)) - .collect(); - sm.apply_chunk(&early).await.expect("apply early entries"); - - let stale_snapshot_dir = dir.path().join("stale_snapshot"); - let stale_last_included = LogId { term: 1, index: 20 }; - sm.generate_snapshot_data(stale_snapshot_dir.clone(), stale_last_included) - .await - .expect("generate stale snapshot"); - - let later: Vec = (21..=33) - .map(|i| insert_at(format!("key_{i}").as_bytes(), b"later", i)) - .collect(); - sm.apply_chunk(&later).await.expect("apply later entries"); - assert_eq!( - sm.last_applied().index, - 33, - "sanity: advanced past the stale snapshot's index" - ); - - let stale_metadata = SnapshotMetadata { - last_included: Some(stale_last_included), - checksum: Bytes::from_static(&[0; 32]), - }; - let result = sm - .apply_snapshot_from_file(&stale_metadata, stale_snapshot_dir) - .await - .expect("apply_snapshot_from_file should not error even when rejecting a stale snapshot"); - assert!( - matches!(result, SnapshotApplyResult::IgnoredStale { current } if current.index == 33), - "expected IgnoredStale{{current.index: 33}}, got: {result:?}" - ); - assert_eq!( - sm.last_applied().index, - 33, - "must not roll back last_applied" - ); - assert_eq!( - sm.get(b"key_23").expect("read after install"), - Some(Bytes::from_static(b"later")), - "key_23 must survive installing a stale snapshot" - ); -} - -/// A duplicate snapshot (`last_included` == current `last_applied`, same term) is an -/// idempotent no-op — not an error, and must not re-run the destructive install. -#[tokio::test] -async fn test_file_apply_snapshot_from_file_ignores_duplicate_snapshot() { - let dir = TempDir::new().expect("Failed to create temp dir"); - let sm = FileStateMachine::new(dir.path().join("sm")).await.expect("create sm"); - - let entries: Vec = - (1..=5).map(|i| insert_at(format!("key_{i}").as_bytes(), b"v", i)).collect(); - sm.apply_chunk(&entries).await.expect("apply entries"); - - let last_included = LogId { term: 1, index: 5 }; - let snapshot_dir = dir.path().join("snapshot"); - sm.generate_snapshot_data(snapshot_dir.clone(), last_included) - .await - .expect("generate snapshot"); - - let metadata = SnapshotMetadata { - last_included: Some(last_included), - checksum: Bytes::from_static(&[0; 32]), - }; - let result = sm - .apply_snapshot_from_file(&metadata, snapshot_dir) - .await - .expect("duplicate snapshot must not error"); - assert!( - matches!(result, SnapshotApplyResult::IgnoredDuplicate { current } if current.index == 5 && current.term == 1), - "expected IgnoredDuplicate{{current: (5,1)}}, got: {result:?}" - ); - assert_eq!( - sm.last_applied(), - last_included, - "duplicate install must not change last_applied" - ); -} - -/// Same index, DIFFERENT term than current `last_applied` — can't happen under a -/// correctly-functioning Raft cluster (Log Matching Property); must be rejected as a -/// hard inconsistency, not silently ignored or auto-installed. -#[tokio::test] -async fn test_file_apply_snapshot_from_file_rejects_boundary_conflict() { - let dir = TempDir::new().expect("Failed to create temp dir"); - let sm = FileStateMachine::new(dir.path().join("sm")).await.expect("create sm"); - - let entries: Vec = - (1..=5).map(|i| insert_at(format!("key_{i}").as_bytes(), b"v", i)).collect(); - sm.apply_chunk(&entries).await.expect("apply entries"); - assert_eq!(sm.last_applied(), LogId { term: 1, index: 5 }); - - let snapshot_dir = dir.path().join("snapshot"); - sm.generate_snapshot_data(snapshot_dir.clone(), LogId { term: 2, index: 5 }) - .await - .expect("generate snapshot"); - let conflicting_metadata = SnapshotMetadata { - last_included: Some(LogId { term: 2, index: 5 }), - checksum: Bytes::from_static(&[0; 32]), - }; - - let result = sm.apply_snapshot_from_file(&conflicting_metadata, snapshot_dir).await; - assert!( - result.is_err(), - "same index, different term must be rejected as a boundary conflict, not silently accepted" - ); - assert_eq!( - sm.last_applied(), - LogId { term: 1, index: 5 }, - "a rejected boundary conflict must not change last_applied" - ); -} - /// A genuinely newer snapshot (`last_included.index` > current `last_applied`) must /// actually run the destructive install and report `Applied`. #[tokio::test] @@ -819,59 +696,12 @@ async fn test_file_apply_snapshot_from_file_reports_applied_for_newer_snapshot() last_included: Some(last_included), checksum: Bytes::copy_from_slice(&checksum), }; - let result = sm - .apply_snapshot_from_file(&metadata, snapshot_dir) + sm.apply_snapshot_from_file(&metadata, snapshot_dir) .await .expect("newer snapshot must install successfully"); - assert!( - matches!(result, SnapshotApplyResult::Applied { last_included: li } if li == last_included), - "expected Applied{{last_included: {last_included:?}}}, got: {result:?}" - ); assert_eq!(sm.last_applied(), last_included); } -/// #436-adjacent: same as the RocksDB adaptor's -/// `test_apply_snapshot_from_file_rejects_missing_last_included` — `last_included: None` -/// must be rejected outright, not silently fall through to an unconditional install. -#[tokio::test] -async fn test_file_apply_snapshot_from_file_rejects_missing_last_included() { - let dir = TempDir::new().expect("Failed to create temp dir"); - let sm = FileStateMachine::new(dir.path().join("sm")).await.expect("create sm"); - - let early_entries: Vec = - (1..=5).map(|i| insert_at(format!("key_{i}").as_bytes(), b"v", i)).collect(); - sm.apply_chunk(&early_entries).await.expect("apply early entries"); - let snapshot_dir = dir.path().join("snapshot"); - sm.generate_snapshot_data(snapshot_dir.clone(), LogId { term: 1, index: 5 }) - .await - .expect("generate snapshot"); - - let later_entries: Vec = - (6..=8).map(|i| insert_at(format!("key_{i}").as_bytes(), b"v", i)).collect(); - sm.apply_chunk(&later_entries).await.expect("apply later entries"); - assert_eq!(sm.last_applied(), LogId { term: 1, index: 8 }); - - let metadata = SnapshotMetadata { - last_included: None, - checksum: Bytes::from_static(&[0; 32]), - }; - let result = sm.apply_snapshot_from_file(&metadata, snapshot_dir).await; - - assert!( - result.is_err(), - "a snapshot with no boundary must be rejected, not installed — got: {result:?}" - ); - assert!( - sm.get(b"key_6").unwrap().is_some(), - "key_6 (applied after the snapshot boundary) must survive a rejected install" - ); - assert_eq!( - sm.last_applied(), - LogId { term: 1, index: 8 }, - "last_applied must be unchanged after a rejected install" - ); -} - /// A snapshot advances `last_applied` past the follower's pre-snapshot WAL entries. /// If a crash leaves those stale entries in the WAL, replay must skip them /// (`index <= last_applied`) instead of clobbering the newer snapshot content (#442). @@ -904,14 +734,9 @@ async fn test_replay_wal_skips_stale_entries_below_last_applied() { last_included: Some(LogId { term: 1, index: 10 }), checksum: Bytes::copy_from_slice(&checksum), }; - let result = sm - .apply_snapshot_from_file(&metadata, snapshot_dir) + sm.apply_snapshot_from_file(&metadata, snapshot_dir) .await .expect("install snapshot"); - assert!( - matches!(result, SnapshotApplyResult::Applied { last_included } if last_included.index == 10), - "expected Applied{{index:10}}, got: {result:?}" - ); // Simulate the crash window: stale pre-snapshot WAL entries survive while // last_applied is already 10 — the state the old persist order could leave. @@ -1028,12 +853,7 @@ async fn test_file_sm_install_empty_snapshot_clears_existing_data() { last_included: Some(LogId { term: 1, index: 5 }), checksum: Bytes::copy_from_slice(&checksum), }; - let result = receiver.apply_snapshot_from_file(&metadata, snapshot_dir).await.unwrap(); - - assert!( - matches!(result, SnapshotApplyResult::Applied { .. }), - "expected Applied, got: {result:?}" - ); + receiver.apply_snapshot_from_file(&metadata, snapshot_dir).await.unwrap(); assert_eq!( receiver.get(b"stale").unwrap(), None, diff --git a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs index dee6339a..1533fbbe 100644 --- a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs +++ b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs @@ -12,7 +12,6 @@ use d_engine_core::ApplyResult; use d_engine_core::Command; use d_engine_core::Error; use d_engine_core::ScanResult; -use d_engine_core::SnapshotApplyResult; use d_engine_core::SnapshotError; use d_engine_core::StateMachine; use d_engine_core::StorageError; @@ -774,43 +773,11 @@ impl StateMachine for RocksDBStateMachine { &self, metadata: &SnapshotMetadata, snapshot_dir: std::path::PathBuf, - ) -> Result { - // A snapshot with no boundary can't be classified (stale/duplicate/conflict/ - // eligible) at all — reject outright rather than silently falling through to - // an unconditional install (#436). Mirrors prepare_transfer_meta's handling - // of the same missing-field case. + ) -> Result<(), Error> { let incoming = metadata.last_included.ok_or_else(|| { SnapshotError::OperationFailed("Missing last_included in snapshot metadata".into()) })?; - let current = self.last_applied(); - match incoming.index.cmp(¤t.index) { - std::cmp::Ordering::Less => { - // Case 1: incoming index behind current progress — stale, no-op. - info!(?incoming, ?current, "Ignoring stale snapshot"); - return Ok(SnapshotApplyResult::IgnoredStale { current }); - } - std::cmp::Ordering::Equal if incoming.term == current.term => { - // Case 2: same boundary, same term — idempotent duplicate, no-op. - info!(?incoming, ?current, "Ignoring duplicate snapshot"); - return Ok(SnapshotApplyResult::IgnoredDuplicate { current }); - } - std::cmp::Ordering::Equal => { - // Case 3: same index, different term — not a normal stale/duplicate - // case. This is a real inconsistency (bad metadata, wrong cluster, - // or local corruption) — must not be silently ignored or auto-installed. - return Err(SnapshotError::BoundaryConflict { - index: current.index, - local_term: current.term, - incoming_term: incoming.term, - } - .into()); - } - std::cmp::Ordering::Greater => { - // Case 4: incoming index ahead — eligible to install, fall through. - } - } - info!("Applying snapshot from: {:?}", snapshot_dir); // Stop serving — prevents SM reads/writes during restore @@ -829,9 +796,7 @@ impl StateMachine for RocksDBStateMachine { self.is_serving.store(true, Ordering::SeqCst); } - result.map(|()| SnapshotApplyResult::Applied { - last_included: incoming, - }) + result } async fn generate_snapshot_data( diff --git a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine_test.rs b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine_test.rs index 6af5eaf0..3d411fa5 100644 --- a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine_test.rs +++ b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine_test.rs @@ -5,7 +5,6 @@ use async_trait::async_trait; use bytes::Bytes; use d_engine_core::ApplyEntry; use d_engine_core::Command; -use d_engine_core::SnapshotApplyResult; use d_engine_core::file_io::compute_checksum_from_folder_path; use d_engine_core::state_machine_test::StateMachineBuilder; use d_engine_core::state_machine_test::StateMachineTestSuite; @@ -802,167 +801,6 @@ async fn test_snapshot_join_error_reports_cancellation() { // ── stale snapshot install (#436) ───────────────────────────────────── -/// A stale snapshot (`last_included` behind current `last_applied`) must not be -/// installed — doing so rolls back already-applied data. -/// -/// Currently RED: `apply_snapshot_from_file` doesn't check `last_included` against -/// `last_applied` before installing. #436. -#[tokio::test] -async fn test_apply_snapshot_from_file_rejects_stale_snapshot() { - let temp_dir = TempDir::new().expect("Failed to create temp dir"); - let db_path = temp_dir.path().join("test_stale_snapshot"); - let (_storage, state_machine) = - RocksDBUnifiedEngine::open(&db_path).expect("Failed to open unified DB"); - - // 1. Apply entries 1..=20 — the state a snapshot generated "now" would capture. - let early_entries: Vec = (1..=20) - .map(|i| insert_at(format!("key_{i}").as_bytes(), b"early", i)) - .collect(); - state_machine.apply_chunk(&early_entries).await.expect("apply early entries"); - - // 2. Generate a real snapshot reflecting state as of index 20 — genuinely complete - // and correct for its own last_included, just old by the time it gets applied later. - let stale_snapshot_dir = temp_dir.path().join("stale_snapshot"); - let stale_last_included = LogId { term: 1, index: 20 }; - state_machine - .generate_snapshot_data(stale_snapshot_dir.clone(), stale_last_included) - .await - .expect("generate stale snapshot"); - - // 3. Keep advancing past what the snapshot captured — the follower catching up via - // normal AppendEntries while the stale snapshot is still "in flight". - let later_entries: Vec = (21..=33) - .map(|i| insert_at(format!("key_{i}").as_bytes(), b"later", i)) - .collect(); - state_machine.apply_chunk(&later_entries).await.expect("apply later entries"); - - assert_eq!( - state_machine.last_applied().index, - 33, - "sanity: advanced past the stale snapshot's index" - ); - let value_before = state_machine.get(b"key_23").expect("read before install"); - assert_eq!( - value_before, - Some(Bytes::from_static(b"later")), - "sanity: key_23 present before stale install" - ); - - // 4. The stale snapshot (last_included=20) arrives late and gets installed — exactly - // what happened in the flaky integration test. - let stale_metadata = SnapshotMetadata { - last_included: Some(stale_last_included), - checksum: Bytes::from_static(&[0; 32]), - }; - let result = state_machine - .apply_snapshot_from_file(&stale_metadata, stale_snapshot_dir) - .await - .expect("apply_snapshot_from_file should not error even when rejecting a stale snapshot"); - assert!( - matches!(result, SnapshotApplyResult::IgnoredStale { current } if current.index == 33), - "expected IgnoredStale{{current.index: 33}}, got: {result:?}" - ); - - // 5. A stale snapshot must be a no-op: state must not roll back, key_23 must survive. - assert_eq!( - state_machine.last_applied().index, - 33, - "installing a snapshot older than current progress must not roll back last_applied" - ); - let value_after = state_machine.get(b"key_23").expect("read after install"); - assert_eq!( - value_after, - Some(Bytes::from_static(b"later")), - "key_23 (applied after the stale snapshot was generated) must survive installing that stale snapshot" - ); -} - -/// A duplicate snapshot (`last_included` == current `last_applied`, same term) is an -/// idempotent no-op — not an error, and must not re-run the destructive install. -/// -/// #436. -#[tokio::test] -async fn test_apply_snapshot_from_file_ignores_duplicate_snapshot() { - let temp_dir = TempDir::new().expect("Failed to create temp dir"); - let db_path = temp_dir.path().join("test_duplicate_snapshot"); - let (_storage, state_machine) = - RocksDBUnifiedEngine::open(&db_path).expect("Failed to open unified DB"); - - let entries: Vec = - (1..=5).map(|i| insert_at(format!("key_{i}").as_bytes(), b"v", i)).collect(); - state_machine.apply_chunk(&entries).await.expect("apply entries"); - - let last_included = LogId { term: 1, index: 5 }; - let snapshot_dir = temp_dir.path().join("snapshot"); - state_machine - .generate_snapshot_data(snapshot_dir.clone(), last_included) - .await - .expect("generate snapshot"); - - // Same (index, term) as current progress — installing it again must be a no-op, - // not an error, and must not touch the data. - let metadata = SnapshotMetadata { - last_included: Some(last_included), - checksum: Bytes::from_static(&[0; 32]), - }; - let result = state_machine - .apply_snapshot_from_file(&metadata, snapshot_dir) - .await - .expect("duplicate snapshot must not error"); - assert!( - matches!(result, SnapshotApplyResult::IgnoredDuplicate { current } if current.index == 5 && current.term == 1), - "expected IgnoredDuplicate{{current: (5,1)}}, got: {result:?}" - ); - assert_eq!( - state_machine.last_applied(), - last_included, - "duplicate install must not change last_applied" - ); -} - -/// Same index, DIFFERENT term than current `last_applied` — this can't happen under a -/// correctly-functioning Raft cluster (Log Matching Property), so it must be treated as -/// a hard inconsistency: reject with an error, don't silently ignore or auto-install. -/// -/// #436. -#[tokio::test] -async fn test_apply_snapshot_from_file_rejects_boundary_conflict() { - let temp_dir = TempDir::new().expect("Failed to create temp dir"); - let db_path = temp_dir.path().join("test_boundary_conflict"); - let (_storage, state_machine) = - RocksDBUnifiedEngine::open(&db_path).expect("Failed to open unified DB"); - - let entries: Vec = - (1..=5).map(|i| insert_at(format!("key_{i}").as_bytes(), b"v", i)).collect(); - state_machine.apply_chunk(&entries).await.expect("apply entries"); - assert_eq!(state_machine.last_applied(), LogId { term: 1, index: 5 }); - - let snapshot_dir = temp_dir.path().join("snapshot"); - // Same index (5) as current progress, but a different term (2) — a boundary conflict, - // not a duplicate. - state_machine - .generate_snapshot_data(snapshot_dir.clone(), LogId { term: 2, index: 5 }) - .await - .expect("generate snapshot"); - let conflicting_metadata = SnapshotMetadata { - last_included: Some(LogId { term: 2, index: 5 }), - checksum: Bytes::from_static(&[0; 32]), - }; - - let result = state_machine - .apply_snapshot_from_file(&conflicting_metadata, snapshot_dir) - .await; - assert!( - result.is_err(), - "same index, different term must be rejected as a boundary conflict, not silently accepted" - ); - assert_eq!( - state_machine.last_applied(), - LogId { term: 1, index: 5 }, - "a rejected boundary conflict must not change last_applied" - ); -} - /// A genuinely newer snapshot (`last_included.index` > current `last_applied`) must /// actually run the destructive install and report `Applied`. /// @@ -998,63 +836,9 @@ async fn test_apply_snapshot_from_file_reports_applied_for_newer_snapshot() { last_included: Some(last_included), checksum: Bytes::copy_from_slice(&checksum), }; - let result = state_machine + state_machine .apply_snapshot_from_file(&metadata, snapshot_dir) .await .expect("newer snapshot must install successfully"); - assert!( - matches!(result, SnapshotApplyResult::Applied { last_included: li } if li == last_included), - "expected Applied{{last_included: {last_included:?}}}, got: {result:?}" - ); assert_eq!(state_machine.last_applied(), last_included); } - -/// #436-adjacent: `last_included: None` (e.g. a malformed/corrupted message off the -/// wire) must be rejected outright, not silently fall through to an unconditional -/// install. Before the fix, `None` skipped the entire classification `if let` block -/// and installed anyway — this pinned down the found bug; now pins down the fix. -#[tokio::test] -async fn test_apply_snapshot_from_file_rejects_missing_last_included() { - let temp_dir = TempDir::new().expect("Failed to create temp dir"); - let db_path = temp_dir.path().join("test_none_last_included"); - let (_storage, state_machine) = - RocksDBUnifiedEngine::open(&db_path).expect("Failed to open unified DB"); - - // Snapshot captured early, containing only key_1..key_5. - let early_entries: Vec = - (1..=5).map(|i| insert_at(format!("key_{i}").as_bytes(), b"v", i)).collect(); - state_machine.apply_chunk(&early_entries).await.expect("apply early entries"); - let snapshot_dir = temp_dir.path().join("snapshot"); - state_machine - .generate_snapshot_data(snapshot_dir.clone(), LogId { term: 1, index: 5 }) - .await - .expect("generate snapshot"); - - // Node keeps going past the snapshot boundary: key_6..key_8, last_applied=8. - let later_entries: Vec = - (6..=8).map(|i| insert_at(format!("key_{i}").as_bytes(), b"v", i)).collect(); - state_machine.apply_chunk(&later_entries).await.expect("apply later entries"); - assert_eq!(state_machine.last_applied(), LogId { term: 1, index: 8 }); - - let metadata = SnapshotMetadata { - last_included: None, - checksum: Bytes::from_static(&[0; 32]), - }; - let result = state_machine.apply_snapshot_from_file(&metadata, snapshot_dir).await; - - assert!( - result.is_err(), - "a snapshot with no boundary must be rejected, not installed — got: {result:?}" - ); - - // Nothing must have changed: no destructive install happened. - assert!( - state_machine.get(b"key_6").unwrap().is_some(), - "key_6 (applied after the snapshot boundary) must survive a rejected install" - ); - assert_eq!( - state_machine.last_applied(), - LogId { term: 1, index: 8 }, - "last_applied must be unchanged after a rejected install" - ); -} diff --git a/d-engine-server/src/test_utils/mock/mock_node_builder.rs b/d-engine-server/src/test_utils/mock/mock_node_builder.rs index 4f4290f1..ca6aea75 100644 --- a/d-engine-server/src/test_utils/mock/mock_node_builder.rs +++ b/d-engine-server/src/test_utils/mock/mock_node_builder.rs @@ -26,7 +26,6 @@ use d_engine_core::RaftRole; use d_engine_core::RaftStorageHandles; use d_engine_core::SignalParams; use d_engine_core::SmWorkerHandle; -use d_engine_core::SnapshotApplyResult; use d_engine_core::StateMachine; use d_engine_core::StateMachineCommand; use d_engine_core::StateMachineWorker; @@ -634,11 +633,7 @@ pub(crate) fn mock_state_machine() -> MockStateMachine { mock.expect_snapshot_metadata().returning(|| None); mock.expect_persist_last_snapshot_metadata().returning(|_| Ok(())); - mock.expect_apply_snapshot_from_file().returning(|_, _| { - Ok(SnapshotApplyResult::Applied { - last_included: LogId::default(), - }) - }); + mock.expect_apply_snapshot_from_file().returning(|_, _| Ok(())); mock.expect_generate_snapshot_data().returning(|_, _| Ok(())); mock diff --git a/d-engine-server/tests/common/shared_state_machine.rs b/d-engine-server/tests/common/shared_state_machine.rs index 03239fb5..3ded33f5 100644 --- a/d-engine-server/tests/common/shared_state_machine.rs +++ b/d-engine-server/tests/common/shared_state_machine.rs @@ -17,7 +17,6 @@ use d_engine_core::ApplyEntry; use d_engine_core::ApplyResult; use d_engine_core::Error; use d_engine_core::ScanResult; -use d_engine_core::SnapshotApplyResult; use d_engine_core::StateMachine; use d_engine_proto::common::LogId; use d_engine_proto::server::storage::SnapshotMetadata; @@ -71,7 +70,7 @@ impl StateMachine for SharedStateMachine { &self, metadata: &SnapshotMetadata, snapshot_path: std::path::PathBuf, - ) -> Result { + ) -> Result<(), Error> { self.0.apply_snapshot_from_file(metadata, snapshot_path).await } async fn generate_snapshot_data( diff --git a/d-engine/src/docs/server_guide/customize-state-machine.md b/d-engine/src/docs/server_guide/customize-state-machine.md index 58e95f46..2c1d61fc 100644 --- a/d-engine/src/docs/server_guide/customize-state-machine.md +++ b/d-engine/src/docs/server_guide/customize-state-machine.md @@ -102,6 +102,7 @@ impl StateMachine for CustomStateMachine { - **Idempotency**: Handle duplicate entries safely—Raft may resend committed entries - **Snapshot Isolation**: `apply_snapshot_from_file()` must atomically replace state - **Snapshot Checksums**: The engine computes the checksum of the directory your `generate_snapshot_data()` writes, and verifies it before it calls `apply_snapshot_from_file()`. You neither compute nor check one +- **Snapshot Install**: The engine decides whether a snapshot is installed. `apply_snapshot_from_file()` is called only for a snapshot newer than `last_applied()`, and the engine checks afterwards that `last_applied()` equals the snapshot boundary - **Restore On Open**: The engine never asks you to restore state. When your state machine is opened, `last_applied()` must return the last entry whose effects are durable (persist it together with the data it describes), and `snapshot_metadata()` must return the value most recently passed to `persist_last_snapshot_metadata()` - **Concurrency Control**: Use appropriate locking for state mutations - **Resource Cleanup**: Implement `Drop` to ensure proper flush on shutdown diff --git a/examples/sled-cluster/src/sled_state_machine.rs b/examples/sled-cluster/src/sled_state_machine.rs index 5a6f928d..62f8c182 100644 --- a/examples/sled-cluster/src/sled_state_machine.rs +++ b/examples/sled-cluster/src/sled_state_machine.rs @@ -1,17 +1,5 @@ //! It works as KV storage for client business CRUDs. -use crate::{safe_kv, safe_vk_ivec}; -use arc_swap::ArcSwapOption; -use async_trait::async_trait; -use bincode::config; -use bytes::Bytes; -use d_engine::BatchOp; -use d_engine::{ - ApplyEntry, ApplyResult, Command, Result, ScanResult, SnapshotApplyResult, SnapshotError, - StateMachine, StorageError, common::LogId, server_storage::SnapshotMetadata, -}; -use parking_lot::Mutex; -use sled::Batch; use std::path::Path; use std::path::PathBuf; use std::sync::Arc; @@ -19,12 +7,33 @@ use std::sync::atomic::AtomicBool; use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; use std::time::Instant; + +use arc_swap::ArcSwapOption; +use async_trait::async_trait; +use bincode::config; +use bytes::Bytes; +use d_engine::ApplyEntry; +use d_engine::ApplyResult; +use d_engine::BatchOp; +use d_engine::Command; +use d_engine::Result; +use d_engine::ScanResult; +use d_engine::SnapshotError; +use d_engine::StateMachine; +use d_engine::StorageError; +use d_engine::common::LogId; +use d_engine::server_storage::SnapshotMetadata; +use parking_lot::Mutex; +use sled::Batch; use tracing::debug; use tracing::error; use tracing::info; use tracing::trace; use tracing::warn; +use crate::safe_kv; +use crate::safe_vk_ivec; + /// Sled database tree namespaces pub(crate) const STATE_MACHINE_TREE: &str = "_state_machine_tree"; pub(crate) const STATE_MACHINE_META_NAMESPACE: &str = "_state_machine_metadata"; @@ -304,43 +313,17 @@ impl StateMachine for SledStateMachine { &self, metadata: &SnapshotMetadata, decompressed_snapshot_path: PathBuf, - ) -> Result { + ) -> Result<()> { // No internal locking needed — see generate_snapshot_data. `StateMachineWorker` // processes ApplyEntries/InstallSnapshot/CaptureLocalSnapshot one at a time from a // single queue (#436), so this call never races a concurrent snapshot generation // or another install. - // Boundary classification (#436, every StateMachine impl must honor this): stale or - // duplicate is a no-op, same-index-different-term is a conflict error, and a missing - // boundary can't be classified at all — reject it outright (mirrors - // prepare_transfer_meta's handling of the same field). + // The engine only calls this for a snapshot newer than `last_applied()`. let new_last_included = metadata.last_included.ok_or_else(|| { SnapshotError::OperationFailed("Missing last_included in snapshot metadata".into()) })?; - let current = self.last_applied(); - match new_last_included.index.cmp(¤t.index) { - std::cmp::Ordering::Less => { - info!(?new_last_included, ?current, "Ignoring stale snapshot"); - return Ok(SnapshotApplyResult::IgnoredStale { current }); - } - std::cmp::Ordering::Equal if new_last_included.term == current.term => { - info!(?new_last_included, ?current, "Ignoring duplicate snapshot"); - return Ok(SnapshotApplyResult::IgnoredDuplicate { current }); - } - std::cmp::Ordering::Equal => { - return Err(SnapshotError::BoundaryConflict { - index: current.index, - local_term: current.term, - incoming_term: new_last_included.term, - } - .into()); - } - std::cmp::Ordering::Greater => { - // Eligible to install, fall through. - } - } - // `decompressed_snapshot_path` is a temporary directory owned by the caller // (see `PreparedSnapshot`), not this state machine's own `self.path`. Opening a // sled::Db directly there and swapping `self.db` to point at it — the previous @@ -408,9 +391,7 @@ impl StateMachine for SledStateMachine { self.save_hard_state()?; - Ok(SnapshotApplyResult::Applied { - last_included: new_last_included, - }) + Ok(()) } async fn reset(&self) -> Result<()> { diff --git a/examples/sled-cluster/src/state_machine/sled_state_machine_test.rs b/examples/sled-cluster/src/state_machine/sled_state_machine_test.rs index a9b43643..a34fa8f0 100644 --- a/examples/sled-cluster/src/state_machine/sled_state_machine_test.rs +++ b/examples/sled-cluster/src/state_machine/sled_state_machine_test.rs @@ -4,12 +4,17 @@ //! SledStateMachine to that suite and adds sled-specific scenarios not covered //! by the standard suite. +use std::sync::Arc; + use async_trait::async_trait; use bytes::Bytes; +use d_engine::ApplyEntry; +use d_engine::Command; +use d_engine::Error; +use d_engine::StateMachine; use d_engine::common::LogId; -use d_engine::state_machine_test::{StateMachineBuilder, StateMachineTestSuite}; -use d_engine::{ApplyEntry, Command, Error, StateMachine}; -use std::sync::Arc; +use d_engine::state_machine_test::StateMachineBuilder; +use d_engine::state_machine_test::StateMachineTestSuite; use tempfile::TempDir; use crate::SledStateMachine; @@ -80,7 +85,8 @@ async fn test_snapshot_captures_all_current_data() { term: 1, command: Command::Insert { key: Bytes::from(format!("snap_key_{i}")), - value: Bytes::from(format!("snap_val_{i}")), }, + value: Bytes::from(format!("snap_val_{i}")), + }, }) .collect(); sm.apply_chunk(&entries).await.unwrap(); @@ -116,7 +122,8 @@ async fn test_snapshot_does_not_include_post_snapshot_entries() { term: 1, command: Command::Insert { key: Bytes::from_static(b"before"), - value: Bytes::from_static(b"v1"), }, + value: Bytes::from_static(b"v1"), + }, }]; sm.apply_chunk(&pre_entries).await.unwrap(); @@ -131,7 +138,8 @@ async fn test_snapshot_does_not_include_post_snapshot_entries() { term: 1, command: Command::Insert { key: Bytes::from_static(b"after"), - value: Bytes::from_static(b"v2"), }, + value: Bytes::from_static(b"v2"), + }, }]; sm.apply_chunk(&post_entries).await.unwrap(); @@ -162,7 +170,8 @@ async fn test_snapshot_install_content_survives_restart() { term: 1, command: Command::Insert { key: Bytes::from(format!("install_key_{i}")), - value: Bytes::from(format!("install_val_{i}")), }, + value: Bytes::from(format!("install_val_{i}")), + }, }) .collect(); source_sm.apply_chunk(&entries).await.unwrap(); @@ -186,14 +195,10 @@ async fn test_snapshot_install_content_survives_restart() { let node_b_path = node_b_dir.path().to_path_buf(); { let node_b = SledStateMachine::new(&node_b_path, 2).unwrap(); - let result = node_b + node_b .apply_snapshot_from_file(&metadata, export_dir.path().to_path_buf()) .await .unwrap(); - assert!( - matches!(result, d_engine::SnapshotApplyResult::Applied { .. }), - "expected Applied, got: {result:?}" - ); assert_eq!( node_b.get(b"install_key_1").unwrap(), Some(Bytes::from_static(b"install_val_1")), @@ -240,7 +245,8 @@ async fn test_new_recovers_backup_after_interrupted_install() { term: 1, command: Command::Insert { key: Bytes::from_static(b"pre_crash_key"), - value: Bytes::from_static(b"pre_crash_val"), }, + value: Bytes::from_static(b"pre_crash_val"), + }, }]) .await .unwrap(); From 0bb11846534f924691582c2964c59069708bafd1 Mon Sep 17 00:00:00 2001 From: Joshua Chi <112539+JoshuaChi@users.noreply.github.com> Date: Sun, 1 Nov 2026 23:23:17 +0800 Subject: [PATCH 08/19] refactor #400: The engine records the snapshots it holds The engine now keeps its own record of the snapshots a node holds: next to each archive snapshot-{index}-{term}.tar.gz it writes snapshot-{index}-{term}.meta (the snapshot's boundary and checksum), written after the archive is complete and, on a follower, after the snapshot is installed. The latest snapshot is kept in memory and recovered from disk when the node starts. The snapshot trigger policy, the leader's snapshot push and the join response read it instead of asking the state machine. Cleanup goes by the records: it keeps the newest N recorded snapshots and removes older ones together with their records, plus archives that have no record and are not newer than the newest recorded snapshot (such an archive is left behind when a received snapshot is ignored or rejected, and can never be installed or served). The newest recorded snapshot is never deleted. With no recorded snapshot nothing is deleted. Tests: record published after build and after install, recovered after restart, removed with its archive, not published for a rejected snapshot, an unrecorded archive is ignored; cleanup never deletes the newest recorded snapshot; an install still succeeds when its record cannot be written. Remove the now unused CleanupSnapshotMeta, parse_snapshot_dirname and the handler's snapshots_dir field. The state machine's snapshot_metadata and persist_last_snapshot_metadata are still called; they go in the next step. --- CHANGELOG.md | 2 - d-engine-core/src/raft_role/leader_state.rs | 2 +- .../leader_state_test/snapshot_worker_test.rs | 7 +- .../default_state_machine_handler.rs | 147 +---- .../default_state_machine_handler_test.rs | 509 +++++++++++++++++- .../src/state_machine_handler/mod.rs | 8 +- .../state_machine_handler/snapshot_catalog.rs | 238 ++++++++ .../snapshot_catalog_test.rs | 149 +++++ .../src/state_machine_handler/writer.rs | 22 + .../src/test_utils/mock/mock_raft_builder.rs | 11 +- d-engine-server/src/lib.rs | 8 - 11 files changed, 939 insertions(+), 164 deletions(-) create mode 100644 d-engine-core/src/state_machine_handler/snapshot_catalog.rs create mode 100644 d-engine-core/src/state_machine_handler/snapshot_catalog_test.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 4cc6b121..bd9829c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -81,8 +81,6 @@ All notable changes to this project will be documented in this file. - **⚠️ `StateMachine::entry_term()` removed from trait** (#418): Term lookup belongs to the log layer, not the state machine. Custom `StateMachine` implementations must delete this method — it is no longer part of the trait. See [Migration Guide](./MIGRATION_GUIDE.md) for details. -- **⚠️ `StateMachine::apply_snapshot_from_file` return type changed** (#436): Was `Result<()>`, now `Result`. The three outcomes (`Applied`/`IgnoredStale`/`IgnoredDuplicate`) are now explicit instead of every call looking identical on success. Custom `StateMachine` implementations must update the signature and return the matching variant. See [Migration Guide](./MIGRATION_GUIDE.md) for details. - - **⚠️ Learner initial-snapshot PULL path removed** (#436): `Transport::request_snapshot_from_leader` and `StateMachineHandler::apply_snapshot_stream_from_leader` are removed. New learners catch up via the leader's PUSH replication loop (AppendEntries / InstallSnapshot) instead of eagerly pulling a snapshot on startup. See [Migration Guide](./MIGRATION_GUIDE.md) for details. - **KV encoding moved out of `d-engine-core`** (#415): `ClientWriteRequest.command` is now `Option` (pre-serialized). Serialization happens in the server transport layer (embedded/standalone/gRPC handler). Core is encoding-agnostic. diff --git a/d-engine-core/src/raft_role/leader_state.rs b/d-engine-core/src/raft_role/leader_state.rs index cdbce695..9f7f34f3 100644 --- a/d-engine-core/src/raft_role/leader_state.rs +++ b/d-engine-core/src/raft_role/leader_state.rs @@ -3653,7 +3653,7 @@ impl LeaderState { // Peers within an active backoff window (after consecutive failures) are skipped // to protect the leader from repeatedly pushing to unreachable peers. if !requests.snapshot_targets.is_empty() { - let snapshot_metadata = ctx.state_machine().snapshot_metadata(); + let snapshot_metadata = ctx.state_machine_handler().get_latest_snapshot_metadata(); if let Some(metadata) = snapshot_metadata { for peer_id in requests.snapshot_targets { // Respect per-peer backoff window: if recent push attempts failed, diff --git a/d-engine-core/src/raft_role/leader_state_test/snapshot_worker_test.rs b/d-engine-core/src/raft_role/leader_state_test/snapshot_worker_test.rs index 2825b942..8534add6 100644 --- a/d-engine-core/src/raft_role/leader_state_test/snapshot_worker_test.rs +++ b/d-engine-core/src/raft_role/leader_state_test/snapshot_worker_test.rs @@ -59,6 +59,7 @@ use crate::raft_role::role_state::PeerReplicationState; use crate::raft_role::role_state::RaftRoleState; use crate::test_utils::mock::MockTypeConfig; use crate::test_utils::mock::mock_raft_context; +use crate::test_utils::mock::mock_state_machine_handler_with_snapshot; // ── helpers ────────────────────────────────────────────────────────────────── @@ -104,8 +105,6 @@ fn state_machine_with_snapshot() -> MockStateMachine { sm.expect_get().returning(|_| Ok(None)); sm.expect_apply_chunk().returning(|_| Ok(vec![])); sm.expect_last_applied().return_const(LogId::default()); - // Return a valid snapshot so Phase 6 fires the Snapshot task to the worker. - sm.expect_snapshot_metadata().returning(|| Some(stub_snapshot_metadata())); sm.expect_persist_last_snapshot_metadata().returning(|_| Ok(())); sm.expect_apply_snapshot_from_file().returning(|_, _| Ok(())); sm.expect_generate_snapshot_data().returning(|_, _| Ok(())); @@ -138,6 +137,10 @@ fn new_leader_and_ctx( ctx.membership = Arc::new(two_peer_membership()); ctx.storage.state_machine = Arc::new(state_machine_with_snapshot()); + // The engine reports a valid snapshot, so Phase 6 fires the Snapshot task to the worker. + ctx.handlers.state_machine_handler = Arc::new(mock_state_machine_handler_with_snapshot(Some( + stub_snapshot_metadata(), + ))); let mut raft_log = MockRaftLog::new(); raft_log.expect_last_entry_id().returning(|| 15); diff --git a/d-engine-core/src/state_machine_handler/default_state_machine_handler.rs b/d-engine-core/src/state_machine_handler/default_state_machine_handler.rs index 4f8dc9a1..78428dc0 100644 --- a/d-engine-core/src/state_machine_handler/default_state_machine_handler.rs +++ b/d-engine-core/src/state_machine_handler/default_state_machine_handler.rs @@ -20,9 +20,7 @@ use d_engine_proto::server::storage::snapshot_ack::ChunkStatus; use futures::stream::BoxStream; use memmap2::Mmap; use memmap2::MmapOptions; -use tokio::fs; use tokio::fs::File; -use tokio::fs::remove_dir_all; use tokio::fs::remove_file; use tokio::io::AsyncWriteExt; use tokio::io::BufReader; @@ -64,6 +62,7 @@ use crate::file_io::validate_checksum; use crate::file_io::validate_compressed_format; use crate::scoped_timer::ScopedTimer; use crate::state_machine_handler::applied_state::AppliedState; +use crate::state_machine_handler::snapshot_catalog::SnapshotCatalog; /// Unified snapshot metadata with precomputed values #[derive(Debug, Clone)] @@ -91,21 +90,13 @@ where // current_snapshot_version: AtomicU64, snapshot_config: SnapshotConfig, - /// Explicit runtime path — always `data_dir/snapshots` (see #10). Not - /// part of `snapshot_config`; `snapshots_dir` is a runtime identity, not - /// a configurable setting. - snapshots_dir: PathBuf, snapshot_policy: SNP, snapshot_in_progress: AtomicBool, path_mgr: Arc, -} -#[derive(Debug, PartialEq, Hash, Eq, Clone)] -pub(crate) struct CleanupSnapshotMeta { - pub(crate) index: u64, - pub(crate) term: u64, - pub(crate) path: PathBuf, + /// The snapshots this node holds, recorded by the engine itself. + catalog: Arc, } #[async_trait] @@ -321,7 +312,7 @@ where } let last_applied = self.state_machine.last_applied(); - let last_snapshot_metadata = self.state_machine.snapshot_metadata(); + let last_snapshot_metadata = self.catalog.latest(); let last_included = last_snapshot_metadata .and_then(|meta| meta.last_included) @@ -372,19 +363,13 @@ where temp_dir.remove().await?; // Publish metadata only after the archive is durably built (#436). A crash before - // this point leaves the old metadata on disk, so the leader re-sends the snapshot + // this point leaves the old record on disk, so the leader re-sends the snapshot // instead of advertising one whose archive is missing. + self.catalog.publish(&metadata).await?; self.state_machine.persist_last_snapshot_metadata(&metadata)?; debug!(%self.snapshot_config.cleanup_retain_count, "build_local_snapshot: cleanup old versions"); - if let Err(e) = self - .cleanup_snapshot( - self.snapshot_config.cleanup_retain_count, - &self.snapshots_dir, - &self.snapshot_config.snapshots_dir_prefix, - ) - .await - { + if let Err(e) = self.cleanup_snapshot(self.snapshot_config.cleanup_retain_count).await { error!(%e, "clean up old snapshot file failed"); } @@ -395,82 +380,12 @@ where async fn cleanup_snapshot( &self, retain_count: u64, - snapshot_dir: &Path, - snapshot_dir_prefix: &str, ) -> Result<()> { - // Phase 1: Collect and parse snapshots - let mut snapshots = Vec::new(); - - let mut entries = - fs::read_dir(snapshot_dir).await.map_err(|e| StorageError::PathError { - path: snapshot_dir.to_path_buf(), - source: e, - })?; - - while let Some(entry) = entries.next_entry().await.map_err(StorageError::IoError)? { - let path = entry.path(); - debug!(?path, "cleanup_snapshot"); - - if path.extension().is_some_and(|ext| ext == "gz") - && let Some(file_name) = path.file_name().and_then(|n| n.to_str()) - { - let parsed = parse_snapshot_dirname(file_name, snapshot_dir_prefix).or_else(|| { - file_name - .strip_suffix(".tar.gz") - .and_then(|s| parse_snapshot_dirname(s, snapshot_dir_prefix)) - }); - - let (index, term) = if let Some(pair) = parsed { - pair - } else { - continue; - }; - - info!( - "Index: {:>10} | Term: {:>10} | Path: {}", - index, - term, - path.display() - ); - snapshots.push(CleanupSnapshotMeta { index, term, path }); - } - } - - // Phase 2: Sorting and cleaning - if snapshots.len() <= retain_count as usize { - return Ok(()); // No need to clean - } - - // Sort in ascending order by index (earlier snapshots are at the front) - snapshots.sort_by_key(|m| m.index); - - // Phase 4: Difference: Take the remaining elements - // Calculate the split point to be retained - let split_point = snapshots.len() - retain_count as usize; - - for meta in &snapshots[..split_point] { - info!( - "Deleting old snapshot [index={}, term={}] at {}", - meta.index, - meta.term, - meta.path.display() - ); - let file_type = tokio::fs::metadata(&meta.path) - .await - .map_err(StorageError::IoError)? - .file_type(); - - if file_type.is_file() { - remove_file(&meta.path).await.map_err(StorageError::IoError)?; - } else if file_type.is_dir() { - remove_dir_all(&meta.path).await.map_err(StorageError::IoError)?; - } - } - Ok(()) + self.catalog.prune(retain_count).await } fn get_latest_snapshot_metadata(&self) -> Option { - self.state_machine.snapshot_metadata() + self.catalog.latest() } /// Load snapshot data as a stream of chunks (ZERO-COPY) @@ -677,7 +592,10 @@ where snapshot_policy: SNP, ) -> Self { let (applied, applied_notify_rx) = AppliedState::new(last_applied_index); - + let path_mgr = Arc::new(SnapshotPathManager::new( + snapshots_dir.clone(), + snapshot_config.snapshots_dir_prefix.clone(), + )); Self { node_id, applied, @@ -685,11 +603,8 @@ where pending_commit: AtomicU64::new(0), state_machine, snapshot_policy, - path_mgr: Arc::new(SnapshotPathManager::new( - snapshots_dir.clone(), - snapshot_config.snapshots_dir_prefix.clone(), - )), - snapshots_dir, + catalog: Arc::new(SnapshotCatalog::open(path_mgr.clone())), + path_mgr, snapshot_config, snapshot_in_progress: AtomicBool::new(false), @@ -850,34 +765,6 @@ fn zero_copy_bytes_from_mmap( Bytes::copy_from_slice(slice) } -/// Manual parsing file name format: snapshot-{index}-{term} -fn parse_snapshot_dirname( - name: &str, - snapshot_dir_prefix: &str, -) -> Option<(u64, u64)> { - debug!(%name, "parse_snapshot_dirname"); - - // Check prefix and suffix - if !name.starts_with(snapshot_dir_prefix) { - return None; - } - - // Remove fixed parts - let core = &name[9..name.len()]; // "snapshot-".len() = 9, - - // Split version and index - let parts: Vec<&str> = core.splitn(2, '-').collect(); - if parts.len() != 2 { - return None; - } - - // Parse numbers - match (parts[0].parse::(), parts[1].parse::()) { - (Ok(i), Ok(t)) => Some((i, t)), - _ => None, - } -} - /// Constructs a paired reader/writer sharing one `AppliedState` and one `state_machine` /// handle. This is the only place `DefaultStateMachineWriter` should be built alongside /// its reader — `StateMachineWorker` must be the sole holder of the writer half. @@ -899,6 +786,7 @@ pub fn new_reader_writer_pair( snapshots_dir.clone(), snapshot_config.snapshots_dir_prefix.clone(), )); + let catalog = Arc::new(SnapshotCatalog::open(path_mgr.clone())); let reader = DefaultStateMachineHandler { node_id, pending_commit: AtomicU64::new(0), @@ -907,7 +795,7 @@ pub fn new_reader_writer_pair( applied_notify_rx, snapshot_policy, path_mgr: path_mgr.clone(), - snapshots_dir, + catalog: catalog.clone(), snapshot_config, snapshot_in_progress: AtomicBool::new(false), }; @@ -916,6 +804,7 @@ pub fn new_reader_writer_pair( state_machine, applied, path_mgr, + catalog, watch_event_tx, prev_kv_watcher_count, ); diff --git a/d-engine-core/src/state_machine_handler/default_state_machine_handler_test.rs b/d-engine-core/src/state_machine_handler/default_state_machine_handler_test.rs index 73bed761..47128d46 100644 --- a/d-engine-core/src/state_machine_handler/default_state_machine_handler_test.rs +++ b/d-engine-core/src/state_machine_handler/default_state_machine_handler_test.rs @@ -857,6 +857,55 @@ async fn test_prepare_snapshot_stream_persists_file_in_snapshots_dir() { ); } +/// Node C receives a snapshot from the leader, then a third node that is even further behind +/// asks C for a snapshot at the same boundary. +/// +/// Purpose: the archive C kept must actually be servable. `load_snapshot_data` finds it at the +/// path derived from the boundary and streams exactly the bytes that were received. Together +/// with `test_prepare_snapshot_stream_persists_file_in_snapshots_dir` this pins the whole +/// receive-then-forward path: a node that installed a snapshot can later serve it. +#[tokio::test] +#[traced_test] +async fn test_snapshot_received_from_leader_can_be_served_to_another_node() { + let temp_dir = tempdir().unwrap(); + let temp_path = temp_dir.path().join("test_received_snapshot_is_servable"); + create_dir_all(&temp_path).await.unwrap(); + let handler = create_test_handler(&temp_path, None); + + let (compressed_data, metadata) = create_test_compressed_snapshot().await; + let first_chunk = SnapshotChunk { + leader_term: TEST_TERM, + leader_id: TEST_LEADER_ID, + metadata: Some(metadata.clone()), + seq: 0, + total_chunks: 1, + data: Bytes::from(compressed_data.clone()), + chunk_checksum: Bytes::from(crc32fast::hash(&compressed_data).to_be_bytes().to_vec()), + }; + let (ack_tx, _ack_rx) = mpsc::channel::(1); + let (_tx, rx) = mpsc::channel(32); + drop(_tx); + let config = snapshot_config(temp_path.to_path_buf()); + handler + .prepare_snapshot_stream(first_chunk, rx, ack_tx, &config) + .await + .expect("the snapshot should be received"); + + let mut stream = handler + .load_snapshot_data(metadata, TEST_TERM) + .await + .expect("a node that received a snapshot must be able to serve it"); + let mut served = Vec::new(); + while let Some(chunk) = stream.next().await { + served.extend_from_slice(&chunk.expect("every chunk must load").data); + } + + assert_eq!( + served, compressed_data, + "the served bytes must be the archive that was received" + ); +} + const TEST_TERM: u64 = 1; const TEST_LEADER_ID: u32 = 1; @@ -1555,10 +1604,10 @@ fn count_snapshots( // Ignore directory entries that cannot be read let entry = entry.ok()?; // Extract the file name and check the prefix - entry - .file_name() - .to_str() - .and_then(|name| name.starts_with(snapshots_dir_prefix).then_some(())) + // Count archives only: each archive's record (`.meta`) is bookkeeping. + entry.file_name().to_str().and_then(|name| { + (name.starts_with(snapshots_dir_prefix) && !name.ends_with(".meta")).then_some(()) + }) }) .count() } @@ -1574,8 +1623,12 @@ fn get_snapshot_versions(dir: &Path) -> Vec { let name = name.to_str().unwrap(); debug!(%name, "get_snapshot_versions"); + // A snapshot's record (`.meta`) is bookkeeping, not another snapshot. + if name.ends_with(".meta") { + None + } // Handle compressed snapshots - if name.ends_with(".tar.gz") { + else if name.ends_with(".tar.gz") { let base_name = name.trim_end_matches(".tar.gz"); base_name.split('-').nth(1).and_then(|v| v.parse().ok()) } @@ -1606,10 +1659,7 @@ async fn test_cleanup_snapshot_case1() { MockSnapshotPolicy::new(), ); - handler - .cleanup_snapshot(2, temp_dir.path(), &config.snapshots_dir_prefix) - .await - .unwrap(); + handler.cleanup_snapshot(2).await.unwrap(); // Verify remaining snapshots let mut remaining = get_snapshot_versions(temp_dir.path()); @@ -1646,10 +1696,7 @@ async fn test_cleanup_snapshot_case2() { MockSnapshotPolicy::new(), ); - handler - .cleanup_snapshot(2, temp_dir.path(), &config.snapshots_dir_prefix) - .await - .unwrap(); + handler.cleanup_snapshot(2).await.unwrap(); // Verify no deletions let mut remaining = get_snapshot_versions(temp_dir.path()); @@ -1684,10 +1731,7 @@ async fn test_cleanup_snapshot_case3() { MockSnapshotPolicy::new(), ); - handler - .cleanup_snapshot(2, temp_dir.path(), &config.snapshots_dir_prefix) - .await - .unwrap(); + handler.cleanup_snapshot(2).await.unwrap(); //Verify only valid version 1 is deleted let remaining = get_dir_names(temp_dir.path()).await; @@ -1710,6 +1754,22 @@ async fn create_test_files( let mut file = File::create(&path).await.unwrap(); // Write some dummy content file.write_all(b"dummy snapshot data").await.unwrap(); + + // A snapshot the node holds has a record next to its archive. + let metadata = SnapshotMetadata { + last_included: Some(LogId { + index: *id, + term: 1, + }), + checksum: Bytes::from(vec![*id as u8; 32]), + }; + let record = format!("{snapshots_dir_prefix}{id}-1.meta"); + tokio::fs::write( + temp_dir.path().join(record), + prost::Message::encode_to_vec(&metadata), + ) + .await + .unwrap(); } } @@ -3335,3 +3395,418 @@ async fn test_install_prepared_snapshot_state_machine_not_advanced_is_an_error() "a state machine that did not move to the snapshot's boundary must be reported" ); } + +// ── The engine's own record of the snapshots this node holds ───────────────────────────────── +// +// What the latest snapshot is (boundary and checksum) is recorded by the engine next to the +// archive, not by the state machine. The mock state machine below reports no snapshot at all, so +// every answer these tests see comes from the engine. + +mod snapshot_catalog_behavior_tests { + use super::*; + + fn mock_sm_at(index: Arc) -> MockStateMachine { + let mut sm = MockStateMachine::new(); + let position = index.clone(); + sm.expect_last_applied().returning(move || LogId { + index: position.load(std::sync::atomic::Ordering::SeqCst), + term: 2, + }); + sm.expect_generate_snapshot_data().returning(|path, last_included| { + fs::create_dir_all(&path).unwrap(); + fs::write( + path.join("data.bin"), + format!("state at {}", last_included.index), + ) + .unwrap(); + Ok(()) + }); + sm.expect_persist_last_snapshot_metadata().returning(|_| Ok(())); + // The state machine knows nothing about snapshots. + sm.expect_snapshot_metadata().returning(|| None); + sm + } + + fn pair_over( + dir: &Path, + sm: MockStateMachine, + retain: u64, + ) -> ( + DefaultStateMachineHandler, + DefaultStateMachineWriter, + ) { + let mut config = snapshot_config(dir.to_path_buf()); + config.cleanup_retain_count = retain; + new_reader_writer_pair::( + 1, + 0, + Arc::new(sm), + dir.to_path_buf(), + config, + MockSnapshotPolicy::new(), + None, + Arc::new(AtomicUsize::new(0)), + ) + } + + async fn take_snapshot( + handler: &DefaultStateMachineHandler, + writer: &DefaultStateMachineWriter, + ) -> (SnapshotMetadata, PathBuf) { + handler.try_begin_local_snapshot_capture().unwrap(); + let captured = writer.capture_local_snapshot().await.unwrap(); + let built = handler.build_local_snapshot(captured).await.unwrap(); + handler.end_local_snapshot_capture(); + built + } + + fn files_with_extension( + dir: &Path, + extension: &str, + ) -> Vec { + let mut names: Vec = fs::read_dir(dir) + .unwrap() + .map(|e| e.unwrap().path()) + .filter(|p| p.extension().is_some_and(|e| e == extension)) + .map(|p| p.file_name().unwrap().to_string_lossy().into_owned()) + .collect(); + names.sort(); + names + } + + /// Leader A builds a snapshot at log 10. A's state machine keeps no snapshot record. + /// + /// Purpose: the engine reports the snapshot it just built (boundary and checksum), and records + /// it next to the archive, so nothing depends on the state machine remembering it. + #[tokio::test] + #[traced_test] + async fn test_built_snapshot_is_reported_as_the_latest_snapshot() { + let temp_dir = tempfile::tempdir().unwrap(); + let (handler, writer) = pair_over(temp_dir.path(), mock_sm_at(Arc::new(10.into())), 3); + + let (metadata, archive) = take_snapshot(&handler, &writer).await; + + assert_eq!(handler.get_latest_snapshot_metadata(), Some(metadata)); + assert!(archive.exists()); + assert_eq!(files_with_extension(temp_dir.path(), "meta"), vec![ + "snapshot-10-2.meta" + ]); + } + + /// Leader A restarts. Its state machine still reports no snapshot. + /// + /// Purpose: the engine finds the latest snapshot again from what it recorded on disk. If it + /// did not, leader A would have nothing to push to a follower that fell behind. + #[tokio::test] + #[traced_test] + async fn test_latest_snapshot_is_recovered_after_restart() { + let temp_dir = tempfile::tempdir().unwrap(); + let (handler, writer) = pair_over(temp_dir.path(), mock_sm_at(Arc::new(10.into())), 3); + let (metadata, _) = take_snapshot(&handler, &writer).await; + drop((handler, writer)); + + let (restarted, _writer) = pair_over(temp_dir.path(), mock_sm_at(Arc::new(10.into())), 3); + + assert_eq!(restarted.get_latest_snapshot_metadata(), Some(metadata)); + } + + /// Follower C installs a snapshot (its archive was kept when it was received). + /// + /// Purpose: after a successful install the engine reports that snapshot as C's latest, so C + /// can serve it to another node and does not immediately take a snapshot of its own. + #[tokio::test] + #[traced_test] + async fn test_installed_snapshot_is_reported_as_the_latest_snapshot() { + let temp_dir = tempfile::tempdir().unwrap(); + let position = Arc::new(std::sync::atomic::AtomicU64::new(0)); + let mut sm = mock_sm_at(position.clone()); + let installer = position.clone(); + sm.expect_apply_snapshot_from_file().times(1).returning(move |_, _| { + installer.store(5, std::sync::atomic::Ordering::SeqCst); + Ok(()) + }); + let (handler, writer) = pair_over(temp_dir.path(), sm, 3); + // What `prepare_snapshot_stream` leaves behind: the received archive at its final path. + fs::write( + temp_dir.path().join("snapshot-5-2.tar.gz"), + b"received archive", + ) + .unwrap(); + let (dir, metadata) = received_snapshot( + &temp_dir.path().join("work"), + Some(LogId { index: 5, term: 2 }), + ) + .await; + + writer.install_prepared_snapshot(metadata.clone(), dir).await.unwrap(); + + assert_eq!(handler.get_latest_snapshot_metadata(), Some(metadata)); + assert_eq!(files_with_extension(temp_dir.path(), "meta"), vec![ + "snapshot-5-2.meta" + ]); + } + + /// A snapshot that is rejected (here: corrupted in transit) is never reported as the latest. + /// + /// Purpose: only a snapshot the node actually installed is offered to other nodes. + #[tokio::test] + #[traced_test] + async fn test_rejected_snapshot_is_not_reported_as_the_latest_snapshot() { + let temp_dir = tempfile::tempdir().unwrap(); + let mut sm = mock_sm_at(Arc::new(0.into())); + sm.expect_apply_snapshot_from_file().times(0); + let (handler, writer) = pair_over(temp_dir.path(), sm, 3); + fs::write( + temp_dir.path().join("snapshot-5-2.tar.gz"), + b"received archive", + ) + .unwrap(); + let (dir, mut metadata) = received_snapshot( + &temp_dir.path().join("work"), + Some(LogId { index: 5, term: 2 }), + ) + .await; + metadata.checksum = Bytes::from(vec![0; 32]); + + let result = writer.install_prepared_snapshot(metadata, dir).await; + + assert!(result.is_err()); + assert_eq!(handler.get_latest_snapshot_metadata(), None); + assert!(files_with_extension(temp_dir.path(), "meta").is_empty()); + } + + /// An archive with no record next to it (a crash between receiving and installing, or an + /// archive from before the engine kept records) is not a snapshot the node can vouch for. + /// + /// Purpose: it is ignored, not reported with guessed metadata. + #[tokio::test] + #[traced_test] + async fn test_archive_without_a_record_is_ignored() { + let temp_dir = tempfile::tempdir().unwrap(); + fs::write(temp_dir.path().join("snapshot-9-2.tar.gz"), b"archive").unwrap(); + + let (handler, _writer) = pair_over(temp_dir.path(), mock_sm_at(Arc::new(9.into())), 3); + + assert_eq!(handler.get_latest_snapshot_metadata(), None); + } + + fn write_record( + dir: &Path, + index: u64, + term: u64, + ) { + use prost::Message; + fs::write( + dir.join(format!("snapshot-{index}-{term}.tar.gz")), + b"archive", + ) + .unwrap(); + let metadata = SnapshotMetadata { + last_included: Some(LogId { index, term }), + checksum: Bytes::from(vec![index as u8; 32]), + }; + fs::write( + dir.join(format!("snapshot-{index}-{term}.meta")), + metadata.encode_to_vec(), + ) + .unwrap(); + } + + /// Node C holds snapshot 10 (recorded). A snapshot 20 was received but then ignored, so its + /// archive is on disk without a record. Cleanup runs with "keep the newest 1". + /// + /// Purpose: cleanup counts only snapshots the node holds. The newest recorded snapshot is + /// never deleted, so the node can still serve it, and nothing has to be rescanned afterwards. + #[tokio::test] + #[traced_test] + async fn test_cleanup_never_deletes_the_newest_recorded_snapshot() { + let temp_dir = tempfile::tempdir().unwrap(); + write_record(temp_dir.path(), 10, 2); + fs::write( + temp_dir.path().join("snapshot-20-2.tar.gz"), + b"received, then ignored", + ) + .unwrap(); + let (handler, _writer) = pair_over(temp_dir.path(), mock_sm_at(Arc::new(10.into())), 1); + + handler.cleanup_snapshot(1).await.unwrap(); + + assert!(temp_dir.path().join("snapshot-10-2.tar.gz").exists()); + assert!(temp_dir.path().join("snapshot-10-2.meta").exists()); + assert_eq!( + handler.get_latest_snapshot_metadata().and_then(|m| m.last_included), + Some(LogId { index: 10, term: 2 }) + ); + assert!( + temp_dir.path().join("snapshot-20-2.tar.gz").exists(), + "an archive newer than every recorded snapshot may be one being installed" + ); + } + + /// An archive with no record that is older than the newest recorded snapshot. + /// + /// Purpose: it can never be installed or served (an install needs a snapshot newer than the + /// node's position), so cleanup removes it instead of leaving it on disk forever. + #[tokio::test] + #[traced_test] + async fn test_cleanup_removes_an_unrecorded_archive_older_than_the_newest_recorded_snapshot() { + let temp_dir = tempfile::tempdir().unwrap(); + write_record(temp_dir.path(), 30, 2); + fs::write( + temp_dir.path().join("snapshot-10-2.tar.gz"), + b"received, then ignored", + ) + .unwrap(); + let (handler, _writer) = pair_over(temp_dir.path(), mock_sm_at(Arc::new(30.into())), 3); + + handler.cleanup_snapshot(3).await.unwrap(); + + assert!(!temp_dir.path().join("snapshot-10-2.tar.gz").exists()); + assert!(temp_dir.path().join("snapshot-30-2.tar.gz").exists()); + } + + /// A node upgraded from a build that kept no records: its archives have none. + /// + /// Purpose: with no recorded snapshot there is nothing to compare against, so cleanup deletes + /// nothing. The first snapshot the node builds after the upgrade starts the records. + #[tokio::test] + #[traced_test] + async fn test_cleanup_deletes_nothing_when_no_snapshot_is_recorded() { + let temp_dir = tempfile::tempdir().unwrap(); + for index in [10, 20, 30] { + fs::write( + temp_dir.path().join(format!("snapshot-{index}-2.tar.gz")), + b"legacy", + ) + .unwrap(); + } + let (handler, _writer) = pair_over(temp_dir.path(), mock_sm_at(Arc::new(30.into())), 1); + + handler.cleanup_snapshot(1).await.unwrap(); + + assert_eq!(files_with_extension(temp_dir.path(), "gz").len(), 3); + } + + /// Node C is at log 100 and receives a delayed push of a snapshot that ends at log 5. + /// + /// Purpose: pins what really happens to the archive of a snapshot that is received and then + /// ignored: it stays at its final path with no record. That is why cleanup must go by the + /// records, not by counting archive files. + #[tokio::test] + #[traced_test] + async fn test_received_snapshot_that_is_ignored_leaves_an_archive_without_a_record() { + let temp_dir = tempfile::tempdir().unwrap(); + let (handler, writer) = pair_over(temp_dir.path(), mock_sm_at(Arc::new(100.into())), 3); + let (compressed_data, metadata) = create_test_compressed_snapshot().await; + let first_chunk = SnapshotChunk { + leader_term: TEST_TERM, + leader_id: TEST_LEADER_ID, + metadata: Some(metadata.clone()), + seq: 0, + total_chunks: 1, + data: Bytes::from(compressed_data.clone()), + chunk_checksum: Bytes::from(crc32fast::hash(&compressed_data).to_be_bytes().to_vec()), + }; + let (ack_tx, _ack_rx) = mpsc::channel::(1); + let (_tx, rx) = mpsc::channel(32); + drop(_tx); + let config = snapshot_config(temp_dir.path().to_path_buf()); + let prepared = handler + .prepare_snapshot_stream(first_chunk, rx, ack_tx, &config) + .await + .expect("the snapshot should be received"); + + let result = writer + .install_prepared_snapshot( + prepared.metadata.clone(), + prepared.temp_dir.path().to_path_buf(), + ) + .await + .unwrap(); + + assert!( + matches!(result, crate::SnapshotApplyResult::IgnoredStale { .. }), + "got: {result:?}" + ); + assert!( + temp_dir.path().join("snapshot-5-1.tar.gz").exists(), + "the received archive stays on disk" + ); + assert!(files_with_extension(temp_dir.path(), "meta").is_empty()); + assert_eq!(handler.get_latest_snapshot_metadata(), None); + } + + /// Node C installs a snapshot, but writing the record next to its archive fails (here: a + /// non-empty directory sits where the record file must go, standing in for a full disk or a + /// permission error). + /// + /// Purpose: the install has already replaced C's state, so it is still reported as applied. + /// Failing it would leave the commit index behind the state machine and skip the log purge. + /// The failure is logged, and C simply holds no record of this snapshot. + #[tokio::test] + #[traced_test] + async fn test_install_succeeds_even_when_its_record_cannot_be_written() { + let temp_dir = tempfile::tempdir().unwrap(); + let position = Arc::new(std::sync::atomic::AtomicU64::new(0)); + let mut sm = mock_sm_at(position.clone()); + let installer = position.clone(); + sm.expect_apply_snapshot_from_file().times(1).returning(move |_, _| { + installer.store(5, std::sync::atomic::Ordering::SeqCst); + Ok(()) + }); + let (handler, writer) = pair_over(temp_dir.path(), sm, 3); + fs::write( + temp_dir.path().join("snapshot-5-2.tar.gz"), + b"received archive", + ) + .unwrap(); + fs::create_dir_all(temp_dir.path().join("snapshot-5-2.meta")).unwrap(); + fs::write( + temp_dir.path().join("snapshot-5-2.meta").join("blocker"), + b"x", + ) + .unwrap(); + let (dir, metadata) = received_snapshot( + &temp_dir.path().join("work"), + Some(LogId { index: 5, term: 2 }), + ) + .await; + + let result = writer.install_prepared_snapshot(metadata, dir).await.unwrap(); + + assert!( + matches!(result, crate::SnapshotApplyResult::Applied { .. }), + "got: {result:?}" + ); + assert!(logs_contain("Failed to record the installed snapshot")); + assert_eq!(handler.get_latest_snapshot_metadata(), None); + } + + /// Old snapshots are cleaned up, keeping the newest one. + /// + /// Purpose: the record goes with its archive. The newest archive keeps its record, and no + /// record is left behind for an archive that was deleted. + #[tokio::test] + #[traced_test] + async fn test_cleanup_removes_the_record_together_with_the_archive() { + let temp_dir = tempfile::tempdir().unwrap(); + let position = Arc::new(std::sync::atomic::AtomicU64::new(10)); + let (handler, writer) = pair_over(temp_dir.path(), mock_sm_at(position.clone()), 1); + + for index in [10, 20, 30] { + position.store(index, std::sync::atomic::Ordering::SeqCst); + take_snapshot(&handler, &writer).await; + } + + assert_eq!(files_with_extension(temp_dir.path(), "gz"), vec![ + "snapshot-30-2.tar.gz" + ]); + assert_eq!(files_with_extension(temp_dir.path(), "meta"), vec![ + "snapshot-30-2.meta" + ]); + assert_eq!( + handler.get_latest_snapshot_metadata().and_then(|m| m.last_included), + Some(LogId { index: 30, term: 2 }) + ); + } +} diff --git a/d-engine-core/src/state_machine_handler/mod.rs b/d-engine-core/src/state_machine_handler/mod.rs index 5be35566..07effd09 100644 --- a/d-engine-core/src/state_machine_handler/mod.rs +++ b/d-engine-core/src/state_machine_handler/mod.rs @@ -35,6 +35,7 @@ mod applied_state; mod command; mod default_state_machine_handler; mod snapshot_assembler; +mod snapshot_catalog; mod snapshot_policy; mod worker; mod worker_handle; @@ -146,12 +147,11 @@ where captured: crate::CapturedLocalSnapshot, ) -> Result<(SnapshotMetadata, std::path::PathBuf)>; - /// Cleans up old snapshots before specified version + /// Deletes the snapshots this node no longer needs, keeping the newest `retain_count` + /// recorded ones. async fn cleanup_snapshot( &self, - before_version: u64, - snapshot_dir: &std::path::Path, - snapshot_dir_prefix: &str, + retain_count: u64, ) -> crate::Result<()>; /// Retrieves metadata of the latest valid snapshot diff --git a/d-engine-core/src/state_machine_handler/snapshot_catalog.rs b/d-engine-core/src/state_machine_handler/snapshot_catalog.rs new file mode 100644 index 00000000..6b0daea6 --- /dev/null +++ b/d-engine-core/src/state_machine_handler/snapshot_catalog.rs @@ -0,0 +1,238 @@ +//! The engine's own record of the snapshots this node holds. +//! +//! A snapshot is an archive `snapshot-{index}-{term}.tar.gz` in the snapshots directory. Next to +//! it the engine keeps `snapshot-{index}-{term}.meta`: the snapshot's metadata (boundary and +//! checksum). The record is written only once the archive is complete and, on a follower, only +//! once the snapshot has been installed. An archive without a record is not a snapshot this node +//! can vouch for, so it is ignored. + +use std::io::Write; +use std::path::Path; +use std::path::PathBuf; +use std::sync::Arc; + +use d_engine_proto::server::storage::SnapshotMetadata; +use parking_lot::RwLock; +use prost::Message; +use tracing::error; +use tracing::warn; + +use crate::Result; +use crate::SnapshotError; +use crate::SnapshotPathManager; +use crate::StorageError; + +const RECORD_EXTENSION: &str = "meta"; + +/// The newest snapshot this node holds, kept in memory and backed by the records on disk. +#[derive(Debug)] +pub(crate) struct SnapshotCatalog { + path_mgr: Arc, + latest: RwLock>, +} + +impl SnapshotCatalog { + /// Reads the records on disk. Done once, when the node starts. + pub(crate) fn open(path_mgr: Arc) -> Self { + let latest = Self::scan(&path_mgr); + Self { + path_mgr, + latest: RwLock::new(latest), + } + } + + /// Metadata of the newest snapshot this node holds, if any. + pub(crate) fn latest(&self) -> Option { + self.latest.read().clone() + } + + /// Records `metadata` next to the archive of that snapshot, which must already be in place. + /// + /// The record is written to a temporary file, synced, then renamed, so a crash leaves the old + /// record or the new one, never a partial file. + pub(crate) async fn publish( + &self, + metadata: &SnapshotMetadata, + ) -> Result<()> { + let last_included = metadata.last_included.ok_or_else(|| { + SnapshotError::OperationFailed("Missing last_included in snapshot metadata".into()) + })?; + let record = record_path(&self.path_mgr.final_snapshot_path(&last_included)); + let bytes = metadata.encode_to_vec(); + tokio::task::spawn_blocking(move || write_record(&record, &bytes)) + .await + .map_err(|e| StorageError::IoError(std::io::Error::other(e.to_string())))? + .map_err(StorageError::IoError)?; + + let mut latest = self.latest.write(); + let is_newest = latest.as_ref().and_then(|m| m.last_included).is_none_or(|current| { + (last_included.index, last_included.term) >= (current.index, current.term) + }); + if is_newest { + *latest = Some(metadata.clone()); + } + Ok(()) + } + + /// Deletes the snapshots this node no longer needs. + /// + /// Keeps the newest `retain` recorded snapshots and deletes older recorded ones together with + /// their records. Also deletes an archive that has no record and is not newer than the newest + /// recorded snapshot: a snapshot is installed only if it is newer than the node's position, + /// which is at least that of the newest recorded snapshot, so such an archive can never be + /// installed or served. An archive newer than that may be a snapshot that is being installed + /// right now, so it is left alone. With no recorded snapshot nothing is deleted. + /// + /// The newest recorded snapshot is never deleted (`retain` is at least 1), so + /// [`latest`](Self::latest) stays valid. + pub(crate) async fn prune( + &self, + retain: u64, + ) -> Result<()> { + let path_mgr = self.path_mgr.clone(); + tokio::task::spawn_blocking(move || prune_blocking(&path_mgr, retain)) + .await + .map_err(|e| StorageError::IoError(std::io::Error::other(e.to_string())))? + .map_err(|e| StorageError::IoError(e).into()) + } + + fn scan(path_mgr: &SnapshotPathManager) -> Option { + recorded(path_mgr).pop().map(|snapshot| snapshot.metadata) + } +} + +/// A recorded snapshot: its metadata and the archive it describes. +struct RecordedSnapshot { + metadata: SnapshotMetadata, + boundary: (u64, u64), + archive: PathBuf, +} + +/// Every snapshot with a valid record and an archive, oldest first. +fn recorded(path_mgr: &SnapshotPathManager) -> Vec { + // A directory that does not exist yet holds no snapshots. + let Ok(entries) = std::fs::read_dir(&path_mgr.base_dir) else { + return Vec::new(); + }; + let mut snapshots = Vec::new(); + for entry in entries.flatten() { + let path = entry.path(); + if path.extension().is_none_or(|e| e != RECORD_EXTENSION) { + continue; + } + let bytes = match std::fs::read(&path) { + Ok(bytes) => bytes, + Err(e) => { + error!(?path, %e, "ignoring unreadable snapshot record"); + continue; + } + }; + let metadata = match SnapshotMetadata::decode(bytes.as_slice()) { + Ok(metadata) => metadata, + Err(e) => { + error!(?path, %e, "ignoring undecodable snapshot record"); + continue; + } + }; + let Some(last_included) = metadata.last_included else { + error!(?path, "ignoring snapshot record without a boundary"); + continue; + }; + let archive = path_mgr.final_snapshot_path(&last_included); + if record_path(&archive) != path { + warn!( + ?path, + ?last_included, + "ignoring snapshot record whose name disagrees with its boundary" + ); + continue; + } + if !archive.exists() { + continue; + } + snapshots.push(RecordedSnapshot { + metadata, + boundary: (last_included.index, last_included.term), + archive, + }); + } + snapshots.sort_by_key(|snapshot| snapshot.boundary); + snapshots +} + +fn prune_blocking( + path_mgr: &SnapshotPathManager, + retain: u64, +) -> std::io::Result<()> { + let recorded = recorded(path_mgr); + let Some(newest) = recorded.last().map(|snapshot| snapshot.boundary) else { + return Ok(()); + }; + + let delete_from = recorded.len().saturating_sub(retain as usize); + for snapshot in &recorded[..delete_from] { + remove_if_present(&snapshot.archive)?; + remove_if_present(&record_path(&snapshot.archive))?; + } + + // Archives without a record that can no longer matter. + let recorded_archives: Vec<&PathBuf> = recorded.iter().map(|s| &s.archive).collect(); + for entry in std::fs::read_dir(&path_mgr.base_dir)?.flatten() { + let path = entry.path(); + if !entry.file_type()?.is_file() || recorded_archives.contains(&&path) { + continue; + } + let Some(boundary) = archive_boundary(&path, &path_mgr.snapshot_prefix) else { + continue; + }; + if boundary <= newest { + remove_if_present(&path)?; + } + } + Ok(()) +} + +/// `snapshot-{index}-{term}.tar.gz` -> `(index, term)`; anything else is not an archive. +fn archive_boundary( + path: &Path, + prefix: &str, +) -> Option<(u64, u64)> { + let stem = path.file_name()?.to_str()?.strip_suffix(".tar.gz")?; + let (index, term) = stem.strip_prefix(prefix)?.split_once('-')?; + Some((index.parse().ok()?, term.parse().ok()?)) +} + +fn remove_if_present(path: &Path) -> std::io::Result<()> { + match std::fs::remove_file(path) { + Err(e) if e.kind() != std::io::ErrorKind::NotFound => Err(e), + _ => Ok(()), + } +} + +/// `snapshot-5-1.tar.gz` is recorded in `snapshot-5-1.meta`. +fn record_path(archive: &Path) -> PathBuf { + let name = archive.file_name().and_then(|n| n.to_str()).unwrap_or_default(); + let stem = name.strip_suffix(".tar.gz").unwrap_or(name); + archive.with_file_name(format!("{stem}.{RECORD_EXTENSION}")) +} + +fn write_record( + record: &Path, + bytes: &[u8], +) -> std::io::Result<()> { + let temp = record.with_extension(format!("{RECORD_EXTENSION}.tmp")); + { + let mut file = std::fs::File::create(&temp)?; + file.write_all(bytes)?; + file.sync_all()?; + } + std::fs::rename(&temp, record)?; + if let Some(dir) = record.parent() { + std::fs::File::open(dir)?.sync_all()?; + } + Ok(()) +} + +#[cfg(test)] +#[path = "snapshot_catalog_test.rs"] +mod tests; diff --git a/d-engine-core/src/state_machine_handler/snapshot_catalog_test.rs b/d-engine-core/src/state_machine_handler/snapshot_catalog_test.rs new file mode 100644 index 00000000..459d3fb9 --- /dev/null +++ b/d-engine-core/src/state_machine_handler/snapshot_catalog_test.rs @@ -0,0 +1,149 @@ +use std::fs; +use std::path::Path; +use std::sync::Arc; + +use bytes::Bytes; +use d_engine_proto::common::LogId; +use d_engine_proto::server::storage::SnapshotMetadata; +use prost::Message; +use tempfile::tempdir; +use tracing_test::traced_test; + +use super::SnapshotCatalog; +use crate::SnapshotPathManager; + +fn metadata( + index: u64, + term: u64, +) -> SnapshotMetadata { + SnapshotMetadata { + last_included: Some(LogId { index, term }), + checksum: Bytes::from(vec![index as u8; 32]), + } +} + +fn catalog_over(dir: &Path) -> SnapshotCatalog { + SnapshotCatalog::open(Arc::new(SnapshotPathManager::new( + dir.to_path_buf(), + "snapshot-".to_string(), + ))) +} + +/// An archive and the record the engine wrote for it, as a finished snapshot leaves them. +fn write_snapshot( + dir: &Path, + index: u64, + term: u64, +) { + fs::write( + dir.join(format!("snapshot-{index}-{term}.tar.gz")), + b"archive", + ) + .unwrap(); + fs::write( + dir.join(format!("snapshot-{index}-{term}.meta")), + metadata(index, term).encode_to_vec(), + ) + .unwrap(); +} + +/// Purpose: with several snapshots on disk, the newest one is the node's latest. +#[test] +#[traced_test] +fn test_open_picks_the_newest_snapshot() { + let dir = tempdir().unwrap(); + write_snapshot(dir.path(), 10, 1); + write_snapshot(dir.path(), 30, 2); + write_snapshot(dir.path(), 20, 2); + + assert_eq!(catalog_over(dir.path()).latest(), Some(metadata(30, 2))); +} + +/// A record whose archive was deleted (cleanup crashed halfway) describes a snapshot the node no +/// longer has. +/// +/// Purpose: it is ignored, and the newest snapshot that still has its archive is used. +#[test] +#[traced_test] +fn test_open_ignores_a_record_whose_archive_is_gone() { + let dir = tempdir().unwrap(); + write_snapshot(dir.path(), 10, 1); + write_snapshot(dir.path(), 20, 1); + fs::remove_file(dir.path().join("snapshot-20-1.tar.gz")).unwrap(); + + assert_eq!(catalog_over(dir.path()).latest(), Some(metadata(10, 1))); +} + +/// A record that cannot be decoded (a damaged disk) must not stop the node from starting. +/// +/// Purpose: it is ignored and reported, and the other snapshots are still found. +#[test] +#[traced_test] +fn test_open_ignores_an_undecodable_record() { + let dir = tempdir().unwrap(); + write_snapshot(dir.path(), 10, 1); + fs::write(dir.path().join("snapshot-20-1.tar.gz"), b"archive").unwrap(); + fs::write(dir.path().join("snapshot-20-1.meta"), b"not a record").unwrap(); + + assert_eq!(catalog_over(dir.path()).latest(), Some(metadata(10, 1))); + assert!(logs_contain("ignoring undecodable snapshot record")); +} + +/// A record whose file name says one boundary while its content says another. +/// +/// Purpose: it is not trusted: the archive is looked up by the boundary in the name. +#[test] +#[traced_test] +fn test_open_ignores_a_record_whose_name_disagrees_with_its_boundary() { + let dir = tempdir().unwrap(); + fs::write(dir.path().join("snapshot-20-1.tar.gz"), b"archive").unwrap(); + fs::write( + dir.path().join("snapshot-5-1.meta"), + metadata(20, 1).encode_to_vec(), + ) + .unwrap(); + + assert_eq!(catalog_over(dir.path()).latest(), None); +} + +/// Purpose: a directory that does not exist yet holds no snapshots, and is not an error. +#[test] +#[traced_test] +fn test_open_without_a_snapshots_directory_has_no_snapshot() { + let dir = tempdir().unwrap(); + + assert_eq!(catalog_over(&dir.path().join("missing")).latest(), None); +} + +/// A late, older snapshot (for example one received after a newer one was built) is recorded but +/// must not replace the newer one as the latest. +/// +/// Purpose: the latest snapshot never moves backwards. +#[tokio::test] +#[traced_test] +async fn test_publish_does_not_replace_a_newer_snapshot_with_an_older_one() { + let dir = tempdir().unwrap(); + let catalog = catalog_over(dir.path()); + fs::write(dir.path().join("snapshot-30-2.tar.gz"), b"archive").unwrap(); + fs::write(dir.path().join("snapshot-10-1.tar.gz"), b"archive").unwrap(); + + catalog.publish(&metadata(30, 2)).await.unwrap(); + catalog.publish(&metadata(10, 1)).await.unwrap(); + + assert_eq!(catalog.latest(), Some(metadata(30, 2))); + assert!( + dir.path().join("snapshot-10-1.meta").exists(), + "the older snapshot is still recorded" + ); +} + +/// Purpose: a record written by `publish` is found again by a node that starts later. +#[tokio::test] +#[traced_test] +async fn test_published_record_survives_a_restart() { + let dir = tempdir().unwrap(); + fs::write(dir.path().join("snapshot-10-1.tar.gz"), b"archive").unwrap(); + catalog_over(dir.path()).publish(&metadata(10, 1)).await.unwrap(); + + assert_eq!(catalog_over(dir.path()).latest(), Some(metadata(10, 1))); +} diff --git a/d-engine-core/src/state_machine_handler/writer.rs b/d-engine-core/src/state_machine_handler/writer.rs index 3f4cf259..67f99613 100644 --- a/d-engine-core/src/state_machine_handler/writer.rs +++ b/d-engine-core/src/state_machine_handler/writer.rs @@ -8,6 +8,7 @@ use d_engine_proto::server::storage::SnapshotMetadata; use tracing::error; use tracing::info; use tracing::trace; +use tracing::warn; use super::StateMachineWriterOps; use crate::ApplyResult; @@ -25,6 +26,7 @@ use crate::decode_entries; use crate::file_io::compute_checksum_from_folder_path; use crate::scoped_timer::ScopedTimer; use crate::state_machine_handler::applied_state::AppliedState; +use crate::state_machine_handler::snapshot_catalog::SnapshotCatalog; /// The only type allowed to run destructive state-machine mutations — held exclusively /// by `StateMachineWorker`. Shares `state_machine`/`applied` with its paired @@ -44,6 +46,9 @@ where /// build/cleanup (reader side) never drift apart. path_mgr: Arc, + /// Shared with the paired handler: records the snapshots this node holds. + catalog: Arc, + #[cfg(feature = "watch")] watch_event_tx: Option>, #[cfg(feature = "watch")] @@ -56,6 +61,7 @@ impl DefaultStateMachineWriter { state_machine: Arc>, applied: Arc, path_mgr: Arc, + catalog: Arc, #[cfg_attr(not(feature = "watch"), allow(unused_variables))] watch_event_tx: Option< tokio::sync::broadcast::Sender, >, @@ -68,6 +74,7 @@ impl DefaultStateMachineWriter { state_machine, applied, path_mgr, + catalog, #[cfg(feature = "watch")] watch_event_tx, #[cfg(feature = "watch")] @@ -234,6 +241,21 @@ impl StateMachineWriterOps for DefaultStateMachineWriter { } self.applied.advance(incoming.index); + + // The snapshot is installed, so this node now holds it: the archive was kept at its final + // path when it was received. A failure to record it does not undo the install. + let archive = self.path_mgr.final_snapshot_path(&incoming); + if tokio::fs::try_exists(&archive).await.unwrap_or(false) { + if let Err(e) = self.catalog.publish(&metadata).await { + error!(?e, "Failed to record the installed snapshot"); + } + } else { + warn!( + ?archive, + "The installed snapshot's archive is missing; not recorded" + ); + } + Ok(SnapshotApplyResult::Applied { last_included: incoming, }) diff --git a/d-engine-core/src/test_utils/mock/mock_raft_builder.rs b/d-engine-core/src/test_utils/mock/mock_raft_builder.rs index 28ff3c9b..0c93c314 100644 --- a/d-engine-core/src/test_utils/mock/mock_raft_builder.rs +++ b/d-engine-core/src/test_utils/mock/mock_raft_builder.rs @@ -398,13 +398,22 @@ pub fn mock_state_machine() -> MockStateMachine { } pub fn mock_state_machine_handler() -> MockStateMachineHandler { + mock_state_machine_handler_with_snapshot(None) +} + +/// Like [`mock_state_machine_handler`], for a node whose latest snapshot is `latest_snapshot`. +pub fn mock_state_machine_handler_with_snapshot( + latest_snapshot: Option +) -> MockStateMachineHandler { let mut state_machine_handler = MockStateMachineHandler::new(); state_machine_handler.expect_update_pending().returning(|_| {}); // Commit delivery (also run on shutdown) reads the applied position every iteration. state_machine_handler.expect_last_applied().returning(|| 0); state_machine_handler.expect_read_from_state_machine().returning(|_| Ok(vec![])); state_machine_handler.expect_should_snapshot().returning(|_| false); - state_machine_handler.expect_get_latest_snapshot_metadata().returning(|| None); + state_machine_handler + .expect_get_latest_snapshot_metadata() + .returning(move || latest_snapshot.clone()); // handle_create_snapshot (#436 Task #21) always acquires/releases this guard // before it can even reach the Worker round-trip, regardless of whether the // test cares about snapshot creation — give every default-builder test a diff --git a/d-engine-server/src/lib.rs b/d-engine-server/src/lib.rs index 7f1a1984..6598e617 100644 --- a/d-engine-server/src/lib.rs +++ b/d-engine-server/src/lib.rs @@ -166,14 +166,6 @@ pub use d_engine_core::Result; pub use d_engine_core::ScanResult; /// Client-facing prefix scan result, returned by `EmbeddedClient::scan_prefix()`. pub use d_engine_core::ScanResults; -/// Outcome the engine reports for a received snapshot. -/// -/// Distinguishes "the install actually ran" (`Applied`) from two no-op cases that are not -/// errors: the incoming snapshot is behind what's already applied (`IgnoredStale`), or is an -/// idempotent retry of the currently-applied boundary (`IgnoredDuplicate`). The engine decides -/// this itself before it calls [`StateMachine::apply_snapshot_from_file`], which only ever -/// receives a snapshot to install and returns `Result<()>`. -pub use d_engine_core::SnapshotApplyResult; /// Snapshot-specific error type. Return one from [`StateMachine::apply_snapshot_from_file`] to /// report a genuine install failure; stale, duplicate and conflicting snapshots never reach /// that method, because the engine rejects them first. From b0eaf0d89ae25176e7aa678a8611ee8f01d5eb59 Mon Sep 17 00:00:00 2001 From: Joshua Chi <112539+JoshuaChi@users.noreply.github.com> Date: Mon, 2 Nov 2026 15:55:55 +0800 Subject: [PATCH 09/19] fix #400: Export RocksDB snapshots from one frozen view of the database A snapshot exported while writes continued could hold last_applied from a later moment than its data: apply_chunk commits data (sm) and last_applied (sm_meta) in one write batch, but the two column families were exported one after the other from the live database. A node installing such a snapshot skipped the entries in between after a restart, silently. The export now takes a checkpoint of the database, opens that frozen copy, and exports both column families from it, so the data and the position describe the same applied entry. The checkpoint is removed afterwards. State machine writes skip the WAL, so the two column families must reach disk together for the checkpoint to be consistent: atomic_flush is turned on for the database. RocksDB then forces min_write_buffer_number_to_merge to 1 for the state machine column family, so it is set to 1 explicitly. Tests: a generic suite test (restart after a snapshot taken during writes must not have a position ahead of its data) with a correct and a faulty in-memory state machine, a RocksDB test that requires position == data, and a test that the database opens with atomic_flush. --- CHANGELOG.md | 4 +- MIGRATION_GUIDE.md | 5 +- .../leader_state_test/snapshot_worker_test.rs | 1 - .../default_state_machine_handler.rs | 1 - .../default_state_machine_handler_test.rs | 24 - d-engine-core/src/storage/state_machine.rs | 44 +- .../src/storage/state_machine_test.rs | 465 +++++++++++++++--- .../src/test_utils/mock/mock_raft_builder.rs | 3 - .../api/embedded_test/shutdown_order_test.rs | 9 - .../adaptors/file/file_state_machine.rs | 83 ---- .../src/storage/adaptors/rocksdb/mod.rs | 6 +- .../adaptors/rocksdb/rocksdb_state_machine.rs | 180 +++---- .../rocksdb/rocksdb_state_machine_test.rs | 104 ++++ .../src/test_utils/mock/mock_node_builder.rs | 3 - .../tests/common/shared_state_machine.rs | 9 - .../server_guide/customize-state-machine.md | 4 +- .../sled-cluster/src/sled_state_machine.rs | 140 +----- 17 files changed, 617 insertions(+), 468 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bd9829c5..2837a8b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,8 +56,6 @@ All notable changes to this project will be documented in this file. - **File state machine lost empty values on WAL replay** (#464): `replay_wal` treated a zero value length as "no value", so an `Insert` with an empty value was dropped after a restart. It now restores the empty value. -- **File state machine forgot its snapshot metadata on restart** (#400): `persist_last_snapshot_metadata` only updated memory, and `metadata.bin` stores only `last_applied`. The metadata is now written to `snapshot_metadata.bin` (temporary file, `fsync`, rename) and loaded when the state machine opens; a file that cannot be decoded fails the open instead of being ignored. - - **Snapshot checksum ignored files in sub-directories and file boundaries** (#400): see the snapshot checksum entry under Changed. ### Changed @@ -120,6 +118,8 @@ All notable changes to this project will be documented in this file. - **⚠️ `StateMachine::apply_snapshot_from_file` returns `Result<()>`; the engine decides which snapshots are installed** (#400): the engine compares the snapshot's boundary with `last_applied()` before it calls the state machine, so stale, duplicate and conflicting snapshots never reach it, and it checks afterwards that `last_applied()` equals the snapshot's boundary, reporting an error if not. Before, every state machine had to implement this decision itself, and one that did not (d-lmdb) would install a stale snapshot and roll its data back. See [MIGRATION_GUIDE.md](./MIGRATION_GUIDE.md). +- **⚠️ `StateMachine` no longer declares `snapshot_metadata` and `persist_last_snapshot_metadata`; the engine records the snapshots a node holds** (#400): the engine keeps `snapshot-{index}-{term}.meta` (boundary and checksum) next to each snapshot archive, written after the archive is complete and, on a follower, after the snapshot is installed. The snapshot trigger policy, the leader's snapshot push and the join response read it. Cleanup goes by these records and also removes archives without a record that are older than the newest recorded snapshot. Before, every state machine had to store and restore this metadata, and the file state machine, sled and d-lmdb did not do it correctly. Archives created before the upgrade have no record, so the node reports no snapshot until it builds its next one. See [MIGRATION_GUIDE.md](./MIGRATION_GUIDE.md). + - **⚠️ `StateMachine` no longer declares `update_last_applied`, `persist_last_applied`, `update_last_snapshot_metadata`, `save_hard_state`, `flush` and `flush_async`** (#400): the engine never called them. Keep any you still need as methods on your own type. Your state machine must restore `last_applied` and its snapshot metadata when it is opened; the `StateMachine` docs now state this as a durability contract, and `StateMachineTestSuite` checks it. See [MIGRATION_GUIDE.md](./MIGRATION_GUIDE.md). - **⚠️ `StateMachine::generate_snapshot_data` returns `Result<()>`; the engine owns snapshot checksums** (#400): the engine computes the SHA-256 of the exported snapshot directory and verifies it before calling `apply_snapshot_from_file`, which now only ever receives a directory that matches the recorded checksum. A state machine neither returns nor checks a checksum. The checksum now covers every file at any depth together with its relative path and length; before, it covered only top-level files, so the data files RocksDB exports into sub-directories were not protected. Snapshot archives created before the upgrade fail verification, so remove old snapshot archives when upgrading. See [MIGRATION_GUIDE.md](./MIGRATION_GUIDE.md). diff --git a/MIGRATION_GUIDE.md b/MIGRATION_GUIDE.md index c8ae810b..cd3a69a3 100644 --- a/MIGRATION_GUIDE.md +++ b/MIGRATION_GUIDE.md @@ -35,8 +35,9 @@ EmbeddedEngine::start_custom(dir, storage, sm, None).await?; - **Embedded reads (`get_eventual`, lease reads) are answered by the ReadActor**, as they already were in standalone mode, instead of reading the state machine on the caller's thread. Expect one extra channel hop per read. - **Snapshot checksums belong to the engine.** `generate_snapshot_data` now returns `Result<()>`: write the snapshot into the directory you are given and return. The engine computes the checksum of that directory itself, and verifies it before it calls `apply_snapshot_from_file`, so you no longer compute or check a checksum. Delete that code (and any hashing dependency you added only for it). - **`apply_snapshot_from_file` returns `Result<()>` again, and you no longer classify snapshots.** The engine compares the snapshot's boundary with `last_applied()` first and calls you only for a newer snapshot, so delete your stale / duplicate / conflict handling and return `Ok(())` after installing. After your call the engine checks that `last_applied()` equals `metadata.last_included` and reports an error if it does not, so make sure you set it (durably) as part of the install. -- **Six methods left the trait:** `update_last_applied`, `persist_last_applied`, `update_last_snapshot_metadata`, `save_hard_state`, `flush` and `flush_async`. The engine never called them, so deleting them from your `impl StateMachine` block changes no behaviour. Keep any your own code still uses as plain methods on your type. -- **Restore your state when you open it.** The engine does not do it for you: after reopening, `last_applied()` must return the last entry whose effects are durable, and `snapshot_metadata()` must return the value most recently passed to `persist_last_snapshot_metadata()` (which must therefore update memory and write to disk). `StateMachineTestSuite` now checks the snapshot metadata case. +- **Eight methods left the trait:** `update_last_applied`, `persist_last_applied`, `update_last_snapshot_metadata`, `save_hard_state`, `flush`, `flush_async`, `snapshot_metadata` and `persist_last_snapshot_metadata`. The first six were never called by the engine, so deleting them from your `impl StateMachine` block changes no behaviour. The last two are replaced by the engine's own record of the snapshots it holds (see below). Keep any your own code still uses as plain methods on your type. +- **Restore your position when you open the state machine.** The engine does not do it for you: after reopening, `last_applied()` must return the last entry whose effects are durable. You no longer store or return anything about snapshots; delete your snapshot-metadata storage and its loading code. +- **The engine records the snapshots a node holds.** Next to each snapshot archive `snapshot-{index}-{term}.tar.gz` it writes `snapshot-{index}-{term}.meta` (boundary and checksum). After you upgrade, existing archives have no record, so the node reports no snapshot until it builds its next one; archives without a record that are older than the newest recorded snapshot are then removed. - **The snapshot directory must contain only regular files and directories.** The engine rejects symbolic links and other special entries instead of skipping them. - When `stop()` returns, the engine has dropped the state machine. If your state machine releases its resources in `Drop` (a database handle, a lock file), they are free at that point and the same data directory can be reopened. diff --git a/d-engine-core/src/raft_role/leader_state_test/snapshot_worker_test.rs b/d-engine-core/src/raft_role/leader_state_test/snapshot_worker_test.rs index 8534add6..31700d12 100644 --- a/d-engine-core/src/raft_role/leader_state_test/snapshot_worker_test.rs +++ b/d-engine-core/src/raft_role/leader_state_test/snapshot_worker_test.rs @@ -105,7 +105,6 @@ fn state_machine_with_snapshot() -> MockStateMachine { sm.expect_get().returning(|_| Ok(None)); sm.expect_apply_chunk().returning(|_| Ok(vec![])); sm.expect_last_applied().return_const(LogId::default()); - sm.expect_persist_last_snapshot_metadata().returning(|_| Ok(())); sm.expect_apply_snapshot_from_file().returning(|_, _| Ok(())); sm.expect_generate_snapshot_data().returning(|_, _| Ok(())); sm diff --git a/d-engine-core/src/state_machine_handler/default_state_machine_handler.rs b/d-engine-core/src/state_machine_handler/default_state_machine_handler.rs index 78428dc0..936cef53 100644 --- a/d-engine-core/src/state_machine_handler/default_state_machine_handler.rs +++ b/d-engine-core/src/state_machine_handler/default_state_machine_handler.rs @@ -366,7 +366,6 @@ where // this point leaves the old record on disk, so the leader re-sends the snapshot // instead of advertising one whose archive is missing. self.catalog.publish(&metadata).await?; - self.state_machine.persist_last_snapshot_metadata(&metadata)?; debug!(%self.snapshot_config.cleanup_retain_count, "build_local_snapshot: cleanup old versions"); if let Err(e) = self.cleanup_snapshot(self.snapshot_config.cleanup_retain_count).await { diff --git a/d-engine-core/src/state_machine_handler/default_state_machine_handler_test.rs b/d-engine-core/src/state_machine_handler/default_state_machine_handler_test.rs index 47128d46..02befff2 100644 --- a/d-engine-core/src/state_machine_handler/default_state_machine_handler_test.rs +++ b/d-engine-core/src/state_machine_handler/default_state_machine_handler_test.rs @@ -959,7 +959,6 @@ mod create_snapshot_tests { path.ends_with("temp-5-1") && last_included.index == 5 && last_included.term == 1 }) .returning(|_, _| Ok(())); - sm.expect_persist_last_snapshot_metadata().returning(|_| Ok(())); let mut config = snapshot_config(temp_path.to_path_buf()); config.retained_log_entries = 0; @@ -1030,12 +1029,6 @@ mod create_snapshot_tests { fs::create_dir(path.join("state_machine")).unwrap(); Ok(()) }); - // The publish must come after generate_snapshot_data (capture), i.e. during - // build_local_snapshot once the archive exists — never during capture. - sm.expect_persist_last_snapshot_metadata() - .times(1) - .in_sequence(&mut seq) - .returning(|_| Ok(())); let mut config = snapshot_config(temp_path.to_path_buf()); config.retained_log_entries = 0; @@ -1069,12 +1062,6 @@ mod create_snapshot_tests { // Setup slow snapshot generation let (tx, _rx) = tokio::sync::oneshot::channel(); sm.expect_last_applied().returning(|| LogId { term: 1, index: 1 }); - sm.expect_snapshot_metadata().returning(move || { - Some(SnapshotMetadata { - last_included: Some(LogId { index: 1, term: 1 }), - checksum: Bytes::from(vec![1; 8]), - }) - }); sm.expect_generate_snapshot_data().times(1..=2).returning(move |path, _| { // Track invocation count let mut count = counter_clone.lock().unwrap(); @@ -1091,7 +1078,6 @@ mod create_snapshot_tests { Err(SnapshotError::OperationFailed("Concurrency failure".into()).into()) } }); - sm.expect_persist_last_snapshot_metadata().returning(|_| Ok(())); let mut snapshot_policy = MockSnapshotPolicy::new(); snapshot_policy.expect_should_trigger().returning(|_| true); @@ -1171,7 +1157,6 @@ mod create_snapshot_tests { Ok(()) }); - sm.expect_persist_last_snapshot_metadata().returning(|_| Ok(())); let snapshot_dir = temp_path.to_path_buf(); let mut config = snapshot_config(snapshot_dir.clone()); @@ -1253,7 +1238,6 @@ mod create_snapshot_tests { fs::create_dir_all(path).unwrap(); Ok(()) }); - sm.expect_persist_last_snapshot_metadata().returning(|_| Ok(())); let config = snapshot_config(temp_path.clone()); let (handler, writer) = new_reader_writer_pair::( @@ -1354,7 +1338,6 @@ mod create_snapshot_tests { fs::create_dir_all(path).unwrap(); Ok(()) }); - sm.expect_persist_last_snapshot_metadata().returning(|_| Ok(())); let mut config = snapshot_config(temp_path.clone()); config.retained_log_entries = 10; @@ -1430,7 +1413,6 @@ mod create_snapshot_tests { last_included.index == 100 && last_included.term == 2 }) .returning(|_, _| Ok(())); - sm.expect_persist_last_snapshot_metadata().returning(|_| Ok(())); let mut config = snapshot_config(temp_path.to_path_buf()); // retained=60: old code would subtract and land in term 1 territory (index 40) @@ -1491,7 +1473,6 @@ mod create_snapshot_tests { fs::create_dir_all(path).unwrap(); Ok(()) }); - sm.expect_persist_last_snapshot_metadata().returning(|_| Ok(())); let mut config = snapshot_config(temp_path.to_path_buf()); config.retained_log_entries = 10; @@ -1569,7 +1550,6 @@ mod create_snapshot_tests { fs::create_dir_all(&path).unwrap(); Ok(()) }); - sm.expect_persist_last_snapshot_metadata().returning(|_| Ok(())); let config = snapshot_config(snapshot_dir.clone()); let (handler, writer) = new_reader_writer_pair::( @@ -2032,7 +2012,6 @@ async fn test_snapshot_compression() { Ok(()) }); - sm.expect_persist_last_snapshot_metadata().returning(|_| Ok(())); let mut config = snapshot_config(temp_path.to_path_buf()); config.retained_log_entries = 0; @@ -3421,9 +3400,6 @@ mod snapshot_catalog_behavior_tests { .unwrap(); Ok(()) }); - sm.expect_persist_last_snapshot_metadata().returning(|_| Ok(())); - // The state machine knows nothing about snapshots. - sm.expect_snapshot_metadata().returning(|| None); sm } diff --git a/d-engine-core/src/storage/state_machine.rs b/d-engine-core/src/storage/state_machine.rs index c94ca922..11a624af 100644 --- a/d-engine-core/src/storage/state_machine.rs +++ b/d-engine-core/src/storage/state_machine.rs @@ -92,12 +92,22 @@ impl ApplyResult { /// the last entry whose effects are durable. Persist it together with the data it describes (in /// one transaction or write batch where the storage allows), so a restart never replays an entry /// that was already applied. -/// - After reopening, [`snapshot_metadata`](Self::snapshot_metadata) returns the value most -/// recently passed to [`persist_last_snapshot_metadata`](Self::persist_last_snapshot_metadata). -/// The engine publishes snapshot metadata only through that method, so it must update what -/// `snapshot_metadata()` returns and write it durably. +/// The engine keeps its own record of the snapshots it holds, so a state machine stores nothing +/// about snapshots. /// -/// `StateMachineTestSuite` checks these rules; run it against your implementation. +/// `StateMachineTestSuite` checks this rule; run it against your implementation. +/// +/// # `NotServing` +/// +/// While a state machine cannot answer reads correctly, `get` and `get_multi` return +/// `StorageError::NotServing`, for example +/// `Err(StorageError::NotServing("installing a snapshot".into()).into())`. This happens in two +/// situations: the state machine is replacing its state from a snapshot, and it has been closed. +/// +/// It means "not now", not "the data is damaged". The engine answers that read through Raft +/// instead, from the state machine after the install, so the client still gets a correct answer. +/// A simple implementation keeps an `AtomicBool` that it clears just before it starts replacing +/// its state and sets again when it has finished. #[cfg_attr(any(test, feature = "__test_support"), automock)] #[async_trait] pub trait StateMachine: Send + Sync + 'static { @@ -180,18 +190,6 @@ pub trait StateMachine: Send + Sync + 'static { /// Sync operation as it reads from atomic variables. fn last_applied(&self) -> LogId; - /// Retrieves the current snapshot metadata. - /// Sync operation as it reads from in-memory structures. - fn snapshot_metadata(&self) -> Option; - - /// Persists snapshot metadata durably and makes it visible to `snapshot_metadata()`. - /// - /// The engine calls this once per snapshot it builds, after the archive is durably built. - fn persist_last_snapshot_metadata( - &self, - snapshot_metadata: &SnapshotMetadata, - ) -> Result<(), Error>; - /// Installs a snapshot received from the Raft leader. /// /// The engine decides whether to install: this is called only for a snapshot that is newer @@ -199,8 +197,9 @@ pub trait StateMachine: Send + Sync + 'static { /// the archive into and whose checksum it has verified against `metadata`. /// /// The implementation must: - /// 1. Replace its state atomically. A concurrent reader sees the old state, the new state, or - /// `NotServing`, never a mix. + /// 1. Replace its state so that a concurrent reader sees the old state, the new state, or + /// `NotServing` (see the trait documentation), never a mix. Either swap the state in one + /// atomic step, or return `NotServing` from reads for as long as the replacement takes. /// 2. Set `last_applied()` to `metadata.last_included` and make that durable. The engine checks /// this after the call and reports an error if it did not happen. async fn apply_snapshot_from_file( @@ -213,10 +212,9 @@ pub trait StateMachine: Send + Sync + 'static { /// up to the specified `last_included_index`. /// /// Exports the current state (as of `last_included`) to `new_snapshot_dir`. The framework - /// computes the checksum of that directory itself; the state machine does not. Does NOT - /// publish snapshot metadata — the caller publishes it (via - /// `persist_last_snapshot_metadata`) only after the final archive is durably built, so a - /// crash mid-snapshot never leaves metadata pointing at a missing archive. + /// computes the checksum of that directory itself and records the snapshot only once the + /// final archive is durably built, so a crash mid-snapshot never leaves a record pointing at + /// a missing archive; the state machine does neither. /// /// # Concurrency contract (#436) /// May run concurrently with `apply_chunk` (never with `apply_snapshot_from_file` — the two diff --git a/d-engine-core/src/storage/state_machine_test.rs b/d-engine-core/src/storage/state_machine_test.rs index ee30ea79..6d77beb9 100644 --- a/d-engine-core/src/storage/state_machine_test.rs +++ b/d-engine-core/src/storage/state_machine_test.rs @@ -42,7 +42,8 @@ impl StateMachineTestSuite { Self::test_batch_operations(builder.build().await?).await?; Self::test_last_applied_detection(builder.build().await?).await?; Self::test_snapshot_operations(builder.build().await?).await?; - Self::test_snapshot_metadata_survives_reopen(&builder).await?; + Self::test_reads_during_snapshot_install_see_old_new_or_not_serving(&builder).await?; + Self::test_snapshot_taken_during_writes_never_skips_entries_after_restart(&builder).await?; Self::test_drop_flushes_data(&builder).await?; Self::test_drop_persists_last_applied(&builder).await?; @@ -239,6 +240,192 @@ impl StateMachineTestSuite { Ok(()) } + /// A lagging node installs a snapshot while clients keep reading from it. + /// + /// Scenario: the node holds old values for keys 1..=50 and receives a snapshot in which every + /// key holds a new value. Reads that bypass Raft are served straight from the state machine, + /// so they run while the install replaces the state. + /// + /// Expected: every read returns the old value, the new value, or `NotServing` ("not now", + /// see the trait documentation). It never returns "not found", an error of another kind, or + /// any other value: that would show a client a state that never existed. After the install + /// every key holds the new value. + pub async fn test_reads_during_snapshot_install_see_old_new_or_not_serving< + B: StateMachineBuilder, + >( + builder: &B + ) -> Result<(), Error> { + use std::sync::Mutex; + use std::sync::atomic::AtomicBool; + use std::sync::atomic::AtomicUsize; + use std::sync::atomic::Ordering; + + const KEYS: u64 = 300; + const READ_KEYS: u64 = 50; + let key = |i: u64| Bytes::from(format!("key-{i}")); + let entries = |range: std::ops::RangeInclusive, value: &'static [u8]| { + range + .map(|i| create_insert_entry(i, key(i), Bytes::from_static(value))) + .collect::>() + }; + + // 1. A snapshot of the state in which every key holds the new value. + let sm = builder.build().await?; + sm.apply_chunk(&entries(1..=KEYS, b"new")).await?; + let temp_dir = TempDir::new()?; + let snapshot_dir = temp_dir.path().join("snapshot"); + let last_included = LogId { + index: KEYS, + term: 1, + }; + sm.generate_snapshot_data(snapshot_dir.clone(), last_included).await?; + let metadata = SnapshotMetadata { + last_included: Some(last_included), + checksum: Bytes::from(vec![0; 32]), + }; + + // 2. The node is behind: it only has the old values for the first keys. + sm.reset().await?; + sm.apply_chunk(&entries(1..=READ_KEYS, b"old")).await?; + + // 3. Readers keep reading while the snapshot is installed. + let stop = Arc::new(AtomicBool::new(false)); + let reads = Arc::new(AtomicUsize::new(0)); + let violations = Arc::new(Mutex::new(Vec::::new())); + let readers: Vec<_> = (0..4) + .map(|_| { + let (sm, stop, reads, violations) = + (sm.clone(), stop.clone(), reads.clone(), violations.clone()); + std::thread::spawn(move || { + while !stop.load(Ordering::Relaxed) { + for i in 1..=READ_KEYS { + let seen = match sm.get(&key(i)) { + Ok(Some(v)) if v == "old" || v == "new" => None, + Ok(Some(other)) => { + Some(format!("key {i}: unexpected value {other:?}")) + } + Ok(None) => Some(format!("key {i}: reported as not found")), + Err(e) if e.is_not_serving() => None, + Err(e) => Some(format!("key {i}: {e:?}")), + }; + reads.fetch_add(1, Ordering::Relaxed); + if let Some(violation) = seen { + violations.lock().unwrap().push(violation); + } + } + } + }) + }) + .collect(); + while reads.load(Ordering::Relaxed) == 0 { + tokio::task::yield_now().await; + } + + let installed = sm.apply_snapshot_from_file(&metadata, snapshot_dir).await; + + stop.store(true, Ordering::Relaxed); + for reader in readers { + reader.join().expect("a reader thread panicked"); + } + installed?; + + let violations = violations.lock().unwrap(); + assert!( + violations.is_empty(), + "reads during the snapshot install saw a state that never existed, for example: {:?} \ + ({} reads, {} violations)", + &violations[..violations.len().min(3)], + reads.load(Ordering::Relaxed), + violations.len() + ); + for i in 1..=KEYS { + assert_eq!( + sm.get(&key(i))?, + Some(Bytes::from_static(b"new")), + "key {i} must hold the new value after the install" + ); + } + Ok(()) + } + + /// A snapshot taken while orders keep arriving must not make a restarted node skip orders. + /// + /// Scenario: an order service keeps writing. Order number `n` is log entry `n`, and the key + /// `orders` always holds the number of the latest order. The leader takes a snapshot while the + /// writes continue and a lagging node installs it. The lagging node then restarts. + /// + /// Expected: after the restart the node's position is not ahead of its data. A position ahead + /// of the data (say "applied up to order 17197" while the data stops at order 15993) makes + /// the node continue from order 17198, so orders 15994 to 17197 are never replayed and are + /// lost without any error. + /// + /// The position may be behind the data: the engine then replays a few orders that the data + /// already contains, which is harmless for a key-value store. + pub async fn test_snapshot_taken_during_writes_never_skips_entries_after_restart< + B: StateMachineBuilder, + >( + builder: &B + ) -> Result<(), Error> { + use std::sync::atomic::AtomicBool; + use std::sync::atomic::Ordering; + + for attempt in 0..3 { + // 1. Orders keep arriving: entry n sets `orders` to n, one entry at a time. + let sm = builder.build().await?; + sm.reset().await?; + let stop = Arc::new(AtomicBool::new(false)); + let writer = { + let (sm, stop) = (sm.clone(), stop.clone()); + tokio::spawn(async move { + let mut n = 1u64; + while !stop.load(Ordering::Relaxed) { + let entry = create_insert_entry( + n, + Bytes::from_static(b"orders"), + Bytes::from(n.to_string()), + ); + sm.apply_chunk(&[entry]).await.expect("apply"); + n += 1; + tokio::task::yield_now().await; + } + }) + }; + tokio::time::sleep(Duration::from_millis(100)).await; + + // 2. A snapshot is taken in the middle of the writes. + let temp_dir = TempDir::new()?; + let snapshot_dir = temp_dir.path().join("snapshot"); + sm.generate_snapshot_data(snapshot_dir.clone(), LogId { index: 1, term: 1 }) + .await?; + stop.store(true, Ordering::Relaxed); + writer.await.expect("the writer must not panic"); + + // 3. A lagging node (an empty one) installs it and restarts. + sm.reset().await?; + let metadata = SnapshotMetadata { + last_included: Some(LogId { index: 1, term: 1 }), + checksum: Bytes::from(vec![0; 32]), + }; + sm.apply_snapshot_from_file(&metadata, snapshot_dir).await?; + sm.close()?; + drop(sm); + let restarted = builder.build().await?; + + let position = restarted.last_applied().index; + let data: u64 = restarted + .get(b"orders")? + .map(|v| String::from_utf8_lossy(&v).parse().expect("a number")) + .unwrap_or(0); + assert!( + position <= data, + "attempt {attempt}: after the restart the node believes it applied up to {position}, \ + but its data only reaches {data}: entries {}..={position} would never be replayed", + data + 1 + ); + } + Ok(()) + } + /// Test last applied index detection pub async fn test_last_applied_detection( state_machine: Arc @@ -307,10 +494,6 @@ impl StateMachineTestSuite { .await?; let checksum = crate::file_io::compute_checksum_from_folder_path(&snapshot_dir).await?; - // (#436) generate_snapshot_data no longer publishes snapshot metadata — that - // moved to build_local_snapshot (the handler), after the archive is durable. - // So snapshot_metadata() is expected to remain unchanged here. - // Apply snapshot (simulate receiving from leader) // let snapshot_path = snapshot_dir.join("snapshot.bin"); let metadata = SnapshotMetadata { @@ -374,47 +557,6 @@ impl StateMachineTestSuite { Ok(()) } - /// Snapshot metadata published through `persist_last_snapshot_metadata` must survive a - /// restart. - /// - /// # Scenario - /// A node builds a snapshot and the engine publishes its metadata, once the archive is - /// durably built, by calling `persist_last_snapshot_metadata`. The node then restarts. - /// - /// # Expected - /// The reopened state machine reports exactly that metadata from `snapshot_metadata()`. - /// Otherwise the node forgets which snapshot it holds and can no longer tell followers - /// where its snapshot ends. - pub async fn test_snapshot_metadata_survives_reopen( - builder: &B - ) -> Result<(), Error> { - let sm = builder.build().await?; - sm.apply_chunk(&[ - create_insert_entry(1, Bytes::from_static(b"k1"), Bytes::from_static(b"v1")), - create_insert_entry(2, Bytes::from_static(b"k2"), Bytes::from_static(b"v2")), - create_insert_entry(3, Bytes::from_static(b"k3"), Bytes::from_static(b"v3")), - ]) - .await?; - - let metadata = SnapshotMetadata { - last_included: Some(LogId { index: 3, term: 1 }), - checksum: Bytes::from(vec![7; 32]), - }; - sm.persist_last_snapshot_metadata(&metadata)?; - sm.close()?; - // `sm` stays alive until the end of the test: Drop must not be what saves the state. - - let reopened = builder.build().await?; - assert_eq!( - reopened.snapshot_metadata(), - Some(metadata), - "snapshot metadata published with persist_last_snapshot_metadata must survive a \ - restart" - ); - - Ok(()) - } - /// Performance smoke test: apply_chunk baseline /// /// Ensures apply_chunk doesn't have catastrophic performance issues. @@ -1076,7 +1218,6 @@ impl StateMachineTestSuite { Some(Bytes::from(b"value3".to_vec())) ); assert_eq!(state_machine.last_applied(), LogId { index: 3, term: 1 }); - // Note: snapshot_metadata may or may not be None depending on previous tests // The test focuses on verifying reset() clears everything, not initial state // Perform reset @@ -1089,7 +1230,6 @@ impl StateMachineTestSuite { // Verify Raft state is reset assert_eq!(state_machine.last_applied(), LogId { index: 0, term: 0 }); - assert!(state_machine.snapshot_metadata().is_none()); // Verify operational state is maintained: reads above succeeded, so it still serves @@ -1673,17 +1813,6 @@ mod default_scan_prefix_tests { LogId::default() } - fn snapshot_metadata(&self) -> Option { - None - } - - fn persist_last_snapshot_metadata( - &self, - _snapshot_metadata: &SnapshotMetadata, - ) -> Result<(), Error> { - Ok(()) - } - async fn apply_snapshot_from_file( &self, _metadata: &SnapshotMetadata, @@ -1731,3 +1860,221 @@ mod default_scan_prefix_tests { ); } } + +/// The suite tests for a snapshot install and for a snapshot taken during writes must be able to +/// fail. +/// +/// `InMemorySm` is a small state machine with three ways to go wrong, each switched on by a flag: +/// - `faulty`: it installs a snapshot by emptying its state, pausing, and refilling it, so a read +/// in between sees an empty state. +/// - `late_position`: it records the position of a snapshot after it has copied the data, so a +/// write in between leaves the position ahead of the data. Without a flag it replaces its state +/// in one step and exports data and position together. +#[cfg(test)] +mod install_reader_suite_tests { + use std::collections::HashMap; + use std::path::PathBuf; + use std::sync::Arc; + use std::sync::RwLock; + use std::time::Duration; + + use async_trait::async_trait; + use bytes::Bytes; + use d_engine_proto::common::LogId; + use d_engine_proto::server::storage::SnapshotMetadata; + + use super::StateMachineBuilder; + use super::StateMachineTestSuite; + use crate::ApplyEntry; + use crate::Command; + use crate::Error; + use crate::storage::StateMachine; + use crate::storage::state_machine::ApplyResult; + + /// What survives a restart: shared by every state machine the builder creates. + #[derive(Default)] + struct Durable { + data: RwLock>, + applied: RwLock, + } + + struct InMemorySm { + durable: Arc, + faulty: bool, + late_position: bool, + } + + #[async_trait] + impl StateMachine for InMemorySm { + fn get( + &self, + key: &[u8], + ) -> Result, Error> { + Ok(self.durable.data.read().unwrap().get(key).cloned()) + } + + async fn apply_chunk( + &self, + chunk: &[ApplyEntry], + ) -> Result, Error> { + // Data and position change under one lock, like a database transaction. + let mut data = self.durable.data.write().unwrap(); + for entry in chunk { + if let Command::Insert { key, value } = &entry.command { + data.insert(key.clone(), value.clone()); + } + *self.durable.applied.write().unwrap() = LogId { + index: entry.index, + term: entry.term, + }; + } + Ok(chunk.iter().map(|e| ApplyResult::success(e.index)).collect()) + } + + fn last_applied(&self) -> LogId { + *self.durable.applied.read().unwrap() + } + + async fn generate_snapshot_data( + &self, + new_snapshot_dir: PathBuf, + _last_included: LogId, + ) -> Result<(), Error> { + std::fs::create_dir_all(&new_snapshot_dir)?; + // Right: copy the data and read the position under the same lock. + let (lines, mut position) = { + let data = self.durable.data.read().unwrap(); + let lines: Vec = data + .iter() + .map(|(k, v)| { + format!( + "{}={}", + String::from_utf8_lossy(k), + String::from_utf8_lossy(v) + ) + }) + .collect(); + (lines, *self.durable.applied.read().unwrap()) + }; + if self.late_position { + // Wrong: the position is read after the data was copied, and writes may have + // happened in between. + tokio::time::sleep(Duration::from_millis(30)).await; + position = *self.durable.applied.read().unwrap(); + } + std::fs::write(new_snapshot_dir.join("data.txt"), lines.join("\n"))?; + std::fs::write( + new_snapshot_dir.join("position.txt"), + format!("{} {}", position.index, position.term), + )?; + Ok(()) + } + + async fn apply_snapshot_from_file( + &self, + _metadata: &SnapshotMetadata, + snapshot_path: PathBuf, + ) -> Result<(), Error> { + let text = std::fs::read_to_string(snapshot_path.join("data.txt"))?; + let new_data: HashMap = text + .lines() + .filter_map(|l| l.split_once('=')) + .map(|(k, v)| (Bytes::from(k.to_string()), Bytes::from(v.to_string()))) + .collect(); + let position = std::fs::read_to_string(snapshot_path.join("position.txt"))?; + let (index, term) = position.split_once(' ').expect("index and term"); + let position = LogId { + index: index.parse().unwrap(), + term: term.parse().unwrap(), + }; + if self.faulty { + // Wrong: readers can run between the clear and the refill. + self.durable.data.write().unwrap().clear(); + tokio::time::sleep(Duration::from_millis(20)).await; + *self.durable.data.write().unwrap() = new_data; + } else { + // Right: the whole state is replaced in one step. + *self.durable.data.write().unwrap() = new_data; + } + *self.durable.applied.write().unwrap() = position; + Ok(()) + } + + async fn reset(&self) -> Result<(), Error> { + self.durable.data.write().unwrap().clear(); + *self.durable.applied.write().unwrap() = LogId::default(); + Ok(()) + } + } + + #[derive(Default)] + struct InMemoryBuilder { + durable: Arc, + faulty: bool, + late_position: bool, + } + + #[async_trait] + impl StateMachineBuilder for InMemoryBuilder { + async fn build(&self) -> Result, Error> { + Ok(Arc::new(InMemorySm { + durable: self.durable.clone(), + faulty: self.faulty, + late_position: self.late_position, + })) + } + + async fn cleanup(&self) -> Result<(), Error> { + Ok(()) + } + } + + /// Purpose: a state machine that replaces its state in one step passes. + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn test_reader_suite_accepts_a_state_machine_that_swaps_its_state_atomically() { + StateMachineTestSuite::test_reads_during_snapshot_install_see_old_new_or_not_serving( + &InMemoryBuilder::default(), + ) + .await + .expect("an atomic install must pass"); + } + + /// Purpose: a state machine that empties its state and then refills it is caught, because a + /// reader that runs in between sees keys as not found. + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + #[should_panic(expected = "saw a state that never existed")] + async fn test_reader_suite_rejects_a_state_machine_that_empties_its_state_during_install() { + let _ = + StateMachineTestSuite::test_reads_during_snapshot_install_see_old_new_or_not_serving( + &InMemoryBuilder { + faulty: true, + ..Default::default() + }, + ) + .await; + } + + /// Purpose: a state machine that exports its data and position together passes. + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn test_snapshot_suite_accepts_a_state_machine_that_exports_data_and_position_together() { + StateMachineTestSuite::test_snapshot_taken_during_writes_never_skips_entries_after_restart( + &InMemoryBuilder::default(), + ) + .await + .expect("a consistent export must pass"); + } + + /// Purpose: a state machine that records the position after copying the data is caught: after + /// the restart its position is ahead of its data. + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + #[should_panic(expected = "would never be replayed")] + async fn test_snapshot_suite_rejects_a_state_machine_whose_exported_position_runs_ahead() { + let _ = StateMachineTestSuite::test_snapshot_taken_during_writes_never_skips_entries_after_restart( + &InMemoryBuilder { + late_position: true, + ..Default::default() + }, + ) + .await; + } +} diff --git a/d-engine-core/src/test_utils/mock/mock_raft_builder.rs b/d-engine-core/src/test_utils/mock/mock_raft_builder.rs index 0c93c314..4f2d1f25 100644 --- a/d-engine-core/src/test_utils/mock/mock_raft_builder.rs +++ b/d-engine-core/src/test_utils/mock/mock_raft_builder.rs @@ -388,9 +388,6 @@ pub fn mock_state_machine() -> MockStateMachine { mock.expect_last_applied().return_const(LogId::default()); - mock.expect_snapshot_metadata().returning(|| None); - mock.expect_persist_last_snapshot_metadata().returning(|_| Ok(())); - mock.expect_apply_snapshot_from_file().returning(|_, _| Ok(())); mock.expect_generate_snapshot_data().returning(|_, _| Ok(())); diff --git a/d-engine-server/src/api/embedded_test/shutdown_order_test.rs b/d-engine-server/src/api/embedded_test/shutdown_order_test.rs index 87b38c2f..c92e9017 100644 --- a/d-engine-server/src/api/embedded_test/shutdown_order_test.rs +++ b/d-engine-server/src/api/embedded_test/shutdown_order_test.rs @@ -83,15 +83,6 @@ impl StateMachine for SlowApplyProbe { fn last_applied(&self) -> LogId { self.inner.last_applied() } - fn snapshot_metadata(&self) -> Option { - self.inner.snapshot_metadata() - } - fn persist_last_snapshot_metadata( - &self, - snapshot_metadata: &SnapshotMetadata, - ) -> Result<(), Error> { - self.inner.persist_last_snapshot_metadata(snapshot_metadata) - } async fn apply_snapshot_from_file( &self, metadata: &SnapshotMetadata, diff --git a/d-engine-server/src/storage/adaptors/file/file_state_machine.rs b/d-engine-server/src/storage/adaptors/file/file_state_machine.rs index 228f0e15..d57ea276 100644 --- a/d-engine-server/src/storage/adaptors/file/file_state_machine.rs +++ b/d-engine-server/src/storage/adaptors/file/file_state_machine.rs @@ -74,7 +74,6 @@ use d_engine_core::StorageError; use d_engine_proto::common::LogId; use d_engine_proto::server::storage::SnapshotMetadata; use parking_lot::RwLock; -use prost::Message; use tokio::fs; use tokio::fs::File; use tokio::fs::OpenOptions; @@ -86,9 +85,6 @@ use tracing::error; use tracing::info; use tracing::warn; -/// File holding the snapshot metadata published by `persist_last_snapshot_metadata`. -const SNAPSHOT_METADATA_FILE: &str = "snapshot_metadata.bin"; - type FileStateMachineDataType = RwLock>; /// WAL operation codes for fixed-size encoding @@ -218,7 +214,6 @@ pub struct FileStateMachine { // Raft state with disk persistence last_applied_index: AtomicU64, last_applied_term: AtomicU64, - last_snapshot_metadata: RwLock>, // Operational state running: AtomicBool, @@ -251,7 +246,6 @@ impl FileStateMachine { data: RwLock::new(HashMap::new()), last_applied_index: AtomicU64::new(0), last_applied_term: AtomicU64::new(0), - last_snapshot_metadata: RwLock::new(None), running: AtomicBool::new(true), wal_entries_since_checkpoint: AtomicU64::new(0), last_checkpoint: Mutex::new(Instant::now()), @@ -269,9 +263,6 @@ impl FileStateMachine { // Load last applied index and term from metadata file self.load_metadata().await?; - // Load the snapshot metadata published by the last persist_last_snapshot_metadata - self.load_snapshot_metadata().await?; - // Load key-value data from data file self.load_data().await?; @@ -282,40 +273,6 @@ impl FileStateMachine { Ok(()) } - /// Loads the snapshot metadata file, if one was ever persisted. - /// - /// A file that exists but cannot be decoded is an error: silently starting without it would - /// make the node forget which snapshot it holds. - async fn load_snapshot_metadata(&self) -> Result<(), Error> { - let path = self.data_dir.join(SNAPSHOT_METADATA_FILE); - if !path.exists() { - return Ok(()); - } - let bytes = fs::read(&path).await?; - let metadata = SnapshotMetadata::decode(bytes.as_slice()).map_err(|e| { - StorageError::StateMachineError(format!("corrupt {SNAPSHOT_METADATA_FILE}: {e}")) - })?; - *self.last_snapshot_metadata.write() = Some(metadata); - Ok(()) - } - - /// Writes the snapshot metadata durably: to a temporary file first, then renamed over the - /// real one, so a crash leaves either the old file or the new one, never a partial file. - fn persist_snapshot_metadata( - &self, - metadata: &SnapshotMetadata, - ) -> Result<(), Error> { - let path = self.data_dir.join(SNAPSHOT_METADATA_FILE); - let temp_path = self.data_dir.join(format!("{SNAPSHOT_METADATA_FILE}.tmp")); - { - let mut file = std::fs::File::create(&temp_path)?; - file.write_all(&metadata.encode_to_vec())?; - file.sync_all()?; - } - std::fs::rename(&temp_path, &path)?; - std::fs::File::open(&self.data_dir)?.sync_all()?; - Ok(()) - } /// Loads metadata from disk async fn load_metadata(&self) -> Result<(), Error> { let metadata_path = self.data_dir.join("metadata.bin"); @@ -675,22 +632,10 @@ impl FileStateMachine { self.persist_metadata() } - pub(crate) fn update_last_snapshot_metadata( - &self, - snapshot_metadata: &SnapshotMetadata, - ) -> Result<(), Error> { - *self.last_snapshot_metadata.write() = Some(snapshot_metadata.clone()); - Ok(()) - } - pub(crate) fn save_hard_state(&self) -> Result<(), Error> { let last_applied = self.last_applied(); self.persist_last_applied(last_applied)?; - if let Some(last_snapshot_metadata) = self.snapshot_metadata() { - self.persist_last_snapshot_metadata(&last_snapshot_metadata)?; - } - self.flush()?; Ok(()) } @@ -824,11 +769,6 @@ impl FileStateMachine { self.last_applied_index.store(0, Ordering::SeqCst); self.last_applied_term.store(0, Ordering::SeqCst); - { - let mut snapshot_metadata = self.last_snapshot_metadata.write(); - *snapshot_metadata = None; - } - // Clear all persisted files self.clear_data_file().await?; self.clear_metadata_file().await?; @@ -866,16 +806,8 @@ impl FileStateMachine { // Write default values (0 for both index and term) file.write_all(&0u64.to_be_bytes()).await?; file.write_all(&0u64.to_be_bytes()).await?; - file.flush().await?; - // The snapshot metadata is part of the state being reset - match fs::remove_file(self.data_dir.join(SNAPSHOT_METADATA_FILE)).await { - Ok(()) => {} - Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} - Err(e) => return Err(e.into()), - } - Ok(()) } @@ -1173,18 +1105,6 @@ impl StateMachine for FileStateMachine { } } - fn snapshot_metadata(&self) -> Option { - self.last_snapshot_metadata.read().clone() - } - - fn persist_last_snapshot_metadata( - &self, - snapshot_metadata: &SnapshotMetadata, - ) -> Result<(), Error> { - self.update_last_snapshot_metadata(snapshot_metadata)?; - self.persist_snapshot_metadata(snapshot_metadata) - } - async fn apply_snapshot_from_file( &self, metadata: &SnapshotMetadata, @@ -1290,9 +1210,6 @@ impl StateMachine for FileStateMachine { *data = new_data; } - // Update metadata - *self.last_snapshot_metadata.write() = Some(metadata.clone()); - self.update_last_applied(incoming); // Persist to disk diff --git a/d-engine-server/src/storage/adaptors/rocksdb/mod.rs b/d-engine-server/src/storage/adaptors/rocksdb/mod.rs index 81ac2866..54e28b66 100644 --- a/d-engine-server/src/storage/adaptors/rocksdb/mod.rs +++ b/d-engine-server/src/storage/adaptors/rocksdb/mod.rs @@ -42,6 +42,10 @@ pub(super) fn base_db_options() -> Options { // but provides no durability guarantee. Pairs well with Level 3 (flush_wal) if added. opts.set_wal_bytes_per_sync(1024 * 1024); opts.set_manual_wal_flush(true); + // State machine writes skip the WAL, so its data and `last_applied` column families must reach + // disk together; otherwise a flush of one without the other leaves `last_applied` ahead of the + // data. Checkpoints rely on the same guarantee. + opts.set_atomic_flush(true); opts.set_max_background_jobs(2); opts.set_max_open_files(5000); opts.set_use_direct_io_for_flush_and_compaction(true); @@ -75,7 +79,7 @@ pub(super) fn sm_cf_options(cache: &Cache) -> Options { let mut opts = Options::default(); opts.set_write_buffer_size(64 * 1024 * 1024); opts.set_max_write_buffer_number(4); - opts.set_min_write_buffer_number_to_merge(2); + opts.set_min_write_buffer_number_to_merge(1); opts.set_compression_type(rocksdb::DBCompressionType::Lz4); opts.set_bottommost_compression_type(rocksdb::DBCompressionType::Zstd); diff --git a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs index 1533fbbe..a1cba613 100644 --- a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs +++ b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs @@ -17,7 +17,6 @@ use d_engine_core::StateMachine; use d_engine_core::StorageError; use d_engine_proto::common::LogId; use d_engine_proto::server::storage::SnapshotMetadata; -use parking_lot::RwLock; use rocksdb::Cache; use rocksdb::ColumnFamilyDescriptor; use rocksdb::DB; @@ -43,7 +42,6 @@ use super::STATE_MACHINE_META_CF; const LAST_APPLIED_INDEX_KEY: &[u8] = b"last_applied_index"; const LAST_APPLIED_TERM_KEY: &[u8] = b"last_applied_term"; -const SNAPSHOT_METADATA_KEY: &[u8] = b"snapshot_metadata"; thread_local! { // SM WAL is redundant — Raft log WAL guarantees entry durability. @@ -87,7 +85,6 @@ pub struct RocksDBStateMachine { is_serving: AtomicBool, last_applied_index: AtomicU64, last_applied_term: AtomicU64, - last_snapshot_metadata: RwLock>, } /// Returns the lexicographic successor of `prefix` for use as an iterator upper bound. @@ -129,14 +126,12 @@ impl RocksDBStateMachine { let db_arc = Arc::new(db); let (last_applied_index, last_applied_term) = Self::load_state_machine_metadata(&db_arc)?; - let last_snapshot_metadata = Self::load_snapshot_metadata(&db_arc)?; Ok(Self { db: ArcSwapOption::new(Some(db_arc)), is_serving: AtomicBool::new(true), last_applied_index: AtomicU64::new(last_applied_index), last_applied_term: AtomicU64::new(last_applied_term), - last_snapshot_metadata: RwLock::new(last_snapshot_metadata), }) } @@ -146,14 +141,11 @@ impl RocksDBStateMachine { /// `STATE_MACHINE_CF` and `STATE_MACHINE_META_CF` column families open. pub(super) fn from_shared_db(db: Arc) -> Result { let (last_applied_index, last_applied_term) = Self::load_state_machine_metadata(&db)?; - let last_snapshot_metadata = Self::load_snapshot_metadata(&db)?; - Ok(Self { db: ArcSwapOption::new(Some(db)), is_serving: AtomicBool::new(true), last_applied_index: AtomicU64::new(last_applied_index), last_applied_term: AtomicU64::new(last_applied_term), - last_snapshot_metadata: RwLock::new(last_snapshot_metadata), }) } @@ -235,23 +227,6 @@ impl RocksDBStateMachine { Ok((index, term)) } - fn load_snapshot_metadata(db: &Arc) -> Result, Error> { - let cf = db - .cf_handle(STATE_MACHINE_META_CF) - .ok_or_else(|| StorageError::DbError("State machine meta CF not found".to_string()))?; - - match db - .get_cf(&cf, SNAPSHOT_METADATA_KEY) - .map_err(|e| StorageError::DbError(e.to_string()))? - { - Some(bytes) => { - let metadata = bincode::deserialize(&bytes).map_err(StorageError::BincodeError)?; - Ok(Some(metadata)) - } - None => Ok(None), - } - } - fn persist_state_machine_metadata(&self) -> Result<(), Error> { let index = self.last_applied_index.load(Ordering::SeqCst); let term = self.last_applied_term.load(Ordering::SeqCst); @@ -269,23 +244,6 @@ impl RocksDBStateMachine { }) } - fn persist_snapshot_metadata(&self) -> Result<(), Error> { - let snapshot = self.last_snapshot_metadata.read().clone(); - self.with_db(|db| { - let cf = db.cf_handle(STATE_MACHINE_META_CF).ok_or_else(|| { - Error::System(d_engine_core::SystemError::Storage(StorageError::DbError( - "State machine meta CF not found".to_string(), - ))) - })?; - if let Some(metadata) = &snapshot { - let bytes = bincode::serialize(metadata).map_err(StorageError::BincodeError)?; - db.put_cf(&cf, SNAPSHOT_METADATA_KEY, bytes) - .map_err(|e| StorageError::DbError(e.to_string()))?; - } - Ok(()) - }) - } - // ===== Snapshot restore helpers ===== /// Restore SM state from a snapshot (CF export format). @@ -293,7 +251,6 @@ impl RocksDBStateMachine { /// Uses CF export: drop+import, O(1), no tombstones. async fn restore_from_snapshot( &self, - metadata: &SnapshotMetadata, last_included: LogId, snapshot_dir: &std::path::Path, ) -> Result<(), Error> { @@ -301,7 +258,6 @@ impl RocksDBStateMachine { info!("Snapshot restore complete"); - *self.last_snapshot_metadata.write() = Some(metadata.clone()); self.update_last_applied(last_included); self.is_serving.store(true, Ordering::SeqCst); @@ -364,6 +320,78 @@ impl RocksDBStateMachine { Ok(()) } + /// Exports the `sm` and `sm_meta` column families from one frozen view of the database. + /// + /// `apply_chunk` keeps writing while a snapshot is taken, and it commits data (`sm`) and + /// `last_applied` (`sm_meta`) in one write batch. Exporting the two column families from the + /// live database one after the other would let writes land in between, so the exported + /// `last_applied` could be ahead of the exported data: a node that installs such a snapshot + /// would skip the entries in between forever. + /// + /// A checkpoint is a frozen copy of the database (hard links to immutable SST files, made after + /// flushing the memtables, which RocksDB does for every column family together because + /// `atomic_flush` is on). Both column families are exported from that copy, which no writer + /// touches, so the exported data and position describe the same applied entry. The checkpoint + /// is a sibling of `snapshot_dir`, not inside it, so it never becomes part of the snapshot, + /// and it is removed afterwards. + fn export_frozen_view( + db: &DB, + snapshot_dir: &std::path::Path, + ) -> Result<(), Error> { + let name = snapshot_dir + .file_name() + .ok_or_else(|| StorageError::DbError("snapshot dir has no name".to_string()))? + .to_string_lossy(); + let checkpoint_dir = snapshot_dir.with_file_name(format!(".{name}.checkpoint")); + if checkpoint_dir.exists() { + std::fs::remove_dir_all(&checkpoint_dir)?; + } + let result = Self::create_checkpoint_and_export(db, &checkpoint_dir, snapshot_dir); + if let Err(e) = std::fs::remove_dir_all(&checkpoint_dir) { + error!("Failed to remove snapshot checkpoint {checkpoint_dir:?}: {e}"); + } + result + } + + fn create_checkpoint_and_export( + db: &DB, + checkpoint_dir: &std::path::Path, + snapshot_dir: &std::path::Path, + ) -> Result<(), Error> { + let to_err = |e: rocksdb::Error| StorageError::DbError(e.to_string()); + rocksdb::checkpoint::Checkpoint::new(db) + .and_then(|c| c.create_checkpoint(checkpoint_dir)) + .map_err(to_err)?; + + // Open the frozen copy with every column family it contains. Exporting needs a writable + // handle; background compaction is off so that nothing rewrites the files being exported. + let mut opts = Options::default(); + opts.set_disable_auto_compactions(true); + opts.set_max_background_jobs(1); + let cf_names = DB::list_cf(&opts, checkpoint_dir).map_err(to_err)?; + let frozen = DB::open_cf(&opts, checkpoint_dir, &cf_names).map_err(to_err)?; + let checkpoint = rocksdb::checkpoint::Checkpoint::new(&frozen).map_err(to_err)?; + + let cf_sm = frozen + .cf_handle(STATE_MACHINE_CF) + .ok_or_else(|| StorageError::DbError("SM CF not found".to_string()))?; + let cf_sm_meta = frozen + .cf_handle(STATE_MACHINE_META_CF) + .ok_or_else(|| StorageError::DbError("SM meta CF not found".to_string()))?; + + // export_column_family() creates only the final subdirectory (e.g. "sm"); the parent + // (snapshot_dir) already exists. + let sm_export = checkpoint + .export_column_family(&cf_sm, snapshot_dir.join("sm")) + .map_err(to_err)?; + let sm_meta_export = checkpoint + .export_column_family(&cf_sm_meta, snapshot_dir.join("sm_meta")) + .map_err(to_err)?; + Self::save_cf_export_metadata(&sm_export, &snapshot_dir.join("sm_metadata.bin"))?; + Self::save_cf_export_metadata(&sm_meta_export, &snapshot_dir.join("sm_meta_metadata.bin"))?; + Ok(()) + } + /// Serialize `ExportImportFilesMetaData` to a bincode file at `path`. /// /// `directory` is excluded; on restore it is reconstructed from the snapshot path. @@ -503,20 +531,6 @@ impl RocksDBStateMachine { self.last_applied_term.store(last_applied.term, Ordering::SeqCst); } - pub(crate) fn update_last_snapshot_metadata( - &self, - snapshot_metadata: &SnapshotMetadata, - ) -> Result<(), Error> { - *self.last_snapshot_metadata.write() = Some(snapshot_metadata.clone()); - Ok(()) - } - - pub(crate) fn save_hard_state(&self) -> Result<(), Error> { - // last_applied_index/term are now persisted atomically with each - // apply_chunk() batch (see apply_chunk()) — no longer written here. - self.persist_snapshot_metadata() - } - /// Flushes the WAL and both state machine column families to disk. pub fn flush(&self) -> Result<(), Error> { self.with_db(|db| { @@ -587,7 +601,6 @@ impl StateMachine for RocksDBStateMachine { } }; record("flush", self.flush()); - record("save hard state", self.save_hard_state()); if let Some(db) = self.db.load().as_deref() { db.cancel_all_background_work(true); } @@ -757,18 +770,6 @@ impl StateMachine for RocksDBStateMachine { } } - fn snapshot_metadata(&self) -> Option { - self.last_snapshot_metadata.read().clone() - } - - fn persist_last_snapshot_metadata( - &self, - snapshot_metadata: &SnapshotMetadata, - ) -> Result<(), Error> { - self.update_last_snapshot_metadata(snapshot_metadata)?; - self.persist_snapshot_metadata() - } - async fn apply_snapshot_from_file( &self, metadata: &SnapshotMetadata, @@ -783,7 +784,7 @@ impl StateMachine for RocksDBStateMachine { // Stop serving — prevents SM reads/writes during restore self.is_serving.store(false, Ordering::SeqCst); - let result = self.restore_from_snapshot(metadata, incoming, &snapshot_dir).await; + let result = self.restore_from_snapshot(incoming, &snapshot_dir).await; if let Err(ref e) = result { error!( @@ -816,38 +817,7 @@ impl StateMachine for RocksDBStateMachine { let dir = new_snapshot_dir.clone(); tokio::task::spawn_blocking(move || -> Result<(), Error> { std::fs::create_dir_all(&dir)?; - { - let checkpoint = rocksdb::checkpoint::Checkpoint::new(db.as_ref()) - .map_err(|e| StorageError::DbError(e.to_string()))?; - let cf_sm = db - .cf_handle(STATE_MACHINE_CF) - .ok_or_else(|| StorageError::DbError("SM CF not found".to_string()))?; - let cf_sm_meta = db - .cf_handle(STATE_MACHINE_META_CF) - .ok_or_else(|| StorageError::DbError("SM meta CF not found".to_string()))?; - - // Flush both CFs before export: export_column_family only captures SST files, - // not MemTable data. Without this, small CFs (e.g. sm_meta with only a few - // metadata keys) may never have been flushed and would export 0 files. - // Synchronous flush (FlushOptions::wait = true by default) is intentional: - // we must wait for flush to complete before export or in-flight writes are lost. - let flush_opts = rocksdb::FlushOptions::default(); - db.flush_cf_opt(&cf_sm, &flush_opts) - .map_err(|e| StorageError::DbError(e.to_string()))?; - db.flush_cf_opt(&cf_sm_meta, &flush_opts) - .map_err(|e| StorageError::DbError(e.to_string()))?; - - let sm_export = checkpoint - .export_column_family(&cf_sm, dir.join("sm")) - .map_err(|e| StorageError::DbError(e.to_string()))?; - let sm_meta_export = checkpoint - .export_column_family(&cf_sm_meta, dir.join("sm_meta")) - .map_err(|e| StorageError::DbError(e.to_string()))?; - - Self::save_cf_export_metadata(&sm_export, &dir.join("sm_metadata.bin"))?; - Self::save_cf_export_metadata(&sm_meta_export, &dir.join("sm_meta_metadata.bin"))?; - } // checkpoint and CF handles dropped here - Ok(()) + Self::export_frozen_view(&db, &dir) }) .await .map_err(Self::map_snapshot_join_error)??; @@ -875,10 +845,8 @@ impl StateMachine for RocksDBStateMachine { self.last_applied_index.store(0, Ordering::SeqCst); self.last_applied_term.store(0, Ordering::SeqCst); - *self.last_snapshot_metadata.write() = None; self.persist_state_machine_metadata()?; - self.persist_snapshot_metadata()?; info!("RocksDB state machine reset completed"); Ok(()) diff --git a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine_test.rs b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine_test.rs index 3d411fa5..afea7192 100644 --- a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine_test.rs +++ b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine_test.rs @@ -842,3 +842,107 @@ async fn test_apply_snapshot_from_file_reports_applied_for_newer_snapshot() { .expect("newer snapshot must install successfully"); assert_eq!(state_machine.last_applied(), last_included); } + +/// Purpose: a snapshot exported while orders keep arriving holds the data and the position of the +/// same applied entry, so after a restart the position equals the data exactly. +/// +/// Orders service: entry `n` sets `orders` to `n`, so the value of `orders` is the entry the data +/// has reached. `data` and `last_applied` are committed in one write batch, which means a +/// consistent export always has `last_applied == orders`. A position larger than `orders` means the +/// two column families were exported at different moments: the node would skip the entries in +/// between. A position smaller than `orders` would mean entries are replayed on top of data that +/// already contains them. The generic suite only checks `position <= data`; RocksDB exports from +/// one frozen view, so it has to be exact. +#[tokio::test] +async fn test_snapshot_taken_during_writes_exports_position_equal_to_data() { + for attempt in 0..3 { + let source_dir = TempDir::new().expect("source dir"); + let (source_storage, source) = + RocksDBUnifiedEngine::open(source_dir.path()).expect("open source"); + let source = Arc::new(source); + + let stop = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let writer = { + let source = source.clone(); + let stop = stop.clone(); + tokio::spawn(async move { + let mut n = 1u64; + while !stop.load(std::sync::atomic::Ordering::SeqCst) { + source + .apply_chunk(&[insert_at(b"orders", n.to_string().as_bytes(), n)]) + .await + .expect("apply"); + n += 1; + tokio::task::yield_now().await; + } + }) + }; + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + + let snapshot_dir = source_dir.path().join("snapshot-out"); + source + .generate_snapshot_data(snapshot_dir.clone(), LogId { index: 1, term: 1 }) + .await + .expect("generate snapshot"); + stop.store(true, std::sync::atomic::Ordering::SeqCst); + writer.await.expect("writer"); + + let target_dir = TempDir::new().expect("target dir"); + let target_path = target_dir.path().join("target"); + { + let (_target_storage, target) = + RocksDBUnifiedEngine::open(&target_path).expect("open target"); + let checksum = + compute_checksum_from_folder_path(&snapshot_dir).await.expect("checksum"); + let metadata = SnapshotMetadata { + last_included: Some(LogId { index: 1, term: 1 }), + checksum: Bytes::copy_from_slice(&checksum), + }; + target.apply_snapshot_from_file(&metadata, snapshot_dir).await.expect("install"); + target.close().expect("close"); + } + drop(source); + drop(source_storage); + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + + let (_restarted_storage, restarted) = + RocksDBUnifiedEngine::open(&target_path).expect("reopen target"); + let position = restarted.last_applied().index; + let data: u64 = restarted + .get(b"orders") + .expect("get") + .map(|v| String::from_utf8_lossy(&v).parse().expect("number")) + .unwrap_or(0); + assert_eq!( + position, data, + "attempt {attempt}: the exported position {position} and data {data} must describe the same entry" + ); + } +} + +/// Purpose: the database runs with `atomic_flush`. State machine writes skip the WAL and put data +/// and `last_applied` in different column families; only an atomic flush guarantees the two reach +/// disk together, which both snapshot export (checkpoint) and crash recovery rely on. Without it +/// `last_applied` can be on disk while the data it describes is lost. +/// +/// RocksDB records the options a database was opened with in its `OPTIONS-*` file, so the setting +/// is read back from there instead of being assumed. +#[tokio::test] +async fn test_state_machine_database_opens_with_atomic_flush() { + let dir = TempDir::new().expect("dir"); + let (_storage, _sm) = RocksDBUnifiedEngine::open(dir.path()).expect("open"); + + let options_file = std::fs::read_dir(dir.path()) + .expect("read db dir") + .filter_map(Result::ok) + .map(|e| e.path()) + .filter(|p| p.file_name().is_some_and(|n| n.to_string_lossy().starts_with("OPTIONS-"))) + .max() + .expect("RocksDB writes an OPTIONS file when it opens a database"); + let text = std::fs::read_to_string(options_file).expect("read options"); + + assert!( + text.contains("atomic_flush=true"), + "atomic_flush must be on for the state machine database" + ); +} diff --git a/d-engine-server/src/test_utils/mock/mock_node_builder.rs b/d-engine-server/src/test_utils/mock/mock_node_builder.rs index ca6aea75..ebbec971 100644 --- a/d-engine-server/src/test_utils/mock/mock_node_builder.rs +++ b/d-engine-server/src/test_utils/mock/mock_node_builder.rs @@ -630,9 +630,6 @@ pub(crate) fn mock_state_machine() -> MockStateMachine { mock.expect_last_applied().return_const(LogId::default()); - mock.expect_snapshot_metadata().returning(|| None); - mock.expect_persist_last_snapshot_metadata().returning(|_| Ok(())); - mock.expect_apply_snapshot_from_file().returning(|_, _| Ok(())); mock.expect_generate_snapshot_data().returning(|_, _| Ok(())); diff --git a/d-engine-server/tests/common/shared_state_machine.rs b/d-engine-server/tests/common/shared_state_machine.rs index 3ded33f5..ba333469 100644 --- a/d-engine-server/tests/common/shared_state_machine.rs +++ b/d-engine-server/tests/common/shared_state_machine.rs @@ -57,15 +57,6 @@ impl StateMachine for SharedStateMachine { fn last_applied(&self) -> LogId { self.0.last_applied() } - fn snapshot_metadata(&self) -> Option { - self.0.snapshot_metadata() - } - fn persist_last_snapshot_metadata( - &self, - snapshot_metadata: &SnapshotMetadata, - ) -> Result<(), Error> { - self.0.persist_last_snapshot_metadata(snapshot_metadata) - } async fn apply_snapshot_from_file( &self, metadata: &SnapshotMetadata, diff --git a/d-engine/src/docs/server_guide/customize-state-machine.md b/d-engine/src/docs/server_guide/customize-state-machine.md index 2c1d61fc..b6f5e3f4 100644 --- a/d-engine/src/docs/server_guide/customize-state-machine.md +++ b/d-engine/src/docs/server_guide/customize-state-machine.md @@ -103,7 +103,7 @@ impl StateMachine for CustomStateMachine { - **Snapshot Isolation**: `apply_snapshot_from_file()` must atomically replace state - **Snapshot Checksums**: The engine computes the checksum of the directory your `generate_snapshot_data()` writes, and verifies it before it calls `apply_snapshot_from_file()`. You neither compute nor check one - **Snapshot Install**: The engine decides whether a snapshot is installed. `apply_snapshot_from_file()` is called only for a snapshot newer than `last_applied()`, and the engine checks afterwards that `last_applied()` equals the snapshot boundary -- **Restore On Open**: The engine never asks you to restore state. When your state machine is opened, `last_applied()` must return the last entry whose effects are durable (persist it together with the data it describes), and `snapshot_metadata()` must return the value most recently passed to `persist_last_snapshot_metadata()` +- **Restore On Open**: The engine never asks you to restore state. When your state machine is opened, `last_applied()` must return the last entry whose effects are durable (persist it together with the data it describes). You store nothing about snapshots: the engine records the snapshots it holds itself - **Concurrency Control**: Use appropriate locking for state mutations - **Resource Cleanup**: Implement `Drop` to ensure proper flush on shutdown @@ -115,8 +115,6 @@ impl StateMachine for CustomStateMachine { | `get()` | Read value by key | Sync | High | | `apply_chunk()` | Apply committed entries | Async | Critical | | `last_applied()` | Get last applied index | Sync | High | -| `snapshot_metadata()` | Get current snapshot metadata | Sync | Medium | -| `persist_last_snapshot_metadata()` | Persist snapshot metadata | Sync | Medium | | `apply_snapshot_from_file()` | Replace state with snapshot | Async | Critical | | `generate_snapshot_data()` | Export a snapshot into the given directory (the engine checksums it) | Async | High | | `reset()` | Reset to initial state | Async | Medium | diff --git a/examples/sled-cluster/src/sled_state_machine.rs b/examples/sled-cluster/src/sled_state_machine.rs index 62f8c182..4c3b8cb1 100644 --- a/examples/sled-cluster/src/sled_state_machine.rs +++ b/examples/sled-cluster/src/sled_state_machine.rs @@ -10,7 +10,6 @@ use std::time::Instant; use arc_swap::ArcSwapOption; use async_trait::async_trait; -use bincode::config; use bytes::Bytes; use d_engine::ApplyEntry; use d_engine::ApplyResult; @@ -23,7 +22,6 @@ use d_engine::StateMachine; use d_engine::StorageError; use d_engine::common::LogId; use d_engine::server_storage::SnapshotMetadata; -use parking_lot::Mutex; use sled::Batch; use tracing::debug; use tracing::error; @@ -37,14 +35,11 @@ use crate::safe_vk_ivec; /// Sled database tree namespaces pub(crate) const STATE_MACHINE_TREE: &str = "_state_machine_tree"; pub(crate) const STATE_MACHINE_META_NAMESPACE: &str = "_state_machine_metadata"; -pub(crate) const STATE_SNAPSHOT_METADATA_TREE: &str = "_snapshot_metadata"; /// Sled entry key namespaces pub(crate) const STATE_MACHINE_META_KEY_LAST_APPLIED_INDEX: &str = "_raft_last_applied_index"; pub(crate) const STATE_MACHINE_META_KEY_LAST_APPLIED_TERM: &str = "_raft_last_applied_term"; -pub(crate) const LAST_SNAPSHOT_METADATA_KEY: &str = "_raft_last_snapshot_metadata"; - pub struct SledStateMachine { node_id: u32, @@ -64,11 +59,6 @@ pub struct SledStateMachine { /// The last submitted log index and term (atomic operation ensures lock-free) last_applied_index: AtomicU64, last_applied_term: AtomicU64, - - /// Snapshot metadata - last_included_index: AtomicU64, // load from STATE_SNAPSHOT_METADATA_TREE - last_included_term: AtomicU64, - last_snapshot_checksum: Mutex>, //SHA-256 } impl std::fmt::Debug for SledStateMachine { @@ -115,25 +105,6 @@ impl StateMachine for SledStateMachine { } } - fn snapshot_metadata(&self) -> Option { - let last_included_index = self.last_included_index.load(Ordering::SeqCst); - if last_included_index > 0 { - Some(SnapshotMetadata { - last_included: Some(LogId { - index: self.last_included_index.load(Ordering::SeqCst), - term: self.last_included_term.load(Ordering::SeqCst), - }), - checksum: self - .last_snapshot_checksum - .lock() - .as_ref() - .map(|arr| Bytes::copy_from_slice(arr))?, - }) - } else { - None - } - } - fn get( &self, key_buffer: &[u8], @@ -245,23 +216,6 @@ impl StateMachine for SledStateMachine { Ok(results) } - fn persist_last_snapshot_metadata( - &self, - last_snapshot_metadata: &SnapshotMetadata, - ) -> Result<()> { - debug!(%self.node_id, ?last_snapshot_metadata, "persist_last_snapshot_metadata"); - // The engine only ever calls this method, so it must also update the in-memory copy that - // `snapshot_metadata()` reads and that `close()` writes back. - self.update_last_snapshot_metadata(last_snapshot_metadata)?; - let db = self.db()?; - let tree = db - .open_tree(STATE_SNAPSHOT_METADATA_TREE) - .map_err(|e| StorageError::DbError(e.to_string()))?; - self.persist_last_snapshot_metadata_with_tree(tree, last_snapshot_metadata)?; - - Ok(()) - } - async fn generate_snapshot_data( &self, new_snapshot_dir: PathBuf, @@ -387,7 +341,6 @@ impl StateMachine for SledStateMachine { "Updating Raft metadata and indexes" ); self.update_last_applied(new_last_included); - self.update_last_snapshot_metadata(metadata)?; self.save_hard_state()?; @@ -396,11 +349,7 @@ impl StateMachine for SledStateMachine { async fn reset(&self) -> Result<()> { let db = self.db()?; - for tree_name in &[ - STATE_MACHINE_TREE, - STATE_MACHINE_META_NAMESPACE, - STATE_SNAPSHOT_METADATA_TREE, - ] { + for tree_name in &[STATE_MACHINE_TREE, STATE_MACHINE_META_NAMESPACE] { db.open_tree(tree_name) .map_err(|e| StorageError::DbError(e.to_string()))? .clear() @@ -408,9 +357,6 @@ impl StateMachine for SledStateMachine { } self.last_applied_index.store(0, Ordering::SeqCst); self.last_applied_term.store(0, Ordering::SeqCst); - self.last_included_index.store(0, Ordering::SeqCst); - self.last_included_term.store(0, Ordering::SeqCst); - *self.last_snapshot_checksum.lock() = None; Ok(()) } @@ -462,10 +408,6 @@ impl SledStateMachine { let (last_applied_index, last_applied_term) = Self::load_state_machine_metadata(&state_machine_meta_tree)?; - let snapshot_meta_tree = db - .open_tree(STATE_SNAPSHOT_METADATA_TREE) - .map_err(|e| StorageError::DbError(e.to_string()))?; - let sm = SledStateMachine { db: ArcSwapOption::from(Some(db)), path, @@ -474,10 +416,6 @@ impl SledStateMachine { last_applied_index: AtomicU64::new(0), last_applied_term: AtomicU64::new(0), - - last_included_index: AtomicU64::new(0), - last_included_term: AtomicU64::new(0), - last_snapshot_checksum: Mutex::new(None), }; // Important to sync the last applied index into memory @@ -486,17 +424,6 @@ impl SledStateMachine { term: last_applied_term, }); - // Update last snapshot metadata - if let Some(last_snapshot_metadata) = Self::load_snapshot_metadata(&snapshot_meta_tree)? { - trace!( - ?last_snapshot_metadata, - "Updating last snapshot metadata from local database when node starts" - ); - sm.update_last_snapshot_metadata(&last_snapshot_metadata)?; - } else { - info!("No snapshot metadata found in DB"); - } - Ok(sm) } @@ -516,34 +443,6 @@ impl SledStateMachine { Ok((index, term)) } - pub(super) fn load_snapshot_metadata(tree: &sled::Tree) -> Result> { - if let Ok(Some(v)) = tree.get(LAST_SNAPSHOT_METADATA_KEY) { - info!( - "found SnapshotMetadata from DB with key: {}", - LAST_SNAPSHOT_METADATA_KEY - ); - let v = v.to_vec(); - - let config = config::standard(); - match bincode::serde::decode_from_slice::(&v, config) { - Ok((last_snapshot_metadata, _)) => { - info!( - "load_snapshot_metadata: last_included={:?}", - last_snapshot_metadata.last_included - ); - return Ok(Some(last_snapshot_metadata)); - } - Err(e) => { - return Err(SnapshotError::OperationFailed(format!( - "state:load_snapshot_metadata deserialize error. {e}" - )) - .into()); - } - } - } - Ok(None) - } - pub(super) fn apply_batch( &self, batch: Batch, @@ -570,23 +469,6 @@ impl SledStateMachine { .ok_or_else(|| StorageError::NotServing("state machine stopped".into()).into()) } - pub(super) fn persist_last_snapshot_metadata_with_tree( - &self, - tree: sled::Tree, - last_snapshot_metadata: &SnapshotMetadata, - ) -> Result<()> { - let config = config::standard(); - let v = bincode::serde::encode_to_vec(last_snapshot_metadata, config) - .map_err(|e| StorageError::DbError(e.to_string()))?; - - tree.insert(LAST_SNAPSHOT_METADATA_KEY, v) - .map_err(|e| StorageError::DbError(e.to_string()))?; - info!("persist_last_snapshot_metadata_with_tree successfully!"); - - tree.flush().map_err(|e| StorageError::DbError(e.to_string()))?; - Ok(()) - } - pub(crate) fn update_last_applied( &self, last_applied: LogId, @@ -620,30 +502,10 @@ impl SledStateMachine { Ok(()) } - pub(crate) fn update_last_snapshot_metadata( - &self, - snapshot_metadata: &SnapshotMetadata, - ) -> Result<()> { - debug!(%self.node_id, ?snapshot_metadata, "update_last_snapshot_metadata"); - let last_included = snapshot_metadata.last_included.unwrap(); - self.last_included_index.store(last_included.index, Ordering::SeqCst); - self.last_included_term.store(last_included.term, Ordering::SeqCst); - let checksum = snapshot_metadata - .checksum_array() - .map_err(|e| SnapshotError::OperationFailed(format!("{e:?}")))?; - *self.last_snapshot_checksum.lock() = Some(checksum); - - Ok(()) - } - pub(crate) fn save_hard_state(&self) -> Result<()> { let last_applied = self.last_applied(); self.persist_last_applied(last_applied)?; - if let Some(last_snapshot_metadata) = self.snapshot_metadata() { - self.persist_last_snapshot_metadata(&last_snapshot_metadata)?; - } - self.flush()?; Ok(()) } From 10575073c9421ea38d25c914ea840c11847067a4 Mon Sep 17 00:00:00 2001 From: Joshua Chi <112539+JoshuaChi@users.noreply.github.com> Date: Mon, 2 Nov 2026 17:02:40 +0800 Subject: [PATCH 10/19] fix #400: Label snapshots with the position the state machine exports The engine used to read last_applied() and then ask the state machine to export. The export runs concurrently with apply_chunk, so the label could be older than the data: a snapshot labelled 100 held the state at 105, and a node installing it replayed entries on top of data that already contained them. generate_snapshot_data(dir) now returns the position of the last applied entry its exported data contains, read from the same view as the data, and no longer takes last_included. The engine labels the snapshot with it. It reads last_applied() before and after the export and rejects a position outside that range: the snapshot is discarded and the node keeps running. The check only catches a plainly wrong position; the contract and the trait documentation say where a correct one comes from (one read transaction, a checkpoint, or a lock shared with apply_chunk). RocksDB reads the position from the frozen checkpoint copy it exports from. The file state machine now updates last_applied under the same write lock as the data and reads both under one read lock; before, a snapshot could see the data without the position. The sled example and d-lmdb only adapt to the new signature and report the position read before the copy, which stays at or behind the data; their position is not yet written with the data. Tests: the engine labels with the returned position and rejects a position before the export started or one the node has not reached (each fails if that behaviour is reverted); the suite and the RocksDB and file tests check the returned position of a quiescent state machine; mock-based snapshot tests report their position instead of receiving a label. --- CHANGELOG.md | 2 +- MIGRATION_GUIDE.md | 40 ++-- .../leader_state_test/snapshot_worker_test.rs | 2 +- .../default_state_machine_handler_test.rs | 220 +++++++++++++----- .../src/state_machine_handler/writer.rs | 27 ++- d-engine-core/src/storage/state_machine.rs | 44 +++- .../src/storage/state_machine_test.rs | 45 ++-- .../src/test_utils/mock/mock_raft_builder.rs | 2 +- .../api/embedded_test/shutdown_order_test.rs | 5 +- .../adaptors/file/file_state_machine.rs | 25 +- .../adaptors/file/file_state_machine_test.rs | 16 +- .../adaptors/rocksdb/rocksdb_state_machine.rs | 17 +- .../rocksdb/rocksdb_state_machine_test.rs | 8 +- .../src/test_utils/mock/mock_node_builder.rs | 2 +- .../tests/common/shared_state_machine.rs | 5 +- .../server_guide/customize-state-machine.md | 22 +- .../sled-cluster/src/sled_state_machine.rs | 21 +- .../state_machine/sled_state_machine_test.rs | 6 +- 18 files changed, 324 insertions(+), 185 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2837a8b7..ef90a7af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -122,7 +122,7 @@ All notable changes to this project will be documented in this file. - **⚠️ `StateMachine` no longer declares `update_last_applied`, `persist_last_applied`, `update_last_snapshot_metadata`, `save_hard_state`, `flush` and `flush_async`** (#400): the engine never called them. Keep any you still need as methods on your own type. Your state machine must restore `last_applied` and its snapshot metadata when it is opened; the `StateMachine` docs now state this as a durability contract, and `StateMachineTestSuite` checks it. See [MIGRATION_GUIDE.md](./MIGRATION_GUIDE.md). -- **⚠️ `StateMachine::generate_snapshot_data` returns `Result<()>`; the engine owns snapshot checksums** (#400): the engine computes the SHA-256 of the exported snapshot directory and verifies it before calling `apply_snapshot_from_file`, which now only ever receives a directory that matches the recorded checksum. A state machine neither returns nor checks a checksum. The checksum now covers every file at any depth together with its relative path and length; before, it covered only top-level files, so the data files RocksDB exports into sub-directories were not protected. Snapshot archives created before the upgrade fail verification, so remove old snapshot archives when upgrading. See [MIGRATION_GUIDE.md](./MIGRATION_GUIDE.md). +- **⚠️ `StateMachine::generate_snapshot_data(dir)` returns `Result`; the engine owns snapshot checksums** (#400): the method no longer takes the `last_included` argument and returns the position of the last entry its exported data contains, read from the same consistent view as the data. The engine labels the snapshot with it and rejects a position outside what the node applied while exporting. The engine computes the SHA-256 of the exported snapshot directory and verifies it before calling `apply_snapshot_from_file`, which now only ever receives a directory that matches the recorded checksum. A state machine neither returns nor checks a checksum. The checksum now covers every file at any depth together with its relative path and length; before, it covered only top-level files, so the data files RocksDB exports into sub-directories were not protected. Snapshot archives created before the upgrade fail verification, so remove old snapshot archives when upgrading. See [MIGRATION_GUIDE.md](./MIGRATION_GUIDE.md). - **⚠️ The engine takes the state machine by value** (#400): `EmbeddedEngine::start_custom`/`start_node`, `StandaloneEngine::run_custom`/`start_node` take `SM` instead of `Arc`; the `Arc` is created inside the engine. Nothing outside the engine holds the state machine, so `stop()` returns only after it is dropped and its resources are released. Embedded reads now go through the ReadActor like standalone reads (`EmbeddedReadHandle` is removed). `EmbeddedEngine` and `EmbeddedClient` no longer have type parameters. See [MIGRATION_GUIDE.md](./MIGRATION_GUIDE.md). diff --git a/MIGRATION_GUIDE.md b/MIGRATION_GUIDE.md index cd3a69a3..9019e34f 100644 --- a/MIGRATION_GUIDE.md +++ b/MIGRATION_GUIDE.md @@ -33,7 +33,7 @@ EmbeddedEngine::start_custom(dir, storage, sm, None).await?; - **`EmbeddedEngine` and `EmbeddedClient` no longer have type parameters.** Write `EmbeddedEngine` and `EmbeddedClient`. `DefaultEmbeddedEngine` and `DefaultEmbeddedClient` still exist and are now plain aliases of them. - **If you kept a clone of your `Arc` to read local data directly** (for example `DLmdb` reading its own LMDB), you can no longer clone the instance the engine uses. Share a handle to your storage instead: put the handles (LMDB `Env`, sled `Db`, `Arc`) behind an `Arc` inside your state machine type, derive `Clone` on it, give the engine one clone and keep the other. - **Embedded reads (`get_eventual`, lease reads) are answered by the ReadActor**, as they already were in standalone mode, instead of reading the state machine on the caller's thread. Expect one extra channel hop per read. -- **Snapshot checksums belong to the engine.** `generate_snapshot_data` now returns `Result<()>`: write the snapshot into the directory you are given and return. The engine computes the checksum of that directory itself, and verifies it before it calls `apply_snapshot_from_file`, so you no longer compute or check a checksum. Delete that code (and any hashing dependency you added only for it). +- **Snapshot checksums belong to the engine.** `generate_snapshot_data(dir)` now returns `Result` and no longer takes `last_included`: write the snapshot into the directory you are given and return the position of the last applied entry the exported data contains, read from the same consistent view as the data (not from a separate counter, because `apply_chunk` keeps running). The engine labels the snapshot with it. The engine computes the checksum of that directory itself, and verifies it before it calls `apply_snapshot_from_file`, so you no longer compute or check a checksum. Delete that code (and any hashing dependency you added only for it). - **`apply_snapshot_from_file` returns `Result<()>` again, and you no longer classify snapshots.** The engine compares the snapshot's boundary with `last_applied()` first and calls you only for a newer snapshot, so delete your stale / duplicate / conflict handling and return `Ok(())` after installing. After your call the engine checks that `last_applied()` equals `metadata.last_included` and reports an error if it does not, so make sure you set it (durably) as part of the install. - **Eight methods left the trait:** `update_last_applied`, `persist_last_applied`, `update_last_snapshot_metadata`, `save_hard_state`, `flush`, `flush_async`, `snapshot_metadata` and `persist_last_snapshot_metadata`. The first six were never called by the engine, so deleting them from your `impl StateMachine` block changes no behaviour. The last two are replaced by the engine's own record of the snapshots it holds (see below). Keep any your own code still uses as plain methods on your type. - **Restore your position when you open the state machine.** The engine does not do it for you: after reopening, `last_applied()` must return the last entry whose effects are durable. You no longer store or return anything about snapshots; delete your snapshot-metadata storage and its loading code. @@ -47,14 +47,14 @@ EmbeddedEngine::start_custom(dir, storage, sm, None).await?; v0.3.0 nodes and clients cannot talk to v0.2.4 ones. There is no rolling upgrade: stop the whole cluster, upgrade every node, then upgrade every client. -| Proto item | v0.2.4 | v0.3.0 | -| ------------------------------- | ---------------------------- | ------------------------------------------------------------------ | -| `ClientResponse.metadata` | field 4 | field 5 (field 4 is now `scan_data`) | -| `HandleClientScan` response | `ScanResponse` | `ClientResponse`; the scan payload is the `ScanResults` message | -| `ChunkStatus` | `ACCEPTED = 0` … `FAILED = 4` | `CHUNK_STATUS_UNSPECIFIED = 0`, `CHUNK_STATUS_ACCEPTED = 1` … `CHUNK_STATUS_FAILED = 5` | -| `WriteCommand.Insert.ttl_secs` | field 3 | removed, number 3 and the name are reserved | -| `StreamSnapshot` RPC | present | removed (see the learner snapshot section below) | -| `PreVote` RPC | - | new | +| Proto item | v0.2.4 | v0.3.0 | +| ------------------------------ | ----------------------------- | --------------------------------------------------------------------------------------- | +| `ClientResponse.metadata` | field 4 | field 5 (field 4 is now `scan_data`) | +| `HandleClientScan` response | `ScanResponse` | `ClientResponse`; the scan payload is the `ScanResults` message | +| `ChunkStatus` | `ACCEPTED = 0` … `FAILED = 4` | `CHUNK_STATUS_UNSPECIFIED = 0`, `CHUNK_STATUS_ACCEPTED = 1` … `CHUNK_STATUS_FAILED = 5` | +| `WriteCommand.Insert.ttl_secs` | field 3 | removed, number 3 and the name are reserved | +| `StreamSnapshot` RPC | present | removed (see the learner snapshot section below) | +| `PreVote` RPC | - | new | Clients in other languages: regenerate your stubs from the v0.3.0 `.proto` files. Rust users get this by upgrading the crate. @@ -182,12 +182,12 @@ grep "WAL" /var/log/d-engine.log ## Timeline -| Version | WAL Format | Wire Protocol | Migration Required | -| ------------- | ------------------- | --------------------- | ------------------------------------------------------------------------------------------------------------------------------- | -| v0.1.x | Relative TTL | Compatible | - | -| v0.2.0–v0.2.2 | Absolute expiration | Compatible | ✅ Yes (clear WAL from v0.1.x) | -| v0.2.3 | Same as v0.2.0+ | **Incompatible** | ✅ Yes (protobuf enum changes + API changes) | -| v0.2.4 | Same as v0.2.0+ | Compatible (additive) | ✅ Yes (delete `snapshot/` — format changed to CF export) | +| Version | WAL Format | Wire Protocol | Migration Required | +| ------------- | ---------------------- | ------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| v0.1.x | Relative TTL | Compatible | - | +| v0.2.0–v0.2.2 | Absolute expiration | Compatible | ✅ Yes (clear WAL from v0.1.x) | +| v0.2.3 | Same as v0.2.0+ | **Incompatible** | ✅ Yes (protobuf enum changes + API changes) | +| v0.2.4 | Same as v0.2.0+ | Compatible (additive) | ✅ Yes (delete `snapshot/` — format changed to CF export) | | v0.3.0 | File SM format changed | **Incompatible** (proto field and enum renumbering; upgrade all nodes and clients together) | ⚠️ Yes (TTL removed; fresh data directory; custom SM: remove `entry_term()`, `lease_background_cleanup`, rename `close_storage()` to `close()`; use `start_node` instead of `start_custom_with_config`) | --- @@ -410,11 +410,11 @@ fn apply_chunk(&mut self, entries: &[ApplyEntry]) -> Result<(), ...> ### What Changed -| Before (v0.2.4) | After (v0.3.0) | -| ---------------------------------------------------------------- | ----------------------------------------------------------------- | -| `fn stop(&self) -> Result<(), Error>` (required) | **Removed** | -| `fn close_storage(&self)` (default no-op) | **Renamed** `fn close(&self) -> Result<(), Error>` (default `Ok`) | -| `[raft.storage]` accepted as an alias of `[raft.state_machine]` | Use `[raft.state_machine]` | +| Before (v0.2.4) | After (v0.3.0) | +| --------------------------------------------------------------- | ----------------------------------------------------------------- | +| `fn stop(&self) -> Result<(), Error>` (required) | **Removed** | +| `fn close_storage(&self)` (default no-op) | **Renamed** `fn close(&self) -> Result<(), Error>` (default `Ok`) | +| `[raft.storage]` accepted as an alias of `[raft.state_machine]` | Use `[raft.state_machine]` | `stop()` was never called by d-engine. `close()` is now the one end-of-life hook. d-engine calls it once, as the last step of the state machine worker. When the worker drains cleanly, every committed entry has been applied by then; if diff --git a/d-engine-core/src/raft_role/leader_state_test/snapshot_worker_test.rs b/d-engine-core/src/raft_role/leader_state_test/snapshot_worker_test.rs index 31700d12..08dcc452 100644 --- a/d-engine-core/src/raft_role/leader_state_test/snapshot_worker_test.rs +++ b/d-engine-core/src/raft_role/leader_state_test/snapshot_worker_test.rs @@ -106,7 +106,7 @@ fn state_machine_with_snapshot() -> MockStateMachine { sm.expect_apply_chunk().returning(|_| Ok(vec![])); sm.expect_last_applied().return_const(LogId::default()); sm.expect_apply_snapshot_from_file().returning(|_, _| Ok(())); - sm.expect_generate_snapshot_data().returning(|_, _| Ok(())); + sm.expect_generate_snapshot_data().returning(|_| Ok(LogId::default())); sm } diff --git a/d-engine-core/src/state_machine_handler/default_state_machine_handler_test.rs b/d-engine-core/src/state_machine_handler/default_state_machine_handler_test.rs index 02befff2..6f3dda8d 100644 --- a/d-engine-core/src/state_machine_handler/default_state_machine_handler_test.rs +++ b/d-engine-core/src/state_machine_handler/default_state_machine_handler_test.rs @@ -943,22 +943,26 @@ mod create_snapshot_tests { // Mock state machine behavior let mut seq = Sequence::new(); sm.expect_last_applied() - .times(1) + .times(1) // before the export .in_sequence(&mut seq) .returning(|| LogId { index: 5, term: 1 }); sm.expect_generate_snapshot_data() .times(1) - .withf(|path, last_included| { - debug!(?path, ?last_included); + .withf(|path| { + debug!(?path); // Create the directory structure correctly fs::create_dir_all(path.clone()).unwrap(); let db_path = path.join("state_machine"); fs::create_dir(&db_path).unwrap(); - path.ends_with("temp-5-1") && last_included.index == 5 && last_included.term == 1 + path.ends_with("temp-5-1") }) - .returning(|_, _| Ok(())); + .returning(|_| Ok(LogId { index: 5, term: 1 })); + sm.expect_last_applied() + .times(1) // after the export + .in_sequence(&mut seq) + .returning(|| LogId { index: 5, term: 1 }); let mut config = snapshot_config(temp_path.to_path_buf()); config.retained_log_entries = 0; @@ -1018,17 +1022,21 @@ mod create_snapshot_tests { let mut seq = Sequence::new(); sm.expect_last_applied() - .times(1) + .times(1) // before the export .in_sequence(&mut seq) .returning(|| LogId { index: 5, term: 1 }); sm.expect_generate_snapshot_data() .times(1) .in_sequence(&mut seq) - .returning(|path, _| { + .returning(|path| { fs::create_dir_all(path.clone()).unwrap(); fs::create_dir(path.join("state_machine")).unwrap(); - Ok(()) + Ok(LogId { index: 5, term: 1 }) }); + sm.expect_last_applied() + .times(1) // after the export + .in_sequence(&mut seq) + .returning(|| LogId { index: 5, term: 1 }); let mut config = snapshot_config(temp_path.to_path_buf()); config.retained_log_entries = 0; @@ -1062,7 +1070,7 @@ mod create_snapshot_tests { // Setup slow snapshot generation let (tx, _rx) = tokio::sync::oneshot::channel(); sm.expect_last_applied().returning(|| LogId { term: 1, index: 1 }); - sm.expect_generate_snapshot_data().times(1..=2).returning(move |path, _| { + sm.expect_generate_snapshot_data().times(1..=2).returning(move |path| { // Track invocation count let mut count = counter_clone.lock().unwrap(); *count += 1; @@ -1073,7 +1081,7 @@ mod create_snapshot_tests { fs::create_dir_all(path.clone()).unwrap(); let db_path = path.join("state_machine"); fs::create_dir(&db_path).unwrap(); - Ok(()) + Ok(LogId { term: 1, index: 1 }) } else { Err(SnapshotError::OperationFailed("Concurrency failure".into()).into()) } @@ -1143,19 +1151,20 @@ mod create_snapshot_tests { let temp_path = temp_dir.path().join("test_create_snapshot_case3"); let mut sm = MockStateMachine::new(); - let mut count = 0; - sm.expect_last_applied().returning(move || { - count += 1; - LogId { - term: 1, - index: count, - } + // Every export moves the state machine one entry further, so each snapshot gets the next + // index: 1, 2, 3, 4. + let applied = Arc::new(std::sync::atomic::AtomicU64::new(0)); + let applied_for_read = applied.clone(); + sm.expect_last_applied().returning(move || LogId { + term: 1, + index: applied_for_read.load(std::sync::atomic::Ordering::SeqCst), }); - sm.expect_generate_snapshot_data().returning(|path, _| { + sm.expect_generate_snapshot_data().returning(move |path| { debug!(?path, "expect_generate_snapshot_data"); std::fs::create_dir_all(path).expect("Failed to create directory"); - Ok(()) + let index = applied.fetch_add(1, std::sync::atomic::Ordering::SeqCst) + 1; + Ok(LogId { term: 1, index }) }); let snapshot_dir = temp_path.to_path_buf(); @@ -1202,7 +1211,7 @@ mod create_snapshot_tests { // Setup failing snapshot generation sm.expect_last_applied().returning(|| LogId { term: 1, index: 1 }); sm.expect_generate_snapshot_data() - .returning(|_, _| Err(SnapshotError::OperationFailed("test failure".into()).into())); + .returning(|_| Err(SnapshotError::OperationFailed("test failure".into()).into())); let mut config = snapshot_config(temp_path.to_path_buf()); config.retained_log_entries = 0; @@ -1234,9 +1243,9 @@ mod create_snapshot_tests { let mut sm = MockStateMachine::new(); sm.expect_last_applied().returning(|| LogId { term: 1, index: 5 }); - sm.expect_generate_snapshot_data().returning(|path, _| { + sm.expect_generate_snapshot_data().returning(|path| { fs::create_dir_all(path).unwrap(); - Ok(()) + Ok(LogId { term: 1, index: 5 }) }); let config = snapshot_config(temp_path.clone()); @@ -1272,7 +1281,7 @@ mod create_snapshot_tests { sm.expect_last_applied().returning(|| LogId { term: 1, index: 1 }); sm.expect_generate_snapshot_data() - .returning(|_, _| Err(SnapshotError::OperationFailed("test error".into()).into())); + .returning(|_| Err(SnapshotError::OperationFailed("test error".into()).into())); let config = snapshot_config(temp_path.clone()); let (handler, writer) = new_reader_writer_pair::( @@ -1329,15 +1338,13 @@ mod create_snapshot_tests { term: 1, index: 100, }); - sm.expect_generate_snapshot_data() - .times(1) - .withf(|_path, last_included| { - last_included.index == 100 && last_included.term == 1 // NOT 90 + sm.expect_generate_snapshot_data().times(1).returning(|path| { + fs::create_dir_all(path).unwrap(); + Ok(LogId { + term: 1, + index: 100, }) - .returning(|path, _| { - fs::create_dir_all(path).unwrap(); - Ok(()) - }); + }); let mut config = snapshot_config(temp_path.clone()); config.retained_log_entries = 10; @@ -1403,16 +1410,16 @@ mod create_snapshot_tests { term: 2, index: 100, }); - sm.expect_generate_snapshot_data() - .times(1) - .withf(|path, last_included| { - fs::create_dir_all(path).unwrap(); - // Old code: index = 100 - 60 = 40, term = 2 (copied from last_applied.term) - // → forged LogId { index: 40, term: 2 } — entry 40 was in term 1, not term 2 - // New code: last_included = last_applied = { index: 100, term: 2 } — truthful - last_included.index == 100 && last_included.term == 2 + // The state machine reports { index: 100, term: 2 }. Subtracting retained_log_entries + // (60) and keeping the term would forge { index: 40, term: 2 }, an entry that was in term + // 1; the snapshot must carry exactly what the state machine reported. + sm.expect_generate_snapshot_data().times(1).returning(|path| { + fs::create_dir_all(path).unwrap(); + Ok(LogId { + term: 2, + index: 100, }) - .returning(|_, _| Ok(())); + }); let mut config = snapshot_config(temp_path.to_path_buf()); // retained=60: old code would subtract and land in term 1 territory (index 40) @@ -1466,13 +1473,10 @@ mod create_snapshot_tests { let mut sm = MockStateMachine::new(); sm.expect_last_applied().returning(|| LogId { term: 0, index: 0 }); - sm.expect_generate_snapshot_data() - .times(1) - .withf(|_, last_included| last_included.index == 0 && last_included.term == 0) - .returning(|path, _| { - fs::create_dir_all(path).unwrap(); - Ok(()) - }); + sm.expect_generate_snapshot_data().times(1).returning(|path| { + fs::create_dir_all(path).unwrap(); + Ok(LogId { term: 0, index: 0 }) + }); let mut config = snapshot_config(temp_path.to_path_buf()); config.retained_log_entries = 10; @@ -1503,7 +1507,7 @@ mod create_snapshot_tests { let mut sm = MockStateMachine::new(); sm.expect_last_applied().returning(|| LogId { term: 1, index: 5 }); - sm.expect_generate_snapshot_data().returning(|path, _| { + sm.expect_generate_snapshot_data().returning(|path| { // Simulate a partial write before the failure (e.g. checkpoint killed mid-copy). fs::create_dir_all(&path).unwrap(); Err(SnapshotError::OperationFailed("simulated crash".into()).into()) @@ -1539,7 +1543,7 @@ mod create_snapshot_tests { let mut sm = MockStateMachine::new(); sm.expect_last_applied().returning(|| LogId { term: 1, index: 5 }); - sm.expect_generate_snapshot_data().returning(|path, _| { + sm.expect_generate_snapshot_data().returning(|path| { // Real engines (e.g. RocksDB checkpoint) fail with "File exists" if `path` // is already present — mirror that here instead of using the idempotent // `create_dir_all`, otherwise this test can't distinguish "cleaned up" from @@ -1548,7 +1552,7 @@ mod create_snapshot_tests { return Err(SnapshotError::OperationFailed("mkdir: File exists".into()).into()); } fs::create_dir_all(&path).unwrap(); - Ok(()) + Ok(LogId { term: 1, index: 5 }) }); let config = snapshot_config(snapshot_dir.clone()); @@ -2000,7 +2004,7 @@ async fn test_snapshot_compression() { // Mock state machine to create test data sm.expect_last_applied().returning(|| LogId { index: 10, term: 2 }); - sm.expect_generate_snapshot_data().returning(|path, _| { + sm.expect_generate_snapshot_data().returning(|path| { // Create the directory structure correctly fs::create_dir_all(path.clone()).unwrap(); // Create test files in the temp directory @@ -2010,7 +2014,7 @@ async fn test_snapshot_compression() { std::fs::write(&file1, "This is a test file").unwrap(); std::fs::write(&file2, vec![0u8; 1024]).unwrap(); - Ok(()) + Ok(LogId { index: 10, term: 2 }) }); let mut config = snapshot_config(temp_path.to_path_buf()); @@ -2984,6 +2988,99 @@ fn writer_with_mock_state_machine( ) } +/// Mock state machine whose `last_applied()` is `before` until the export runs and `after` once it +/// has, and whose export reports `exported`. +fn mock_sm_applying_during_export( + before: u64, + after: u64, + exported: u64, +) -> MockStateMachine { + let position = Arc::new(std::sync::atomic::AtomicU64::new(before)); + let mut sm = MockStateMachine::new(); + let reader = position.clone(); + sm.expect_last_applied().returning(move || LogId { + index: reader.load(std::sync::atomic::Ordering::SeqCst), + term: 2, + }); + sm.expect_generate_snapshot_data().returning(move |path| { + fs::create_dir_all(&path).unwrap(); + fs::write(path.join("data.bin"), b"state").unwrap(); + // Entries keep being applied while the export runs. + position.store(after, std::sync::atomic::Ordering::SeqCst); + Ok(LogId { + index: exported, + term: 2, + }) + }); + sm +} + +/// Orders service: the node applies entries 10..=12 while its state machine exports, and the export +/// holds everything up to entry 11. +/// +/// Purpose: the snapshot is labelled with the position the state machine reports for what it +/// exported (11), not with the position the engine saw before the export started (10), which would +/// make the label older than the data. +#[tokio::test] +async fn test_capture_local_snapshot_is_labelled_with_the_position_the_state_machine_exported() { + let temp_dir = tempfile::tempdir().unwrap(); + let sm = mock_sm_applying_during_export(10, 12, 11); + let (_handler, writer) = writer_with_mock_state_machine(sm, temp_dir.path()); + + let captured = writer.capture_local_snapshot().await.unwrap(); + + assert_eq!( + captured.metadata.last_included, + Some(LogId { index: 11, term: 2 }) + ); +} + +/// The state machine reports an export at entry 7 although the node had applied up to 10 before the +/// export started. +/// +/// Purpose: a position older than anything the node applied cannot describe an export made now, so +/// the snapshot is rejected and its directory removed; the node keeps running. +#[tokio::test] +async fn test_capture_local_snapshot_rejects_a_position_before_the_export_started() { + let temp_dir = tempfile::tempdir().unwrap(); + let sm = mock_sm_applying_during_export(10, 12, 7); + let (_handler, writer) = writer_with_mock_state_machine(sm, temp_dir.path()); + + let result = writer.capture_local_snapshot().await; + + assert!( + result.is_err(), + "a position before the export started must be rejected" + ); + assert!( + std::fs::read_dir(temp_dir.path()).unwrap().next().is_none(), + "the rejected export must not be left on disk" + ); +} + +/// The state machine reports an export at entry 15 although the node had applied only up to 12 when +/// the export finished. +/// +/// Purpose: a position the node has not reached yet cannot be right, so the snapshot is rejected +/// and its directory removed. +#[tokio::test] +async fn test_capture_local_snapshot_rejects_a_position_the_node_has_not_reached() { + let temp_dir = tempfile::tempdir().unwrap(); + let sm = mock_sm_applying_during_export(10, 12, 15); + let (_handler, writer) = writer_with_mock_state_machine(sm, temp_dir.path()); + + let result = writer.capture_local_snapshot().await; + + assert!( + result.is_err(), + "a position the node has not reached must be rejected" + ); + assert!( + std::fs::read_dir(temp_dir.path()).unwrap().next().is_none(), + "the rejected export must not be left on disk" + ); +} + /// A leader exports a snapshot for a lagging follower and records the checksum the follower will /// verify against. The state machine only writes files; it never returns a checksum. /// @@ -2995,11 +3092,11 @@ async fn test_capture_local_snapshot_records_checksum_of_exported_directory() { let temp_dir = tempfile::tempdir().unwrap(); let mut sm = MockStateMachine::new(); sm.expect_last_applied().returning(|| LogId { index: 10, term: 2 }); - sm.expect_generate_snapshot_data().returning(|path, _| { + sm.expect_generate_snapshot_data().returning(|path| { fs::create_dir_all(path.clone()).unwrap(); fs::write(path.join("a.bin"), b"alpha").unwrap(); fs::write(path.join("b.bin"), b"beta").unwrap(); - Ok(()) + Ok(LogId { index: 10, term: 2 }) }); let (_handler, writer) = writer_with_mock_state_machine(sm, temp_dir.path()); @@ -3391,14 +3488,15 @@ mod snapshot_catalog_behavior_tests { index: position.load(std::sync::atomic::Ordering::SeqCst), term: 2, }); - sm.expect_generate_snapshot_data().returning(|path, last_included| { + let exported = index.clone(); + sm.expect_generate_snapshot_data().returning(move |path| { + let at = LogId { + index: exported.load(std::sync::atomic::Ordering::SeqCst), + term: 2, + }; fs::create_dir_all(&path).unwrap(); - fs::write( - path.join("data.bin"), - format!("state at {}", last_included.index), - ) - .unwrap(); - Ok(()) + fs::write(path.join("data.bin"), format!("state at {}", at.index)).unwrap(); + Ok(at) }); sm } diff --git a/d-engine-core/src/state_machine_handler/writer.rs b/d-engine-core/src/state_machine_handler/writer.rs index 67f99613..5c448589 100644 --- a/d-engine-core/src/state_machine_handler/writer.rs +++ b/d-engine-core/src/state_machine_handler/writer.rs @@ -261,8 +261,11 @@ impl StateMachineWriterOps for DefaultStateMachineWriter { }) } async fn capture_local_snapshot(&self) -> Result { - let last_included = self.state_machine.last_applied(); - let temp_path = self.path_mgr.temp_work_path(&last_included); + // The position is read before and after the export only to bound what the state machine + // may return; the label comes from the state machine, which reads it from the view it + // exported. + let before = self.state_machine.last_applied(); + let temp_path = self.path_mgr.temp_work_path(&before); // A crashed/killed previous attempt at this exact index can leave temp_path // behind; without removing it first, generate_snapshot_data fails with @@ -278,14 +281,22 @@ impl StateMachineWriterOps for DefaultStateMachineWriter { // generate_snapshot_data only exports the state; it does not publish snapshot // metadata. The metadata is published later, once the archive is durably built. - if let Err(e) = self - .state_machine - .generate_snapshot_data(temp_path.clone(), last_included) - .await - { + let exported = match self.state_machine.generate_snapshot_data(temp_path.clone()).await { + Ok(position) => position, + Err(e) => { + let _ = tokio::fs::remove_dir_all(&temp_path).await; + return Err(e); + } + }; + let after = self.state_machine.last_applied(); + if exported.index < before.index || exported.index > after.index { let _ = tokio::fs::remove_dir_all(&temp_path).await; - return Err(e); + return Err(SnapshotError::OperationFailed(format!( + "state machine exported a snapshot at {exported:?}, outside the range {before:?}..={after:?} that it applied while exporting" + )) + .into()); } + let last_included = exported; // The framework, not the state machine, checksums the exported directory. let checksum = match compute_checksum_from_folder_path(&temp_path).await { diff --git a/d-engine-core/src/storage/state_machine.rs b/d-engine-core/src/storage/state_machine.rs index 11a624af..aca6e96c 100644 --- a/d-engine-core/src/storage/state_machine.rs +++ b/d-engine-core/src/storage/state_machine.rs @@ -208,13 +208,40 @@ pub trait StateMachine: Send + Sync + 'static { snapshot_path: std::path::PathBuf, ) -> Result<(), Error>; - /// Generates a snapshot of the state machine's current key-value entries - /// up to the specified `last_included_index`. + /// Exports the state machine's current state into `new_snapshot_dir` and returns the position + /// of the last applied entry that the exported state contains. /// /// Exports the current state (as of `last_included`) to `new_snapshot_dir`. The framework - /// computes the checksum of that directory itself and records the snapshot only once the - /// final archive is durably built, so a crash mid-snapshot never leaves a record pointing at - /// a missing archive; the state machine does neither. + /// The engine checksums the directory itself, labels the snapshot with the returned position, + /// and records the snapshot only once the final archive is durably built, so a crash + /// mid-snapshot never leaves a record pointing at a missing archive; the state machine does + /// none of that. `new_snapshot_dir` exists and is empty. Put any regular files in it, with + /// names of your choice; the engine archives the whole directory and hands it back to + /// [`apply_snapshot_from_file`](Self::apply_snapshot_from_file) unchanged. + /// + /// # The position must describe the exported data + /// + /// Read the position from the same consistent view the data is read from, never from a + /// separate in-memory counter: `apply_chunk` keeps running during the export, so a counter read + /// a moment earlier or later describes a different state. A snapshot whose position is ahead of + /// its data makes the node that installs it skip entries for good; one whose position is + /// behind makes it replay entries on top of data that already contains them, which gives a + /// different result for operations such as compare-and-swap. + /// + /// The engine reads [`last_applied`](Self::last_applied) before and after the call and rejects + /// a position outside that range; the snapshot is then discarded and the node keeps running. + /// That check only catches a position that is plainly wrong, so it does not replace the + /// consistency described above. + /// + /// # Exporting a consistent view + /// + /// Any of these works: + /// - read everything inside one read transaction or snapshot of the storage engine, position + /// included (MVCC databases); + /// - take a checkpoint of the storage engine and export from that frozen copy; + /// - hold a read lock while copying the data and reading the position, with `apply_chunk` + /// taking the write lock and updating the position under it. + /// /// /// # Concurrency contract (#436) /// May run concurrently with `apply_chunk` (never with `apply_snapshot_from_file` — the two @@ -222,15 +249,10 @@ pub trait StateMachine: Send + Sync + 'static { /// `apply_chunk` runs mid-export, the result must reflect strictly before or strictly after /// it, never a mix — a torn read across related keys silently corrupts data. Verified by /// `test_generate_snapshot_data_is_atomic_under_concurrent_apply`. - /// - /// # Arguments - /// * `new_snapshot_dir` - Temporary path to store the snapshot data. - /// * `last_included` - Index and term of the last log entry the snapshot includes. async fn generate_snapshot_data( &self, new_snapshot_dir: std::path::PathBuf, - last_included: LogId, - ) -> Result<(), Error>; + ) -> Result; /// Resets the state machine to its initial state. /// Async operation as it may involve cleaning up files and data. diff --git a/d-engine-core/src/storage/state_machine_test.rs b/d-engine-core/src/storage/state_machine_test.rs index 6d77beb9..2ea20c5f 100644 --- a/d-engine-core/src/storage/state_machine_test.rs +++ b/d-engine-core/src/storage/state_machine_test.rs @@ -274,11 +274,15 @@ impl StateMachineTestSuite { sm.apply_chunk(&entries(1..=KEYS, b"new")).await?; let temp_dir = TempDir::new()?; let snapshot_dir = temp_dir.path().join("snapshot"); - let last_included = LogId { - index: KEYS, - term: 1, - }; - sm.generate_snapshot_data(snapshot_dir.clone(), last_included).await?; + let last_included = sm.generate_snapshot_data(snapshot_dir.clone()).await?; + assert_eq!( + last_included, + LogId { + index: KEYS, + term: 1 + }, + "the exported position must be the last entry applied before the export" + ); let metadata = SnapshotMetadata { last_included: Some(last_included), checksum: Bytes::from(vec![0; 32]), @@ -395,8 +399,7 @@ impl StateMachineTestSuite { // 2. A snapshot is taken in the middle of the writes. let temp_dir = TempDir::new()?; let snapshot_dir = temp_dir.path().join("snapshot"); - sm.generate_snapshot_data(snapshot_dir.clone(), LogId { index: 1, term: 1 }) - .await?; + sm.generate_snapshot_data(snapshot_dir.clone()).await?; stop.store(true, Ordering::Relaxed); writer.await.expect("the writer must not panic"); @@ -488,10 +491,12 @@ impl StateMachineTestSuite { let snapshot_dir = temp_dir.path().join("snapshot"); // Generate snapshot - let last_included = LogId { index: 3, term: 1 }; - state_machine - .generate_snapshot_data(snapshot_dir.clone(), last_included) - .await?; + let last_included = state_machine.generate_snapshot_data(snapshot_dir.clone()).await?; + assert_eq!( + last_included, + LogId { index: 3, term: 1 }, + "the exported position must be the last entry applied before the export" + ); let checksum = crate::file_io::compute_checksum_from_folder_path(&snapshot_dir).await?; // Apply snapshot (simulate receiving from leader) @@ -827,7 +832,11 @@ impl StateMachineTestSuite { let temp_dir = TempDir::new()?; let snapshot_dir = temp_dir.path().join("snapshot"); - sm.generate_snapshot_data(snapshot_dir.clone(), last_included).await?; + let exported = sm.generate_snapshot_data(snapshot_dir.clone()).await?; + assert_eq!( + exported, last_included, + "the exported position must be the last entry applied before the export" + ); let checksum = crate::file_io::compute_checksum_from_folder_path(&snapshot_dir).await?; // Fresh node receiving this snapshot for the first time (current == default), @@ -1639,7 +1648,7 @@ impl StateMachineTestSuite { let export_fut = { let live = live.clone(); let dir = snapshot_dir.clone(); - async move { live.generate_snapshot_data(dir, last_included).await } + async move { live.generate_snapshot_data(dir).await } }; let apply_fut = { let live = live.clone(); @@ -1824,9 +1833,8 @@ mod default_scan_prefix_tests { async fn generate_snapshot_data( &self, _new_snapshot_dir: std::path::PathBuf, - _last_included: LogId, - ) -> Result<(), Error> { - Ok(()) + ) -> Result { + Ok(LogId::default()) } async fn reset(&self) -> Result<(), Error> { @@ -1938,8 +1946,7 @@ mod install_reader_suite_tests { async fn generate_snapshot_data( &self, new_snapshot_dir: PathBuf, - _last_included: LogId, - ) -> Result<(), Error> { + ) -> Result { std::fs::create_dir_all(&new_snapshot_dir)?; // Right: copy the data and read the position under the same lock. let (lines, mut position) = { @@ -1967,7 +1974,7 @@ mod install_reader_suite_tests { new_snapshot_dir.join("position.txt"), format!("{} {}", position.index, position.term), )?; - Ok(()) + Ok(position) } async fn apply_snapshot_from_file( diff --git a/d-engine-core/src/test_utils/mock/mock_raft_builder.rs b/d-engine-core/src/test_utils/mock/mock_raft_builder.rs index 4f2d1f25..5cf0354c 100644 --- a/d-engine-core/src/test_utils/mock/mock_raft_builder.rs +++ b/d-engine-core/src/test_utils/mock/mock_raft_builder.rs @@ -389,7 +389,7 @@ pub fn mock_state_machine() -> MockStateMachine { mock.expect_last_applied().return_const(LogId::default()); mock.expect_apply_snapshot_from_file().returning(|_, _| Ok(())); - mock.expect_generate_snapshot_data().returning(|_, _| Ok(())); + mock.expect_generate_snapshot_data().returning(|_| Ok(LogId::default())); mock } diff --git a/d-engine-server/src/api/embedded_test/shutdown_order_test.rs b/d-engine-server/src/api/embedded_test/shutdown_order_test.rs index c92e9017..b7ec96ae 100644 --- a/d-engine-server/src/api/embedded_test/shutdown_order_test.rs +++ b/d-engine-server/src/api/embedded_test/shutdown_order_test.rs @@ -93,9 +93,8 @@ impl StateMachine for SlowApplyProbe { async fn generate_snapshot_data( &self, new_snapshot_dir: std::path::PathBuf, - last_included: LogId, - ) -> Result<(), Error> { - self.inner.generate_snapshot_data(new_snapshot_dir, last_included).await + ) -> Result { + self.inner.generate_snapshot_data(new_snapshot_dir).await } async fn reset(&self) -> Result<(), Error> { self.inner.reset().await diff --git a/d-engine-server/src/storage/adaptors/file/file_state_machine.rs b/d-engine-server/src/storage/adaptors/file/file_state_machine.rs index d57ea276..fb4e7036 100644 --- a/d-engine-server/src/storage/adaptors/file/file_state_machine.rs +++ b/d-engine-server/src/storage/adaptors/file/file_state_machine.rs @@ -1079,15 +1079,18 @@ impl StateMachine for FileStateMachine { } } } + + // last_applied changes under the same write lock as the data, so a snapshot that + // reads both under the read lock never sees one without the other. + if let Some(log_id) = highest_log_id { + debug!("State machine - updated last_applied: {:?}", log_id); + self.update_last_applied(log_id); + } } // Lock released immediately - no awaits inside! - // PHASE 4: Update last applied index and conditionally checkpoint. + // PHASE 4: Conditionally checkpoint. // WAL (written in PHASE 2) is the primary crash-safety path. // Checkpoint snapshots full data periodically to bound WAL replay time on recovery. - if let Some(log_id) = highest_log_id { - debug!("State machine - updated last_applied: {:?}", log_id); - self.update_last_applied(log_id); - } self.wal_entries_since_checkpoint.fetch_add(chunk_len as u64, Ordering::Relaxed); @@ -1225,8 +1228,7 @@ impl StateMachine for FileStateMachine { async fn generate_snapshot_data( &self, new_snapshot_dir: std::path::PathBuf, - _last_included: LogId, - ) -> Result<(), Error> { + ) -> Result { // Create snapshot directory fs::create_dir_all(&new_snapshot_dir).await?; @@ -1234,9 +1236,12 @@ impl StateMachine for FileStateMachine { let snapshot_path = new_snapshot_dir.join("snapshot.bin"); let mut file = File::create(&snapshot_path).await?; - let data_copy: HashMap = { + // The data and the position are read under one read lock; apply_chunk updates both under + // the write lock, so they describe the same applied entry. + let (data_copy, position): (HashMap, LogId) = { let data = self.data.read(); - data.iter().map(|(k, (v, t))| (k.clone(), (v.clone(), *t))).collect() + let copy = data.iter().map(|(k, (v, t))| (k.clone(), (v.clone(), *t))).collect(); + (copy, self.last_applied()) }; // Batch serialize into a single buffer — eliminates per-entry async yield overhead. @@ -1258,7 +1263,7 @@ impl StateMachine for FileStateMachine { info!("Snapshot generated at {:?}", snapshot_path); - Ok(()) + Ok(position) } async fn reset(&self) -> Result<(), Error> { diff --git a/d-engine-server/src/storage/adaptors/file/file_state_machine_test.rs b/d-engine-server/src/storage/adaptors/file/file_state_machine_test.rs index 22cc144e..e77070a1 100644 --- a/d-engine-server/src/storage/adaptors/file/file_state_machine_test.rs +++ b/d-engine-server/src/storage/adaptors/file/file_state_machine_test.rs @@ -680,11 +680,12 @@ async fn test_file_apply_snapshot_from_file_reports_applied_for_newer_snapshot() (1..=5).map(|i| insert_at(format!("key_{i}").as_bytes(), b"v", i)).collect(); sm.apply_chunk(&entries).await.expect("apply entries"); - let last_included = LogId { term: 1, index: 5 }; let snapshot_dir = dir.path().join("snapshot"); - sm.generate_snapshot_data(snapshot_dir.clone(), last_included) + let last_included = sm + .generate_snapshot_data(snapshot_dir.clone()) .await .expect("generate snapshot"); + assert_eq!(last_included, LogId { term: 1, index: 5 }); sm.reset().await.expect("reset"); assert_eq!(sm.last_applied(), LogId::default()); @@ -722,7 +723,7 @@ async fn test_replay_wal_skips_stale_entries_below_last_applied() { gen_sm.apply_chunk(&newer).await.expect("apply generator entries"); let snapshot_dir = dir.path().join("snapshot"); gen_sm - .generate_snapshot_data(snapshot_dir.clone(), LogId { term: 1, index: 10 }) + .generate_snapshot_data(snapshot_dir.clone()) .await .expect("generate snapshot"); @@ -818,9 +819,7 @@ async fn test_file_sm_snapshot_contains_only_kv_records() { sm.apply_chunk(&[insert_at(b"k", b"v", 1)]).await.unwrap(); let snapshot_dir = dir.path().join("snapshot"); - sm.generate_snapshot_data(snapshot_dir.clone(), LogId { term: 1, index: 1 }) - .await - .unwrap(); + sm.generate_snapshot_data(snapshot_dir.clone()).await.unwrap(); let len = std::fs::metadata(snapshot_dir.join("snapshot.bin")).unwrap().len(); // key_len(8) + "k"(1) + value_len(8) + "v"(1) + term(8) @@ -837,10 +836,7 @@ async fn test_file_sm_install_empty_snapshot_clears_existing_data() { let source = FileStateMachine::new(dir.path().join("source")).await.unwrap(); let snapshot_dir = dir.path().join("snapshot"); - source - .generate_snapshot_data(snapshot_dir.clone(), LogId { term: 1, index: 5 }) - .await - .unwrap(); + source.generate_snapshot_data(snapshot_dir.clone()).await.unwrap(); let checksum = d_engine_core::file_io::compute_checksum_from_folder_path(&snapshot_dir) .await .unwrap(); diff --git a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs index a1cba613..7ca6ec58 100644 --- a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs +++ b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs @@ -199,7 +199,7 @@ impl RocksDBStateMachine { .ok_or_else(|| StorageError::NotServing("state machine stopped".into()).into()) } - fn load_state_machine_metadata(db: &Arc) -> Result<(u64, u64), Error> { + fn load_state_machine_metadata(db: &DB) -> Result<(u64, u64), Error> { let cf = db .cf_handle(STATE_MACHINE_META_CF) .ok_or_else(|| StorageError::DbError("State machine meta CF not found".to_string()))?; @@ -337,7 +337,7 @@ impl RocksDBStateMachine { fn export_frozen_view( db: &DB, snapshot_dir: &std::path::Path, - ) -> Result<(), Error> { + ) -> Result { let name = snapshot_dir .file_name() .ok_or_else(|| StorageError::DbError("snapshot dir has no name".to_string()))? @@ -357,7 +357,7 @@ impl RocksDBStateMachine { db: &DB, checkpoint_dir: &std::path::Path, snapshot_dir: &std::path::Path, - ) -> Result<(), Error> { + ) -> Result { let to_err = |e: rocksdb::Error| StorageError::DbError(e.to_string()); rocksdb::checkpoint::Checkpoint::new(db) .and_then(|c| c.create_checkpoint(checkpoint_dir)) @@ -389,7 +389,9 @@ impl RocksDBStateMachine { .map_err(to_err)?; Self::save_cf_export_metadata(&sm_export, &snapshot_dir.join("sm_metadata.bin"))?; Self::save_cf_export_metadata(&sm_meta_export, &snapshot_dir.join("sm_meta_metadata.bin"))?; - Ok(()) + // The position is read from the frozen copy, the same view the data was exported from. + let (index, term) = Self::load_state_machine_metadata(&frozen)?; + Ok(LogId { index, term }) } /// Serialize `ExportImportFilesMetaData` to a bincode file at `path`. @@ -803,8 +805,7 @@ impl StateMachine for RocksDBStateMachine { async fn generate_snapshot_data( &self, new_snapshot_dir: std::path::PathBuf, - _last_included: LogId, - ) -> Result<(), Error> { + ) -> Result { // Export only SM CFs: compact SST-only export, no Raft log data. // export_column_family() creates only the final subdirectory (e.g. "sm"), so the parent // (new_snapshot_dir) must already exist before calling it. @@ -815,7 +816,7 @@ impl StateMachine for RocksDBStateMachine { // writes (#315). let db = self.load_db()?; // Arc: Send, safe to move into spawn_blocking let dir = new_snapshot_dir.clone(); - tokio::task::spawn_blocking(move || -> Result<(), Error> { + let position = tokio::task::spawn_blocking(move || -> Result { std::fs::create_dir_all(&dir)?; Self::export_frozen_view(&db, &dir) }) @@ -823,7 +824,7 @@ impl StateMachine for RocksDBStateMachine { .map_err(Self::map_snapshot_join_error)??; info!("Snapshot generated at {:?}", new_snapshot_dir); - Ok(()) + Ok(position) } async fn reset(&self) -> Result<(), Error> { diff --git a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine_test.rs b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine_test.rs index afea7192..124fd0ad 100644 --- a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine_test.rs +++ b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine_test.rs @@ -818,12 +818,12 @@ async fn test_apply_snapshot_from_file_reports_applied_for_newer_snapshot() { (1..=5).map(|i| insert_at(format!("key_{i}").as_bytes(), b"v", i)).collect(); state_machine.apply_chunk(&entries).await.expect("apply entries"); - let last_included = LogId { term: 1, index: 5 }; let snapshot_dir = temp_dir.path().join("snapshot"); - state_machine - .generate_snapshot_data(snapshot_dir.clone(), last_included) + let last_included = state_machine + .generate_snapshot_data(snapshot_dir.clone()) .await .expect("generate snapshot"); + assert_eq!(last_included, LogId { term: 1, index: 5 }); // Fall behind, then receive the (now newer-than-current) snapshot. state_machine.reset().await.expect("reset"); @@ -881,7 +881,7 @@ async fn test_snapshot_taken_during_writes_exports_position_equal_to_data() { let snapshot_dir = source_dir.path().join("snapshot-out"); source - .generate_snapshot_data(snapshot_dir.clone(), LogId { index: 1, term: 1 }) + .generate_snapshot_data(snapshot_dir.clone()) .await .expect("generate snapshot"); stop.store(true, std::sync::atomic::Ordering::SeqCst); diff --git a/d-engine-server/src/test_utils/mock/mock_node_builder.rs b/d-engine-server/src/test_utils/mock/mock_node_builder.rs index ebbec971..57d9a12c 100644 --- a/d-engine-server/src/test_utils/mock/mock_node_builder.rs +++ b/d-engine-server/src/test_utils/mock/mock_node_builder.rs @@ -631,7 +631,7 @@ pub(crate) fn mock_state_machine() -> MockStateMachine { mock.expect_last_applied().return_const(LogId::default()); mock.expect_apply_snapshot_from_file().returning(|_, _| Ok(())); - mock.expect_generate_snapshot_data().returning(|_, _| Ok(())); + mock.expect_generate_snapshot_data().returning(|_| Ok(LogId::default())); mock } diff --git a/d-engine-server/tests/common/shared_state_machine.rs b/d-engine-server/tests/common/shared_state_machine.rs index ba333469..840b5844 100644 --- a/d-engine-server/tests/common/shared_state_machine.rs +++ b/d-engine-server/tests/common/shared_state_machine.rs @@ -67,9 +67,8 @@ impl StateMachine for SharedStateMachine { async fn generate_snapshot_data( &self, new_snapshot_dir: std::path::PathBuf, - last_included: LogId, - ) -> Result<(), Error> { - self.0.generate_snapshot_data(new_snapshot_dir, last_included).await + ) -> Result { + self.0.generate_snapshot_data(new_snapshot_dir).await } async fn reset(&self) -> Result<(), Error> { self.0.reset().await diff --git a/d-engine/src/docs/server_guide/customize-state-machine.md b/d-engine/src/docs/server_guide/customize-state-machine.md index b6f5e3f4..9ee90cff 100644 --- a/d-engine/src/docs/server_guide/customize-state-machine.md +++ b/d-engine/src/docs/server_guide/customize-state-machine.md @@ -102,6 +102,7 @@ impl StateMachine for CustomStateMachine { - **Idempotency**: Handle duplicate entries safely—Raft may resend committed entries - **Snapshot Isolation**: `apply_snapshot_from_file()` must atomically replace state - **Snapshot Checksums**: The engine computes the checksum of the directory your `generate_snapshot_data()` writes, and verifies it before it calls `apply_snapshot_from_file()`. You neither compute nor check one +- **Snapshot Position**: `generate_snapshot_data()` returns the position of the last applied entry the exported data contains. Read it from the same consistent view the data comes from (one read transaction, a checkpoint, or a read lock that `apply_chunk` also takes when it updates the position), never from a separate counter: `apply_chunk` keeps running during the export - **Snapshot Install**: The engine decides whether a snapshot is installed. `apply_snapshot_from_file()` is called only for a snapshot newer than `last_applied()`, and the engine checks afterwards that `last_applied()` equals the snapshot boundary - **Restore On Open**: The engine never asks you to restore state. When your state machine is opened, `last_applied()` must return the last entry whose effects are durable (persist it together with the data it describes). You store nothing about snapshots: the engine records the snapshots it holds itself - **Concurrency Control**: Use appropriate locking for state mutations @@ -109,15 +110,15 @@ impl StateMachine for CustomStateMachine { ## 3. StateMachine API Reference -| Method | Purpose | Sync/Async | Criticality | -| ---------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | ----------- | -| `close()` | Final step at shutdown: persist what must survive a restart and release exclusive OS resources (e.g. DB lock file), even if persisting fails; returns the first error. Idempotent. Default `Ok(())`. | Sync | Medium | -| `get()` | Read value by key | Sync | High | -| `apply_chunk()` | Apply committed entries | Async | Critical | -| `last_applied()` | Get last applied index | Sync | High | -| `apply_snapshot_from_file()` | Replace state with snapshot | Async | Critical | -| `generate_snapshot_data()` | Export a snapshot into the given directory (the engine checksums it) | Async | High | -| `reset()` | Reset to initial state | Async | Medium | +| Method | Purpose | Sync/Async | Criticality | +| ---------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | ----------- | +| `close()` | Final step at shutdown: persist what must survive a restart and release exclusive OS resources (e.g. DB lock file), even if persisting fails; returns the first error. Idempotent. Default `Ok(())`. | Sync | Medium | +| `get()` | Read value by key | Sync | High | +| `apply_chunk()` | Apply committed entries | Async | Critical | +| `last_applied()` | Get last applied index | Sync | High | +| `apply_snapshot_from_file()` | Replace state with snapshot | Async | Critical | +| `generate_snapshot_data()` | Export a snapshot into the given directory and return the position of the last entry it contains (the engine checksums it) | Async | High | +| `reset()` | Reset to initial state | Async | Medium | ## 4. Testing Your Implementation @@ -153,7 +154,8 @@ async fn test_custom_state_machine() -> Result<(), Error> { // Test snapshot operations let temp_dir = TempDir::new()?; - sm.generate_snapshot_data(temp_dir.path().to_path_buf(), LogId::new(1, 1)).await?; + let position = sm.generate_snapshot_data(temp_dir.path().to_path_buf()).await?; + assert_eq!(position.index, 1); sm.close()?; Ok(()) diff --git a/examples/sled-cluster/src/sled_state_machine.rs b/examples/sled-cluster/src/sled_state_machine.rs index 4c3b8cb1..f617acf0 100644 --- a/examples/sled-cluster/src/sled_state_machine.rs +++ b/examples/sled-cluster/src/sled_state_machine.rs @@ -219,12 +219,13 @@ impl StateMachine for SledStateMachine { async fn generate_snapshot_data( &self, new_snapshot_dir: PathBuf, - _last_included: LogId, - ) -> Result<()> { - // No internal locking needed: `StateMachineWorker` is the sole caller of - // generate_snapshot_data/apply_snapshot_from_file/apply_chunk, and it processes - // them one at a time from a single queue (#436) — concurrent snapshot - // generation from this trait's methods cannot happen. + ) -> Result { + // Known limits of this example: the tree is iterated while `apply_chunk` may run, so the + // copy is not a point-in-time view, and the position below comes from memory, not from the + // exported data. Reading it before the copy keeps it at or behind the data. A state machine + // that must export a consistent view has to exclude `apply_chunk` while it copies and read + // the position under the same lock (see `StateMachine::generate_snapshot_data`). + let position = self.last_applied(); // 2. Create a new state machine database instance let new_db = init_sled_state_machine_db(&new_snapshot_dir) @@ -260,7 +261,7 @@ impl StateMachine for SledStateMachine { // Make sure flush into disk new_db.flush().map_err(|e| StorageError::DbError(e.to_string()))?; - Ok(()) + Ok(position) } async fn apply_snapshot_from_file( @@ -268,10 +269,8 @@ impl StateMachine for SledStateMachine { metadata: &SnapshotMetadata, decompressed_snapshot_path: PathBuf, ) -> Result<()> { - // No internal locking needed — see generate_snapshot_data. `StateMachineWorker` - // processes ApplyEntries/InstallSnapshot/CaptureLocalSnapshot one at a time from a - // single queue (#436), so this call never races a concurrent snapshot generation - // or another install. + // `StateMachineWorker` never runs this concurrently with `apply_chunk`, a snapshot + // export or another install (#436). // The engine only calls this for a snapshot newer than `last_applied()`. let new_last_included = metadata.last_included.ok_or_else(|| { diff --git a/examples/sled-cluster/src/state_machine/sled_state_machine_test.rs b/examples/sled-cluster/src/state_machine/sled_state_machine_test.rs index a34fa8f0..e9e6bdc5 100644 --- a/examples/sled-cluster/src/state_machine/sled_state_machine_test.rs +++ b/examples/sled-cluster/src/state_machine/sled_state_machine_test.rs @@ -92,7 +92,7 @@ async fn test_snapshot_captures_all_current_data() { sm.apply_chunk(&entries).await.unwrap(); let snap_dir = temp.path().join("snapshot"); - sm.generate_snapshot_data(snap_dir.clone(), LogId { index: 5, term: 1 }) + sm.generate_snapshot_data(snap_dir.clone()) .await .unwrap(); @@ -128,7 +128,7 @@ async fn test_snapshot_does_not_include_post_snapshot_entries() { sm.apply_chunk(&pre_entries).await.unwrap(); let snap_dir = temp.path().join("partial_snapshot"); - sm.generate_snapshot_data(snap_dir.clone(), LogId { index: 1, term: 1 }) + sm.generate_snapshot_data(snap_dir.clone()) .await .unwrap(); @@ -179,7 +179,7 @@ async fn test_snapshot_install_content_survives_restart() { let last_included = LogId { index: 3, term: 1 }; let export_dir = TempDir::new().unwrap(); source_sm - .generate_snapshot_data(export_dir.path().to_path_buf(), last_included) + .generate_snapshot_data(export_dir.path().to_path_buf()) .await .unwrap(); From 6e63696ec22da71f059963bd01cf0d654c8bb94c Mon Sep 17 00:00:00 2001 From: Joshua Chi <112539+JoshuaChi@users.noreply.github.com> Date: Mon, 2 Nov 2026 17:47:51 +0800 Subject: [PATCH 11/19] fix #400: Stop the read path before the state machine is closed; slim close() EmbeddedEngine::stop() aborted the read actor after the state machine worker had closed the state machine, so a read could reach a state machine that was closing: with a close() that takes 300 ms and clients reading meanwhile, over 330,000 reads arrived after close() had started. The read actor is now stopped and awaited first, so the engine calls nothing on a state machine once close() has started. That makes it possible to drop the rule that a closed state machine must answer NotServing to every operation. close() now only has to persist what must survive a restart and release exclusive resources; it may be called more than once and later calls return Ok. The trait documentation says not to rely on close() for crash safety: when shutdown_apply_timeout_ms expires it runs after the node has returned, or not at all if the process exits first. Tests: a shutdown test with a slow close() and continuous reads (fails with the old order); the suite's close test no longer requires NotServing after close, and test_operations_fail_after_close is removed. State machines' own tests of what they do after close are unchanged. --- CHANGELOG.md | 2 +- MIGRATION_GUIDE.md | 5 +- d-engine-core/src/storage/state_machine.rs | 37 ++++++----- .../src/storage/state_machine_test.rs | 62 ++---------------- .../api/embedded_test/shutdown_order_test.rs | 65 +++++++++++++++++++ d-engine-server/src/node/mod.rs | 22 +++---- .../server_guide/customize-state-machine.md | 28 ++++---- 7 files changed, 123 insertions(+), 98 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ef90a7af..7e8f913b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -62,7 +62,7 @@ All notable changes to this project will be documented in this file. - **⚠️ Wire protocol is not compatible with v0.2.4**: `ClientResponse.metadata` moved from field 4 to 5 (field 4 is now `scan_data`); `HandleClientScan` returns `ClientResponse` (the `ScanResponse` message is now `ScanResults`); the `ChunkStatus` values are renumbered with `CHUNK_STATUS_UNSPECIFIED = 0`; `WriteCommand.Insert.ttl_secs` (field 3) is reserved; new RPC `PreVote`. Do not run v0.2.4 and v0.3.0 nodes or clients together. See `MIGRATION_GUIDE.md`. -- **⚠️ `StateMachine::stop()` removed; `StateMachine::close_storage()` renamed to `close()`** (#455): `stop()` had no caller in d-engine. `close()` is now the single, permanent, idempotent end-of-life hook and also persists what must survive a restart (lease state). It returns `Result<(), Error>`: release your resources even if persisting fails, and return the first error; the node logs it at ERROR level. Custom implementations: delete `stop()`, rename `close_storage()` to `close()`, and move any lease persistence from `stop()` into `close()`. See `MIGRATION_GUIDE.md`. +- **⚠️ `StateMachine::stop()` removed; `StateMachine::close_storage()` renamed to `close()`** (#455): `stop()` had no caller in d-engine. `close()` is now the single end-of-life hook, may be called more than once, and also persists what must survive a restart (lease state). The engine stops its read path before calling it and calls nothing on the state machine afterwards, so a state machine no longer has to answer "not serving" once closed; `EmbeddedEngine::stop()` now stops the read actor before the state machine worker. It returns `Result<(), Error>`: release your resources even if persisting fails, and return the first error; the node logs it at ERROR level. Custom implementations: delete `stop()`, rename `close_storage()` to `close()`, and move any lease persistence from `stop()` into `close()`. See `MIGRATION_GUIDE.md`. - **State machine worker lifecycle** (#455): the `NodeBuilder` only assembles the worker; `Node::run()` starts it and stops it. `StateMachineWorker::new` no longer takes a shutdown signal. diff --git a/MIGRATION_GUIDE.md b/MIGRATION_GUIDE.md index 9019e34f..ca696e3a 100644 --- a/MIGRATION_GUIDE.md +++ b/MIGRATION_GUIDE.md @@ -419,6 +419,9 @@ fn apply_chunk(&mut self, entries: &[ApplyEntry]) -> Result<(), ...> `stop()` was never called by d-engine. `close()` is now the one end-of-life hook. d-engine calls it once, as the last step of the state machine worker. When the worker drains cleanly, every committed entry has been applied by then; if `shutdown_apply_timeout_ms` expires first, shutdown continues and committed entries may remain unapplied until the next start. +d-engine stops serving reads before it calls `close()` and calls nothing on the state machine afterwards, so you do not need to +define what a closed state machine does. If the timeout expires, `close()` runs after the node has returned, or not at all if the +process exits first: do not rely on it for crash safety, and persist `last_applied` together with the data it describes. Because you may keep your own `Arc` to the state machine, do not rely on `Drop` alone. `close()` returns an error if persisting fails: release your resources anyway and return the first error. The node logs it at ERROR level. @@ -427,7 +430,7 @@ fails: release your resources anyway and return the first error. The node logs i 1. Delete your `fn stop(...)`. 2. Rename `fn close_storage(&self)` to `fn close(&self) -> Result<(), Error>`. 3. If your `stop()` persisted something that must survive a restart (for example lease state), do it in `close()`. -4. `close()` must be synchronous and idempotent: after it returns, every operation should return a "not serving" error. +4. `close()` must be synchronous and may be called more than once (a later call returns `Ok(())`). You no longer have to make operations return a "not serving" error after it. 5. In config files, rename `[raft.storage]` to `[raft.state_machine]`. ```rust diff --git a/d-engine-core/src/storage/state_machine.rs b/d-engine-core/src/storage/state_machine.rs index aca6e96c..b5470f95 100644 --- a/d-engine-core/src/storage/state_machine.rs +++ b/d-engine-core/src/storage/state_machine.rs @@ -99,10 +99,9 @@ impl ApplyResult { /// /// # `NotServing` /// -/// While a state machine cannot answer reads correctly, `get` and `get_multi` return -/// `StorageError::NotServing`, for example -/// `Err(StorageError::NotServing("installing a snapshot".into()).into())`. This happens in two -/// situations: the state machine is replacing its state from a snapshot, and it has been closed. +/// While a state machine is replacing its state from a snapshot and cannot answer reads +/// correctly, `get` and `get_multi` return `StorageError::NotServing`, for example +/// `Err(StorageError::NotServing("installing a snapshot".into()).into())`. /// /// It means "not now", not "the data is damaged". The engine answers that read through Raft /// instead, from the state machine after the install, so the client still gets a correct answer. @@ -111,20 +110,26 @@ impl ApplyResult { #[cfg_attr(any(test, feature = "__test_support"), automock)] #[async_trait] pub trait StateMachine: Send + Sync + 'static { - /// Permanently closes the state machine: stops serving, persists what must survive a - /// restart (e.g. data, metadata), flushes, and releases exclusive resources - /// (e.g. a DB handle or its lock file). + /// Persists what must survive a restart (e.g. data, metadata), flushes, and releases exclusive + /// resources (e.g. a DB handle or its lock file). /// - /// Called once by the state machine worker as its last act, after everything queued for - /// it has been applied. The engine owns the state machine, but a state machine that shares - /// its storage handle with the developer (a cheap `Clone`) is not dropped when the engine - /// stops, so this must not rely on `Drop`. Must be synchronous and idempotent; afterwards - /// every operation returns `NotServing`. + /// Called once by the state machine worker as its last act, after the engine has stopped its + /// read path. The engine calls nothing on the state machine once `close()` has started, so you + /// do not need to define what a closed state machine does when it is asked to read or write. + /// The engine owns the state machine, but one that shares its storage handle with the + /// developer (a cheap `Clone`) is not dropped when the engine stops, so this must not rely on + /// `Drop`. Must be synchronous and may be called more than once; later calls return `Ok(())`. /// /// When it returns, reopening the same storage restores every applied entry and a /// `last_applied` that is not older than the one before the call, and every exclusive /// resource is released. If persisting fails, still release the resources and return the - /// first error. A second call returns `Ok(())`. + /// first error. + /// + /// Do not rely on `close()` for crash safety. When `shutdown_apply_timeout_ms` expires the + /// node stops waiting for the worker: `close()` then runs after the node has returned, or not + /// at all if the process exits first. A restart must be correct without it, which is why + /// `last_applied` has to be persisted together with the data it describes. + /// /// Default is a no-op, for implementations with nothing to persist or release. fn close(&self) -> Result<(), Error> { Ok(()) @@ -133,8 +138,8 @@ pub trait StateMachine: Send + Sync + 'static { /// Retrieves a value by key from the state machine. /// /// Called concurrently from the read path, so it must be thread-safe and cheap. Return - /// `StorageError::NotServing` while the state machine cannot answer (for example during a - /// snapshot install, or after `close()`): the engine then routes the read through Raft. + /// `StorageError::NotServing` while the state machine cannot answer (during a snapshot + /// install): the engine then routes the read through Raft. fn get( &self, key_buffer: &[u8], @@ -211,7 +216,6 @@ pub trait StateMachine: Send + Sync + 'static { /// Exports the state machine's current state into `new_snapshot_dir` and returns the position /// of the last applied entry that the exported state contains. /// - /// Exports the current state (as of `last_included`) to `new_snapshot_dir`. The framework /// The engine checksums the directory itself, labels the snapshot with the returned position, /// and records the snapshot only once the final archive is durably built, so a crash /// mid-snapshot never leaves a record pointing at a missing archive; the state machine does @@ -242,7 +246,6 @@ pub trait StateMachine: Send + Sync + 'static { /// - hold a read lock while copying the data and reading the position, with `apply_chunk` /// taking the write lock and updating the position under it. /// - /// /// # Concurrency contract (#436) /// May run concurrently with `apply_chunk` (never with `apply_snapshot_from_file` — the two /// are mutually exclusive by construction). Must still export an atomically-pinned view: if diff --git a/d-engine-core/src/storage/state_machine_test.rs b/d-engine-core/src/storage/state_machine_test.rs index 2ea20c5f..d5361d65 100644 --- a/d-engine-core/src/storage/state_machine_test.rs +++ b/d-engine-core/src/storage/state_machine_test.rs @@ -35,7 +35,7 @@ pub trait StateMachineBuilder: Send + Sync { impl StateMachineTestSuite { /// Run all state machine tests pub async fn run_all_tests(builder: B) -> Result<(), Error> { - Self::test_close_is_idempotent_and_stops_serving(builder.build().await?).await?; + Self::test_close_may_be_called_twice(builder.build().await?).await?; Self::test_basic_kv_operations(builder.build().await?).await?; Self::test_apply_chunk_functionality(builder.build().await?).await?; Self::test_cas_operations(builder.build().await?).await?; @@ -52,7 +52,6 @@ impl StateMachineTestSuite { Self::test_batch_survives_reopen(&builder).await?; Self::test_ungraceful_shutdown_recovery(&builder).await?; Self::test_close_persists_without_drop(&builder).await?; - Self::test_operations_fail_after_close(builder.build().await?).await?; Self::test_reset_operation(builder.build().await?).await?; Self::test_get_multi_returns_values_in_key_order(builder.build().await?).await?; @@ -78,24 +77,17 @@ impl StateMachineTestSuite { Ok(()) } - /// Test close lifecycle: a freshly built state machine already serves, and answers - /// `NotServing` once `close`d. `close` is permanent and idempotent (#455), so there is no - /// restart. - pub async fn test_close_is_idempotent_and_stops_serving( + /// Test close lifecycle: a freshly built state machine already serves, and `close` may be + /// called more than once. The engine calls nothing on a state machine after `close`, so what a + /// closed one answers is not part of the contract. + pub async fn test_close_may_be_called_twice( state_machine: Arc ) -> Result<(), Error> { - let is_closed = |sm: &Arc| { - sm.get(b"lifecycle-probe") - .is_err_and(|err| format!("{err:?}").contains("NotServing")) - }; - // Serving as soon as it is built: there is no separate start step - assert!(!is_closed(&state_machine)); + assert!(state_machine.get(b"lifecycle-probe").is_ok()); state_machine.close()?; - assert!(is_closed(&state_machine)); - state_machine.close()?; // idempotent - assert!(is_closed(&state_machine)); + state_machine.close()?; // a second call returns Ok Ok(()) } @@ -1146,46 +1138,6 @@ impl StateMachineTestSuite { Ok(()) } - /// After `close()` every operation must fail with `NotServing`, including writes. - /// - /// Business scenario: the application retains the state machine after the node has shut - /// down. A late read must not answer from state that is no longer maintained, and a late - /// `apply_chunk` must not mutate a state machine that already persisted its final state. - pub async fn test_operations_fail_after_close(sm: Arc) -> Result<(), Error> { - sm.apply_chunk(&[create_insert_entry( - 1, - Bytes::from_static(b"serving_key"), - Bytes::from_static(b"v"), - )]) - .await?; - - sm.close()?; - - let not_serving = |err: Error| format!("{err:?}").contains("NotServing"); - assert!( - not_serving(sm.get(b"serving_key").expect_err("get must fail after close")), - "get after close must return NotServing" - ); - assert!( - not_serving(sm.scan_prefix(b"serving").expect_err("scan_prefix must fail after close")), - "scan_prefix after close must return NotServing" - ); - let late_write = sm - .apply_chunk(&[create_insert_entry( - 2, - Bytes::from_static(b"late_key"), - Bytes::from_static(b"v"), - )]) - .await - .expect_err("apply_chunk must fail after close"); - assert!( - not_serving(late_write), - "apply_chunk after close must return NotServing" - ); - - Ok(()) - } - /// /// This test verifies that the reset operation: /// 1. Clears all data from memory diff --git a/d-engine-server/src/api/embedded_test/shutdown_order_test.rs b/d-engine-server/src/api/embedded_test/shutdown_order_test.rs index b7ec96ae..840b446d 100644 --- a/d-engine-server/src/api/embedded_test/shutdown_order_test.rs +++ b/d-engine-server/src/api/embedded_test/shutdown_order_test.rs @@ -31,6 +31,9 @@ use crate::storage::FileStorageEngine; struct SlowApplyProbe { inner: FileStateMachine, apply_delay: Duration, + /// How long `close()` takes after it has marked the storage closed (a large database + /// flushing). + close_delay: Duration, flags: Arc, } @@ -42,10 +45,20 @@ struct ProbeFlags { storage_closed: AtomicBool, /// Entries whose apply started or finished after `close_storage()` had been called. applied_after_close: AtomicUsize, + /// Reads (`get`, `scan_prefix`) that reached the state machine after `close()` had started. + reads_after_close: AtomicUsize, /// Set when the probe (the state machine) is dropped. dropped: AtomicBool, } +impl SlowApplyProbe { + fn note_read(&self) { + if self.flags.storage_closed.load(Ordering::SeqCst) { + self.flags.reads_after_close.fetch_add(1, Ordering::SeqCst); + } + } +} + impl Drop for SlowApplyProbe { fn drop(&mut self) { self.flags.dropped.store(true, Ordering::SeqCst); @@ -70,6 +83,7 @@ impl StateMachine for SlowApplyProbe { fn close(&self) -> Result<(), Error> { self.flags.storage_closed.store(true, Ordering::SeqCst); + std::thread::sleep(self.close_delay); self.inner.close() } @@ -78,6 +92,7 @@ impl StateMachine for SlowApplyProbe { &self, key_buffer: &[u8], ) -> Result, Error> { + self.note_read(); self.inner.get(key_buffer) } fn last_applied(&self) -> LogId { @@ -103,6 +118,7 @@ impl StateMachine for SlowApplyProbe { &self, prefix: &[u8], ) -> Result { + self.note_read(); self.inner.scan_prefix(prefix) } } @@ -126,6 +142,7 @@ async fn test_stop_closes_state_machine_storage_only_after_in_flight_applies_fin let probe = SlowApplyProbe { inner: FileStateMachine::new(temp_dir.path().join("sm")).await.expect("sm"), apply_delay: Duration::from_millis(300), + close_delay: Duration::ZERO, flags: Arc::clone(&flags), }; @@ -176,6 +193,7 @@ async fn test_stop_drops_state_machine_before_returning() { let probe = SlowApplyProbe { inner: FileStateMachine::new(temp_dir.path().join("sm")).await.expect("sm"), apply_delay: Duration::ZERO, + close_delay: Duration::ZERO, flags: Arc::clone(&flags), }; @@ -200,3 +218,50 @@ async fn test_stop_drops_state_machine_before_returning() { "stop() returned while the state machine was still referenced by an internal holder" ); } + +/// Business scenario: a single-node embedded service is serving reads when its operator stops it. +/// Closing a large state machine takes a while (it flushes everything to disk), and clients keep +/// reading during that time. +/// +/// Purpose: the engine calls nothing on the state machine after `close()` has started, so an +/// implementation need not define what a closed state machine does when asked to read. The read +/// path must therefore be stopped before the state machine is closed. +#[tokio::test] +async fn test_stop_sends_no_reads_to_the_state_machine_once_it_is_closing() { + unsafe { + std::env::remove_var("CONFIG_PATH"); + } + let temp_dir = tempfile::tempdir().expect("temp dir"); + let storage = + Arc::new(FileStorageEngine::new(temp_dir.path().join("storage")).expect("storage")); + let flags = Arc::new(ProbeFlags::default()); + let probe = SlowApplyProbe { + inner: FileStateMachine::new(temp_dir.path().join("sm")).await.expect("sm"), + apply_delay: Duration::ZERO, + close_delay: Duration::from_millis(300), + flags: Arc::clone(&flags), + }; + + let engine = EmbeddedEngine::start_custom(temp_dir.path(), storage, probe, None) + .await + .expect("start engine"); + engine.wait_ready(Duration::from_secs(5)).await.expect("leader"); + let client = engine.client(); + client.put("key", "value").await.expect("put"); + + // Clients keep reading until the engine refuses them. + let reader = { + let client = client.clone(); + tokio::spawn(async move { while client.get_eventual("key").await.is_ok() {} }) + }; + tokio::time::sleep(Duration::from_millis(50)).await; + + engine.stop().await.expect("stop"); + let _ = tokio::time::timeout(Duration::from_secs(2), reader).await; + + assert_eq!( + flags.reads_after_close.load(Ordering::SeqCst), + 0, + "a read reached the state machine after close() had started" + ); +} diff --git a/d-engine-server/src/node/mod.rs b/d-engine-server/src/node/mod.rs index a9603cab..230d7847 100644 --- a/d-engine-server/src/node/mod.rs +++ b/d-engine-server/src/node/mod.rs @@ -212,9 +212,17 @@ where } } - // Shutdown in reverse startup order: stop the sm worker first so its Arc clone is - // dropped before we return, letting the storage backend release any exclusive - // resources it holds. + // Stop the read path first: the state machine worker closes the state machine as its last + // act, and the engine calls nothing on a state machine after `close()` has started. Reads + // that arrive during shutdown fail instead of reaching a state machine that is closing. + let ra_handle = self.read_actor_handle.lock().unwrap().take(); + if let Some(ra_handle) = ra_handle { + ra_handle.abort(); + let _ = ra_handle.await; + } + + // Then stop the sm worker, so its Arc clone is dropped before we return, letting the + // storage backend release any exclusive resources it holds. let timeout = Duration::from_millis(self.node_config.raft.state_machine.shutdown_apply_timeout_ms); if let Err(e) = self.sm_worker.stop_and_join(timeout).await { @@ -223,14 +231,6 @@ where committed entries may be unapplied until restart" ); } - - // Abort the ReadActor last. The worker has already closed the state machine, and the - // ReadActor only serves reads, so there is nothing left to wait for. - let ra_handle = self.read_actor_handle.lock().unwrap().take(); - if let Some(ra_handle) = ra_handle { - ra_handle.abort(); - let _ = ra_handle.await; - } } /// Learner bootstrap: skip cluster ready check, join after warmup. diff --git a/d-engine/src/docs/server_guide/customize-state-machine.md b/d-engine/src/docs/server_guide/customize-state-machine.md index 9ee90cff..e9544ccd 100644 --- a/d-engine/src/docs/server_guide/customize-state-machine.md +++ b/d-engine/src/docs/server_guide/customize-state-machine.md @@ -28,10 +28,12 @@ impl StateMachine for CustomStateMachine { // close() has a default no-op implementation. // Override it if your backend must persist something at shutdown (e.g. data, metadata) or // holds an exclusive OS resource (a lock file, an open DB handle). d-engine calls it once, - // as the worker's last step: after every committed entry has been applied when the worker - // drains cleanly, but shutdown continues without that guarantee if - // `shutdown_apply_timeout_ms` expires. It must be synchronous and idempotent, release its - // resources even if persisting fails, and return the first error. + // as the worker's last step, after it has stopped serving reads: nothing is called on the + // state machine once close() has started. It must be synchronous, release its resources even + // if persisting fails, and return the first error; a second call returns Ok(()). + // Do not rely on it for crash safety: if `shutdown_apply_timeout_ms` expires it runs after the + // node has returned, or not at all if the process exits first. A restart must be correct + // without it, so persist last_applied together with the data it describes. // If you share your storage handle with the application (a cloneable state machine), the // state machine is not dropped when the engine stops, so do not rely on Drop alone. // @@ -110,15 +112,15 @@ impl StateMachine for CustomStateMachine { ## 3. StateMachine API Reference -| Method | Purpose | Sync/Async | Criticality | -| ---------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | ----------- | -| `close()` | Final step at shutdown: persist what must survive a restart and release exclusive OS resources (e.g. DB lock file), even if persisting fails; returns the first error. Idempotent. Default `Ok(())`. | Sync | Medium | -| `get()` | Read value by key | Sync | High | -| `apply_chunk()` | Apply committed entries | Async | Critical | -| `last_applied()` | Get last applied index | Sync | High | -| `apply_snapshot_from_file()` | Replace state with snapshot | Async | Critical | -| `generate_snapshot_data()` | Export a snapshot into the given directory and return the position of the last entry it contains (the engine checksums it) | Async | High | -| `reset()` | Reset to initial state | Async | Medium | +| Method | Purpose | Sync/Async | Criticality | +| ---------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | ----------- | +| `close()` | Last step at shutdown: persist what must survive a restart and release exclusive OS resources (e.g. DB lock file), even if persisting fails; returns the first error. May be called more than once. Nothing is called after it starts. Default `Ok(())`. | Sync | Medium | +| `get()` | Read value by key | Sync | High | +| `apply_chunk()` | Apply committed entries | Async | Critical | +| `last_applied()` | Get last applied index | Sync | High | +| `apply_snapshot_from_file()` | Replace state with snapshot | Async | Critical | +| `generate_snapshot_data()` | Export a snapshot into the given directory and return the position of the last entry it contains (the engine checksums it) | Async | High | +| `reset()` | Reset to initial state | Async | Medium | ## 4. Testing Your Implementation From de0314403a2477f4d7b5b2c843d38f988c13c5cb Mon Sep 17 00:00:00 2001 From: Joshua Chi <112539+JoshuaChi@users.noreply.github.com> Date: Mon, 2 Nov 2026 19:28:20 +0800 Subject: [PATCH 12/19] fix #400: Keep the applied position together with the data across a crash A crash must leave last_applied equal to the last entry whose effects are in the data; otherwise the engine applies entries again that the data already contains, which gives a different result for operations such as compare-and-swap. The suite's crash test was a drop(), which runs Drop and hid this. The suite now has test_restart_after_crash_finds_position_and_data_of_the_same_entry: 200 orders are applied, then a copy of the storage as it is on disk (no close, no Drop) is reopened, and the position must equal the entry the data reached. StateMachineBuilder::build_from_crash_image provides the copy (default None, the test is skipped); copy_storage_dir copies a directory. A correct and a faulty in-memory state machine show that the test can fail. The file state machine failed it: replaying the WAL restored the data but never moved last_applied, then cleared the WAL, so a restart reported the position of the last checkpoint. Replay now advances last_applied to the last replayed record and writes data and position out before the WAL is cleared. The sled example applies a chunk and the new position in one transaction over its data and metadata trees; the example's unused apply_batch is removed. RocksDB passes because its writes skip the WAL and its two column families are flushed together. --- CHANGELOG.md | 6 + .../src/storage/state_machine_test.rs | 177 +++++++++++++++++- .../storage/adaptors/file/file_engine_test.rs | 8 + .../adaptors/file/file_state_machine.rs | 16 ++ .../adaptors/file/file_state_machine_test.rs | 8 + .../rocksdb/rocksdb_state_machine_test.rs | 15 ++ .../rocksdb/rocksdb_unified_engine_test.rs | 14 ++ .../sled-cluster/src/sled_state_machine.rs | 168 +++++++++-------- .../state_machine/sled_state_machine_test.rs | 20 +- 9 files changed, 335 insertions(+), 97 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e8f913b..0e11fe11 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,8 @@ All notable changes to this project will be documented in this file. - **`raft.state_machine.shutdown_apply_timeout_ms`** (#455, default `5000`): how long shutdown waits for the state machine worker to apply the entries already committed. On timeout it logs an error and shuts down anyway; a restart replays what was not applied. +- **Crash-recovery test for state machines** (#400): `StateMachineTestSuite` checks that after a crash (no `close()`, no `Drop`) a restart reports a `last_applied` equal to the last entry whose effects are in the data. `StateMachineBuilder` has a new method `build_from_crash_image`, with a default implementation that returns `None`; implement it to open a copy of your storage as it is on disk, otherwise the crash test is skipped. `copy_storage_dir` copies a directory for that purpose. + ### Fixed - **🛑 Snapshot data corruption — label/data mismatch** (#418): `last_included` in snapshot metadata was computed by subtracting `retained_log_entries` from `last_applied`, but the snapshot data already reflected the full `last_applied` state. Followers installing such snapshots would re-apply entries 91–100 after restoring state through index 100, causing double-application and permanent cluster state divergence. Also fixes a fabricated `LogId` where `last_included.term` was copied from `last_applied.term` instead of queried from the log. Snapshot label is now always truthful: `last_included == last_applied`. @@ -58,6 +60,10 @@ All notable changes to this project will be documented in this file. - **Snapshot checksum ignored files in sub-directories and file boundaries** (#400): see the snapshot checksum entry under Changed. +- **RocksDB snapshots could hold a position ahead of their data** (#400): a snapshot exported while writes continued took `sm` (data) and `sm_meta` (`last_applied`) from the live database one after the other, so a node installing it could report a position newer than its data and skip the entries in between for good. The export now takes a checkpoint of the database and exports both column families from that frozen copy, and the database is opened with `atomic_flush` so the two column families always reach disk together. A snapshot export is slower (about 100 ms on a small database, measured with a test database; measure on your data). The engine also labels every snapshot with the position the state machine reports for its exported data. + +- **File state machine: `last_applied` after crash recovery** (#400): replaying the write-ahead log restored the data but did not move `last_applied`, and then cleared the log, so a restart reported the position of the last checkpoint and the engine applied entries again that the data already contained (a different result for compare-and-swap). Replay now advances `last_applied` to the last replayed record and writes data and position out before the log is cleared. The sled example and d-lmdb now commit the applied position in the same transaction as the data. + ### Changed - **⚠️ Wire protocol is not compatible with v0.2.4**: `ClientResponse.metadata` moved from field 4 to 5 (field 4 is now `scan_data`); `HandleClientScan` returns `ClientResponse` (the `ScanResponse` message is now `ScanResults`); the `ChunkStatus` values are renumbered with `CHUNK_STATUS_UNSPECIFIED = 0`; `WriteCommand.Insert.ttl_secs` (field 3) is reserved; new RPC `PreVote`. Do not run v0.2.4 and v0.3.0 nodes or clients together. See `MIGRATION_GUIDE.md`. diff --git a/d-engine-core/src/storage/state_machine_test.rs b/d-engine-core/src/storage/state_machine_test.rs index d5361d65..8438a97c 100644 --- a/d-engine-core/src/storage/state_machine_test.rs +++ b/d-engine-core/src/storage/state_machine_test.rs @@ -28,10 +28,37 @@ pub trait StateMachineBuilder: Send + Sync { /// Create a new StateMachine instance for testing async fn build(&self) -> Result, Error>; + /// Opens a state machine on a copy of the storage exactly as it is on disk right now, without + /// closing or dropping the live one: what a restart after a crash (the process killed, nothing + /// flushed, no `close()`, no `Drop`) would find. Return `None` if the builder cannot do this; + /// the crash test is then skipped. [`copy_storage_dir`] copies a directory. + async fn build_from_crash_image(&self) -> Result>, Error> { + Ok(None) + } + /// Clean up any resources after testing async fn cleanup(&self) -> Result<(), Error>; } +/// Copies `from` and everything under it to `to`; for implementing +/// [`StateMachineBuilder::build_from_crash_image`] on a directory-based storage. +pub fn copy_storage_dir( + from: &std::path::Path, + to: &std::path::Path, +) -> std::io::Result<()> { + std::fs::create_dir_all(to)?; + for entry in std::fs::read_dir(from)? { + let entry = entry?; + let target = to.join(entry.file_name()); + if entry.file_type()?.is_dir() { + copy_storage_dir(&entry.path(), &target)?; + } else { + std::fs::copy(entry.path(), target)?; + } + } + Ok(()) +} + impl StateMachineTestSuite { /// Run all state machine tests pub async fn run_all_tests(builder: B) -> Result<(), Error> { @@ -44,6 +71,7 @@ impl StateMachineTestSuite { Self::test_snapshot_operations(builder.build().await?).await?; Self::test_reads_during_snapshot_install_see_old_new_or_not_serving(&builder).await?; Self::test_snapshot_taken_during_writes_never_skips_entries_after_restart(&builder).await?; + Self::test_restart_after_crash_finds_position_and_data_of_the_same_entry(&builder).await?; Self::test_drop_flushes_data(&builder).await?; Self::test_drop_persists_last_applied(&builder).await?; @@ -421,6 +449,57 @@ impl StateMachineTestSuite { Ok(()) } + /// A node is killed while orders keep arriving: nothing is flushed, `close()` and `Drop` never + /// run. After the restart the position it reports must be exactly the entry its data has + /// reached. + /// + /// Orders service: entry `n` sets `orders` to `n`, so the value of `orders` is the entry the + /// data has reached. The engine starts replaying the log right after the reported position. A + /// position behind the data (typically: the position is only written at `close()`) makes it + /// replay entries the data already contains, which gives a different result for operations + /// such as compare-and-swap; a position ahead of the data skips entries for good. Both are + /// avoided by committing the position in the same transaction as the data. Storage that loses + /// both (no write-ahead log, nothing flushed yet) reports position 0 and an empty state, + /// which is fine: the engine replays everything. + /// + /// Skipped, with a message, when the builder does not implement + /// [`StateMachineBuilder::build_from_crash_image`]. + pub async fn test_restart_after_crash_finds_position_and_data_of_the_same_entry< + B: StateMachineBuilder, + >( + builder: &B + ) -> Result<(), Error> { + let sm = builder.build().await?; + sm.reset().await?; + for n in 1..=200u64 { + sm.apply_chunk(&[create_insert_entry( + n, + Bytes::from_static(b"orders"), + Bytes::from(n.to_string()), + )]) + .await?; + } + + // The node is killed here: no close(), no drop. + let Some(restarted) = builder.build_from_crash_image().await? else { + eprintln!("crash image not supported by this builder; crash test skipped"); + return Ok(()); + }; + let position = restarted.last_applied().index; + let data: u64 = restarted + .get(b"orders")? + .map(|v| String::from_utf8_lossy(&v).parse().expect("a number")) + .unwrap_or(0); + + assert_eq!( + position, data, + "after a crash the reported position {position} and the data (entry {data}) must \ + describe the same entry: the position has to be committed with the data, not only \ + at close()" + ); + Ok(()) + } + /// Test last applied index detection pub async fn test_last_applied_detection( state_machine: Arc @@ -1858,10 +1937,41 @@ mod install_reader_suite_tests { applied: RwLock, } + impl Durable { + /// What a crash leaves on disk: a copy of what is durable right now. + fn copy(&self) -> Durable { + Durable { + data: RwLock::new(self.data.read().unwrap().clone()), + applied: RwLock::new(*self.applied.read().unwrap()), + } + } + } + struct InMemorySm { durable: Arc, faulty: bool, late_position: bool, + /// Wrong: the position is kept in memory and only made durable by `close()`. + position_on_close_only: bool, + pending_position: RwLock, + } + + impl InMemorySm { + fn new( + durable: Arc, + faulty: bool, + late_position: bool, + position_on_close_only: bool, + ) -> Self { + let pending_position = RwLock::new(*durable.applied.read().unwrap()); + Self { + durable, + faulty, + late_position, + position_on_close_only, + pending_position, + } + } } #[async_trait] @@ -1883,16 +1993,32 @@ mod install_reader_suite_tests { if let Command::Insert { key, value } = &entry.command { data.insert(key.clone(), value.clone()); } - *self.durable.applied.write().unwrap() = LogId { + let at = LogId { index: entry.index, term: entry.term, }; + if self.position_on_close_only { + *self.pending_position.write().unwrap() = at; + } else { + *self.durable.applied.write().unwrap() = at; + } } Ok(chunk.iter().map(|e| ApplyResult::success(e.index)).collect()) } fn last_applied(&self) -> LogId { - *self.durable.applied.read().unwrap() + if self.position_on_close_only { + *self.pending_position.read().unwrap() + } else { + *self.durable.applied.read().unwrap() + } + } + + fn close(&self) -> Result<(), Error> { + if self.position_on_close_only { + *self.durable.applied.write().unwrap() = *self.pending_position.read().unwrap(); + } + Ok(()) } async fn generate_snapshot_data( @@ -1956,12 +2082,14 @@ mod install_reader_suite_tests { *self.durable.data.write().unwrap() = new_data; } *self.durable.applied.write().unwrap() = position; + *self.pending_position.write().unwrap() = position; Ok(()) } async fn reset(&self) -> Result<(), Error> { self.durable.data.write().unwrap().clear(); *self.durable.applied.write().unwrap() = LogId::default(); + *self.pending_position.write().unwrap() = LogId::default(); Ok(()) } } @@ -1971,16 +2099,27 @@ mod install_reader_suite_tests { durable: Arc, faulty: bool, late_position: bool, + position_on_close_only: bool, } #[async_trait] impl StateMachineBuilder for InMemoryBuilder { async fn build(&self) -> Result, Error> { - Ok(Arc::new(InMemorySm { - durable: self.durable.clone(), - faulty: self.faulty, - late_position: self.late_position, - })) + Ok(Arc::new(InMemorySm::new( + self.durable.clone(), + self.faulty, + self.late_position, + self.position_on_close_only, + ))) + } + + async fn build_from_crash_image(&self) -> Result>, Error> { + Ok(Some(Arc::new(InMemorySm::new( + Arc::new(self.durable.copy()), + self.faulty, + self.late_position, + self.position_on_close_only, + )))) } async fn cleanup(&self) -> Result<(), Error> { @@ -2036,4 +2175,28 @@ mod install_reader_suite_tests { ) .await; } + + /// Purpose: a state machine that commits the position with the data passes the crash test. + #[tokio::test] + async fn test_crash_suite_accepts_a_state_machine_that_commits_position_with_data() { + StateMachineTestSuite::test_restart_after_crash_finds_position_and_data_of_the_same_entry( + &InMemoryBuilder::default(), + ) + .await + .expect("position committed with the data must pass"); + } + + /// Purpose: a state machine that only makes the position durable at `close()` is caught: after + /// a crash its data is there but its position is not. + #[tokio::test] + #[should_panic(expected = "must describe the same entry")] + async fn test_crash_suite_rejects_a_state_machine_that_persists_position_only_at_close() { + let _ = StateMachineTestSuite::test_restart_after_crash_finds_position_and_data_of_the_same_entry( + &InMemoryBuilder { + position_on_close_only: true, + ..Default::default() + }, + ) + .await; + } } diff --git a/d-engine-server/src/storage/adaptors/file/file_engine_test.rs b/d-engine-server/src/storage/adaptors/file/file_engine_test.rs index 1d0b20dc..0b8ccfca 100644 --- a/d-engine-server/src/storage/adaptors/file/file_engine_test.rs +++ b/d-engine-server/src/storage/adaptors/file/file_engine_test.rs @@ -5,6 +5,7 @@ use d_engine_core::Error; use d_engine_core::StateMachine; use d_engine_core::state_machine_test::StateMachineBuilder; use d_engine_core::state_machine_test::StateMachineTestSuite; +use d_engine_core::state_machine_test::copy_storage_dir; use d_engine_core::storage_engine_test::StorageEngineBuilder; use d_engine_core::storage_engine_test::StorageEngineTestSuite; use tempfile::TempDir; @@ -49,6 +50,13 @@ impl StateMachineBuilder for FileStateMachineBuilder { )) } + async fn build_from_crash_image(&self) -> Result>, Error> { + let image = self.temp_dir.path().join("state_machine_crash_image"); + let _ = std::fs::remove_dir_all(&image); + copy_storage_dir(&self.temp_dir.path().join("state_machine"), &image)?; + Ok(Some(Arc::new(FileStateMachine::new(image).await?))) + } + async fn cleanup(&self) -> Result<(), Error> { // TempDir automatically cleans up when dropped, so no need for explicit cleanup Ok(()) diff --git a/d-engine-server/src/storage/adaptors/file/file_state_machine.rs b/d-engine-server/src/storage/adaptors/file/file_state_machine.rs index fb4e7036..0bb3d0fb 100644 --- a/d-engine-server/src/storage/adaptors/file/file_state_machine.rs +++ b/d-engine-server/src/storage/adaptors/file/file_state_machine.rs @@ -416,6 +416,10 @@ impl FileStateMachine { let mut pos = 0; let mut operations = Vec::new(); let mut replayed_count = 0; + // The position of the last record replayed: the WAL carries the index and term of every + // entry, and the data replayed below is only consistent with the position once it + // reaches that record. + let mut replayed_up_to: Option = None; let last_applied = self.last_applied_index.load(Ordering::SeqCst); @@ -493,6 +497,7 @@ impl FileStateMachine { } operations.push((op_code, key, value, term)); + replayed_up_to = Some(LogId { index, term }); replayed_count += 1; } @@ -540,6 +545,17 @@ impl FileStateMachine { replayed_count, applied_count, ); + // The replayed entries are now part of the data, so last_applied moves to the last of + // them; otherwise a restart would report the position of the last checkpoint and the + // engine would apply entries again that the data already contains. The WAL is the only + // durable copy of these entries until the next checkpoint, so data and position are + // written out before it is cleared. + if let Some(position) = replayed_up_to { + self.update_last_applied(position); + self.persist_data_async().await?; + self.persist_metadata_async().await?; + } + // Unconditionally clear WAL after replay. load_data() already restored the last // checkpoint; WAL is only the post-checkpoint delta. Even if 0 entries were applied // (e.g. truncated tail only), the WAL is stale. Keeping it would cause infinite diff --git a/d-engine-server/src/storage/adaptors/file/file_state_machine_test.rs b/d-engine-server/src/storage/adaptors/file/file_state_machine_test.rs index e77070a1..fd6600ac 100644 --- a/d-engine-server/src/storage/adaptors/file/file_state_machine_test.rs +++ b/d-engine-server/src/storage/adaptors/file/file_state_machine_test.rs @@ -9,6 +9,7 @@ use d_engine_core::StateMachine; use d_engine_core::file_io::compute_checksum_from_folder_path; use d_engine_core::storage::state_machine_test::StateMachineBuilder; use d_engine_core::storage::state_machine_test::StateMachineTestSuite; +use d_engine_core::storage::state_machine_test::copy_storage_dir; use d_engine_proto::common::LogId; use d_engine_proto::server::storage::SnapshotMetadata; use tempfile::TempDir; @@ -109,6 +110,13 @@ impl StateMachineBuilder for FileStateMachineBuilder { Ok(Arc::new(sm)) } + async fn build_from_crash_image(&self) -> Result>, Error> { + let image = self.temp_dir.path().join("file_sm_crash_image"); + let _ = std::fs::remove_dir_all(&image); + copy_storage_dir(&self.temp_dir.path().join("file_sm"), &image)?; + Ok(Some(Arc::new(FileStateMachine::new(image).await?))) + } + async fn cleanup(&self) -> Result<(), Error> { // TempDir automatically cleans up on drop Ok(()) diff --git a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine_test.rs b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine_test.rs index 124fd0ad..ff71cf95 100644 --- a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine_test.rs +++ b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine_test.rs @@ -8,6 +8,7 @@ use d_engine_core::Command; use d_engine_core::file_io::compute_checksum_from_folder_path; use d_engine_core::state_machine_test::StateMachineBuilder; use d_engine_core::state_machine_test::StateMachineTestSuite; +use d_engine_core::state_machine_test::copy_storage_dir; use d_engine_proto::common::LogId; use d_engine_proto::server::storage::SnapshotMetadata; use tempfile::TempDir; @@ -21,6 +22,8 @@ struct RocksDBStateMachineBuilder { temp_dir: TempDir, // Partner storage kept alive alongside SM; must be dropped before the next build(). storage: Mutex>, + // Partner storage of the crash image; kept alive while the image is in use. + image_storage: Mutex>, } impl RocksDBStateMachineBuilder { @@ -28,6 +31,7 @@ impl RocksDBStateMachineBuilder { Self { temp_dir: TempDir::new().expect("Failed to create temp dir"), storage: Mutex::new(None), + image_storage: Mutex::new(None), } } } @@ -47,8 +51,19 @@ impl StateMachineBuilder for RocksDBStateMachineBuilder { Ok(Arc::new(sm)) } + async fn build_from_crash_image(&self) -> Result>, Error> { + *self.image_storage.lock().unwrap() = None; + let image = self.temp_dir.path().join("rocksdb_sm_crash_image"); + let _ = std::fs::remove_dir_all(&image); + copy_storage_dir(&self.temp_dir.path().join("rocksdb_sm"), &image)?; + let (storage, sm) = RocksDBUnifiedEngine::open(&image)?; + *self.image_storage.lock().unwrap() = Some(storage); + Ok(Some(Arc::new(sm))) + } + async fn cleanup(&self) -> Result<(), Error> { *self.storage.lock().unwrap() = None; + *self.image_storage.lock().unwrap() = None; let delay = if std::env::var("CI").is_ok() { std::time::Duration::from_millis(500) } else { diff --git a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_unified_engine_test.rs b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_unified_engine_test.rs index 263be8f1..817331b2 100644 --- a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_unified_engine_test.rs +++ b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_unified_engine_test.rs @@ -7,6 +7,7 @@ use d_engine_core::LogStore; use d_engine_core::StorageEngine; use d_engine_core::state_machine_test::StateMachineBuilder; use d_engine_core::state_machine_test::StateMachineTestSuite; +use d_engine_core::state_machine_test::copy_storage_dir; use d_engine_core::storage_engine_test::StorageEngineBuilder; use d_engine_core::storage_engine_test::StorageEngineTestSuite; use tempfile::TempDir; @@ -83,6 +84,8 @@ struct UnifiedStateMachineBuilder { temp_dir: TempDir, // Partner storage kept alive alongside SM; must be dropped before the next build(). storage: Mutex>, + // Partner storage of the crash image; kept alive while the image is in use. + image_storage: Mutex>, } impl UnifiedStateMachineBuilder { @@ -90,6 +93,7 @@ impl UnifiedStateMachineBuilder { Self { temp_dir: TempDir::new().expect("temp dir"), storage: Mutex::new(None), + image_storage: Mutex::new(None), } } } @@ -108,8 +112,18 @@ impl StateMachineBuilder for UnifiedStateMachineBuilder { Ok(Arc::new(sm)) } + async fn build_from_crash_image(&self) -> Result>, Error> { + *self.image_storage.lock().unwrap() = None; + let image = TempDir::new()?.keep(); + copy_storage_dir(self.temp_dir.path(), &image)?; + let (storage, sm) = RocksDBUnifiedEngine::open(&image)?; + *self.image_storage.lock().unwrap() = Some(storage); + Ok(Some(Arc::new(sm))) + } + async fn cleanup(&self) -> Result<(), Error> { *self.storage.lock().unwrap() = None; + *self.image_storage.lock().unwrap() = None; let delay = if std::env::var("CI").is_ok() { std::time::Duration::from_millis(500) } else { diff --git a/examples/sled-cluster/src/sled_state_machine.rs b/examples/sled-cluster/src/sled_state_machine.rs index f617acf0..d953947e 100644 --- a/examples/sled-cluster/src/sled_state_machine.rs +++ b/examples/sled-cluster/src/sled_state_machine.rs @@ -22,7 +22,7 @@ use d_engine::StateMachine; use d_engine::StorageError; use d_engine::common::LogId; use d_engine::server_storage::SnapshotMetadata; -use sled::Batch; +use sled::Transactional; use tracing::debug; use tracing::error; use tracing::info; @@ -124,12 +124,9 @@ impl StateMachine for SledStateMachine { chunk: &[ApplyEntry], ) -> Result> { trace!("Applying chunk: {} entries", chunk.len()); - self.db()?; + let db = self.db()?; let mut highest_log_id: Option = None; - let mut batch = Batch::default(); - let mut results = Vec::with_capacity(chunk.len()); - for entry in chunk { if let Some(prev) = &highest_log_id { assert!( @@ -143,75 +140,97 @@ impl StateMachine for SledStateMachine { index: entry.index, term: entry.term, }); + } + let Some(position) = highest_log_id else { + return Ok(Vec::new()); + }; - match &entry.command { - Command::Noop => { - debug!("NOOP at index {}", entry.index); - results.push(ApplyResult::success(entry.index)); - } - Command::Insert { key, value, .. } => { - batch.insert(key.as_ref(), value.as_ref()); - results.push(ApplyResult::success(entry.index)); - } - Command::Delete { key } => { - batch.remove(key.as_ref()); - results.push(ApplyResult::success(entry.index)); - } - Command::CompareAndSwap { - key, - expected, - value: new_value, - } => { - // Flush pending batch before CAS (Sled batch doesn't support atomic CAS) - self.apply_batch(std::mem::take(&mut batch))?; - - // Perform native Sled CAS - let tree = self.current_tree()?; - let old = expected.as_ref().map(|v| v.as_ref()); - - let cas_success = - match tree.compare_and_swap(key.as_ref(), old, Some(new_value.as_ref())) { - Ok(Ok(_)) => true, - Ok(Err(_)) => false, - Err(e) => { - error!("CAS error at index {}: {:?}", entry.index, e); - return Err(StorageError::DbError(e.to_string()).into()); + // The whole chunk and the new position are one transaction over the data tree and the + // metadata tree: after a crash the position is exactly the last entry whose effects are in + // the data. A compare-and-swap reads and writes inside the same transaction, so it sees + // the entries applied before it in this chunk. sled may run the closure more than once on + // a conflict, so it builds its results from scratch every time. + let data_tree = self.current_tree()?; + let meta_tree = db + .open_tree(STATE_MACHINE_META_NAMESPACE) + .map_err(|e| StorageError::DbError(e.to_string()))?; + + let results = (&data_tree, &meta_tree) + .transaction(|(data, meta)| { + let mut results = Vec::with_capacity(chunk.len()); + for entry in chunk { + match &entry.command { + Command::Noop => { + debug!("NOOP at index {}", entry.index); + results.push(ApplyResult::success(entry.index)); + } + Command::Insert { key, value, .. } => { + data.insert(key.as_ref(), value.as_ref())?; + results.push(ApplyResult::success(entry.index)); + } + Command::Delete { key } => { + data.remove(key.as_ref())?; + results.push(ApplyResult::success(entry.index)); + } + Command::CompareAndSwap { + key, + expected, + value: new_value, + } => { + let current = data.get(key.as_ref())?; + let cas_success = match (¤t, expected) { + (None, None) => true, + (Some(current), Some(expected)) => { + current.as_ref() == expected.as_ref() + } + _ => false, + }; + if cas_success { + data.insert(key.as_ref(), new_value.as_ref())?; } - }; - - debug!( - "CAS at index {}: key={:?}, success={}", - entry.index, - String::from_utf8_lossy(key), - cas_success - ); - results.push(if cas_success { - ApplyResult::success(entry.index) - } else { - ApplyResult::failure(entry.index) - }); - } - Command::Batch { ops } => { - ops.iter().for_each(|op| match op { - BatchOp::Insert { key, value } => { - batch.insert(key.as_ref(), value.as_ref()) + debug!( + "CAS at index {}: key={:?}, success={}", + entry.index, + String::from_utf8_lossy(key), + cas_success + ); + results.push(if cas_success { + ApplyResult::success(entry.index) + } else { + ApplyResult::failure(entry.index) + }); } - BatchOp::Delete { key } => { - batch.remove(key.as_ref()); + Command::Batch { ops } => { + for op in ops { + match op { + BatchOp::Insert { key, value } => { + data.insert(key.as_ref(), value.as_ref())?; + } + BatchOp::Delete { key } => { + data.remove(key.as_ref())?; + } + } + } + results.push(ApplyResult::success(entry.index)); } - }); - results.push(ApplyResult::success(entry.index)); + } } - } - } - - // Apply final batch - self.apply_batch(batch)?; - - // Update last applied index - if let Some(log_id) = highest_log_id { - self.update_last_applied(log_id); - } + meta.insert( + STATE_MACHINE_META_KEY_LAST_APPLIED_INDEX.as_bytes(), + safe_kv(position.index), + )?; + meta.insert( + STATE_MACHINE_META_KEY_LAST_APPLIED_TERM.as_bytes(), + safe_kv(position.term), + )?; + Ok(results) + }) + .map_err(|e: sled::transaction::TransactionError| { + error!("state_machine apply_chunk failed: {}", e); + StorageError::DbError(e.to_string()) + })?; + + self.update_last_applied(position); Ok(results) } @@ -442,17 +461,6 @@ impl SledStateMachine { Ok((index, term)) } - pub(super) fn apply_batch( - &self, - batch: Batch, - ) -> Result<()> { - if let Err(e) = self.current_tree()?.apply_batch(batch) { - error!("state_machine apply_batch failed: {}", e); - return Err(StorageError::DbError(e.to_string()).into()); - } - Ok(()) - } - // Dynamically obtain the current Tree (lock-free cache optimization) #[inline] fn current_tree(&self) -> Result { diff --git a/examples/sled-cluster/src/state_machine/sled_state_machine_test.rs b/examples/sled-cluster/src/state_machine/sled_state_machine_test.rs index e9e6bdc5..c8cf9575 100644 --- a/examples/sled-cluster/src/state_machine/sled_state_machine_test.rs +++ b/examples/sled-cluster/src/state_machine/sled_state_machine_test.rs @@ -15,6 +15,7 @@ use d_engine::StateMachine; use d_engine::common::LogId; use d_engine::state_machine_test::StateMachineBuilder; use d_engine::state_machine_test::StateMachineTestSuite; +use d_engine::state_machine_test::copy_storage_dir; use tempfile::TempDir; use crate::SledStateMachine; @@ -45,6 +46,12 @@ impl StateMachineBuilder for SledBuilder { Ok(Arc::new(sm)) } + async fn build_from_crash_image(&self) -> Result>, Error> { + let image = TempDir::new()?.keep(); + copy_storage_dir(self.temp_dir.path(), &image)?; + Ok(Some(Arc::new(SledStateMachine::new(&image, 1)?))) + } + async fn cleanup(&self) -> Result<(), Error> { Ok(()) } @@ -92,9 +99,7 @@ async fn test_snapshot_captures_all_current_data() { sm.apply_chunk(&entries).await.unwrap(); let snap_dir = temp.path().join("snapshot"); - sm.generate_snapshot_data(snap_dir.clone()) - .await - .unwrap(); + sm.generate_snapshot_data(snap_dir.clone()).await.unwrap(); // Open the snapshot as a fresh SM and verify all 5 keys were captured. let snap_sm = SledStateMachine::new(&snap_dir, 1).unwrap(); @@ -128,9 +133,7 @@ async fn test_snapshot_does_not_include_post_snapshot_entries() { sm.apply_chunk(&pre_entries).await.unwrap(); let snap_dir = temp.path().join("partial_snapshot"); - sm.generate_snapshot_data(snap_dir.clone()) - .await - .unwrap(); + sm.generate_snapshot_data(snap_dir.clone()).await.unwrap(); // Write another key AFTER snapshot let post_entries = vec![ApplyEntry { @@ -178,10 +181,7 @@ async fn test_snapshot_install_content_survives_restart() { let last_included = LogId { index: 3, term: 1 }; let export_dir = TempDir::new().unwrap(); - source_sm - .generate_snapshot_data(export_dir.path().to_path_buf()) - .await - .unwrap(); + source_sm.generate_snapshot_data(export_dir.path().to_path_buf()).await.unwrap(); // The engine verifies the checksum before it calls the state machine, which only stores it, // so any 32-byte value will do here. From e53ba63a96286e87870ca7015bf60a111aea9882 Mon Sep 17 00:00:00 2001 From: Joshua Chi <112539+JoshuaChi@users.noreply.github.com> Date: Mon, 2 Nov 2026 20:18:47 +0800 Subject: [PATCH 13/19] refactor #400: Remove StateMachine::reset; builders start tests on empty storage The engine never called StateMachine::reset; only tests used it, to get an empty state machine because StateMachineBuilder::build() reopened the same storage every time. That also let data written by one suite test leak into the next. The method is removed from the trait and from the file, RocksDB and sled state machines (with the helpers only it used), which leaves five required methods. StateMachineBuilder now has build() return a state machine on new, empty storage (a new directory per call) and a new required method reopen() that opens the storage the last build() used, for restart scenarios. The suite uses them in place of reset(): installing a snapshot goes to a freshly built empty node, restart tests call reopen(), and the 200-round atomic-snapshot test installs every round's export into one separate empty state machine, oldest first. test_reset_operation is removed; test_snapshot_operations takes the builder because it needs a second node. Docs: the method is dropped from the customization guide, whose builder example gains reopen(); MIGRATION_GUIDE and CHANGELOG describe both changes, and the MIGRATION note that a closed state machine must answer "not serving" is corrected to match the close() contract. --- CHANGELOG.md | 2 + MIGRATION_GUIDE.md | 2 +- d-engine-core/src/storage/state_machine.rs | 4 - .../src/storage/state_machine_test.rs | 199 ++++++------------ .../api/embedded_test/shutdown_order_test.rs | 3 - .../storage/adaptors/file/file_engine_test.rs | 30 ++- .../adaptors/file/file_state_machine.rs | 66 ------ .../adaptors/file/file_state_machine_test.rs | 25 ++- .../adaptors/rocksdb/rocksdb_state_machine.rs | 44 ---- .../rocksdb/rocksdb_state_machine_test.rs | 40 +++- .../rocksdb/rocksdb_unified_engine_test.rs | 32 ++- .../tests/common/shared_state_machine.rs | 3 - .../server_guide/customize-state-machine.md | 8 +- .../sled-cluster/src/sled_state_machine.rs | 13 -- .../state_machine/sled_state_machine_test.rs | 19 +- 15 files changed, 186 insertions(+), 304 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e11fe11..1350ad9e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -134,6 +134,8 @@ All notable changes to this project will be documented in this file. ### Removed +- **`StateMachine::reset` removed** (#400): the engine never called it; only tests used it. `StateMachineBuilder::build()` now returns a state machine on new, empty storage, and the new required method `reopen()` opens the storage the last `build()` used (a restart), so the suite no longer wipes a state machine before a test. `test_reset_operation` is removed from `StateMachineTestSuite`; `test_snapshot_operations` now takes the builder. See `MIGRATION_GUIDE.md`. + - **TTL and per-key lease** (#464): d-engine now does consensus and replication only. Removed `put_with_ttl`, the `Lease` trait, `TtlLease`, the `[raft.state_machine.lease]` config section and `StateMachine::lease_background_cleanup`. `Command::Insert` no longer has `ttl_secs`. In the proto, `WriteCommand.Insert.ttl_secs` (field 3) is removed and the number is reserved. Key expiry is now the job of the layer above d-engine. The leader read lease (`LeaseRead`) is a different feature and is unchanged. - **File state machine storage format** (#464): The WAL record no longer has an `expire_at` slot, and `snapshot.bin` no longer ends with a TTL section. See [MIGRATION_GUIDE.md](./MIGRATION_GUIDE.md). diff --git a/MIGRATION_GUIDE.md b/MIGRATION_GUIDE.md index ca696e3a..f262ea00 100644 --- a/MIGRATION_GUIDE.md +++ b/MIGRATION_GUIDE.md @@ -10,7 +10,7 @@ - Remove calls to `put_with_ttl` and the `ttl_secs` field of `Command::Insert`. Implement key expiry in the layer above d-engine. - Delete the `[raft.state_machine.lease]` section from your config files. -- Custom state machines: remove `lease_background_cleanup`, `start`, `is_running`, `len` and `is_empty` from your `StateMachine` impl; the trait no longer declares them. `start` was called once when the node booted. Do that setup in your constructor (a state machine must be ready to serve as soon as it is built). The engine no longer asks whether the state machine is running: a state machine that cannot serve a read, for example while it installs a snapshot or after `close()`, must return `StorageError::NotServing` from `get` and `get_multi`. If you need an entry count, keep `len` as a plain method on your own type. +- Custom state machines: remove `lease_background_cleanup`, `start`, `is_running`, `len`, `is_empty` and `reset` from your `StateMachine` impl; the trait no longer declares them. `start` was called once when the node booted. Do that setup in your constructor (a state machine must be ready to serve as soon as it is built). The engine no longer asks whether the state machine is running: a state machine that cannot serve a read while it installs a snapshot must return `StorageError::NotServing` from `get` and `get_multi`. If you need an entry count, keep `len` as a plain method on your own type. The engine never called `reset`; if you used it in your own tests, wipe your storage directory or build a new state machine on an empty one instead. `StateMachineBuilder` (test support) now has `build()` return a state machine on new, empty storage and a new required method `reopen()` for restarts. - Proto: `WriteCommand.Insert.ttl_secs` (field 3) is removed. Regenerate your client code. - The file state machine storage format changed. Start with a fresh data directory; old files are not supported. - RocksDB state machine: keys written with `put_with_ttl` under v0.2.x are kept forever after the upgrade. v0.3.0 neither reads the old `ttl_state` entry nor deletes expired keys. Before upgrading, either delete the keys you no longer want, or record their deadlines and have your own expiry layer delete them. Or start with a fresh data directory. diff --git a/d-engine-core/src/storage/state_machine.rs b/d-engine-core/src/storage/state_machine.rs index b5470f95..ae37ac25 100644 --- a/d-engine-core/src/storage/state_machine.rs +++ b/d-engine-core/src/storage/state_machine.rs @@ -257,10 +257,6 @@ pub trait StateMachine: Send + Sync + 'static { new_snapshot_dir: std::path::PathBuf, ) -> Result; - /// Resets the state machine to its initial state. - /// Async operation as it may involve cleaning up files and data. - async fn reset(&self) -> Result<(), Error>; - /// Returns all `(key, value)` pairs whose key starts with `prefix`, plus /// the `last_applied_index` at the time of the scan. /// diff --git a/d-engine-core/src/storage/state_machine_test.rs b/d-engine-core/src/storage/state_machine_test.rs index 8438a97c..7277f0cb 100644 --- a/d-engine-core/src/storage/state_machine_test.rs +++ b/d-engine-core/src/storage/state_machine_test.rs @@ -25,13 +25,20 @@ pub struct StateMachineTestSuite; /// Builder trait for creating StateMachine instances for testing #[async_trait] pub trait StateMachineBuilder: Send + Sync { - /// Create a new StateMachine instance for testing + /// Creates a state machine on new, empty storage. Every call starts from nothing: whatever an + /// earlier call wrote is not visible, so a test never has to wipe a state machine first. async fn build(&self) -> Result, Error>; - /// Opens a state machine on a copy of the storage exactly as it is on disk right now, without - /// closing or dropping the live one: what a restart after a crash (the process killed, nothing - /// flushed, no `close()`, no `Drop`) would find. Return `None` if the builder cannot do this; - /// the crash test is then skipped. [`copy_storage_dir`] copies a directory. + /// Opens a state machine on the storage the last `build()` or `reopen()` used, with whatever + /// that storage holds now: a restart. The instance returned by that call has been closed or + /// dropped first. + async fn reopen(&self) -> Result, Error>; + + /// Opens a state machine on a copy of the storage the last `build()` or `reopen()` used, + /// exactly as it is on disk right now, without closing or dropping the live one: what a restart + /// after a crash (the process killed, nothing flushed, no `close()`, no `Drop`) would find. + /// Return `None` if the builder cannot do this; the crash test is then skipped. + /// [`copy_storage_dir`] copies a directory. async fn build_from_crash_image(&self) -> Result>, Error> { Ok(None) } @@ -68,7 +75,7 @@ impl StateMachineTestSuite { Self::test_cas_operations(builder.build().await?).await?; Self::test_batch_operations(builder.build().await?).await?; Self::test_last_applied_detection(builder.build().await?).await?; - Self::test_snapshot_operations(builder.build().await?).await?; + Self::test_snapshot_operations(&builder).await?; Self::test_reads_during_snapshot_install_see_old_new_or_not_serving(&builder).await?; Self::test_snapshot_taken_during_writes_never_skips_entries_after_restart(&builder).await?; Self::test_restart_after_crash_finds_position_and_data_of_the_same_entry(&builder).await?; @@ -80,7 +87,6 @@ impl StateMachineTestSuite { Self::test_batch_survives_reopen(&builder).await?; Self::test_ungraceful_shutdown_recovery(&builder).await?; Self::test_close_persists_without_drop(&builder).await?; - Self::test_reset_operation(builder.build().await?).await?; Self::test_get_multi_returns_values_in_key_order(builder.build().await?).await?; Self::test_get_multi_absent_keys_return_none(builder.build().await?).await?; @@ -290,11 +296,12 @@ impl StateMachineTestSuite { }; // 1. A snapshot of the state in which every key holds the new value. - let sm = builder.build().await?; - sm.apply_chunk(&entries(1..=KEYS, b"new")).await?; + let source = builder.build().await?; + source.apply_chunk(&entries(1..=KEYS, b"new")).await?; let temp_dir = TempDir::new()?; let snapshot_dir = temp_dir.path().join("snapshot"); - let last_included = sm.generate_snapshot_data(snapshot_dir.clone()).await?; + let last_included = source.generate_snapshot_data(snapshot_dir.clone()).await?; + drop(source); assert_eq!( last_included, LogId { @@ -309,7 +316,7 @@ impl StateMachineTestSuite { }; // 2. The node is behind: it only has the old values for the first keys. - sm.reset().await?; + let sm = builder.build().await?; sm.apply_chunk(&entries(1..=READ_KEYS, b"old")).await?; // 3. Readers keep reading while the snapshot is installed. @@ -396,7 +403,6 @@ impl StateMachineTestSuite { for attempt in 0..3 { // 1. Orders keep arriving: entry n sets `orders` to n, one entry at a time. let sm = builder.build().await?; - sm.reset().await?; let stop = Arc::new(AtomicBool::new(false)); let writer = { let (sm, stop) = (sm.clone(), stop.clone()); @@ -424,15 +430,16 @@ impl StateMachineTestSuite { writer.await.expect("the writer must not panic"); // 3. A lagging node (an empty one) installs it and restarts. - sm.reset().await?; + drop(sm); + let lagging = builder.build().await?; let metadata = SnapshotMetadata { last_included: Some(LogId { index: 1, term: 1 }), checksum: Bytes::from(vec![0; 32]), }; - sm.apply_snapshot_from_file(&metadata, snapshot_dir).await?; - sm.close()?; - drop(sm); - let restarted = builder.build().await?; + lagging.apply_snapshot_from_file(&metadata, snapshot_dir).await?; + lagging.close()?; + drop(lagging); + let restarted = builder.reopen().await?; let position = restarted.last_applied().index; let data: u64 = restarted @@ -470,7 +477,6 @@ impl StateMachineTestSuite { builder: &B ) -> Result<(), Error> { let sm = builder.build().await?; - sm.reset().await?; for n in 1..=200u64 { sm.apply_chunk(&[create_insert_entry( n, @@ -504,7 +510,6 @@ impl StateMachineTestSuite { pub async fn test_last_applied_detection( state_machine: Arc ) -> Result<(), Error> { - assert!(state_machine.reset().await.is_ok()); // Initial state assert_eq!(state_machine.last_applied(), LogId { index: 0, term: 0 }); @@ -534,9 +539,11 @@ impl StateMachineTestSuite { } /// Test snapshot operations - pub async fn test_snapshot_operations( - state_machine: Arc + pub async fn test_snapshot_operations( + builder: &B ) -> Result<(), Error> { + let state_machine = builder.build().await?; + // Add some test data let entries = vec![ create_insert_entry( @@ -577,8 +584,9 @@ impl StateMachineTestSuite { checksum: Bytes::from(checksum.to_vec()), }; - //Reset State Machine to make sure it is fresh - state_machine.reset().await?; + // The node that receives the snapshot starts empty. + drop(state_machine); + let state_machine = builder.build().await?; assert_eq!(state_machine.get(b"key1")?, None); assert_eq!(state_machine.get(b"key2")?, None); assert_eq!(state_machine.get(b"key3")?, None); @@ -767,7 +775,7 @@ impl StateMachineTestSuite { tokio::time::sleep(tokio::time::Duration::from_millis(20)).await; // Step 2: Reopen state machine from same path - let sm2 = builder.build().await?; + let sm2 = builder.reopen().await?; // Step 3: Verify data survived let value1 = sm2.get(b"drop_test_key1")?; @@ -841,7 +849,7 @@ impl StateMachineTestSuite { tokio::time::sleep(tokio::time::Duration::from_millis(50)).await; // Step 2: Reopen - let sm2 = builder.build().await?; + let sm2 = builder.reopen().await?; // Step 3: Critical assertion - last_applied must be persisted by Drop assert_eq!( @@ -912,7 +920,8 @@ impl StateMachineTestSuite { // Fresh node receiving this snapshot for the first time (current == default), // so the install is strictly "Greater" — eligible, not a stale/duplicate no-op. - sm.reset().await?; + drop(sm); + let sm = builder.build().await?; assert_eq!(sm.last_applied(), LogId::default()); let metadata = SnapshotMetadata { @@ -932,7 +941,7 @@ impl StateMachineTestSuite { tokio::time::sleep(Duration::from_millis(50)).await; - let sm2 = builder.build().await?; + let sm2 = builder.reopen().await?; assert_eq!( sm2.last_applied(), @@ -996,7 +1005,7 @@ impl StateMachineTestSuite { } // Step 2: Reopen and verify - let sm2 = builder.build().await?; + let sm2 = builder.reopen().await?; // Verify all data assert_eq!( @@ -1059,7 +1068,7 @@ impl StateMachineTestSuite { tokio::time::sleep(Duration::from_millis(50)).await; // Step 2: reopen from same path — WAL must replay correctly - let sm2 = builder.build().await?; + let sm2 = builder.reopen().await?; assert_eq!( sm2.get(b"bk1")?, @@ -1133,7 +1142,7 @@ impl StateMachineTestSuite { tokio::time::sleep(tokio::time::Duration::from_millis(50)).await; // Step 2: Simulate restart - create new instance from same path - let sm2 = builder.build().await?; + let sm2 = builder.reopen().await?; // Step 3: Verify crash recovery - both data and metadata let value1 = sm2.get(b"crash_key1")?; @@ -1199,7 +1208,7 @@ impl StateMachineTestSuite { sm.close()?; // `sm` stays alive until the end of the test: Drop must not be what saves the state. - let sm2 = builder.build().await?; + let sm2 = builder.reopen().await?; assert_eq!( sm2.last_applied().index, @@ -1217,91 +1226,6 @@ impl StateMachineTestSuite { Ok(()) } - /// - /// This test verifies that the reset operation: - /// 1. Clears all data from memory - /// 2. Resets Raft state to initial values - /// 3. Clears all persisted files - /// 4. Maintains operational state (running status) - pub async fn test_reset_operation(state_machine: Arc) -> Result<(), Error> { - // Add test data - let entries = vec![ - create_insert_entry( - 1, - Bytes::from(b"key1".to_vec()), - Bytes::from(b"value1".to_vec()), - ), - create_insert_entry( - 2, - Bytes::from(b"key2".to_vec()), - Bytes::from(b"value2".to_vec()), - ), - create_insert_entry( - 3, - Bytes::from(b"key3".to_vec()), - Bytes::from(b"value3".to_vec()), - ), - ]; - state_machine.apply_chunk(&entries).await?; - - // Verify data exists - assert_eq!( - state_machine.get(b"key1")?, - Some(Bytes::from(b"value1".to_vec())) - ); - assert_eq!( - state_machine.get(b"key2")?, - Some(Bytes::from(b"value2".to_vec())) - ); - assert_eq!( - state_machine.get(b"key3")?, - Some(Bytes::from(b"value3".to_vec())) - ); - assert_eq!(state_machine.last_applied(), LogId { index: 3, term: 1 }); - // The test focuses on verifying reset() clears everything, not initial state - - // Perform reset - state_machine.reset().await?; - - // Verify all data is cleared - assert!(state_machine.get(b"key1")?.is_none()); - assert!(state_machine.get(b"key2")?.is_none()); - assert!(state_machine.get(b"key3")?.is_none()); - - // Verify Raft state is reset - assert_eq!(state_machine.last_applied(), LogId { index: 0, term: 0 }); - - // Verify operational state is maintained: reads above succeeded, so it still serves - - // Test that we can add new data after reset - let new_entries = vec![ - create_insert_entry( - 1, - Bytes::from(b"new_key1".to_vec()), - Bytes::from(b"new_value1".to_vec()), - ), - create_insert_entry( - 2, - Bytes::from(b"new_key2".to_vec()), - Bytes::from(b"new_value2".to_vec()), - ), - ]; - state_machine.apply_chunk(&new_entries).await?; - - // Verify new data exists - assert_eq!( - state_machine.get(b"new_key1")?, - Some(Bytes::from(b"new_value1".to_vec())) - ); - assert_eq!( - state_machine.get(b"new_key2")?, - Some(Bytes::from(b"new_value2".to_vec())) - ); - assert_eq!(state_machine.last_applied(), LogId { index: 2, term: 1 }); - - Ok(()) - } - // ── get_multi tests ─────────────────────────────────────────────────────── /// Verify that get_multi preserves input key order in its output. @@ -1662,7 +1586,7 @@ impl StateMachineTestSuite { // Racing phase: keep every round's exported directory around (don't verify yet) — // `StateMachineBuilder` implementations (e.g. RocksDB) only support one live handle - // per path, so verifying via reset+reload here would clobber `live` mid-race. + // per path, so verifying on `live` here would clobber it mid-race. let mut rounds = Vec::with_capacity(200); let mut next_index = 3u64; for round_num in 0u32..200 { @@ -1714,18 +1638,20 @@ impl StateMachineTestSuite { prev_version = new_version; } - // Verification phase: now that racing is done, reuse `live` sequentially - // (reset + reload) to inspect what each round's export actually captured. + // Verification phase: now that racing is done, install every round's export, oldest first, + // into a separate empty state machine and inspect what the export actually captured. Each + // snapshot is newer than the previous one and replaces the whole state, so the rounds do + // not affect each other. + let target = builder.build().await?; for r in &rounds { - live.reset().await?; let metadata = SnapshotMetadata { last_included: Some(r.last_included), checksum: r.checksum.clone(), }; - live.apply_snapshot_from_file(&metadata, r.snapshot_dir.clone()).await?; + target.apply_snapshot_from_file(&metadata, r.snapshot_dir.clone()).await?; - let got_addr = live.get(&addr_key)?; - let got_version = live.get(&version_key)?; + let got_addr = target.get(&addr_key)?; + let got_version = target.get(&version_key)?; let matches_before = got_addr.as_ref() == Some(&r.before.0) && got_version.as_ref() == Some(&r.before.1); @@ -1868,9 +1794,6 @@ mod default_scan_prefix_tests { Ok(LogId::default()) } - async fn reset(&self) -> Result<(), Error> { - Ok(()) - } // scan_prefix intentionally omitted → exercises the trait default implementation } @@ -2085,18 +2008,11 @@ mod install_reader_suite_tests { *self.pending_position.write().unwrap() = position; Ok(()) } - - async fn reset(&self) -> Result<(), Error> { - self.durable.data.write().unwrap().clear(); - *self.durable.applied.write().unwrap() = LogId::default(); - *self.pending_position.write().unwrap() = LogId::default(); - Ok(()) - } } #[derive(Default)] struct InMemoryBuilder { - durable: Arc, + durable: std::sync::Mutex>, faulty: bool, late_position: bool, position_on_close_only: bool, @@ -2105,8 +2021,19 @@ mod install_reader_suite_tests { #[async_trait] impl StateMachineBuilder for InMemoryBuilder { async fn build(&self) -> Result, Error> { + let fresh = Arc::new(Durable::default()); + *self.durable.lock().unwrap() = fresh.clone(); + Ok(Arc::new(InMemorySm::new( + fresh, + self.faulty, + self.late_position, + self.position_on_close_only, + ))) + } + + async fn reopen(&self) -> Result, Error> { Ok(Arc::new(InMemorySm::new( - self.durable.clone(), + self.durable.lock().unwrap().clone(), self.faulty, self.late_position, self.position_on_close_only, @@ -2115,7 +2042,7 @@ mod install_reader_suite_tests { async fn build_from_crash_image(&self) -> Result>, Error> { Ok(Some(Arc::new(InMemorySm::new( - Arc::new(self.durable.copy()), + Arc::new(self.durable.lock().unwrap().copy()), self.faulty, self.late_position, self.position_on_close_only, diff --git a/d-engine-server/src/api/embedded_test/shutdown_order_test.rs b/d-engine-server/src/api/embedded_test/shutdown_order_test.rs index 840b446d..d5e3b67f 100644 --- a/d-engine-server/src/api/embedded_test/shutdown_order_test.rs +++ b/d-engine-server/src/api/embedded_test/shutdown_order_test.rs @@ -111,9 +111,6 @@ impl StateMachine for SlowApplyProbe { ) -> Result { self.inner.generate_snapshot_data(new_snapshot_dir).await } - async fn reset(&self) -> Result<(), Error> { - self.inner.reset().await - } fn scan_prefix( &self, prefix: &[u8], diff --git a/d-engine-server/src/storage/adaptors/file/file_engine_test.rs b/d-engine-server/src/storage/adaptors/file/file_engine_test.rs index 0b8ccfca..4e433b4b 100644 --- a/d-engine-server/src/storage/adaptors/file/file_engine_test.rs +++ b/d-engine-server/src/storage/adaptors/file/file_engine_test.rs @@ -37,14 +37,32 @@ impl StorageEngineBuilder for FileStorageEngineBuilder { struct FileStateMachineBuilder { temp_dir: TempDir, // Keep the temp dir alive for the duration of the test + // How many times `build()` ran: every build gets its own directory. + builds: std::sync::atomic::AtomicUsize, +} + +impl FileStateMachineBuilder { + /// The directory the last `build()` used. + fn current_dir(&self) -> std::path::PathBuf { + let n = self.builds.load(std::sync::atomic::Ordering::SeqCst); + self.temp_dir.path().join(format!("state_machine_{n}")) + } } #[async_trait] impl StateMachineBuilder for FileStateMachineBuilder { async fn build(&self) -> Result, Error> { - let state_machine_path = self.temp_dir.path().join("state_machine"); + self.builds.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Ok(Arc::new( + FileStateMachine::new(self.current_dir()) + .await + .expect("Expect file init successfully"), + )) + } + + async fn reopen(&self) -> Result, Error> { Ok(Arc::new( - FileStateMachine::new(state_machine_path) + FileStateMachine::new(self.current_dir()) .await .expect("Expect file init successfully"), )) @@ -53,7 +71,7 @@ impl StateMachineBuilder for FileStateMachineBuilder { async fn build_from_crash_image(&self) -> Result>, Error> { let image = self.temp_dir.path().join("state_machine_crash_image"); let _ = std::fs::remove_dir_all(&image); - copy_storage_dir(&self.temp_dir.path().join("state_machine"), &image)?; + copy_storage_dir(&self.current_dir(), &image)?; Ok(Some(Arc::new(FileStateMachine::new(image).await?))) } @@ -74,5 +92,9 @@ async fn test_file_storage_engine() -> Result<(), Error> { #[traced_test] async fn test_file_state_machine() -> Result<(), Error> { let temp_dir = TempDir::new()?; - StateMachineTestSuite::run_all_tests(FileStateMachineBuilder { temp_dir }).await + StateMachineTestSuite::run_all_tests(FileStateMachineBuilder { + temp_dir, + builds: std::sync::atomic::AtomicUsize::new(0), + }) + .await } diff --git a/d-engine-server/src/storage/adaptors/file/file_state_machine.rs b/d-engine-server/src/storage/adaptors/file/file_state_machine.rs index 0bb3d0fb..d6f67da6 100644 --- a/d-engine-server/src/storage/adaptors/file/file_state_machine.rs +++ b/d-engine-server/src/storage/adaptors/file/file_state_machine.rs @@ -765,68 +765,6 @@ impl FileStateMachine { Ok(()) } - /// Resets the state machine to its initial empty state - /// - /// This method: - /// 1. Clears all in-memory data - /// 2. Resets Raft state to initial values - /// 3. Clears all persisted files - /// 4. Maintains operational state (running status, node ID) - pub async fn reset(&self) -> Result<(), Error> { - info!("Resetting state machine"); - - // Clear in-memory data - { - let mut data = self.data.write(); - data.clear(); - } - - // Reset Raft state - self.last_applied_index.store(0, Ordering::SeqCst); - self.last_applied_term.store(0, Ordering::SeqCst); - - // Clear all persisted files - self.clear_data_file().await?; - self.clear_metadata_file().await?; - self.clear_wal_async().await?; - - info!("State machine reset completed"); - Ok(()) - } - - /// Clears the data file - async fn clear_data_file(&self) -> Result<(), Error> { - let data_path = self.data_dir.join("state.data"); - let mut file = OpenOptions::new() - .write(true) - .create(true) - .truncate(true) - .open(data_path) - .await?; - - file.set_len(0).await?; - file.flush().await?; - Ok(()) - } - - /// Clears the metadata file - async fn clear_metadata_file(&self) -> Result<(), Error> { - let metadata_path = self.data_dir.join("metadata.bin"); - let mut file = OpenOptions::new() - .write(true) - .create(true) - .truncate(true) - .open(metadata_path) - .await?; - - // Write default values (0 for both index and term) - file.write_all(&0u64.to_be_bytes()).await?; - file.write_all(&0u64.to_be_bytes()).await?; - file.flush().await?; - - Ok(()) - } - /// Reads are served only while running. After `close()` the state machine may still be /// reachable through a handle the developer shares, and it must not answer from state that /// is no longer maintained. @@ -1282,10 +1220,6 @@ impl StateMachine for FileStateMachine { Ok(position) } - async fn reset(&self) -> Result<(), Error> { - self.reset().await - } - fn scan_prefix( &self, prefix: &[u8], diff --git a/d-engine-server/src/storage/adaptors/file/file_state_machine_test.rs b/d-engine-server/src/storage/adaptors/file/file_state_machine_test.rs index fd6600ac..f3e9bf71 100644 --- a/d-engine-server/src/storage/adaptors/file/file_state_machine_test.rs +++ b/d-engine-server/src/storage/adaptors/file/file_state_machine_test.rs @@ -91,29 +91,40 @@ async fn test_file_sm_close_twice_is_safe() { /// Builder for FileStateMachine test instances struct FileStateMachineBuilder { temp_dir: TempDir, + // How many times `build()` ran: every build gets its own directory. + builds: std::sync::atomic::AtomicUsize, } impl FileStateMachineBuilder { fn new() -> Self { Self { temp_dir: TempDir::new().expect("Failed to create temp dir"), + builds: std::sync::atomic::AtomicUsize::new(0), } } + + /// The directory the last `build()` used. + fn current_dir(&self) -> std::path::PathBuf { + let n = self.builds.load(std::sync::atomic::Ordering::SeqCst); + self.temp_dir.path().join(format!("file_sm_{n}")) + } } #[async_trait] impl StateMachineBuilder for FileStateMachineBuilder { async fn build(&self) -> Result, Error> { - // Use fixed path to support restart recovery testing - let path = self.temp_dir.path().to_path_buf().join("file_sm"); - let sm = FileStateMachine::new(path).await?; - Ok(Arc::new(sm)) + self.builds.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Ok(Arc::new(FileStateMachine::new(self.current_dir()).await?)) + } + + async fn reopen(&self) -> Result, Error> { + Ok(Arc::new(FileStateMachine::new(self.current_dir()).await?)) } async fn build_from_crash_image(&self) -> Result>, Error> { let image = self.temp_dir.path().join("file_sm_crash_image"); let _ = std::fs::remove_dir_all(&image); - copy_storage_dir(&self.temp_dir.path().join("file_sm"), &image)?; + copy_storage_dir(&self.current_dir(), &image)?; Ok(Some(Arc::new(FileStateMachine::new(image).await?))) } @@ -695,7 +706,9 @@ async fn test_file_apply_snapshot_from_file_reports_applied_for_newer_snapshot() .expect("generate snapshot"); assert_eq!(last_included, LogId { term: 1, index: 5 }); - sm.reset().await.expect("reset"); + // The node that receives the snapshot starts empty. + drop(sm); + let sm = FileStateMachine::new(dir.path().join("sm_behind")).await.expect("create sm"); assert_eq!(sm.last_applied(), LogId::default()); let checksum = compute_checksum_from_folder_path(&snapshot_dir) diff --git a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs index 7ca6ec58..d451e931 100644 --- a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs +++ b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs @@ -27,7 +27,6 @@ use rocksdb::IteratorMode; use rocksdb::LiveFile; use rocksdb::Options; use rocksdb::ReadOptions; -use rocksdb::WriteBatch; use rocksdb::WriteBatchWithIndex; use rocksdb::WriteOptions; use serde::Deserialize; @@ -227,23 +226,6 @@ impl RocksDBStateMachine { Ok((index, term)) } - fn persist_state_machine_metadata(&self) -> Result<(), Error> { - let index = self.last_applied_index.load(Ordering::SeqCst); - let term = self.last_applied_term.load(Ordering::SeqCst); - self.with_db(|db| { - let cf = db.cf_handle(STATE_MACHINE_META_CF).ok_or_else(|| { - Error::System(d_engine_core::SystemError::Storage(StorageError::DbError( - "State machine meta CF not found".to_string(), - ))) - })?; - db.put_cf(&cf, LAST_APPLIED_INDEX_KEY, index.to_be_bytes()) - .map_err(|e| StorageError::DbError(e.to_string()))?; - db.put_cf(&cf, LAST_APPLIED_TERM_KEY, term.to_be_bytes()) - .map_err(|e| StorageError::DbError(e.to_string()))?; - Ok(()) - }) - } - // ===== Snapshot restore helpers ===== /// Restore SM state from a snapshot (CF export format). @@ -827,32 +809,6 @@ impl StateMachine for RocksDBStateMachine { Ok(position) } - async fn reset(&self) -> Result<(), Error> { - self.with_db(|db| { - let cf = db.cf_handle(STATE_MACHINE_CF).ok_or_else(|| { - Error::System(d_engine_core::SystemError::Storage(StorageError::DbError( - "State machine CF not found".to_string(), - ))) - })?; - let mut batch = WriteBatch::default(); - let iter = db.iterator_cf(&cf, IteratorMode::Start); - for item in iter { - let (key, _) = item.map_err(|e| StorageError::DbError(e.to_string()))?; - batch.delete_cf(&cf, &key); - } - db.write(&batch).map_err(|e| StorageError::DbError(e.to_string()))?; - Ok(()) - })?; - - self.last_applied_index.store(0, Ordering::SeqCst); - self.last_applied_term.store(0, Ordering::SeqCst); - - self.persist_state_machine_metadata()?; - - info!("RocksDB state machine reset completed"); - Ok(()) - } - fn scan_prefix( &self, prefix: &[u8], diff --git a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine_test.rs b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine_test.rs index ff71cf95..e52d2814 100644 --- a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine_test.rs +++ b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine_test.rs @@ -24,6 +24,8 @@ struct RocksDBStateMachineBuilder { storage: Mutex>, // Partner storage of the crash image; kept alive while the image is in use. image_storage: Mutex>, + // How many times `build()` ran: every build gets its own directory. + builds: std::sync::atomic::AtomicUsize, } impl RocksDBStateMachineBuilder { @@ -32,30 +34,43 @@ impl RocksDBStateMachineBuilder { temp_dir: TempDir::new().expect("Failed to create temp dir"), storage: Mutex::new(None), image_storage: Mutex::new(None), + builds: std::sync::atomic::AtomicUsize::new(0), } } + + /// The directory the last `build()` used. + fn current_dir(&self) -> std::path::PathBuf { + let n = self.builds.load(std::sync::atomic::Ordering::SeqCst); + self.temp_dir.path().join(format!("rocksdb_sm_{n}")) + } + + fn open_current(&self) -> Result, Error> { + let (storage, sm) = RocksDBUnifiedEngine::open(self.current_dir())?; + *self.storage.lock().unwrap() = Some(storage); + Ok(Arc::new(sm)) + } } #[async_trait] impl StateMachineBuilder for RocksDBStateMachineBuilder { async fn build(&self) -> Result, Error> { + *self.storage.lock().unwrap() = None; + self.builds.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + self.open_current() + } + + async fn reopen(&self) -> Result, Error> { // Drop old storage so Arc refcount falls to 0 and RocksDB releases the lock. - { - *self.storage.lock().unwrap() = None; - } + *self.storage.lock().unwrap() = None; tokio::time::sleep(std::time::Duration::from_millis(50)).await; - - let path = self.temp_dir.path().join("rocksdb_sm"); - let (storage, sm) = RocksDBUnifiedEngine::open(&path)?; - *self.storage.lock().unwrap() = Some(storage); - Ok(Arc::new(sm)) + self.open_current() } async fn build_from_crash_image(&self) -> Result>, Error> { *self.image_storage.lock().unwrap() = None; let image = self.temp_dir.path().join("rocksdb_sm_crash_image"); let _ = std::fs::remove_dir_all(&image); - copy_storage_dir(&self.temp_dir.path().join("rocksdb_sm"), &image)?; + copy_storage_dir(&self.current_dir(), &image)?; let (storage, sm) = RocksDBUnifiedEngine::open(&image)?; *self.image_storage.lock().unwrap() = Some(storage); Ok(Some(Arc::new(sm))) @@ -840,8 +855,11 @@ async fn test_apply_snapshot_from_file_reports_applied_for_newer_snapshot() { .expect("generate snapshot"); assert_eq!(last_included, LogId { term: 1, index: 5 }); - // Fall behind, then receive the (now newer-than-current) snapshot. - state_machine.reset().await.expect("reset"); + // Fall behind (a node that starts empty), then receive the (now newer-than-current) snapshot. + drop(state_machine); + drop(_storage); + let (_storage, state_machine) = + RocksDBUnifiedEngine::open(temp_dir.path().join("test_behind")).expect("open behind"); assert_eq!(state_machine.last_applied(), LogId::default()); let checksum = compute_checksum_from_folder_path(&snapshot_dir) diff --git a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_unified_engine_test.rs b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_unified_engine_test.rs index 817331b2..9089ee6b 100644 --- a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_unified_engine_test.rs +++ b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_unified_engine_test.rs @@ -86,6 +86,8 @@ struct UnifiedStateMachineBuilder { storage: Mutex>, // Partner storage of the crash image; kept alive while the image is in use. image_storage: Mutex>, + // How many times `build()` ran: every build gets its own directory. + builds: std::sync::atomic::AtomicUsize, } impl UnifiedStateMachineBuilder { @@ -94,28 +96,42 @@ impl UnifiedStateMachineBuilder { temp_dir: TempDir::new().expect("temp dir"), storage: Mutex::new(None), image_storage: Mutex::new(None), + builds: std::sync::atomic::AtomicUsize::new(0), } } + + /// The directory the last `build()` used. + fn current_dir(&self) -> std::path::PathBuf { + let n = self.builds.load(std::sync::atomic::Ordering::SeqCst); + self.temp_dir.path().join(format!("unified_{n}")) + } + + fn open_current(&self) -> Result, Error> { + let (storage, sm) = RocksDBUnifiedEngine::open(self.current_dir())?; + *self.storage.lock().unwrap() = Some(storage); + Ok(Arc::new(sm)) + } } #[async_trait] impl StateMachineBuilder for UnifiedStateMachineBuilder { async fn build(&self) -> Result, Error> { + *self.storage.lock().unwrap() = None; + self.builds.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + self.open_current() + } + + async fn reopen(&self) -> Result, Error> { // Drop old storage so Arc refcount falls to 0 and RocksDB releases the lock. - { - *self.storage.lock().unwrap() = None; - } + *self.storage.lock().unwrap() = None; tokio::time::sleep(std::time::Duration::from_millis(50)).await; - - let (storage, sm) = RocksDBUnifiedEngine::open(self.temp_dir.path())?; - *self.storage.lock().unwrap() = Some(storage); - Ok(Arc::new(sm)) + self.open_current() } async fn build_from_crash_image(&self) -> Result>, Error> { *self.image_storage.lock().unwrap() = None; let image = TempDir::new()?.keep(); - copy_storage_dir(self.temp_dir.path(), &image)?; + copy_storage_dir(&self.current_dir(), &image)?; let (storage, sm) = RocksDBUnifiedEngine::open(&image)?; *self.image_storage.lock().unwrap() = Some(storage); Ok(Some(Arc::new(sm))) diff --git a/d-engine-server/tests/common/shared_state_machine.rs b/d-engine-server/tests/common/shared_state_machine.rs index 840b5844..b9c7ca86 100644 --- a/d-engine-server/tests/common/shared_state_machine.rs +++ b/d-engine-server/tests/common/shared_state_machine.rs @@ -70,9 +70,6 @@ impl StateMachine for SharedStateMachine { ) -> Result { self.0.generate_snapshot_data(new_snapshot_dir).await } - async fn reset(&self) -> Result<(), Error> { - self.0.reset().await - } fn scan_prefix( &self, prefix: &[u8], diff --git a/d-engine/src/docs/server_guide/customize-state-machine.md b/d-engine/src/docs/server_guide/customize-state-machine.md index e9544ccd..a18f79d7 100644 --- a/d-engine/src/docs/server_guide/customize-state-machine.md +++ b/d-engine/src/docs/server_guide/customize-state-machine.md @@ -120,7 +120,6 @@ impl StateMachine for CustomStateMachine { | `last_applied()` | Get last applied index | Sync | High | | `apply_snapshot_from_file()` | Replace state with snapshot | Async | Critical | | `generate_snapshot_data()` | Export a snapshot into the given directory and return the position of the last entry it contains (the engine checksums it) | Async | High | -| `reset()` | Reset to initial state | Async | Medium | ## 4. Testing Your Implementation @@ -180,10 +179,15 @@ async fn test_performance() -> Result<(), Error> { #[async_trait] impl StateMachineBuilder for MyStateMachineBuilder { async fn build(&self) -> Result, Error> { - // Create your state machine + // A state machine on new, empty storage (a new directory on every call) Ok(Arc::new(MyStateMachine::new().await?)) } + async fn reopen(&self) -> Result, Error> { + // The storage the last build() used, with whatever it holds now (a restart) + Ok(Arc::new(MyStateMachine::open_existing().await?)) + } + async fn cleanup(&self) -> Result<(), Error> { Ok(()) } diff --git a/examples/sled-cluster/src/sled_state_machine.rs b/examples/sled-cluster/src/sled_state_machine.rs index d953947e..f5eab73e 100644 --- a/examples/sled-cluster/src/sled_state_machine.rs +++ b/examples/sled-cluster/src/sled_state_machine.rs @@ -365,19 +365,6 @@ impl StateMachine for SledStateMachine { Ok(()) } - async fn reset(&self) -> Result<()> { - let db = self.db()?; - for tree_name in &[STATE_MACHINE_TREE, STATE_MACHINE_META_NAMESPACE] { - db.open_tree(tree_name) - .map_err(|e| StorageError::DbError(e.to_string()))? - .clear() - .map_err(|e| StorageError::DbError(e.to_string()))?; - } - self.last_applied_index.store(0, Ordering::SeqCst); - self.last_applied_term.store(0, Ordering::SeqCst); - Ok(()) - } - fn scan_prefix( &self, prefix: &[u8], diff --git a/examples/sled-cluster/src/state_machine/sled_state_machine_test.rs b/examples/sled-cluster/src/state_machine/sled_state_machine_test.rs index c8cf9575..0ff6cffc 100644 --- a/examples/sled-cluster/src/state_machine/sled_state_machine_test.rs +++ b/examples/sled-cluster/src/state_machine/sled_state_machine_test.rs @@ -29,26 +29,39 @@ use crate::SledStateMachine; /// so that data written by the first instance is readable by the second. struct SledBuilder { temp_dir: TempDir, + // How many times `build()` ran: every build gets its own directory. + builds: std::sync::atomic::AtomicUsize, } impl SledBuilder { fn new() -> Self { Self { temp_dir: TempDir::new().expect("create TempDir"), + builds: std::sync::atomic::AtomicUsize::new(0), } } + + /// The directory the last `build()` used. + fn current_dir(&self) -> std::path::PathBuf { + let n = self.builds.load(std::sync::atomic::Ordering::SeqCst); + self.temp_dir.path().join(format!("sled_{n}")) + } } #[async_trait] impl StateMachineBuilder for SledBuilder { async fn build(&self) -> Result, Error> { - let sm = SledStateMachine::new(self.temp_dir.path(), 1)?; - Ok(Arc::new(sm)) + self.builds.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Ok(Arc::new(SledStateMachine::new(self.current_dir(), 1)?)) + } + + async fn reopen(&self) -> Result, Error> { + Ok(Arc::new(SledStateMachine::new(self.current_dir(), 1)?)) } async fn build_from_crash_image(&self) -> Result>, Error> { let image = TempDir::new()?.keep(); - copy_storage_dir(self.temp_dir.path(), &image)?; + copy_storage_dir(&self.current_dir(), &image)?; Ok(Some(Arc::new(SledStateMachine::new(&image, 1)?))) } From 06b47d39aae370eadadc805c980d572f96b833ac Mon Sep 17 00:00:00 2001 From: Joshua Chi <112539+JoshuaChi@users.noreply.github.com> Date: Mon, 2 Nov 2026 21:25:42 +0800 Subject: [PATCH 14/19] fix #400: integration test fix --- .../bank_transfer_invariant_embedded.rs | 91 +++++++++++-------- 1 file changed, 53 insertions(+), 38 deletions(-) diff --git a/d-engine-server/tests/cas_operations/bank_transfer_invariant_embedded.rs b/d-engine-server/tests/cas_operations/bank_transfer_invariant_embedded.rs index 0b6eaad8..a8579662 100644 --- a/d-engine-server/tests/cas_operations/bank_transfer_invariant_embedded.rs +++ b/d-engine-server/tests/cas_operations/bank_transfer_invariant_embedded.rs @@ -18,6 +18,8 @@ const BANK_KEY: &[u8] = b"bank"; const INITIAL_BALANCE: u64 = 1_000; const TRANSFERS_PER_WORKER: usize = 50; const TRANSFER_AMOUNT: u64 = 1; +// The transfers take a few seconds; this only has to turn a hang into a failure. +const WORKER_DEADLINE: Duration = Duration::from_secs(90); // Pack 3 account balances into a single u64 (21 bits each, max ~2M per account). fn pack(balances: [u64; 3]) -> [u8; 8] { @@ -77,14 +79,17 @@ async fn test_concurrent_transfers_preserve_bank_invariant() tokio::fs::create_dir_all(&db_path).await?; let (storage, state_machine) = RocksDBUnifiedEngine::open(&db_path)?; - let config_path = format!("/tmp/d-engine-bank-node{node_id}.toml"); + // In this test's own temp dir: a fixed path under /tmp is shared by every process that + // runs this test at the same time, and each would then start its nodes from another + // run's ports and data directory. + let config_path = temp_dir.path().join(format!("node{node_id}.toml")); tokio::fs::write(&config_path, &config_str).await?; let engine = DefaultEmbeddedEngine::start_custom( &node_db_root, Arc::new(storage), state_machine, - Some(&config_path), + config_path.to_str(), ) .await?; engines.push(engine); @@ -115,47 +120,57 @@ async fn test_concurrent_transfers_preserve_bank_invariant() let to = (worker_id + 1) % 3; worker_handles.push(tokio::spawn(async move { - for _ in 0..TRANSFERS_PER_WORKER { - // CAS retry loop: read current state → compute new → atomic swap. - loop { - let current = match client.get_linearizable(BANK_KEY).await { - Ok(Some(v)) => v, - Ok(None) => panic!("bank key disappeared"), - Err(_) => { - // Transient error (e.g. replication lag); retry. - tokio::time::sleep(Duration::from_millis(10)).await; - continue; - } - }; - - let mut balances = unpack(¤t); - // Circular transfers always have sufficient funds, - // but guard defensively. - if balances[from] < TRANSFER_AMOUNT { - tokio::task::yield_now().await; - continue; - } - balances[from] -= TRANSFER_AMOUNT; - balances[to] += TRANSFER_AMOUNT; - let new_bytes = pack(balances); - - match client - .compare_and_swap(BANK_KEY, Some(current.as_ref()), new_bytes.as_slice()) - .await - { - Ok(true) => break, // committed - Ok(false) => { - // Concurrent update; retry without delay. + // A worker retries on every error, so without a deadline a cluster that never + // recovers would keep it spinning forever and the test would hang instead of fail. + tokio::time::timeout(WORKER_DEADLINE, async move { + for _ in 0..TRANSFERS_PER_WORKER { + // CAS retry loop: read current state → compute new → atomic swap. + loop { + let current = match client.get_linearizable(BANK_KEY).await { + Ok(Some(v)) => v, + Ok(None) => panic!("bank key disappeared"), + Err(_) => { + // Transient error (e.g. replication lag); retry. + tokio::time::sleep(Duration::from_millis(10)).await; + continue; + } + }; + + let mut balances = unpack(¤t); + // Circular transfers always have sufficient funds, + // but guard defensively. + if balances[from] < TRANSFER_AMOUNT { tokio::task::yield_now().await; + continue; } - Err(_) => { - // Transient error; brief backoff. - tokio::time::sleep(Duration::from_millis(10)).await; + balances[from] -= TRANSFER_AMOUNT; + balances[to] += TRANSFER_AMOUNT; + let new_bytes = pack(balances); + + match client + .compare_and_swap( + BANK_KEY, + Some(current.as_ref()), + new_bytes.as_slice(), + ) + .await + { + Ok(true) => break, // committed + Ok(false) => { + // Concurrent update; retry without delay. + tokio::task::yield_now().await; + } + Err(_) => { + // Transient error; brief backoff. + tokio::time::sleep(Duration::from_millis(10)).await; + } } } } - } - Ok::<_, ClientApiError>(()) + Ok::<_, ClientApiError>(()) + }) + .await + .expect("transfers did not finish before the deadline: the cluster stopped serving") })); } From 6096399674abdb178a9ea5c1f7d121320b2bcfc4 Mon Sep 17 00:00:00 2001 From: Joshua Chi <112539+JoshuaChi@users.noreply.github.com> Date: Mon, 2 Nov 2026 23:23:14 +0800 Subject: [PATCH 15/19] fix #400: Address PR review feedback - map_error_response now keeps StorageIoError, DataCorruption and RetryRequired for reads on the Raft command path, as the ReadActor fast path already does. - The snapshot-during-writes suite test labels the snapshot with the exported position, so it can fail for a contract-following state machine. - The run_custom doc example passes the state machine by value. --- .../src/storage/state_machine_test.rs | 7 +++++-- d-engine-server/src/api/read_handle.rs | 5 +++++ d-engine-server/src/api/read_handle_test.rs | 18 ++++++++++++++++++ d-engine-server/src/api/standalone.rs | 2 +- 4 files changed, 29 insertions(+), 3 deletions(-) diff --git a/d-engine-core/src/storage/state_machine_test.rs b/d-engine-core/src/storage/state_machine_test.rs index 7277f0cb..95a69c59 100644 --- a/d-engine-core/src/storage/state_machine_test.rs +++ b/d-engine-core/src/storage/state_machine_test.rs @@ -425,7 +425,7 @@ impl StateMachineTestSuite { // 2. A snapshot is taken in the middle of the writes. let temp_dir = TempDir::new()?; let snapshot_dir = temp_dir.path().join("snapshot"); - sm.generate_snapshot_data(snapshot_dir.clone()).await?; + let exported = sm.generate_snapshot_data(snapshot_dir.clone()).await?; stop.store(true, Ordering::Relaxed); writer.await.expect("the writer must not panic"); @@ -433,7 +433,10 @@ impl StateMachineTestSuite { drop(sm); let lagging = builder.build().await?; let metadata = SnapshotMetadata { - last_included: Some(LogId { index: 1, term: 1 }), + // The engine labels the snapshot with the position the export reports, and the + // installing state machine adopts that label as its `last_applied`, so a position + // ahead of the data shows up after the restart. + last_included: Some(exported), checksum: Bytes::from(vec![0; 32]), }; lagging.apply_snapshot_from_file(&metadata, snapshot_dir).await?; diff --git a/d-engine-server/src/api/read_handle.rs b/d-engine-server/src/api/read_handle.rs index 7148b6ea..f72cb6ff 100644 --- a/d-engine-server/src/api/read_handle.rs +++ b/d-engine-server/src/api/read_handle.rs @@ -94,6 +94,11 @@ pub(crate) fn map_error_response( }; not_leader_error(leader_id, leader_address, retry_after_ms) } + // A state machine read failure keeps its own code and its fixed message, as on the + // ReadActor fast path (`read_failure`). + ErrorCode::StorageIoError | ErrorCode::DataCorruption | ErrorCode::RetryRequired => { + ClientApiError::from(error) + } _ => server_error(format!("Error code: {error:?}")), } } diff --git a/d-engine-server/src/api/read_handle_test.rs b/d-engine-server/src/api/read_handle_test.rs index 6017937b..00441b14 100644 --- a/d-engine-server/src/api/read_handle_test.rs +++ b/d-engine-server/src/api/read_handle_test.rs @@ -770,6 +770,24 @@ mod error_helper_tests { } } + /// A read that goes through the Raft command channel (linearizable reads, eventual reads on a + /// follower, a fast-path read that fell back after `NotServing`) reports a state machine read + /// failure with one of these three codes. + /// + /// Purpose: the client must still be able to tell "retry" from "storage failure" from "the data + /// may be damaged", as it can on the fast path; mapping them to a generic error would hide it. + #[test] + fn test_map_error_response_keeps_each_read_failure_code_distinct() { + for code in [ + ErrorCode::StorageIoError, + ErrorCode::DataCorruption, + ErrorCode::RetryRequired, + ] { + let err = map_error_response(code, None, None); + assert_eq!(err.code(), code, "{code:?} must reach the client unchanged"); + } + } + // ─── extract_read_payload ──────────────────────────────────────────────── #[test] diff --git a/d-engine-server/src/api/standalone.rs b/d-engine-server/src/api/standalone.rs index 6d462b27..f7a14d6b 100644 --- a/d-engine-server/src/api/standalone.rs +++ b/d-engine-server/src/api/standalone.rs @@ -133,7 +133,7 @@ impl StandaloneEngine { /// # Example /// ```ignore /// let storage = Arc::new(MyCustomStorage::new()?); - /// let sm = Arc::new(MyCustomStateMachine::new()?); + /// let sm = MyCustomStateMachine::new()?; /// /// let (shutdown_tx, shutdown_rx) = watch::channel(()); /// StandaloneEngine::run_custom("./data/my-node", storage, sm, shutdown_rx, Some("config.toml")).await?; From e42e6fa73a535f9d68335d60fa255f5ab782ec9e Mon Sep 17 00:00:00 2001 From: Joshua Chi <112539+JoshuaChi@users.noreply.github.com> Date: Wed, 4 Nov 2026 15:32:40 +0800 Subject: [PATCH 16/19] refactor #400: StateMachine methods return StateMachineError A state machine reports any failure with one opaque type that carries a message, instead of d_engine::Error. A failed apply_chunk stops the node; a failed read fails only that read. A state machine no longer reports "not serving" while it installs a snapshot: a concurrent read must see the old state, the new state, or fail. The read actor returns a state machine error to the caller instead of retrying through the Raft path, which reads the same state machine; only an invalid lease falls back. The gRPC fast path no longer sends internal error text, and the read error codes that a state machine can no longer produce are removed from the read path. A linearizable-read test gets a 2 s write timeout so it no longer fails on a slow disk. The v0.3.0 changelog entries for #400 are shortened. --- CHANGELOG.md | 24 ++--- MIGRATION_GUIDE.md | 2 +- d-engine-core/src/errors.rs | 20 ++++ .../leader_state_test/client_read_test.rs | 10 +- .../default_state_machine_handler_test.rs | 19 ++-- .../src/state_machine_handler/writer.rs | 6 +- d-engine-core/src/storage/state_machine.rs | 84 +++++++++++------ .../src/storage/state_machine_test.rs | 57 ++++++------ .../api/embedded_test/shutdown_order_test.rs | 14 +-- d-engine-server/src/api/read_handle.rs | 43 +++------ d-engine-server/src/api/read_handle_test.rs | 93 +++---------------- d-engine-server/src/lib.rs | 6 +- .../grpc/grpc_raft_service_fast_path_test.rs | 48 ++++++---- d-engine-server/src/read_actor.rs | 13 +-- d-engine-server/src/read_actor_test.rs | 21 +++-- .../adaptors/file/file_state_machine.rs | 24 ++--- .../adaptors/file/file_state_machine_test.rs | 30 +++--- .../adaptors/rocksdb/rocksdb_state_machine.rs | 62 ++++++------- .../rocksdb/rocksdb_state_machine_test.rs | 10 +- .../tests/common/shared_state_machine.rs | 16 ++-- .../linearizable_read_consistency_embedded.rs | 3 + .../server_guide/customize-state-machine.md | 8 +- d-engine/src/lib.rs | 2 + .../sled-cluster/src/sled_state_machine.rs | 45 ++++----- 24 files changed, 305 insertions(+), 355 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1350ad9e..004bb326 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,7 +26,7 @@ All notable changes to this project will be documented in this file. - **`raft.state_machine.shutdown_apply_timeout_ms`** (#455, default `5000`): how long shutdown waits for the state machine worker to apply the entries already committed. On timeout it logs an error and shuts down anyway; a restart replays what was not applied. -- **Crash-recovery test for state machines** (#400): `StateMachineTestSuite` checks that after a crash (no `close()`, no `Drop`) a restart reports a `last_applied` equal to the last entry whose effects are in the data. `StateMachineBuilder` has a new method `build_from_crash_image`, with a default implementation that returns `None`; implement it to open a copy of your storage as it is on disk, otherwise the crash test is skipped. `copy_storage_dir` copies a directory for that purpose. +- **Crash-recovery test for state machines** (#400): `StateMachineTestSuite` now checks that a restart after a crash reports a position that matches the data. Implement `StateMachineBuilder::build_from_crash_image` to enable it. ### Fixed @@ -58,17 +58,17 @@ All notable changes to this project will be documented in this file. - **File state machine lost empty values on WAL replay** (#464): `replay_wal` treated a zero value length as "no value", so an `Insert` with an empty value was dropped after a restart. It now restores the empty value. -- **Snapshot checksum ignored files in sub-directories and file boundaries** (#400): see the snapshot checksum entry under Changed. +- **RocksDB snapshot could be ahead of its data** (#400): a snapshot taken during writes could make a node skip entries after it installed it. Snapshots are now exported from one consistent copy. -- **RocksDB snapshots could hold a position ahead of their data** (#400): a snapshot exported while writes continued took `sm` (data) and `sm_meta` (`last_applied`) from the live database one after the other, so a node installing it could report a position newer than its data and skip the entries in between for good. The export now takes a checkpoint of the database and exports both column families from that frozen copy, and the database is opened with `atomic_flush` so the two column families always reach disk together. A snapshot export is slower (about 100 ms on a small database, measured with a test database; measure on your data). The engine also labels every snapshot with the position the state machine reports for its exported data. +- **File state machine reported an old position after a crash** (#400): recovery now restores `last_applied` together with the data. -- **File state machine: `last_applied` after crash recovery** (#400): replaying the write-ahead log restored the data but did not move `last_applied`, and then cleared the log, so a restart reported the position of the last checkpoint and the engine applied entries again that the data already contained (a different result for compare-and-swap). Replay now advances `last_applied` to the last replayed record and writes data and position out before the log is cleared. The sled example and d-lmdb now commit the applied position in the same transaction as the data. +- **Snapshot checksum now covers every file** (#400), including sub-directories. Delete snapshot archives made by older versions when you upgrade. ### Changed - **⚠️ Wire protocol is not compatible with v0.2.4**: `ClientResponse.metadata` moved from field 4 to 5 (field 4 is now `scan_data`); `HandleClientScan` returns `ClientResponse` (the `ScanResponse` message is now `ScanResults`); the `ChunkStatus` values are renumbered with `CHUNK_STATUS_UNSPECIFIED = 0`; `WriteCommand.Insert.ttl_secs` (field 3) is reserved; new RPC `PreVote`. Do not run v0.2.4 and v0.3.0 nodes or clients together. See `MIGRATION_GUIDE.md`. -- **⚠️ `StateMachine::stop()` removed; `StateMachine::close_storage()` renamed to `close()`** (#455): `stop()` had no caller in d-engine. `close()` is now the single end-of-life hook, may be called more than once, and also persists what must survive a restart (lease state). The engine stops its read path before calling it and calls nothing on the state machine afterwards, so a state machine no longer has to answer "not serving" once closed; `EmbeddedEngine::stop()` now stops the read actor before the state machine worker. It returns `Result<(), Error>`: release your resources even if persisting fails, and return the first error; the node logs it at ERROR level. Custom implementations: delete `stop()`, rename `close_storage()` to `close()`, and move any lease persistence from `stop()` into `close()`. See `MIGRATION_GUIDE.md`. +- **⚠️ `StateMachine::stop()` removed; `StateMachine::close_storage()` renamed to `close()`** (#455): `stop()` had no caller in d-engine. `close()` is now the single, permanent, idempotent end-of-life hook and also persists what must survive a restart (lease state). It returns `Result<(), Error>`: release your resources even if persisting fails, and return the first error; the node logs it at ERROR level. Custom implementations: delete `stop()`, rename `close_storage()` to `close()`, and move any lease persistence from `stop()` into `close()`. See `MIGRATION_GUIDE.md`. - **State machine worker lifecycle** (#455): the `NodeBuilder` only assembles the worker; `Node::run()` starts it and stops it. `StateMachineWorker::new` no longer takes a shutdown signal. @@ -85,6 +85,8 @@ All notable changes to this project will be documented in this file. - **⚠️ `StateMachine::entry_term()` removed from trait** (#418): Term lookup belongs to the log layer, not the state machine. Custom `StateMachine` implementations must delete this method — it is no longer part of the trait. See [Migration Guide](./MIGRATION_GUIDE.md) for details. +- **⚠️ `StateMachine::apply_snapshot_from_file` return type changed** (#436): Was `Result<()>`, now `Result`. The three outcomes (`Applied`/`IgnoredStale`/`IgnoredDuplicate`) are now explicit instead of every call looking identical on success. Custom `StateMachine` implementations must update the signature and return the matching variant. See [Migration Guide](./MIGRATION_GUIDE.md) for details. + - **⚠️ Learner initial-snapshot PULL path removed** (#436): `Transport::request_snapshot_from_leader` and `StateMachineHandler::apply_snapshot_stream_from_leader` are removed. New learners catch up via the leader's PUSH replication loop (AppendEntries / InstallSnapshot) instead of eagerly pulling a snapshot on startup. See [Migration Guide](./MIGRATION_GUIDE.md) for details. - **KV encoding moved out of `d-engine-core`** (#415): `ClientWriteRequest.command` is now `Option` (pre-serialized). Serialization happens in the server transport layer (embedded/standalone/gRPC handler). Core is encoding-agnostic. @@ -122,20 +124,14 @@ All notable changes to this project will be documented in this file. "MemFirst"` or `"DiskFirst"` are silently ignored, not an error — remove the field, `flush_policy` is the only persistence knob now. -- **⚠️ `StateMachine::apply_snapshot_from_file` returns `Result<()>`; the engine decides which snapshots are installed** (#400): the engine compares the snapshot's boundary with `last_applied()` before it calls the state machine, so stale, duplicate and conflicting snapshots never reach it, and it checks afterwards that `last_applied()` equals the snapshot's boundary, reporting an error if not. Before, every state machine had to implement this decision itself, and one that did not (d-lmdb) would install a stale snapshot and roll its data back. See [MIGRATION_GUIDE.md](./MIGRATION_GUIDE.md). - -- **⚠️ `StateMachine` no longer declares `snapshot_metadata` and `persist_last_snapshot_metadata`; the engine records the snapshots a node holds** (#400): the engine keeps `snapshot-{index}-{term}.meta` (boundary and checksum) next to each snapshot archive, written after the archive is complete and, on a follower, after the snapshot is installed. The snapshot trigger policy, the leader's snapshot push and the join response read it. Cleanup goes by these records and also removes archives without a record that are older than the newest recorded snapshot. Before, every state machine had to store and restore this metadata, and the file state machine, sled and d-lmdb did not do it correctly. Archives created before the upgrade have no record, so the node reports no snapshot until it builds its next one. See [MIGRATION_GUIDE.md](./MIGRATION_GUIDE.md). +- **⚠️ `StateMachine` trait slimmed from 17 methods to 8** (#400, #455, #469): `start`, `stop`, `is_running`, `len`, `is_empty`, `reset` and the snapshot-metadata methods are removed, `close_storage` is now `close`, methods return `StateMachineError`, and `generate_snapshot_data(dir)` returns the position of what it exported. The engine now owns snapshot checksums, records and install decisions. See [MIGRATION_GUIDE.md](./MIGRATION_GUIDE.md). -- **⚠️ `StateMachine` no longer declares `update_last_applied`, `persist_last_applied`, `update_last_snapshot_metadata`, `save_hard_state`, `flush` and `flush_async`** (#400): the engine never called them. Keep any you still need as methods on your own type. Your state machine must restore `last_applied` and its snapshot metadata when it is opened; the `StateMachine` docs now state this as a durability contract, and `StateMachineTestSuite` checks it. See [MIGRATION_GUIDE.md](./MIGRATION_GUIDE.md). +- **⚠️ The engine takes the state machine by value** (#400): `start_custom`, `start_node` and `run_custom` take `SM` instead of `Arc`, and `EmbeddedEngine` and `EmbeddedClient` no longer have type parameters. -- **⚠️ `StateMachine::generate_snapshot_data(dir)` returns `Result`; the engine owns snapshot checksums** (#400): the method no longer takes the `last_included` argument and returns the position of the last entry its exported data contains, read from the same consistent view as the data. The engine labels the snapshot with it and rejects a position outside what the node applied while exporting. The engine computes the SHA-256 of the exported snapshot directory and verifies it before calling `apply_snapshot_from_file`, which now only ever receives a directory that matches the recorded checksum. A state machine neither returns nor checks a checksum. The checksum now covers every file at any depth together with its relative path and length; before, it covered only top-level files, so the data files RocksDB exports into sub-directories were not protected. Snapshot archives created before the upgrade fail verification, so remove old snapshot archives when upgrading. See [MIGRATION_GUIDE.md](./MIGRATION_GUIDE.md). - -- **⚠️ The engine takes the state machine by value** (#400): `EmbeddedEngine::start_custom`/`start_node`, `StandaloneEngine::run_custom`/`start_node` take `SM` instead of `Arc`; the `Arc` is created inside the engine. Nothing outside the engine holds the state machine, so `stop()` returns only after it is dropped and its resources are released. Embedded reads now go through the ReadActor like standalone reads (`EmbeddedReadHandle` is removed). `EmbeddedEngine` and `EmbeddedClient` no longer have type parameters. See [MIGRATION_GUIDE.md](./MIGRATION_GUIDE.md). +- **⚠️ `StateMachineBuilder` (test support)** (#400): `build()` returns new empty storage and the new `reopen()` restarts it. ### Removed -- **`StateMachine::reset` removed** (#400): the engine never called it; only tests used it. `StateMachineBuilder::build()` now returns a state machine on new, empty storage, and the new required method `reopen()` opens the storage the last `build()` used (a restart), so the suite no longer wipes a state machine before a test. `test_reset_operation` is removed from `StateMachineTestSuite`; `test_snapshot_operations` now takes the builder. See `MIGRATION_GUIDE.md`. - - **TTL and per-key lease** (#464): d-engine now does consensus and replication only. Removed `put_with_ttl`, the `Lease` trait, `TtlLease`, the `[raft.state_machine.lease]` config section and `StateMachine::lease_background_cleanup`. `Command::Insert` no longer has `ttl_secs`. In the proto, `WriteCommand.Insert.ttl_secs` (field 3) is removed and the number is reserved. Key expiry is now the job of the layer above d-engine. The leader read lease (`LeaseRead`) is a different feature and is unchanged. - **File state machine storage format** (#464): The WAL record no longer has an `expire_at` slot, and `snapshot.bin` no longer ends with a TTL section. See [MIGRATION_GUIDE.md](./MIGRATION_GUIDE.md). diff --git a/MIGRATION_GUIDE.md b/MIGRATION_GUIDE.md index f262ea00..d138071f 100644 --- a/MIGRATION_GUIDE.md +++ b/MIGRATION_GUIDE.md @@ -10,7 +10,7 @@ - Remove calls to `put_with_ttl` and the `ttl_secs` field of `Command::Insert`. Implement key expiry in the layer above d-engine. - Delete the `[raft.state_machine.lease]` section from your config files. -- Custom state machines: remove `lease_background_cleanup`, `start`, `is_running`, `len`, `is_empty` and `reset` from your `StateMachine` impl; the trait no longer declares them. `start` was called once when the node booted. Do that setup in your constructor (a state machine must be ready to serve as soon as it is built). The engine no longer asks whether the state machine is running: a state machine that cannot serve a read while it installs a snapshot must return `StorageError::NotServing` from `get` and `get_multi`. If you need an entry count, keep `len` as a plain method on your own type. The engine never called `reset`; if you used it in your own tests, wipe your storage directory or build a new state machine on an empty one instead. `StateMachineBuilder` (test support) now has `build()` return a state machine on new, empty storage and a new required method `reopen()` for restarts. +- Custom state machines: remove `lease_background_cleanup`, `start`, `is_running`, `len`, `is_empty` and `reset` from your `StateMachine` impl; the trait no longer declares them. `start` was called once when the node booted. Do that setup in your constructor (a state machine must be ready to serve as soon as it is built). The engine no longer asks whether the state machine is running: while a state machine installs a snapshot, a concurrent read must return the old state, the new state, or fail, never a mix. If you need an entry count, keep `len` as a plain method on your own type. The engine never called `reset`; if you used it in your own tests, wipe your storage directory or build a new state machine on an empty one instead. `StateMachineBuilder` (test support) now has `build()` return a state machine on new, empty storage and a new required method `reopen()` for restarts. **Every `StateMachine` method now returns `StateMachineError` instead of `d_engine::Error`** (#469): report any failure with `StateMachineError::new("what went wrong")`, or let `?` convert an `io::Error`; the engine stops the node when `apply_chunk` fails and fails only the read when a read fails. `StorageError::NotServing` is no longer part of the contract for a state machine. - Proto: `WriteCommand.Insert.ttl_secs` (field 3) is removed. Regenerate your client code. - The file state machine storage format changed. Start with a fresh data directory; old files are not supported. - RocksDB state machine: keys written with `put_with_ttl` under v0.2.x are kept forever after the upgrade. v0.3.0 neither reads the old `ttl_state` entry nor deletes expired keys. Before upgrading, either delete the keys you no longer want, or record their deadlines and have your own expiry layer delete them. Or start with a fresh data directory. diff --git a/d-engine-core/src/errors.rs b/d-engine-core/src/errors.rs index 7271a201..2809c03d 100644 --- a/d-engine-core/src/errors.rs +++ b/d-engine-core/src/errors.rs @@ -9,6 +9,8 @@ use std::time::Duration; use config::ConfigError; use tokio::task::JoinError; +use crate::StateMachineError; + /// Unified result type used throughout `StateMachine`/`StorageEngine`/`LogStore`/`MetaStore` /// trait signatures — equivalent to `std::result::Result`. pub type Result = std::result::Result; @@ -871,3 +873,21 @@ impl From for Error { StorageError::IoError(e).into() } } + +impl From for Error { + /// A state machine failure reaches the engine as a state machine error with its message; the + /// engine never needs to look inside it. + fn from(error: StateMachineError) -> Self { + Error::System(SystemError::Storage(StorageError::StateMachineError( + error.to_string(), + ))) + } +} + +impl From for StateMachineError { + /// Lets the state machines that ship with d-engine, which share helpers with the rest of the + /// engine, return their internal errors with `?`. The message is kept. + fn from(error: Error) -> Self { + StateMachineError::new(error.to_string()) + } +} diff --git a/d-engine-core/src/raft_role/leader_state_test/client_read_test.rs b/d-engine-core/src/raft_role/leader_state_test/client_read_test.rs index 2ce3a622..471f9dfc 100644 --- a/d-engine-core/src/raft_role/leader_state_test/client_read_test.rs +++ b/d-engine-core/src/raft_role/leader_state_test/client_read_test.rs @@ -20,6 +20,7 @@ use crate::MockStateMachineHandler; use crate::RaftNodeConfig; use crate::ReadConsistencyPolicy; use crate::ScanResult; +use crate::StateMachineError; use crate::client::ClientReadRequest; use crate::client::ClientResponsePayload; use crate::client::ErrorCode; @@ -2298,16 +2299,11 @@ async fn test_scan_cmd_leader_delivers_result() { #[tokio::test] #[traced_test] async fn test_scan_cmd_leader_propagates_sm_error() { - use crate::Error; - use crate::StorageError; use crate::test_utils::mock_state_machine; let mut sm = mock_state_machine(); - sm.expect_scan_prefix().returning(|_prefix| { - Err(Error::System(crate::SystemError::Storage( - StorageError::StateMachineError("scan failed".into()), - ))) - }); + sm.expect_scan_prefix() + .returning(|_prefix| Err(StateMachineError::new("scan failed"))); let (_graceful_tx, graceful_rx) = watch::channel(()); let context = MockBuilder::new(graceful_rx) diff --git a/d-engine-core/src/state_machine_handler/default_state_machine_handler_test.rs b/d-engine-core/src/state_machine_handler/default_state_machine_handler_test.rs index 6f3dda8d..7b1f5af6 100644 --- a/d-engine-core/src/state_machine_handler/default_state_machine_handler_test.rs +++ b/d-engine-core/src/state_machine_handler/default_state_machine_handler_test.rs @@ -40,11 +40,10 @@ use super::StateMachineWriterOps; #[cfg(feature = "watch")] use super::broadcast_watch_events; use super::new_reader_writer_pair; -use crate::Error; use crate::MockSnapshotPolicy; use crate::MockStateMachine; use crate::MockTypeConfig; -use crate::SnapshotError; +use crate::StateMachineError; use crate::test_utils::create_test_compressed_snapshot; use crate::test_utils::snapshot_config; @@ -218,7 +217,7 @@ mod apply_chunk_test { if apply_chunk_error { state_machine .expect_apply_chunk() - .returning(|_| Err(Error::Fatal("Test error".to_string()))); + .returning(|_| Err(StateMachineError::new("Test error"))); } else { state_machine.expect_apply_chunk().returning(|_| Ok(vec![])); } @@ -337,7 +336,7 @@ mod apply_chunk_test { let (watch_tx, mut watch_rx) = tokio::sync::broadcast::channel(16); let mut sm = MockStateMachine::new(); sm.expect_apply_chunk() - .returning(|_| Err(Error::Fatal("batch failed".to_string()))); + .returning(|_| Err(StateMachineError::new("batch failed"))); let (handler, writer) = new_reader_writer_pair::( 1, @@ -1083,7 +1082,7 @@ mod create_snapshot_tests { fs::create_dir(&db_path).unwrap(); Ok(LogId { term: 1, index: 1 }) } else { - Err(SnapshotError::OperationFailed("Concurrency failure".into()).into()) + Err(StateMachineError::new("Concurrency failure")) } }); @@ -1211,7 +1210,7 @@ mod create_snapshot_tests { // Setup failing snapshot generation sm.expect_last_applied().returning(|| LogId { term: 1, index: 1 }); sm.expect_generate_snapshot_data() - .returning(|_| Err(SnapshotError::OperationFailed("test failure".into()).into())); + .returning(|_| Err(StateMachineError::new("test failure"))); let mut config = snapshot_config(temp_path.to_path_buf()); config.retained_log_entries = 0; @@ -1281,7 +1280,7 @@ mod create_snapshot_tests { sm.expect_last_applied().returning(|| LogId { term: 1, index: 1 }); sm.expect_generate_snapshot_data() - .returning(|_| Err(SnapshotError::OperationFailed("test error".into()).into())); + .returning(|_| Err(StateMachineError::new("test error"))); let config = snapshot_config(temp_path.clone()); let (handler, writer) = new_reader_writer_pair::( @@ -1510,7 +1509,7 @@ mod create_snapshot_tests { sm.expect_generate_snapshot_data().returning(|path| { // Simulate a partial write before the failure (e.g. checkpoint killed mid-copy). fs::create_dir_all(&path).unwrap(); - Err(SnapshotError::OperationFailed("simulated crash".into()).into()) + Err(StateMachineError::new("simulated crash")) }); let config = snapshot_config(snapshot_dir.clone()); @@ -1549,7 +1548,7 @@ mod create_snapshot_tests { // `create_dir_all`, otherwise this test can't distinguish "cleaned up" from // "never checked". if path.exists() { - return Err(SnapshotError::OperationFailed("mkdir: File exists".into()).into()); + return Err(StateMachineError::new("mkdir: File exists")); } fs::create_dir_all(&path).unwrap(); Ok(LogId { term: 1, index: 5 }) @@ -2924,7 +2923,7 @@ fn test_read_from_state_machine_returns_the_state_machine_error() { let mut state_machine = MockStateMachine::new(); state_machine .expect_get_multi() - .returning(|_| Err(crate::StorageError::DbError("disk failure".into()).into())); + .returning(|_| Err(StateMachineError::new("disk failure"))); let handler = handler_over(state_machine, temp_dir.path()); let err = handler diff --git a/d-engine-core/src/state_machine_handler/writer.rs b/d-engine-core/src/state_machine_handler/writer.rs index 5c448589..868ca8be 100644 --- a/d-engine-core/src/state_machine_handler/writer.rs +++ b/d-engine-core/src/state_machine_handler/writer.rs @@ -179,7 +179,7 @@ impl StateMachineWriterOps for DefaultStateMachineWriter { } } - apply_result + apply_result.map_err(Into::into) } async fn install_prepared_snapshot( @@ -285,7 +285,7 @@ impl StateMachineWriterOps for DefaultStateMachineWriter { Ok(position) => position, Err(e) => { let _ = tokio::fs::remove_dir_all(&temp_path).await; - return Err(e); + return Err(e.into()); } }; let after = self.state_machine.last_applied(); @@ -317,6 +317,6 @@ impl StateMachineWriterOps for DefaultStateMachineWriter { } fn close(&self) -> Result<()> { - self.state_machine.close() + self.state_machine.close().map_err(Into::into) } } diff --git a/d-engine-core/src/storage/state_machine.rs b/d-engine-core/src/storage/state_machine.rs index ae37ac25..216edb42 100644 --- a/d-engine-core/src/storage/state_machine.rs +++ b/d-engine-core/src/storage/state_machine.rs @@ -10,8 +10,6 @@ use d_engine_proto::server::storage::SnapshotMetadata; use mockall::automock; use crate::ApplyEntry; -use crate::Error; -use crate::StorageError; /// All `(key, value)` pairs returned by a prefix scan, plus the revision anchor. /// @@ -73,6 +71,32 @@ impl ApplyResult { } } +/// A failure reported by a [`StateMachine`]. The message says what went wrong. +/// +/// Return it for any failure: a full disk, an I/O error, data that failed a checksum. The engine +/// does not need to know which: when applying entries or installing a snapshot fails, the node +/// stops and recovers from its replicas; when a read fails, only that read fails. +/// +/// Build one from a message, `StateMachineError::new("disk full")`, or from any error type that +/// implements [`std::error::Error`]; an [`std::io::Error`] converts with `?`. The error chain is +/// kept for the log. +#[derive(Debug, thiserror::Error)] +#[error("state machine error: {0}")] +pub struct StateMachineError(#[source] Box); + +impl StateMachineError { + /// Wraps a message or an error. + pub fn new(error: impl Into>) -> Self { + Self(error.into()) + } +} + +impl From for StateMachineError { + fn from(error: std::io::Error) -> Self { + Self::new(error) + } +} + /// State machine trait for Raft consensus /// /// # Thread Safety Requirements @@ -97,16 +121,18 @@ impl ApplyResult { /// /// `StateMachineTestSuite` checks this rule; run it against your implementation. /// -/// # `NotServing` +/// # Errors +/// +/// Every method reports a failure with [`StateMachineError`]. A failure in `apply_chunk` stops the +/// node, which then recovers from its replicas: an entry that was committed cannot be skipped, so +/// the node must not go on without it. A failed `get`, `get_multi` or `scan_prefix` fails only that +/// read. /// -/// While a state machine is replacing its state from a snapshot and cannot answer reads -/// correctly, `get` and `get_multi` return `StorageError::NotServing`, for example -/// `Err(StorageError::NotServing("installing a snapshot".into()).into())`. +/// # Reads while a snapshot is installed /// -/// It means "not now", not "the data is damaged". The engine answers that read through Raft -/// instead, from the state machine after the install, so the client still gets a correct answer. -/// A simple implementation keeps an `AtomicBool` that it clears just before it starts replacing -/// its state and sets again when it has finished. +/// A read can run while `apply_snapshot_from_file` replaces the state. It must return the old +/// state, the new state, or fail with an error; it must never return a mix of the two, because +/// that is a state that never existed. #[cfg_attr(any(test, feature = "__test_support"), automock)] #[async_trait] pub trait StateMachine: Send + Sync + 'static { @@ -131,19 +157,18 @@ pub trait StateMachine: Send + Sync + 'static { /// `last_applied` has to be persisted together with the data it describes. /// /// Default is a no-op, for implementations with nothing to persist or release. - fn close(&self) -> Result<(), Error> { + fn close(&self) -> Result<(), StateMachineError> { Ok(()) } /// Retrieves a value by key from the state machine. /// - /// Called concurrently from the read path, so it must be thread-safe and cheap. Return - /// `StorageError::NotServing` while the state machine cannot answer (during a snapshot - /// install): the engine then routes the read through Raft. + /// Called concurrently from the read path, so it must be thread-safe and cheap. Return an + /// error if the value cannot be read; only this read fails. fn get( &self, key_buffer: &[u8], - ) -> Result, Error>; + ) -> Result, StateMachineError>; /// Retrieves multiple values by key from the state machine in a single call. /// @@ -174,7 +199,7 @@ pub trait StateMachine: Send + Sync + 'static { fn get_multi( &self, keys: &[Bytes], - ) -> Result>, Error> { + ) -> Result>, StateMachineError> { keys.iter().map(|k| self.get(k)).collect() } @@ -189,7 +214,7 @@ pub trait StateMachine: Send + Sync + 'static { async fn apply_chunk( &self, chunk: &[ApplyEntry], - ) -> Result, Error>; + ) -> Result, StateMachineError>; /// Gets the last applied log index and term. /// Sync operation as it reads from atomic variables. @@ -202,16 +227,16 @@ pub trait StateMachine: Send + Sync + 'static { /// the archive into and whose checksum it has verified against `metadata`. /// /// The implementation must: - /// 1. Replace its state so that a concurrent reader sees the old state, the new state, or - /// `NotServing` (see the trait documentation), never a mix. Either swap the state in one - /// atomic step, or return `NotServing` from reads for as long as the replacement takes. + /// 1. Replace its state so that a concurrent reader sees the old state, the new state, or a + /// failed read, never a mix (see "Reads while a snapshot is installed" in the trait + /// documentation). /// 2. Set `last_applied()` to `metadata.last_included` and make that durable. The engine checks /// this after the call and reports an error if it did not happen. async fn apply_snapshot_from_file( &self, metadata: &SnapshotMetadata, snapshot_path: std::path::PathBuf, - ) -> Result<(), Error>; + ) -> Result<(), StateMachineError>; /// Exports the state machine's current state into `new_snapshot_dir` and returns the position /// of the last applied entry that the exported state contains. @@ -219,8 +244,9 @@ pub trait StateMachine: Send + Sync + 'static { /// The engine checksums the directory itself, labels the snapshot with the returned position, /// and records the snapshot only once the final archive is durably built, so a crash /// mid-snapshot never leaves a record pointing at a missing archive; the state machine does - /// none of that. `new_snapshot_dir` exists and is empty. Put any regular files in it, with - /// names of your choice; the engine archives the whole directory and hands it back to + /// none of that. `new_snapshot_dir` does not exist yet: create it (`create_dir_all`), then put + /// any regular files in it, with names of your choice; the engine archives the whole directory + /// and hands it back to /// [`apply_snapshot_from_file`](Self::apply_snapshot_from_file) unchanged. /// /// # The position must describe the exported data @@ -255,7 +281,7 @@ pub trait StateMachine: Send + Sync + 'static { async fn generate_snapshot_data( &self, new_snapshot_dir: std::path::PathBuf, - ) -> Result; + ) -> Result; /// Returns all `(key, value)` pairs whose key starts with `prefix`, plus /// the `last_applied_index` at the time of the scan. @@ -268,12 +294,10 @@ pub trait StateMachine: Send + Sync + 'static { fn scan_prefix( &self, prefix: &[u8], - ) -> Result { + ) -> Result { let _ = prefix; - Err(Error::System(crate::SystemError::Storage( - StorageError::StateMachineError( - "scan_prefix not supported by this state machine".into(), - ), - ))) + Err(StateMachineError::new( + "scan_prefix not supported by this state machine", + )) } } diff --git a/d-engine-core/src/storage/state_machine_test.rs b/d-engine-core/src/storage/state_machine_test.rs index 95a69c59..ff5ebc7a 100644 --- a/d-engine-core/src/storage/state_machine_test.rs +++ b/d-engine-core/src/storage/state_machine_test.rs @@ -76,7 +76,7 @@ impl StateMachineTestSuite { Self::test_batch_operations(builder.build().await?).await?; Self::test_last_applied_detection(builder.build().await?).await?; Self::test_snapshot_operations(&builder).await?; - Self::test_reads_during_snapshot_install_see_old_new_or_not_serving(&builder).await?; + Self::test_reads_during_snapshot_install_see_old_new_or_fail(&builder).await?; Self::test_snapshot_taken_during_writes_never_skips_entries_after_restart(&builder).await?; Self::test_restart_after_crash_finds_position_and_data_of_the_same_entry(&builder).await?; @@ -272,13 +272,10 @@ impl StateMachineTestSuite { /// key holds a new value. Reads that bypass Raft are served straight from the state machine, /// so they run while the install replaces the state. /// - /// Expected: every read returns the old value, the new value, or `NotServing` ("not now", - /// see the trait documentation). It never returns "not found", an error of another kind, or - /// any other value: that would show a client a state that never existed. After the install - /// every key holds the new value. - pub async fn test_reads_during_snapshot_install_see_old_new_or_not_serving< - B: StateMachineBuilder, - >( + /// Expected: every read returns the old value, the new value, or fails. It never returns + /// "not found" or any other value: that would show a client a state that never existed. After + /// the install every key holds the new value. + pub async fn test_reads_during_snapshot_install_see_old_new_or_fail( builder: &B ) -> Result<(), Error> { use std::sync::Mutex; @@ -336,8 +333,8 @@ impl StateMachineTestSuite { Some(format!("key {i}: unexpected value {other:?}")) } Ok(None) => Some(format!("key {i}: reported as not found")), - Err(e) if e.is_not_serving() => None, - Err(e) => Some(format!("key {i}: {e:?}")), + // A failed read is allowed; it shows nothing wrong. + Err(_) => None, }; reads.fetch_add(1, Ordering::Relaxed); if let Some(violation) = seen { @@ -1749,9 +1746,9 @@ mod default_scan_prefix_tests { use d_engine_proto::server::storage::SnapshotMetadata; use crate::ApplyEntry; - use crate::Error; use crate::storage::state_machine::ApplyResult; use crate::storage::state_machine::StateMachine; + use crate::storage::state_machine::StateMachineError; struct MinimalSm { running: AtomicBool, @@ -1759,7 +1756,7 @@ mod default_scan_prefix_tests { #[async_trait] impl StateMachine for MinimalSm { - fn close(&self) -> Result<(), Error> { + fn close(&self) -> Result<(), StateMachineError> { self.running.store(false, Ordering::SeqCst); Ok(()) } @@ -1767,14 +1764,14 @@ mod default_scan_prefix_tests { fn get( &self, _key_buffer: &[u8], - ) -> Result, Error> { + ) -> Result, StateMachineError> { Ok(None) } async fn apply_chunk( &self, _chunk: &[ApplyEntry], - ) -> Result, Error> { + ) -> Result, StateMachineError> { Ok(vec![]) } @@ -1786,14 +1783,14 @@ mod default_scan_prefix_tests { &self, _metadata: &SnapshotMetadata, _snapshot_path: std::path::PathBuf, - ) -> Result<(), Error> { + ) -> Result<(), StateMachineError> { Ok(()) } async fn generate_snapshot_data( &self, _new_snapshot_dir: std::path::PathBuf, - ) -> Result { + ) -> Result { Ok(LogId::default()) } @@ -1854,6 +1851,7 @@ mod install_reader_suite_tests { use crate::Command; use crate::Error; use crate::storage::StateMachine; + use crate::storage::StateMachineError; use crate::storage::state_machine::ApplyResult; /// What survives a restart: shared by every state machine the builder creates. @@ -1905,14 +1903,14 @@ mod install_reader_suite_tests { fn get( &self, key: &[u8], - ) -> Result, Error> { + ) -> Result, StateMachineError> { Ok(self.durable.data.read().unwrap().get(key).cloned()) } async fn apply_chunk( &self, chunk: &[ApplyEntry], - ) -> Result, Error> { + ) -> Result, StateMachineError> { // Data and position change under one lock, like a database transaction. let mut data = self.durable.data.write().unwrap(); for entry in chunk { @@ -1940,7 +1938,7 @@ mod install_reader_suite_tests { } } - fn close(&self) -> Result<(), Error> { + fn close(&self) -> Result<(), StateMachineError> { if self.position_on_close_only { *self.durable.applied.write().unwrap() = *self.pending_position.read().unwrap(); } @@ -1950,7 +1948,7 @@ mod install_reader_suite_tests { async fn generate_snapshot_data( &self, new_snapshot_dir: PathBuf, - ) -> Result { + ) -> Result { std::fs::create_dir_all(&new_snapshot_dir)?; // Right: copy the data and read the position under the same lock. let (lines, mut position) = { @@ -1985,7 +1983,7 @@ mod install_reader_suite_tests { &self, _metadata: &SnapshotMetadata, snapshot_path: PathBuf, - ) -> Result<(), Error> { + ) -> Result<(), StateMachineError> { let text = std::fs::read_to_string(snapshot_path.join("data.txt"))?; let new_data: HashMap = text .lines() @@ -2060,7 +2058,7 @@ mod install_reader_suite_tests { /// Purpose: a state machine that replaces its state in one step passes. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn test_reader_suite_accepts_a_state_machine_that_swaps_its_state_atomically() { - StateMachineTestSuite::test_reads_during_snapshot_install_see_old_new_or_not_serving( + StateMachineTestSuite::test_reads_during_snapshot_install_see_old_new_or_fail( &InMemoryBuilder::default(), ) .await @@ -2072,14 +2070,13 @@ mod install_reader_suite_tests { #[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[should_panic(expected = "saw a state that never existed")] async fn test_reader_suite_rejects_a_state_machine_that_empties_its_state_during_install() { - let _ = - StateMachineTestSuite::test_reads_during_snapshot_install_see_old_new_or_not_serving( - &InMemoryBuilder { - faulty: true, - ..Default::default() - }, - ) - .await; + let _ = StateMachineTestSuite::test_reads_during_snapshot_install_see_old_new_or_fail( + &InMemoryBuilder { + faulty: true, + ..Default::default() + }, + ) + .await; } /// Purpose: a state machine that exports its data and position together passes. diff --git a/d-engine-server/src/api/embedded_test/shutdown_order_test.rs b/d-engine-server/src/api/embedded_test/shutdown_order_test.rs index d5e3b67f..7b482b33 100644 --- a/d-engine-server/src/api/embedded_test/shutdown_order_test.rs +++ b/d-engine-server/src/api/embedded_test/shutdown_order_test.rs @@ -15,9 +15,9 @@ use async_trait::async_trait; use bytes::Bytes; use d_engine_core::ApplyEntry; use d_engine_core::ApplyResult; -use d_engine_core::Error; use d_engine_core::ScanResult; use d_engine_core::StateMachine; +use d_engine_core::StateMachineError; use d_engine_proto::common::LogId; use d_engine_proto::server::storage::SnapshotMetadata; @@ -70,7 +70,7 @@ impl StateMachine for SlowApplyProbe { async fn apply_chunk( &self, chunk: &[ApplyEntry], - ) -> Result, Error> { + ) -> Result, StateMachineError> { self.flags.apply_started.store(true, Ordering::SeqCst); let closed_at_start = self.flags.storage_closed.load(Ordering::SeqCst); tokio::time::sleep(self.apply_delay).await; @@ -81,7 +81,7 @@ impl StateMachine for SlowApplyProbe { result } - fn close(&self) -> Result<(), Error> { + fn close(&self) -> Result<(), StateMachineError> { self.flags.storage_closed.store(true, Ordering::SeqCst); std::thread::sleep(self.close_delay); self.inner.close() @@ -91,7 +91,7 @@ impl StateMachine for SlowApplyProbe { fn get( &self, key_buffer: &[u8], - ) -> Result, Error> { + ) -> Result, StateMachineError> { self.note_read(); self.inner.get(key_buffer) } @@ -102,19 +102,19 @@ impl StateMachine for SlowApplyProbe { &self, metadata: &SnapshotMetadata, snapshot_path: std::path::PathBuf, - ) -> Result<(), Error> { + ) -> Result<(), StateMachineError> { self.inner.apply_snapshot_from_file(metadata, snapshot_path).await } async fn generate_snapshot_data( &self, new_snapshot_dir: std::path::PathBuf, - ) -> Result { + ) -> Result { self.inner.generate_snapshot_data(new_snapshot_dir).await } fn scan_prefix( &self, prefix: &[u8], - ) -> Result { + ) -> Result { self.note_read(); self.inner.scan_prefix(prefix) } diff --git a/d-engine-server/src/api/read_handle.rs b/d-engine-server/src/api/read_handle.rs index f72cb6ff..cb821ed4 100644 --- a/d-engine-server/src/api/read_handle.rs +++ b/d-engine-server/src/api/read_handle.rs @@ -1,7 +1,8 @@ //! ReadHandle — read routing for standalone (gRPC) mode. //! //! Routes reads through the ReadActor channel for Eventual/LeaseRead fast path, -//! falling back to `cmd_tx` (Raft readIndex) for LinearizableRead or on error. +//! falling back to `cmd_tx` (Raft readIndex) for LinearizableRead, when a lease is no longer valid, +//! or when the ReadActor cannot answer. A state machine read failure is returned to the caller. //! //! For embedded mode, use `EmbeddedReadHandle` which calls the SM directly //! without any channel overhead. @@ -94,11 +95,6 @@ pub(crate) fn map_error_response( }; not_leader_error(leader_id, leader_address, retry_after_ms) } - // A state machine read failure keeps its own code and its fixed message, as on the - // ReadActor fast path (`read_failure`). - ErrorCode::StorageIoError | ErrorCode::DataCorruption | ErrorCode::RetryRequired => { - ClientApiError::from(error) - } _ => server_error(format!("Error code: {error:?}")), } } @@ -129,29 +125,14 @@ pub(crate) fn extract_read_payload( /// Turns a state machine read failure into a client error without internal detail. /// The original error goes to the log and to a counter, so operators still see the cause. pub(crate) fn read_failure(e: &Error) -> ClientApiError { - let code = ErrorCode::for_read_failure(e); - // The failure class a dashboard groups by. - let kind = match code { - ErrorCode::StorageIoError => "storage_io", - ErrorCode::DataCorruption => "data_corruption", - ErrorCode::RetryRequired => "not_serving", - _ => "other", - }; - error!(error = ?e, kind, "state machine read failed"); - metrics::counter!("server.read.errors", "kind" => kind).increment(1); + error!(error = ?e, "state machine read failed"); + metrics::counter!("server.read.errors").increment(1); - match code { - // Fixed, friendly messages from the error code, e.g. "Storage I/O error" - ErrorCode::StorageIoError | ErrorCode::DataCorruption | ErrorCode::RetryRequired => { - ClientApiError::from(code) - } - _ => ClientApiError::Business { - code, - message: - "Read failed due to an internal error. Please retry or contact the administrator." - .to_string(), - required_action: None, - }, + ClientApiError::Business { + code: ErrorCode::Uncategorized, + message: "Read failed due to an internal error. Please retry or contact the administrator." + .to_string(), + required_action: None, } } @@ -183,7 +164,7 @@ impl ReadHandle { /// # Routing rules /// | Policy | Path | On failure | /// |----------------|----------------------------|-----------------------| - /// | Eventual/Lease | ReadActor (fast path) | fall through to cmd_tx| + /// | Eventual/Lease | ReadActor (fast path) | cmd_tx only if the lease is invalid or the ReadActor cannot answer | /// | Linearizable | cmd_tx always | — | /// | SmError | direct return | no fallback | /// @@ -236,8 +217,8 @@ impl ReadHandle { match tokio::time::timeout(timeout, reply_rx).await { Ok(Ok(Ok(values))) => return Ok(values), Ok(Ok(Err(ReadActorError::SmError(e)))) => return Err(read_failure(&e)), - // LeaseInvalid / SmUnavailable → fall through to cmd_tx - Ok(Ok(Err(ReadActorError::LeaseInvalid | ReadActorError::SmUnavailable))) => {} + // LeaseInvalid → fall through to cmd_tx + Ok(Ok(Err(ReadActorError::LeaseInvalid))) => {} // ReadActor exited without replying → fall through Ok(Err(_)) => {} // ReadActor stalled beyond timeout → fall through to cmd_tx diff --git a/d-engine-server/src/api/read_handle_test.rs b/d-engine-server/src/api/read_handle_test.rs index 00441b14..d77ca1f8 100644 --- a/d-engine-server/src/api/read_handle_test.rs +++ b/d-engine-server/src/api/read_handle_test.rs @@ -1,7 +1,7 @@ /// Unit tests for ReadHandle routing logic. /// /// ReadHandle is the single source of truth for read routing: -/// Eventual/LeaseRead → ReadActor fast path; fallback to cmd_tx on LeaseInvalid/SmUnavailable +/// Eventual/LeaseRead → ReadActor fast path; fallback to cmd_tx on LeaseInvalid only /// LinearizableRead → cmd_tx always /// SmError → returned directly, NO cmd_tx fallback /// @@ -17,10 +17,8 @@ mod read_handle_tests { use bytes::Bytes; use d_engine_core::ClientApiError; use d_engine_core::ClientCmd; - use d_engine_core::Error; use d_engine_core::MockStateMachine; use d_engine_core::ReadLease; - use d_engine_core::StorageError; use d_engine_core::client::ClientResponse; use d_engine_core::client::ErrorCode; use d_engine_core::client::KvEntry; @@ -68,13 +66,14 @@ mod read_handle_tests { fn sm_unavailable() -> MockStateMachine { let mut sm = MockStateMachine::new(); sm.expect_get_multi() - .returning(|_| Err(d_engine_core::StorageError::NotServing("stopped".into()).into())); + .returning(|_| Err(d_engine_core::StateMachineError::new("stopped"))); sm } fn sm_with_error() -> MockStateMachine { let mut sm = MockStateMachine::new(); - sm.expect_get_multi().returning(|_| Err(Error::Fatal("disk failure".into()))); + sm.expect_get_multi() + .returning(|_| Err(d_engine_core::StateMachineError::new("disk failure"))); sm } @@ -130,15 +129,16 @@ mod read_handle_tests { handle.await.unwrap(); } - /// Eventual, SM unavailable → SmUnavailable → fallback to cmd_tx. - /// cmd_rx dropped: error is Network (channel-closed), proving cmd_tx fallback triggered. + /// Eventual read, SM cannot serve: the read fails with the state machine's own error. The Raft + /// path would read the same local state machine again, so it is not tried. + /// cmd_rx dropped: a fallback would show up as a Network (channel-closed) error. #[tokio::test] - async fn test_eventual_sm_unavailable_falls_back_to_cmd_tx() { + async fn test_eventual_sm_not_serving_fails_without_trying_cmd_tx() { let (rh, handle) = make_handle(sm_unavailable(), valid_lease(), true).await; let result = rh.get(b"k", ReadConsistencyPolicy::EventualConsistency, 1, TIMEOUT).await; assert!( - matches!(result, Err(ClientApiError::Network { .. })), - "SmUnavailable must fall back to cmd_tx (Network error), got: {:?}", + matches!(&result, Err(e) if e.code() == ErrorCode::Uncategorized), + "the state machine's error must reach the caller, got: {:?}", result ); drop(rh); @@ -192,14 +192,15 @@ mod read_handle_tests { handle.await.unwrap(); } - /// LeaseRead, SM unavailable (even with valid lease) → SmUnavailable → fallback to cmd_tx. + /// LeaseRead with a valid lease, SM cannot serve: the read fails with the state machine's own + /// error; only an invalid lease sends a read to the Raft path. #[tokio::test] - async fn test_lease_read_sm_unavailable_falls_back_to_cmd_tx() { + async fn test_lease_read_sm_not_serving_fails_without_trying_cmd_tx() { let (rh, handle) = make_handle(sm_unavailable(), valid_lease(), true).await; let result = rh.get(b"k", ReadConsistencyPolicy::LeaseRead, 1, TIMEOUT).await; assert!( - matches!(result, Err(ClientApiError::Network { .. })), - "SM unavailable must fall back to cmd_tx, got: {:?}", + matches!(&result, Err(e) if e.code() == ErrorCode::Uncategorized), + "the state machine's error must reach the caller, got: {:?}", result ); drop(rh); @@ -244,52 +245,6 @@ mod read_handle_tests { handle.await.unwrap(); } - /// Storage failures keep a storage error code, so a client can tell a storage problem from - /// an internal bug, and the message stays free of engine detail. - #[tokio::test] - async fn test_sm_error_storage_failures_keep_their_storage_code() { - let cases: [(fn() -> Error, ErrorCode); 3] = [ - ( - || StorageError::DbError("rocksdb: secret detail".into()).into(), - ErrorCode::StorageIoError, - ), - ( - || StorageError::IoError(std::io::Error::other("secret detail")).into(), - ErrorCode::StorageIoError, - ), - ( - || { - StorageError::DataCorruption { - location: "secret detail".into(), - } - .into() - }, - ErrorCode::DataCorruption, - ), - ]; - - for (make_error, expected) in cases { - let mut sm = MockStateMachine::new(); - sm.expect_get_multi().returning(move |_| Err(make_error())); - let (rh, handle) = make_handle(sm, valid_lease(), true).await; - - let result = rh.get(b"k", ReadConsistencyPolicy::EventualConsistency, 1, TIMEOUT).await; - - match result { - Err(ClientApiError::Storage { code, message }) => { - assert_eq!(code, expected); - assert!( - !message.contains("secret detail"), - "detail leaked: {message}" - ); - } - other => panic!("expected a Storage error for {expected:?}, got: {other:?}"), - } - drop(rh); - handle.await.unwrap(); - } - } - // ── Linearizable — always cmd_tx ───────────────────────────────────────── /// LinearizableRead must never touch ReadActor — always routed to cmd_tx. @@ -770,24 +725,6 @@ mod error_helper_tests { } } - /// A read that goes through the Raft command channel (linearizable reads, eventual reads on a - /// follower, a fast-path read that fell back after `NotServing`) reports a state machine read - /// failure with one of these three codes. - /// - /// Purpose: the client must still be able to tell "retry" from "storage failure" from "the data - /// may be damaged", as it can on the fast path; mapping them to a generic error would hide it. - #[test] - fn test_map_error_response_keeps_each_read_failure_code_distinct() { - for code in [ - ErrorCode::StorageIoError, - ErrorCode::DataCorruption, - ErrorCode::RetryRequired, - ] { - let err = map_error_response(code, None, None); - assert_eq!(err.code(), code, "{code:?} must reach the client unchanged"); - } - } - // ─── extract_read_payload ──────────────────────────────────────────────── #[test] diff --git a/d-engine-server/src/lib.rs b/d-engine-server/src/lib.rs index 6598e617..372335c0 100644 --- a/d-engine-server/src/lib.rs +++ b/d-engine-server/src/lib.rs @@ -166,14 +166,14 @@ pub use d_engine_core::Result; pub use d_engine_core::ScanResult; /// Client-facing prefix scan result, returned by `EmbeddedClient::scan_prefix()`. pub use d_engine_core::ScanResults; -/// Snapshot-specific error type. Return one from [`StateMachine::apply_snapshot_from_file`] to -/// report a genuine install failure; stale, duplicate and conflicting snapshots never reach -/// that method, because the engine rejects them first. +/// Snapshot-specific error type (internal to the engine). pub use d_engine_core::SnapshotError; /// Storage trait for implementing custom storage backends /// /// Implement this trait to create your own storage engine. pub use d_engine_core::StateMachine; +/// The error a [`StateMachine`] returns for any failure. +pub use d_engine_core::StateMachineError; /// Storage trait for implementing custom storage backends /// /// Implement this trait to create your own storage engine. diff --git a/d-engine-server/src/network/grpc/grpc_raft_service_fast_path_test.rs b/d-engine-server/src/network/grpc/grpc_raft_service_fast_path_test.rs index 71d3eb26..19ebc695 100644 --- a/d-engine-server/src/network/grpc/grpc_raft_service_fast_path_test.rs +++ b/d-engine-server/src/network/grpc/grpc_raft_service_fast_path_test.rs @@ -2,7 +2,7 @@ /// /// Routing rules under test: /// Eventual/LeaseRead (any key count) → ReadActor fast path; fallback on -/// LeaseInvalid/SmUnavailable LinearizableRead → cmd_tx always +/// LeaseInvalid LinearizableRead → cmd_tx always /// SmError → error returned directly (no cmd_tx fallback) /// /// Proof technique (identical to read_handle_test.rs): @@ -20,6 +20,7 @@ mod grpc_fast_path_tests { use d_engine_core::ReadLease; use d_engine_core::now_ms; use d_engine_proto::client::ClientReadRequest; + use d_engine_proto::client::ClientResponse; use d_engine_proto::client::ReadConsistencyPolicy as ProtoPolicy; use d_engine_proto::client::client_response::SuccessResult; use d_engine_proto::client::raft_client_service_server::RaftClientService; @@ -56,14 +57,14 @@ mod grpc_fast_path_tests { fn sm_unavailable() -> MockStateMachine { let mut sm = MockStateMachine::new(); sm.expect_get_multi() - .returning(|_| Err(d_engine_core::StorageError::NotServing("stopped".into()).into())); + .returning(|_| Err(d_engine_core::StateMachineError::new("stopped"))); sm } fn sm_with_error() -> MockStateMachine { let mut sm = MockStateMachine::new(); sm.expect_get_multi() - .returning(|_| Err(d_engine_core::Error::Fatal("disk failure".into()))); + .returning(|_| Err(d_engine_core::StateMachineError::new("disk failure"))); sm } @@ -165,20 +166,18 @@ mod grpc_fast_path_tests { handle.await.unwrap(); } - /// Eventual, SM unavailable → SmUnavailable → fallback to cmd_tx. - /// cmd_rx dropped: error proves fallback was triggered. + /// Eventual, SM fails: the client gets an internal error, not + /// the result of trying the Raft path, which would read the same state machine again. + /// cmd_rx dropped: a fallback would come back as a channel-closed error. #[tokio::test] - async fn test_eventual_sm_unavailable_falls_back_to_cmd_tx() { + async fn test_eventual_sm_failure_reaches_client_as_internal_error() { let (node, handle) = make_node(sm_unavailable(), valid_lease(), true).await; let result = node .handle_client_read(Request::new(req(b"k", ProtoPolicy::EventualConsistency))) .await; - assert!( - result.is_err(), - "SM unavailable must fall back to closed cmd_tx" - ); + assert_read_fails_with_internal_error(result); drop(node); handle.await.unwrap(); } @@ -214,17 +213,14 @@ mod grpc_fast_path_tests { handle.await.unwrap(); } - /// LeaseRead, SM unavailable (even valid lease) → SmUnavailable → fallback to cmd_tx. + /// LeaseRead with a valid lease, SM cannot serve: the client gets the state machine's error. #[tokio::test] - async fn test_lease_read_sm_unavailable_falls_back_to_cmd_tx() { + async fn test_lease_read_sm_failure_reaches_client_as_internal_error() { let (node, handle) = make_node(sm_unavailable(), valid_lease(), true).await; let result = node.handle_client_read(Request::new(req(b"k", ProtoPolicy::LeaseRead))).await; - assert!( - result.is_err(), - "SM unavailable must fall back to closed cmd_tx" - ); + assert_read_fails_with_internal_error(result); drop(node); handle.await.unwrap(); } @@ -251,11 +247,23 @@ mod grpc_fast_path_tests { handle.await.unwrap(); } - // ── Linearizable — always cmd_tx ───────────────────────────────────────── + // ── Read failure codes reach a remote client ───────────────────────────── + + /// A state machine read failure answers a remote client with a plain internal error: no + /// retry through the Raft path (which would read the same state machine again), and no detail + /// of the failure. + fn assert_read_fails_with_internal_error( + result: Result, tonic::Status> + ) { + let status = result.expect_err("a failed read must be an error"); + assert_eq!(status.code(), tonic::Code::Internal); + assert!( + !status.message().contains("stopped") && !status.message().contains("channel closed"), + "unexpected message: {}", + status.message() + ); + } - /// LinearizableRead must never touch ReadActor — always routed to cmd_tx. - /// No expect_get_multi() on SM: mockall panics if get_multi() is called (implicit safety net). - /// cmd_rx dropped: error proves cmd_tx was used (not fast path). #[tokio::test] async fn test_linearizable_always_uses_cmd_tx() { let sm = MockStateMachine::new(); diff --git a/d-engine-server/src/read_actor.rs b/d-engine-server/src/read_actor.rs index 909b6b75..bf1a1cfd 100644 --- a/d-engine-server/src/read_actor.rs +++ b/d-engine-server/src/read_actor.rs @@ -50,15 +50,12 @@ pub(crate) struct ReadCmd { /// Error returned by the ReadActor to the caller. /// -/// `LeaseInvalid` and `SmUnavailable` are recoverable: fall back to `cmd_tx`. +/// `LeaseInvalid` is recoverable: fall back to `cmd_tx`. /// `SmError` is a hard failure; do NOT retry via cmd_tx. #[derive(Debug)] pub(crate) enum ReadActorError { /// Lease has been revoked or expired. Retry via cmd_tx. LeaseInvalid, - /// The state machine cannot serve reads right now: it is installing a snapshot or has been - /// closed. Retry via cmd_tx, which waits behind the install or reports the shutdown. - SmUnavailable, /// The state machine failed to read. Keeps the original error for logging and /// classification; it must not reach the client as is. SmError(Error), @@ -126,11 +123,5 @@ where _ => return Err(ReadActorError::LeaseInvalid), } - sm.get_multi(&cmd.keys).map_err(|e| { - if e.is_not_serving() { - ReadActorError::SmUnavailable - } else { - ReadActorError::SmError(e) - } - }) + sm.get_multi(&cmd.keys).map_err(|e| ReadActorError::SmError(e.into())) } diff --git a/d-engine-server/src/read_actor_test.rs b/d-engine-server/src/read_actor_test.rs index 90944557..6202377f 100644 --- a/d-engine-server/src/read_actor_test.rs +++ b/d-engine-server/src/read_actor_test.rs @@ -1,7 +1,6 @@ use std::sync::Arc; use bytes::Bytes; -use d_engine_core::Error; use d_engine_core::MockStateMachine; use d_engine_core::ReadLease; use d_engine_core::config::ReadConsistencyPolicy; @@ -20,7 +19,7 @@ fn make_sm_running() -> MockStateMachine { fn make_sm_unavailable() -> MockStateMachine { let mut sm = MockStateMachine::new(); sm.expect_get_multi() - .returning(|_| Err(d_engine_core::StorageError::NotServing("stopped".into()).into())); + .returning(|_| Err(d_engine_core::StateMachineError::new("stopped"))); sm } @@ -145,9 +144,10 @@ async fn test_read_actor_eventual_multi_key_returns_ordered_values() { handle.await.unwrap(); } -/// Eventual read answers SmUnavailable when the SM cannot serve (NotServing) +/// An eventual read is answered with the state machine's own error when the SM cannot serve it. +/// There is no other place to read from, so it is not turned into a "try elsewhere" answer. #[tokio::test] -async fn test_read_actor_eventual_sm_unavailable_returns_error() { +async fn test_read_actor_eventual_sm_not_serving_returns_the_sm_error() { let sm = make_sm_unavailable(); let (tx, rx) = mpsc::channel(8); let handle = tokio::spawn(run_read_actor(rx, valid_lease(), Arc::new(sm), 64)); @@ -156,7 +156,7 @@ async fn test_read_actor_eventual_sm_unavailable_returns_error() { drop(tx); let err = reply_rx.await.unwrap().unwrap_err(); - assert!(matches!(err, ReadActorError::SmUnavailable)); + assert!(matches!(err, ReadActorError::SmError(_)), "got: {err:?}"); handle.await.unwrap(); } @@ -210,10 +210,10 @@ async fn test_read_actor_lease_expired_returns_lease_invalid() { handle.await.unwrap(); } -/// LeaseRead falls back (SmUnavailable, not a hard SmError) when the SM answers NotServing, even if -/// the lease is valid +/// A lease read with a valid lease is answered with the state machine's own error when the SM +/// cannot serve it; only an invalid lease sends the read to the Raft path. #[tokio::test] -async fn test_read_actor_lease_valid_but_sm_unavailable_returns_error() { +async fn test_read_actor_lease_valid_but_sm_not_serving_returns_the_sm_error() { let sm = make_sm_unavailable(); let (tx, rx) = mpsc::channel(8); let handle = tokio::spawn(run_read_actor(rx, valid_lease(), Arc::new(sm), 64)); @@ -222,7 +222,7 @@ async fn test_read_actor_lease_valid_but_sm_unavailable_returns_error() { drop(tx); let err = reply_rx.await.unwrap().unwrap_err(); - assert!(matches!(err, ReadActorError::SmUnavailable)); + assert!(matches!(err, ReadActorError::SmError(_)), "got: {err:?}"); handle.await.unwrap(); } @@ -285,7 +285,8 @@ async fn test_read_actor_eventual_multi_key_uses_get_multi() { #[tokio::test] async fn test_read_actor_sm_get_multi_error_returns_sm_error() { let mut sm = make_sm_running(); - sm.expect_get_multi().returning(|_| Err(Error::Fatal("disk failure".into()))); + sm.expect_get_multi() + .returning(|_| Err(d_engine_core::StateMachineError::new("disk failure"))); let (tx, rx) = mpsc::channel(8); let handle = tokio::spawn(run_read_actor(rx, valid_lease(), Arc::new(sm), 64)); diff --git a/d-engine-server/src/storage/adaptors/file/file_state_machine.rs b/d-engine-server/src/storage/adaptors/file/file_state_machine.rs index d6f67da6..d6324e00 100644 --- a/d-engine-server/src/storage/adaptors/file/file_state_machine.rs +++ b/d-engine-server/src/storage/adaptors/file/file_state_machine.rs @@ -68,8 +68,8 @@ use d_engine_core::ApplyResult; use d_engine_core::Command; use d_engine_core::Error; use d_engine_core::ScanResult; -use d_engine_core::SnapshotError; use d_engine_core::StateMachine; +use d_engine_core::StateMachineError; use d_engine_core::StorageError; use d_engine_proto::common::LogId; use d_engine_proto::server::storage::SnapshotMetadata; @@ -836,20 +836,20 @@ impl Drop for FileStateMachine { #[async_trait] impl StateMachine for FileStateMachine { - fn close(&self) -> Result<(), Error> { + fn close(&self) -> Result<(), StateMachineError> { if !self.running.swap(false, Ordering::SeqCst) { return Ok(()); // already closed } // `Drop` does not run when the engine stops if the developer still holds a shared // handle: persist everything a restart needs here. // No exclusive OS lock file to release. - self.save_hard_state() + self.save_hard_state().map_err(Into::into) } fn get( &self, key_buffer: &[u8], - ) -> Result, Error> { + ) -> Result, StateMachineError> { self.ensure_serving()?; let data = self.data.read(); Ok(data.get(key_buffer).map(|(value, _)| value.clone())) @@ -859,7 +859,7 @@ impl StateMachine for FileStateMachine { async fn apply_chunk( &self, chunk: &[ApplyEntry], - ) -> Result, Error> { + ) -> Result, StateMachineError> { self.ensure_serving()?; let chunk_len = chunk.len(); let mut results = Vec::with_capacity(chunk_len); @@ -1066,10 +1066,10 @@ impl StateMachine for FileStateMachine { &self, metadata: &SnapshotMetadata, snapshot_dir: std::path::PathBuf, - ) -> Result<(), Error> { - let incoming = metadata.last_included.ok_or_else(|| { - SnapshotError::OperationFailed("Missing last_included in snapshot metadata".into()) - })?; + ) -> Result<(), StateMachineError> { + let incoming = metadata + .last_included + .ok_or_else(|| StateMachineError::new("Missing last_included in snapshot metadata"))?; info!("Applying snapshot from file: {:?}", snapshot_dir); @@ -1182,7 +1182,7 @@ impl StateMachine for FileStateMachine { async fn generate_snapshot_data( &self, new_snapshot_dir: std::path::PathBuf, - ) -> Result { + ) -> Result { // Create snapshot directory fs::create_dir_all(&new_snapshot_dir).await?; @@ -1223,7 +1223,7 @@ impl StateMachine for FileStateMachine { fn scan_prefix( &self, prefix: &[u8], - ) -> Result { + ) -> Result { self.ensure_serving()?; let data = self.data.read(); let entries: Vec<(Bytes, Bytes)> = data @@ -1238,7 +1238,7 @@ impl StateMachine for FileStateMachine { fn get_multi( &self, keys: &[Bytes], - ) -> d_engine_core::Result>> { + ) -> Result>, StateMachineError> { self.ensure_serving()?; // Hold read lock for the full batch so all keys come from the same consistent view. // apply_chunk holds the write lock; parking_lot RwLock ensures they don't interleave. diff --git a/d-engine-server/src/storage/adaptors/file/file_state_machine_test.rs b/d-engine-server/src/storage/adaptors/file/file_state_machine_test.rs index f3e9bf71..bcd16aa7 100644 --- a/d-engine-server/src/storage/adaptors/file/file_state_machine_test.rs +++ b/d-engine-server/src/storage/adaptors/file/file_state_machine_test.rs @@ -19,7 +19,7 @@ use crate::storage::adaptors::file::FileStateMachine; // ── close() tests ───────────────────────────────────────────────────────────── -/// close() must stop the SM from serving: reads answer NotServing afterwards. +/// close() must stop the SM from serving: reads fail afterwards. /// FileStateMachine has no exclusive OS lock, so close() is a lifecycle /// signal and flushes state to disk. #[tokio::test] @@ -33,14 +33,13 @@ async fn test_file_sm_close_stops_serving() { sm.close().unwrap(); - let err = sm.get(b"k").expect_err("close must stop serving"); - assert!(format!("{err:?}").contains("NotServing"), "got: {err:?}"); + assert!(sm.get(b"k").is_err(), "close must stop serving"); } /// Business scenario: the node shuts down while the developer still holds a clone of the state /// machine (for example an embedded client reading directly from it). /// -/// Purpose: once closed, every read fails with NotServing instead of answering from state that is +/// Purpose: once closed, every read fails instead of answering from state that is /// no longer maintained. Same contract as the RocksDB state machine. #[tokio::test] async fn test_file_sm_reads_fail_after_close() { @@ -64,17 +63,15 @@ async fn test_file_sm_reads_fail_after_close() { sm.close().unwrap(); - let not_serving = |err: d_engine_core::Error| format!("{err:?}").contains("NotServing"); - assert!(not_serving( - sm.get(b"k").expect_err("get must fail after close") - )); - assert!(not_serving( - sm.get_multi(&[Bytes::from_static(b"k")]) - .expect_err("get_multi must fail after close") - )); - assert!(not_serving( - sm.scan_prefix(b"k").expect_err("scan_prefix must fail after close") - )); + assert!(sm.get(b"k").is_err(), "get must fail after close"); + assert!( + sm.get_multi(&[Bytes::from_static(b"k")]).is_err(), + "get_multi must fail after close" + ); + assert!( + sm.scan_prefix(b"k").is_err(), + "scan_prefix must fail after close" + ); } /// close() is idempotent (the worker and the `Drop` safety net may both close). @@ -84,8 +81,7 @@ async fn test_file_sm_close_twice_is_safe() { let sm = FileStateMachine::new(dir.path().to_path_buf()).await.unwrap(); sm.close().unwrap(); sm.close().unwrap(); - let err = sm.get(b"k").expect_err("still closed after the second close"); - assert!(format!("{err:?}").contains("NotServing"), "got: {err:?}"); + assert!(sm.get(b"k").is_err(), "still closed after the second close"); } /// Builder for FileStateMachine test instances diff --git a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs index d451e931..2a8ecbe9 100644 --- a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs +++ b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs @@ -12,8 +12,8 @@ use d_engine_core::ApplyResult; use d_engine_core::Command; use d_engine_core::Error; use d_engine_core::ScanResult; -use d_engine_core::SnapshotError; use d_engine_core::StateMachine; +use d_engine_core::StateMachineError; use d_engine_core::StorageError; use d_engine_proto::common::LogId; use d_engine_proto::server::storage::SnapshotMetadata; @@ -568,7 +568,7 @@ impl RocksDBStateMachine { impl StateMachine for RocksDBStateMachine { /// Sequence: stop serving → flush → save hard state → cancel background /// work → drop the DB handle (releasing the LOCK). A second call finds no DB and does nothing. - fn close(&self) -> Result<(), Error> { + fn close(&self) -> Result<(), StateMachineError> { if self.db.load().is_none() { return Ok(()); } @@ -577,14 +577,14 @@ impl StateMachine for RocksDBStateMachine { // Every step runs even if an earlier one failed, so the LOCK is always released; // the first error is returned. - let mut first_err: Option = None; - let mut record = |step: &str, result: Result<(), Error>| { + let mut first_err: Option = None; + let mut record = |step: &str, result: Result<(), StateMachineError>| { if let Err(e) = result { error!("close: {step} failed: {e:?}"); first_err.get_or_insert(e); } }; - record("flush", self.flush()); + record("flush", self.flush().map_err(Into::into)); if let Some(db) = self.db.load().as_deref() { db.cancel_all_background_work(true); } @@ -597,15 +597,14 @@ impl StateMachine for RocksDBStateMachine { fn get( &self, key_buffer: &[u8], - ) -> Result, Error> { + ) -> Result, StateMachineError> { // Guard against reads during snapshot restoration // During apply_snapshot_from_file(), is_serving is set to false while the database // is being replaced. This prevents reads from accessing temporary or inconsistent state. if !self.is_serving.load(Ordering::SeqCst) { - return Err(StorageError::NotServing( - "State machine is restoring from snapshot".to_string(), - ) - .into()); + return Err(StateMachineError::new( + "State machine is restoring from snapshot", + )); } self.with_db(|db| { @@ -614,11 +613,12 @@ impl StateMachine for RocksDBStateMachine { "State machine CF not found".to_string(), ))) })?; - match db.get_cf(&cf, key_buffer).map_err(|e| StorageError::DbError(e.to_string()))? { + match db.get_cf(&cf, key_buffer).map_err(StateMachineError::new)? { Some(value) => Ok(Some(Bytes::copy_from_slice(&value))), None => Ok(None), } }) + .map_err(StateMachineError::from) } /// Thread-safe: called serially by the state machine worker @@ -626,20 +626,20 @@ impl StateMachine for RocksDBStateMachine { async fn apply_chunk( &self, chunk: &[ApplyEntry], - ) -> Result, Error> { + ) -> Result, StateMachineError> { // Inline guard: db and cf must share a lifetime across the entire function body // (batch borrows cf, CAS reads borrow db). Using with_db() closure would require // moving the whole async function body into a sync closure — use Guard directly. let guard = self.db.load(); let Some(db) = guard.as_deref() else { - return Err(StorageError::NotServing("state machine stopped".into()).into()); + return Err(StateMachineError::new("state machine stopped")); }; let cf = db .cf_handle(STATE_MACHINE_CF) - .ok_or_else(|| StorageError::DbError("State machine CF not found".to_string()))?; + .ok_or_else(|| StateMachineError::new("State machine CF not found"))?; let meta_cf = db .cf_handle(STATE_MACHINE_META_CF) - .ok_or_else(|| StorageError::DbError("State machine meta CF not found".to_string()))?; + .ok_or_else(|| StateMachineError::new("State machine meta CF not found"))?; let mut batch = WriteBatchWithIndex::new(0, true); let mut highest_index_entry: Option = None; @@ -683,7 +683,7 @@ impl StateMachine for RocksDBStateMachine { // are visible, preventing stale-read linearizability violations. let current_value = batch .get_from_batch_and_db_cf(db, &cf, key, &ReadOptions::default()) - .map_err(|e| StorageError::DbError(format!("CAS read failed: {e}")))?; + .map_err(|e| StateMachineError::new(format!("CAS read failed: {e}")))?; let cas_success = match (current_value, expected) { (Some(current), Some(exp)) => current == exp.as_ref(), @@ -738,7 +738,7 @@ impl StateMachine for RocksDBStateMachine { let _t0 = std::time::Instant::now(); SM_WRITE_OPTS .with(|opts| db.write_wbwi_opt(&batch, opts)) - .map_err(|e| StorageError::DbError(e.to_string()))?; + .map_err(StateMachineError::new)?; if let Some(highest) = highest_index_entry { self.update_last_applied(highest); @@ -758,10 +758,10 @@ impl StateMachine for RocksDBStateMachine { &self, metadata: &SnapshotMetadata, snapshot_dir: std::path::PathBuf, - ) -> Result<(), Error> { - let incoming = metadata.last_included.ok_or_else(|| { - SnapshotError::OperationFailed("Missing last_included in snapshot metadata".into()) - })?; + ) -> Result<(), StateMachineError> { + let incoming = metadata + .last_included + .ok_or_else(|| StateMachineError::new("Missing last_included in snapshot metadata"))?; info!("Applying snapshot from: {:?}", snapshot_dir); @@ -781,13 +781,13 @@ impl StateMachine for RocksDBStateMachine { self.is_serving.store(true, Ordering::SeqCst); } - result + result.map_err(Into::into) } async fn generate_snapshot_data( &self, new_snapshot_dir: std::path::PathBuf, - ) -> Result { + ) -> Result { // Export only SM CFs: compact SST-only export, no Raft log data. // export_column_family() creates only the final subdirectory (e.g. "sm"), so the parent // (new_snapshot_dir) must already exist before calling it. @@ -803,7 +803,7 @@ impl StateMachine for RocksDBStateMachine { Self::export_frozen_view(&db, &dir) }) .await - .map_err(Self::map_snapshot_join_error)??; + .map_err(|e| StateMachineError::new(Self::map_snapshot_join_error(e).to_string()))??; info!("Snapshot generated at {:?}", new_snapshot_dir); Ok(position) @@ -812,19 +812,18 @@ impl StateMachine for RocksDBStateMachine { fn scan_prefix( &self, prefix: &[u8], - ) -> Result { - self.scan_prefix(prefix) + ) -> Result { + self.scan_prefix(prefix).map_err(Into::into) } fn get_multi( &self, keys: &[Bytes], - ) -> Result>, Error> { + ) -> Result>, StateMachineError> { if !self.is_serving.load(Ordering::SeqCst) { - return Err(StorageError::NotServing( - "State machine is restoring from snapshot".to_string(), - ) - .into()); + return Err(StateMachineError::new( + "State machine is restoring from snapshot", + )); } self.with_db(|db| { let cf = db.cf_handle(STATE_MACHINE_CF).ok_or_else(|| { @@ -853,6 +852,7 @@ impl StateMachine for RocksDBStateMachine { }) .collect() }) + .map_err(StateMachineError::from) } } impl Drop for RocksDBStateMachine { diff --git a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine_test.rs b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine_test.rs index e52d2814..65df2000 100644 --- a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine_test.rs +++ b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine_test.rs @@ -545,7 +545,7 @@ async fn test_get_multi_two_keys_still_uses_snapshot_isolation() { // immediately, without waiting for every reference to the state machine to be dropped (a // handle the developer shares may outlive the engine). -/// close() makes reads return a NotServing error, never an empty result. +/// close() makes reads return an error, never an empty result. #[tokio::test] async fn test_close_prevents_reads() { let dir = TempDir::new().unwrap(); @@ -555,10 +555,9 @@ async fn test_close_prevents_reads() { sm.close().unwrap(); - let err = sm.get(b"k").expect_err("get must fail after close"); assert!( - format!("{err:?}").contains("NotServing"), - "a closed state machine must say NotServing, got: {err:?}" + sm.get(b"k").is_err(), + "a closed state machine must fail reads, not answer with an empty result" ); assert!( sm.get_multi(&[Bytes::from_static(b"k")]).is_err(), @@ -574,8 +573,7 @@ fn test_close_twice_is_safe() { let (_storage, sm) = RocksDBUnifiedEngine::open(dir.path()).unwrap(); sm.close().unwrap(); sm.close().unwrap(); - let err = sm.get(b"k").expect_err("still closed after the second close"); - assert!(format!("{err:?}").contains("NotServing"), "got: {err:?}"); + assert!(sm.get(b"k").is_err(), "still closed after the second close"); } /// Drop after close() must not panic or double-flush. diff --git a/d-engine-server/tests/common/shared_state_machine.rs b/d-engine-server/tests/common/shared_state_machine.rs index b9c7ca86..16126436 100644 --- a/d-engine-server/tests/common/shared_state_machine.rs +++ b/d-engine-server/tests/common/shared_state_machine.rs @@ -15,9 +15,9 @@ use async_trait::async_trait; use bytes::Bytes; use d_engine_core::ApplyEntry; use d_engine_core::ApplyResult; -use d_engine_core::Error; use d_engine_core::ScanResult; use d_engine_core::StateMachine; +use d_engine_core::StateMachineError; use d_engine_proto::common::LogId; use d_engine_proto::server::storage::SnapshotMetadata; @@ -33,25 +33,25 @@ impl SharedStateMachine { #[async_trait] impl StateMachine for SharedStateMachine { - fn close(&self) -> Result<(), Error> { + fn close(&self) -> Result<(), StateMachineError> { self.0.close() } fn get( &self, key_buffer: &[u8], - ) -> Result, Error> { + ) -> Result, StateMachineError> { self.0.get(key_buffer) } fn get_multi( &self, keys: &[Bytes], - ) -> Result>, Error> { + ) -> Result>, StateMachineError> { self.0.get_multi(keys) } async fn apply_chunk( &self, chunk: &[ApplyEntry], - ) -> Result, Error> { + ) -> Result, StateMachineError> { self.0.apply_chunk(chunk).await } fn last_applied(&self) -> LogId { @@ -61,19 +61,19 @@ impl StateMachine for SharedStateMachine { &self, metadata: &SnapshotMetadata, snapshot_path: std::path::PathBuf, - ) -> Result<(), Error> { + ) -> Result<(), StateMachineError> { self.0.apply_snapshot_from_file(metadata, snapshot_path).await } async fn generate_snapshot_data( &self, new_snapshot_dir: std::path::PathBuf, - ) -> Result { + ) -> Result { self.0.generate_snapshot_data(new_snapshot_dir).await } fn scan_prefix( &self, prefix: &[u8], - ) -> Result { + ) -> Result { self.0.scan_prefix(prefix) } } diff --git a/d-engine-server/tests/consistent_reads/linearizable_read_consistency_embedded.rs b/d-engine-server/tests/consistent_reads/linearizable_read_consistency_embedded.rs index 3d145d05..c9dc2e63 100644 --- a/d-engine-server/tests/consistent_reads/linearizable_read_consistency_embedded.rs +++ b/d-engine-server/tests/consistent_reads/linearizable_read_consistency_embedded.rs @@ -32,6 +32,9 @@ async fn create_test_engine(test_name: &str) -> (DefaultEmbeddedEngine, TempDir) listen_address = "127.0.0.1:{}" single_node = true +[raft] +general_raft_timeout_duration_in_ms = 2000 + [raft.read_consistency] state_machine_sync_timeout_ms = 2000 "#, diff --git a/d-engine/src/docs/server_guide/customize-state-machine.md b/d-engine/src/docs/server_guide/customize-state-machine.md index a18f79d7..7236d599 100644 --- a/d-engine/src/docs/server_guide/customize-state-machine.md +++ b/d-engine/src/docs/server_guide/customize-state-machine.md @@ -12,7 +12,7 @@ d-engine supports **pluggable state machines** through the `StateMachine` trait. ## 1. Implement the Trait ```rust,ignore -use d_engine::{ApplyEntry, ApplyResult, Command, Error, LogId, SnapshotMetadata, StateMachine}; +use d_engine::{ApplyEntry, ApplyResult, Command, LogId, SnapshotMetadata, StateMachine, StateMachineError}; use async_trait::async_trait; struct CustomStateMachine { @@ -37,18 +37,18 @@ impl StateMachine for CustomStateMachine { // If you share your storage handle with the application (a cloneable state machine), the // state machine is not dropped when the engine stops, so do not rely on Drop alone. // - // fn close(&self) -> Result<(), Error> { + // fn close(&self) -> Result<(), StateMachineError> { // let flushed = self.backend.flush(); // self.backend.close(); // release lock file / DB handle // flushed // } - fn get(&self, key_buffer: &[u8]) -> Result>, Error> { + fn get(&self, key_buffer: &[u8]) -> Result>, StateMachineError> { // Retrieve value by key self.backend.get(key_buffer) } - async fn apply_chunk(&self, chunk: &[ApplyEntry]) -> Result, Error> { + async fn apply_chunk(&self, chunk: &[ApplyEntry]) -> Result, StateMachineError> { // The framework pre-decodes proto bytes — match on Command directly, no prost needed. let mut results = Vec::with_capacity(chunk.len()); for entry in chunk { diff --git a/d-engine/src/lib.rs b/d-engine/src/lib.rs index 8f2d4b53..499f3933 100644 --- a/d-engine/src/lib.rs +++ b/d-engine/src/lib.rs @@ -47,6 +47,8 @@ pub mod prelude { #[cfg(feature = "server")] pub use d_engine_server::StateMachine; #[cfg(feature = "server")] + pub use d_engine_server::StateMachineError; + #[cfg(feature = "server")] pub use d_engine_server::StorageEngine; } diff --git a/examples/sled-cluster/src/sled_state_machine.rs b/examples/sled-cluster/src/sled_state_machine.rs index f5eab73e..9e68ba94 100644 --- a/examples/sled-cluster/src/sled_state_machine.rs +++ b/examples/sled-cluster/src/sled_state_machine.rs @@ -19,6 +19,7 @@ use d_engine::Result; use d_engine::ScanResult; use d_engine::SnapshotError; use d_engine::StateMachine; +use d_engine::StateMachineError; use d_engine::StorageError; use d_engine::common::LogId; use d_engine::server_storage::SnapshotMetadata; @@ -86,7 +87,7 @@ impl Drop for SledStateMachine { #[async_trait] impl StateMachine for SledStateMachine { - fn close(&self) -> Result<()> { + fn close(&self) -> std::result::Result<(), StateMachineError> { if self.db.load().is_none() { return Ok(()); } @@ -95,7 +96,7 @@ impl StateMachine for SledStateMachine { // failed. The lock is released once no other handle to the db is alive. let saved = self.save_hard_state(); self.db.store(None); - saved + saved.map_err(Into::into) } fn last_applied(&self) -> LogId { @@ -108,13 +109,13 @@ impl StateMachine for SledStateMachine { fn get( &self, key_buffer: &[u8], - ) -> Result> { + ) -> std::result::Result, StateMachineError> { match self.current_tree()?.get(key_buffer) { Ok(Some(v)) => Ok(Some(Bytes::copy_from_slice(&v))), Ok(None) => Ok(None), Err(e) => { error!("state_machine get error: {}", e); - Err(StorageError::DbError(e.to_string()).into()) + Err(StateMachineError::new(e.to_string())) } } } @@ -122,7 +123,7 @@ impl StateMachine for SledStateMachine { async fn apply_chunk( &self, chunk: &[ApplyEntry], - ) -> Result> { + ) -> std::result::Result, StateMachineError> { trace!("Applying chunk: {} entries", chunk.len()); let db = self.db()?; @@ -153,7 +154,7 @@ impl StateMachine for SledStateMachine { let data_tree = self.current_tree()?; let meta_tree = db .open_tree(STATE_MACHINE_META_NAMESPACE) - .map_err(|e| StorageError::DbError(e.to_string()))?; + .map_err(|e| StateMachineError::new(e.to_string()))?; let results = (&data_tree, &meta_tree) .transaction(|(data, meta)| { @@ -227,7 +228,7 @@ impl StateMachine for SledStateMachine { }) .map_err(|e: sled::transaction::TransactionError| { error!("state_machine apply_chunk failed: {}", e); - StorageError::DbError(e.to_string()) + StateMachineError::new(e.to_string()) })?; self.update_last_applied(position); @@ -238,7 +239,7 @@ impl StateMachine for SledStateMachine { async fn generate_snapshot_data( &self, new_snapshot_dir: PathBuf, - ) -> Result { + ) -> std::result::Result { // Known limits of this example: the tree is iterated while `apply_chunk` may run, so the // copy is not a point-in-time view, and the position below comes from memory, not from the // exported data. Reading it before the copy keeps it at or behind the data. A state machine @@ -248,7 +249,7 @@ impl StateMachine for SledStateMachine { // 2. Create a new state machine database instance let new_db = init_sled_state_machine_db(&new_snapshot_dir) - .map_err(|e| StorageError::DbError(e.to_string()))?; + .map_err(|e| StateMachineError::new(e.to_string()))?; let exist_db_tree = self.current_tree()?; let new_state_machine_tree = new_tree(&new_db, STATE_MACHINE_TREE)?; @@ -260,7 +261,7 @@ impl StateMachine for SledStateMachine { // as log indices. The SM already reflects state at last_included.index, so // all current entries are valid snapshot content. for item in exist_db_tree.iter() { - let (k, v) = item.map_err(|e| StorageError::DbError(e.to_string()))?; + let (k, v) = item.map_err(|e| StateMachineError::new(e.to_string()))?; batch.insert(k, v); counter += 1; @@ -268,17 +269,17 @@ impl StateMachine for SledStateMachine { if counter % 100 == 0 { new_state_machine_tree .apply_batch(batch) - .map_err(|e| StorageError::DbError(e.to_string()))?; + .map_err(|e| StateMachineError::new(e.to_string()))?; batch = sled::Batch::default(); } } new_state_machine_tree .apply_batch(batch) - .map_err(|e| StorageError::DbError(e.to_string()))?; + .map_err(|e| StateMachineError::new(e.to_string()))?; // Make sure flush into disk - new_db.flush().map_err(|e| StorageError::DbError(e.to_string()))?; + new_db.flush().map_err(|e| StateMachineError::new(e.to_string()))?; Ok(position) } @@ -287,14 +288,14 @@ impl StateMachine for SledStateMachine { &self, metadata: &SnapshotMetadata, decompressed_snapshot_path: PathBuf, - ) -> Result<()> { + ) -> std::result::Result<(), StateMachineError> { // `StateMachineWorker` never runs this concurrently with `apply_chunk`, a snapshot // export or another install (#436). // The engine only calls this for a snapshot newer than `last_applied()`. - let new_last_included = metadata.last_included.ok_or_else(|| { - SnapshotError::OperationFailed("Missing last_included in snapshot metadata".into()) - })?; + let new_last_included = metadata + .last_included + .ok_or_else(|| StateMachineError::new("Missing last_included in snapshot metadata"))?; // `decompressed_snapshot_path` is a temporary directory owned by the caller // (see `PreparedSnapshot`), not this state machine's own `self.path`. Opening a @@ -340,12 +341,12 @@ impl StateMachine for SledStateMachine { Ok(()) }) .await - .map_err(|e| StorageError::DbError(format!("snapshot install task panicked: {e}")))? - .map_err(|e| StorageError::DbError(format!("failed to move snapshot into place: {e}")))?; + .map_err(|e| StateMachineError::new(format!("snapshot install task panicked: {e}")))? + .map_err(|e| StateMachineError::new(format!("failed to move snapshot into place: {e}")))?; debug!("Reopening state machine database at its permanent path"); let db = init_sled_state_machine_db(&self.path) - .map_err(|e| StorageError::DbError(e.to_string()))?; + .map_err(|e| StateMachineError::new(e.to_string()))?; self.db.store(Some(Arc::new(db))); // Best-effort: a leftover `*.pre-install-backup` is harmless (just disk space) @@ -368,11 +369,11 @@ impl StateMachine for SledStateMachine { fn scan_prefix( &self, prefix: &[u8], - ) -> Result { + ) -> std::result::Result { let tree = self.current_tree()?; let mut entries = Vec::new(); for result in tree.scan_prefix(prefix) { - let (k, v) = result.map_err(|e| StorageError::DbError(e.to_string()))?; + let (k, v) = result.map_err(|e| StateMachineError::new(e.to_string()))?; entries.push((Bytes::copy_from_slice(&k), Bytes::copy_from_slice(&v))); } let revision = self.last_applied_index.load(Ordering::SeqCst); From c735c21a6dd8981c0134485fb870be7bf4e5bb05 Mon Sep 17 00:00:00 2001 From: Joshua Chi <112539+JoshuaChi@users.noreply.github.com> Date: Wed, 4 Nov 2026 17:11:12 +0800 Subject: [PATCH 17/19] refactor #400: Drop From for StateMachineError, convert explicitly --- MIGRATION_GUIDE.md | 96 +++++++------------ d-engine-core/src/errors.rs | 8 -- .../adaptors/file/file_state_machine.rs | 14 +-- .../adaptors/rocksdb/rocksdb_state_machine.rs | 15 +-- .../sled-cluster/src/sled_state_machine.rs | 16 ++-- 5 files changed, 55 insertions(+), 94 deletions(-) diff --git a/MIGRATION_GUIDE.md b/MIGRATION_GUIDE.md index d138071f..134d3b72 100644 --- a/MIGRATION_GUIDE.md +++ b/MIGRATION_GUIDE.md @@ -10,15 +10,37 @@ - Remove calls to `put_with_ttl` and the `ttl_secs` field of `Command::Insert`. Implement key expiry in the layer above d-engine. - Delete the `[raft.state_machine.lease]` section from your config files. -- Custom state machines: remove `lease_background_cleanup`, `start`, `is_running`, `len`, `is_empty` and `reset` from your `StateMachine` impl; the trait no longer declares them. `start` was called once when the node booted. Do that setup in your constructor (a state machine must be ready to serve as soon as it is built). The engine no longer asks whether the state machine is running: while a state machine installs a snapshot, a concurrent read must return the old state, the new state, or fail, never a mix. If you need an entry count, keep `len` as a plain method on your own type. The engine never called `reset`; if you used it in your own tests, wipe your storage directory or build a new state machine on an empty one instead. `StateMachineBuilder` (test support) now has `build()` return a state machine on new, empty storage and a new required method `reopen()` for restarts. **Every `StateMachine` method now returns `StateMachineError` instead of `d_engine::Error`** (#469): report any failure with `StateMachineError::new("what went wrong")`, or let `?` convert an `io::Error`; the engine stops the node when `apply_chunk` fails and fails only the read when a read fails. `StorageError::NotServing` is no longer part of the contract for a state machine. +- Custom state machines: see [Upgrading to v0.3.0: your `StateMachine`](#upgrading-to-v030-your-statemachine). - Proto: `WriteCommand.Insert.ttl_secs` (field 3) is removed. Regenerate your client code. - The file state machine storage format changed. Start with a fresh data directory; old files are not supported. - RocksDB state machine: keys written with `put_with_ttl` under v0.2.x are kept forever after the upgrade. v0.3.0 neither reads the old `ttl_state` entry nor deletes expired keys. Before upgrading, either delete the keys you no longer want, or record their deadlines and have your own expiry layer delete them. Or start with a fresh data directory. -- A read that fails in the state machine now returns an error code instead of an empty result: `StorageIoError`, `DataCorruption` (the stored data may be damaged), `RetryRequired` (the state machine is replacing its state; retry) or `Uncategorized`. Before, such a read looked like "key not found". Handle these errors in your client. +- A read that fails in the state machine now returns an error to the client instead of an empty result (before, it looked like "key not found"). Handle it in your client. -## Upgrading to v0.3.0: The Engine Owns the State Machine +## Upgrading to v0.3.0: your `StateMachine` -`start_custom`, `start_node` and `run_custom` (embedded and standalone) now take the state machine by value. The engine creates the `Arc` itself, so by the time `stop()` returns it has dropped the state machine and released whatever it held (a database handle, a lock file). +The trait went from 17 methods to 8, and the engine now owns snapshots. Go through this list. + +| If you have | Do this | +| --- | --- | +| `start`, `is_running`, `len`, `is_empty`, `reset`, `entry_term`, `lease_background_cleanup` | Delete. Do `start`'s setup in your constructor. Keep `len` as a plain method if you need it. | +| `stop`, `close_storage` | Delete `stop`. Rename `close_storage` to `close`. | +| `update_last_applied`, `persist_last_applied`, `update_last_snapshot_metadata`, `save_hard_state`, `flush`, `flush_async` | Delete. The engine never called them. | +| `snapshot_metadata`, `persist_last_snapshot_metadata` | Delete. The engine records the snapshots it holds. | +| Methods that return `d_engine::Error` | Return `StateMachineError` (below). | +| `generate_snapshot_data(dir, last_included)` | Drop `last_included`. Return the position of the last entry the exported data contains. | +| A snapshot checksum, or stale / duplicate / conflict checks in `apply_snapshot_from_file` | Delete. The engine does them and calls you only for a newer snapshot. | +| `Arc` passed to the engine | Pass `MyStateMachine` by value. | + +**Errors.** Report any failure with `StateMachineError::new("what went wrong")`; `?` converts an `io::Error`. A failed `apply_chunk` stops the node. A failed read fails only that read. + +**What your state machine must guarantee** + +1. After a restart, `last_applied()` is the last entry whose effects are durable. Write it in the same transaction as the data. +2. `generate_snapshot_data` creates the directory, exports a consistent view, and returns that view's position (not a separate counter: `apply_chunk` keeps running). +3. While `apply_snapshot_from_file` runs, a read returns the old state, the new state, or an error, never a mix. Afterwards `last_applied()` equals `metadata.last_included`. +4. `close()` persists and releases resources, and may be called twice. Do not rely on it for crash safety: it may not run if shutdown times out. + +**Engine API** ```rust // before @@ -30,16 +52,12 @@ let sm = MyStateMachine::new()?; EmbeddedEngine::start_custom(dir, storage, sm, None).await?; ``` -- **`EmbeddedEngine` and `EmbeddedClient` no longer have type parameters.** Write `EmbeddedEngine` and `EmbeddedClient`. `DefaultEmbeddedEngine` and `DefaultEmbeddedClient` still exist and are now plain aliases of them. -- **If you kept a clone of your `Arc` to read local data directly** (for example `DLmdb` reading its own LMDB), you can no longer clone the instance the engine uses. Share a handle to your storage instead: put the handles (LMDB `Env`, sled `Db`, `Arc`) behind an `Arc` inside your state machine type, derive `Clone` on it, give the engine one clone and keep the other. -- **Embedded reads (`get_eventual`, lease reads) are answered by the ReadActor**, as they already were in standalone mode, instead of reading the state machine on the caller's thread. Expect one extra channel hop per read. -- **Snapshot checksums belong to the engine.** `generate_snapshot_data(dir)` now returns `Result` and no longer takes `last_included`: write the snapshot into the directory you are given and return the position of the last applied entry the exported data contains, read from the same consistent view as the data (not from a separate counter, because `apply_chunk` keeps running). The engine labels the snapshot with it. The engine computes the checksum of that directory itself, and verifies it before it calls `apply_snapshot_from_file`, so you no longer compute or check a checksum. Delete that code (and any hashing dependency you added only for it). -- **`apply_snapshot_from_file` returns `Result<()>` again, and you no longer classify snapshots.** The engine compares the snapshot's boundary with `last_applied()` first and calls you only for a newer snapshot, so delete your stale / duplicate / conflict handling and return `Ok(())` after installing. After your call the engine checks that `last_applied()` equals `metadata.last_included` and reports an error if it does not, so make sure you set it (durably) as part of the install. -- **Eight methods left the trait:** `update_last_applied`, `persist_last_applied`, `update_last_snapshot_metadata`, `save_hard_state`, `flush`, `flush_async`, `snapshot_metadata` and `persist_last_snapshot_metadata`. The first six were never called by the engine, so deleting them from your `impl StateMachine` block changes no behaviour. The last two are replaced by the engine's own record of the snapshots it holds (see below). Keep any your own code still uses as plain methods on your type. -- **Restore your position when you open the state machine.** The engine does not do it for you: after reopening, `last_applied()` must return the last entry whose effects are durable. You no longer store or return anything about snapshots; delete your snapshot-metadata storage and its loading code. -- **The engine records the snapshots a node holds.** Next to each snapshot archive `snapshot-{index}-{term}.tar.gz` it writes `snapshot-{index}-{term}.meta` (boundary and checksum). After you upgrade, existing archives have no record, so the node reports no snapshot until it builds its next one; archives without a record that are older than the newest recorded snapshot are then removed. -- **The snapshot directory must contain only regular files and directories.** The engine rejects symbolic links and other special entries instead of skipping them. -- When `stop()` returns, the engine has dropped the state machine. If your state machine releases its resources in `Drop` (a database handle, a lock file), they are free at that point and the same data directory can be reopened. +- `EmbeddedEngine` and `EmbeddedClient` no longer have type parameters. +- If you read your own storage directly (for example `DLmdb`), share the storage handles behind an `Arc` inside a `Clone` state machine; give the engine one clone and keep the other. +- Embedded reads (`get_eventual`, lease reads) go through the ReadActor, as in standalone mode: one extra channel hop. +- Existing snapshot archives have no `.meta` record, so a node reports no snapshot until it builds its next one. The snapshot directory may contain only regular files and directories. + +**Tests.** `StateMachineBuilder::build()` returns a state machine on new, empty storage. Implement the new `reopen()` to restart the same storage. `build_from_crash_image()` is optional and enables the crash-recovery test. --- @@ -406,48 +424,6 @@ fn apply_chunk(&mut self, entries: &[ApplyEntry]) -> Result<(), ...> --- -## 🚨 For v0.2.4 Users: `StateMachine` Lifecycle Changes in v0.3.0 (#455) - -### What Changed - -| Before (v0.2.4) | After (v0.3.0) | -| --------------------------------------------------------------- | ----------------------------------------------------------------- | -| `fn stop(&self) -> Result<(), Error>` (required) | **Removed** | -| `fn close_storage(&self)` (default no-op) | **Renamed** `fn close(&self) -> Result<(), Error>` (default `Ok`) | -| `[raft.storage]` accepted as an alias of `[raft.state_machine]` | Use `[raft.state_machine]` | - -`stop()` was never called by d-engine. `close()` is now the one end-of-life hook. d-engine calls it once, as the last step of the -state machine worker. When the worker drains cleanly, every committed entry has been applied by then; if -`shutdown_apply_timeout_ms` expires first, shutdown continues and committed entries may remain unapplied until the next start. -d-engine stops serving reads before it calls `close()` and calls nothing on the state machine afterwards, so you do not need to -define what a closed state machine does. If the timeout expires, `close()` runs after the node has returned, or not at all if the -process exits first: do not rely on it for crash safety, and persist `last_applied` together with the data it describes. -Because you may keep your own `Arc` to the state machine, do not rely on `Drop` alone. `close()` returns an error if persisting -fails: release your resources anyway and return the first error. The node logs it at ERROR level. - -### Migration Steps - -1. Delete your `fn stop(...)`. -2. Rename `fn close_storage(&self)` to `fn close(&self) -> Result<(), Error>`. -3. If your `stop()` persisted something that must survive a restart (for example lease state), do it in `close()`. -4. `close()` must be synchronous and may be called more than once (a later call returns `Ok(())`). You no longer have to make operations return a "not serving" error after it. -5. In config files, rename `[raft.storage]` to `[raft.state_machine]`. - -```rust -// Before -fn stop(&self) -> Result<(), Error> { self.persist_ttl()?; Ok(()) } -fn close_storage(&self) { self.db.flush(); } - -// After -fn close(&self) -> Result<(), Error> { - let persisted = self.persist_ttl(); - self.db.flush(); - persisted -} -``` - ---- - ## 🚨 For v0.2.3 Users: API Surface Changes in v0.2.4 (#326) ### What Changed @@ -617,14 +593,6 @@ Compilation will fail with "method not found in trait" if you forget to remove i --- -## For v0.2.4 Users: `StateMachine::apply_snapshot_from_file` Now Receives Only Snapshots To Install (#436, #400) - -The signature is unchanged: it still returns `Result<()>`. What changed is who decides. - -In v0.2.4 every state machine had to decide whether a received snapshot should be installed. In v0.3.0 the engine decides before it calls you, so stale, duplicate and conflicting snapshots (older than `last_applied()`, equal to it, or the same index with a different term) never reach your code. If your implementation compared the snapshot's boundary with its own position, delete that code. After your call the engine checks that `last_applied()` equals `metadata.last_included`, and reports an error if it does not. - ---- - ## For v0.2.4 Users: Learner Initial-Snapshot PULL Path Removed in v0.3.0 (#436) ### What Changed diff --git a/d-engine-core/src/errors.rs b/d-engine-core/src/errors.rs index 2809c03d..7a01a55f 100644 --- a/d-engine-core/src/errors.rs +++ b/d-engine-core/src/errors.rs @@ -883,11 +883,3 @@ impl From for Error { ))) } } - -impl From for StateMachineError { - /// Lets the state machines that ship with d-engine, which share helpers with the rest of the - /// engine, return their internal errors with `?`. The message is kept. - fn from(error: Error) -> Self { - StateMachineError::new(error.to_string()) - } -} diff --git a/d-engine-server/src/storage/adaptors/file/file_state_machine.rs b/d-engine-server/src/storage/adaptors/file/file_state_machine.rs index d6324e00..8a0486b6 100644 --- a/d-engine-server/src/storage/adaptors/file/file_state_machine.rs +++ b/d-engine-server/src/storage/adaptors/file/file_state_machine.rs @@ -768,11 +768,11 @@ impl FileStateMachine { /// Reads are served only while running. After `close()` the state machine may still be /// reachable through a handle the developer shares, and it must not answer from state that /// is no longer maintained. - fn ensure_serving(&self) -> Result<(), Error> { + fn ensure_serving(&self) -> Result<(), StateMachineError> { if self.running.load(Ordering::Acquire) { Ok(()) } else { - Err(StorageError::NotServing("state machine stopped".into()).into()) + Err(StateMachineError::new("state machine stopped")) } } @@ -843,7 +843,7 @@ impl StateMachine for FileStateMachine { // `Drop` does not run when the engine stops if the developer still holds a shared // handle: persist everything a restart needs here. // No exclusive OS lock file to release. - self.save_hard_state().map_err(Into::into) + self.save_hard_state().map_err(StateMachineError::new) } fn get( @@ -1049,7 +1049,7 @@ impl StateMachine for FileStateMachine { self.wal_entries_since_checkpoint.fetch_add(chunk_len as u64, Ordering::Relaxed); if self.should_checkpoint() { - self.checkpoint().await?; + self.checkpoint().await.map_err(StateMachineError::new)?; } Ok(results) @@ -1170,9 +1170,9 @@ impl StateMachine for FileStateMachine { self.update_last_applied(incoming); // Persist to disk - self.persist_data_async().await?; - self.clear_wal_async().await?; - self.persist_metadata_async().await?; + self.persist_data_async().await.map_err(StateMachineError::new)?; + self.clear_wal_async().await.map_err(StateMachineError::new)?; + self.persist_metadata_async().await.map_err(StateMachineError::new)?; info!("Snapshot applied successfully"); diff --git a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs index 2a8ecbe9..b38a85be 100644 --- a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs +++ b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs @@ -584,7 +584,7 @@ impl StateMachine for RocksDBStateMachine { first_err.get_or_insert(e); } }; - record("flush", self.flush().map_err(Into::into)); + record("flush", self.flush().map_err(StateMachineError::new)); if let Some(db) = self.db.load().as_deref() { db.cancel_all_background_work(true); } @@ -618,7 +618,7 @@ impl StateMachine for RocksDBStateMachine { None => Ok(None), } }) - .map_err(StateMachineError::from) + .map_err(StateMachineError::new) } /// Thread-safe: called serially by the state machine worker @@ -781,7 +781,7 @@ impl StateMachine for RocksDBStateMachine { self.is_serving.store(true, Ordering::SeqCst); } - result.map_err(Into::into) + result.map_err(StateMachineError::new) } async fn generate_snapshot_data( @@ -796,14 +796,15 @@ impl StateMachine for RocksDBStateMachine { // run on tokio's blocking thread pool via spawn_blocking, not on an async worker thread. // This prevents snapshot disk I/O from starving the inbound event loop under concurrent // writes (#315). - let db = self.load_db()?; // Arc: Send, safe to move into spawn_blocking + let db = self.load_db().map_err(StateMachineError::new)?; // Arc: Send, safe to move into spawn_blocking let dir = new_snapshot_dir.clone(); let position = tokio::task::spawn_blocking(move || -> Result { std::fs::create_dir_all(&dir)?; Self::export_frozen_view(&db, &dir) }) .await - .map_err(|e| StateMachineError::new(Self::map_snapshot_join_error(e).to_string()))??; + .map_err(|e| StateMachineError::new(Self::map_snapshot_join_error(e).to_string()))? + .map_err(StateMachineError::new)?; info!("Snapshot generated at {:?}", new_snapshot_dir); Ok(position) @@ -813,7 +814,7 @@ impl StateMachine for RocksDBStateMachine { &self, prefix: &[u8], ) -> Result { - self.scan_prefix(prefix).map_err(Into::into) + self.scan_prefix(prefix).map_err(StateMachineError::new) } fn get_multi( @@ -852,7 +853,7 @@ impl StateMachine for RocksDBStateMachine { }) .collect() }) - .map_err(StateMachineError::from) + .map_err(StateMachineError::new) } } impl Drop for RocksDBStateMachine { diff --git a/examples/sled-cluster/src/sled_state_machine.rs b/examples/sled-cluster/src/sled_state_machine.rs index 9e68ba94..9757a443 100644 --- a/examples/sled-cluster/src/sled_state_machine.rs +++ b/examples/sled-cluster/src/sled_state_machine.rs @@ -96,7 +96,7 @@ impl StateMachine for SledStateMachine { // failed. The lock is released once no other handle to the db is alive. let saved = self.save_hard_state(); self.db.store(None); - saved.map_err(Into::into) + saved.map_err(StateMachineError::new) } fn last_applied(&self) -> LogId { @@ -110,7 +110,7 @@ impl StateMachine for SledStateMachine { &self, key_buffer: &[u8], ) -> std::result::Result, StateMachineError> { - match self.current_tree()?.get(key_buffer) { + match self.current_tree().map_err(StateMachineError::new)?.get(key_buffer) { Ok(Some(v)) => Ok(Some(Bytes::copy_from_slice(&v))), Ok(None) => Ok(None), Err(e) => { @@ -125,7 +125,7 @@ impl StateMachine for SledStateMachine { chunk: &[ApplyEntry], ) -> std::result::Result, StateMachineError> { trace!("Applying chunk: {} entries", chunk.len()); - let db = self.db()?; + let db = self.db().map_err(StateMachineError::new)?; let mut highest_log_id: Option = None; for entry in chunk { @@ -151,7 +151,7 @@ impl StateMachine for SledStateMachine { // the data. A compare-and-swap reads and writes inside the same transaction, so it sees // the entries applied before it in this chunk. sled may run the closure more than once on // a conflict, so it builds its results from scratch every time. - let data_tree = self.current_tree()?; + let data_tree = self.current_tree().map_err(StateMachineError::new)?; let meta_tree = db .open_tree(STATE_MACHINE_META_NAMESPACE) .map_err(|e| StateMachineError::new(e.to_string()))?; @@ -251,8 +251,8 @@ impl StateMachine for SledStateMachine { let new_db = init_sled_state_machine_db(&new_snapshot_dir) .map_err(|e| StateMachineError::new(e.to_string()))?; - let exist_db_tree = self.current_tree()?; - let new_state_machine_tree = new_tree(&new_db, STATE_MACHINE_TREE)?; + let exist_db_tree = self.current_tree().map_err(StateMachineError::new)?; + let new_state_machine_tree = new_tree(&new_db, STATE_MACHINE_TREE).map_err(StateMachineError::new)?; let mut batch = sled::Batch::default(); let mut counter = 0; @@ -361,7 +361,7 @@ impl StateMachine for SledStateMachine { ); self.update_last_applied(new_last_included); - self.save_hard_state()?; + self.save_hard_state().map_err(StateMachineError::new)?; Ok(()) } @@ -370,7 +370,7 @@ impl StateMachine for SledStateMachine { &self, prefix: &[u8], ) -> std::result::Result { - let tree = self.current_tree()?; + let tree = self.current_tree().map_err(StateMachineError::new)?; let mut entries = Vec::new(); for result in tree.scan_prefix(prefix) { let (k, v) = result.map_err(|e| StateMachineError::new(e.to_string()))?; From 5afde8e5aed213a885d9b3f34b1329ad3880570e Mon Sep 17 00:00:00 2001 From: Joshua Chi <112539+JoshuaChi@users.noreply.github.com> Date: Wed, 4 Nov 2026 18:25:44 +0800 Subject: [PATCH 18/19] refactor #400: State machine helpers return StateMachineError directly --- .../adaptors/file/file_state_machine.rs | 30 ++-- .../adaptors/rocksdb/rocksdb_state_machine.rs | 148 ++++++++---------- .../sled-cluster/src/sled_state_machine.rs | 65 ++++---- 3 files changed, 112 insertions(+), 131 deletions(-) diff --git a/d-engine-server/src/storage/adaptors/file/file_state_machine.rs b/d-engine-server/src/storage/adaptors/file/file_state_machine.rs index 8a0486b6..e4459d75 100644 --- a/d-engine-server/src/storage/adaptors/file/file_state_machine.rs +++ b/d-engine-server/src/storage/adaptors/file/file_state_machine.rs @@ -570,7 +570,7 @@ impl FileStateMachine { } /// Persists key-value data to disk - fn persist_data(&self) -> Result<(), Error> { + fn persist_data(&self) -> Result<(), StateMachineError> { // Collect data first to minimize lock time let data_copy: HashMap = { let data = self.data.read(); @@ -597,7 +597,7 @@ impl FileStateMachine { } /// Persists key-value data to disk - async fn persist_data_async(&self) -> Result<(), Error> { + async fn persist_data_async(&self) -> Result<(), StateMachineError> { // Collect data under minimal lock scope let data_copy: HashMap = { let data = self.data.read(); @@ -643,12 +643,12 @@ impl FileStateMachine { pub(crate) fn persist_last_applied( &self, last_applied: LogId, - ) -> Result<(), Error> { + ) -> Result<(), StateMachineError> { self.update_last_applied(last_applied); self.persist_metadata() } - pub(crate) fn save_hard_state(&self) -> Result<(), Error> { + pub(crate) fn save_hard_state(&self) -> Result<(), StateMachineError> { let last_applied = self.last_applied(); self.persist_last_applied(last_applied)?; @@ -656,7 +656,7 @@ impl FileStateMachine { Ok(()) } - pub(crate) fn flush(&self) -> Result<(), Error> { + pub(crate) fn flush(&self) -> Result<(), StateMachineError> { self.persist_data()?; self.persist_metadata()?; // self.clear_wal()?; @@ -664,7 +664,7 @@ impl FileStateMachine { } /// Persists metadata to disk - fn persist_metadata(&self) -> Result<(), Error> { + fn persist_metadata(&self) -> Result<(), StateMachineError> { let metadata_path = self.data_dir.join("metadata.bin"); let mut file = std::fs::OpenOptions::new() .write(true) @@ -682,7 +682,7 @@ impl FileStateMachine { Ok(()) } - async fn persist_metadata_async(&self) -> Result<(), Error> { + async fn persist_metadata_async(&self) -> Result<(), StateMachineError> { let metadata_path = self.data_dir.join("metadata.bin"); let mut file = OpenOptions::new() .write(true) @@ -703,7 +703,7 @@ impl FileStateMachine { /// Clears the write-ahead log (called after successful persistence) #[allow(unused)] - fn clear_wal(&self) -> Result<(), Error> { + fn clear_wal(&self) -> Result<(), StateMachineError> { let wal_path = self.data_dir.join("wal.log"); let mut file = std::fs::OpenOptions::new() .write(true) @@ -717,7 +717,7 @@ impl FileStateMachine { } /// Clears the write-ahead log (called after successful persistence) - async fn clear_wal_async(&self) -> Result<(), Error> { + async fn clear_wal_async(&self) -> Result<(), StateMachineError> { let wal_path = self.data_dir.join("wal.log"); let mut file = OpenOptions::new() .write(true) @@ -752,7 +752,7 @@ impl FileStateMachine { /// Checkpoint: snapshot full data + metadata to disk, then clear WAL. /// /// WAL is the primary crash-safety path; checkpoint bounds recovery replay time. - pub(crate) async fn checkpoint(&self) -> Result<(), Error> { + pub(crate) async fn checkpoint(&self) -> Result<(), StateMachineError> { self.persist_data_async().await?; self.persist_metadata_async().await?; self.clear_wal_async().await?; @@ -843,7 +843,7 @@ impl StateMachine for FileStateMachine { // `Drop` does not run when the engine stops if the developer still holds a shared // handle: persist everything a restart needs here. // No exclusive OS lock file to release. - self.save_hard_state().map_err(StateMachineError::new) + self.save_hard_state() } fn get( @@ -1049,7 +1049,7 @@ impl StateMachine for FileStateMachine { self.wal_entries_since_checkpoint.fetch_add(chunk_len as u64, Ordering::Relaxed); if self.should_checkpoint() { - self.checkpoint().await.map_err(StateMachineError::new)?; + self.checkpoint().await?; } Ok(results) @@ -1170,9 +1170,9 @@ impl StateMachine for FileStateMachine { self.update_last_applied(incoming); // Persist to disk - self.persist_data_async().await.map_err(StateMachineError::new)?; - self.clear_wal_async().await.map_err(StateMachineError::new)?; - self.persist_metadata_async().await.map_err(StateMachineError::new)?; + self.persist_data_async().await?; + self.clear_wal_async().await?; + self.persist_metadata_async().await?; info!("Snapshot applied successfully"); diff --git a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs index b38a85be..95316fbd 100644 --- a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs +++ b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs @@ -176,14 +176,14 @@ impl RocksDBStateMachine { fn with_db( &self, f: F, - ) -> Result + ) -> Result where - F: FnOnce(&DB) -> Result, + F: FnOnce(&DB) -> Result, { let guard = self.db.load(); // Guard — no Arc clone match guard.as_deref() { Some(db) => f(db), - None => Err(StorageError::NotServing("state machine stopped".into()).into()), + None => Err(StateMachineError::new("state machine stopped")), } } @@ -192,31 +192,25 @@ impl RocksDBStateMachine { /// Costs 2 atomic ops (Arc increment + future decrement). Only use when the handle /// must be moved into a closure or across an await point. For all other cases prefer /// `with_db()` (zero Arc clone) or the inline Guard pattern. - fn load_db(&self) -> Result, Error> { + fn load_db(&self) -> Result, StateMachineError> { self.db .load_full() - .ok_or_else(|| StorageError::NotServing("state machine stopped".into()).into()) + .ok_or_else(|| StateMachineError::new("state machine stopped")) } - fn load_state_machine_metadata(db: &DB) -> Result<(u64, u64), Error> { + fn load_state_machine_metadata(db: &DB) -> Result<(u64, u64), StateMachineError> { let cf = db .cf_handle(STATE_MACHINE_META_CF) - .ok_or_else(|| StorageError::DbError("State machine meta CF not found".to_string()))?; + .ok_or_else(|| StateMachineError::new("State machine meta CF not found"))?; - let index = match db - .get_cf(&cf, LAST_APPLIED_INDEX_KEY) - .map_err(|e| StorageError::DbError(e.to_string()))? - { + let index = match db.get_cf(&cf, LAST_APPLIED_INDEX_KEY).map_err(StateMachineError::new)? { Some(bytes) if bytes.len() == 8 => u64::from_be_bytes([ bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], ]), _ => 0, }; - let term = match db - .get_cf(&cf, LAST_APPLIED_TERM_KEY) - .map_err(|e| StorageError::DbError(e.to_string()))? - { + let term = match db.get_cf(&cf, LAST_APPLIED_TERM_KEY).map_err(StateMachineError::new)? { Some(bytes) if bytes.len() == 8 => u64::from_be_bytes([ bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], ]), @@ -235,7 +229,7 @@ impl RocksDBStateMachine { &self, last_included: LogId, snapshot_dir: &std::path::Path, - ) -> Result<(), Error> { + ) -> Result<(), StateMachineError> { self.with_db(|db| Self::restore_from_cf_export(db, snapshot_dir))?; info!("Snapshot restore complete"); @@ -254,7 +248,7 @@ impl RocksDBStateMachine { fn restore_from_cf_export( db: &DB, snapshot_dir: &std::path::Path, - ) -> Result<(), Error> { + ) -> Result<(), StateMachineError> { let sm_meta = Self::load_cf_export_metadata( &snapshot_dir.join("sm_metadata.bin"), &snapshot_dir.join("sm"), @@ -272,23 +266,20 @@ impl RocksDBStateMachine { // will be absent on restart. RocksDB does not support CF rename, so true atomicity // is not achievable here. Recovery: the node will restart with missing CFs, return // errors on reads, and wait for the leader to re-send the snapshot (issue #308). - db.drop_cf(STATE_MACHINE_CF).map_err(|e| StorageError::DbError(e.to_string()))?; - db.drop_cf(STATE_MACHINE_META_CF) - .map_err(|e| StorageError::DbError(e.to_string()))?; + db.drop_cf(STATE_MACHINE_CF).map_err(StateMachineError::new)?; + db.drop_cf(STATE_MACHINE_META_CF).map_err(StateMachineError::new)?; // create_column_family_with_import requires at least one SST file. // A CF with no files (e.g. sm_meta before its first flush) must use create_cf instead. if sm_meta.get_files().is_empty() { - db.create_cf(STATE_MACHINE_CF, &cf_opts) - .map_err(|e| StorageError::DbError(e.to_string()))?; + db.create_cf(STATE_MACHINE_CF, &cf_opts).map_err(StateMachineError::new)?; } else { db.create_column_family_with_import(&cf_opts, STATE_MACHINE_CF, &import_opts, &sm_meta) - .map_err(|e| StorageError::DbError(e.to_string()))?; + .map_err(StateMachineError::new)?; } if sm_meta_meta.get_files().is_empty() { - db.create_cf(STATE_MACHINE_META_CF, &cf_opts) - .map_err(|e| StorageError::DbError(e.to_string()))?; + db.create_cf(STATE_MACHINE_META_CF, &cf_opts).map_err(StateMachineError::new)?; } else { db.create_column_family_with_import( &cf_opts, @@ -296,7 +287,7 @@ impl RocksDBStateMachine { &import_opts, &sm_meta_meta, ) - .map_err(|e| StorageError::DbError(e.to_string()))?; + .map_err(StateMachineError::new)?; } Ok(()) @@ -319,10 +310,10 @@ impl RocksDBStateMachine { fn export_frozen_view( db: &DB, snapshot_dir: &std::path::Path, - ) -> Result { + ) -> Result { let name = snapshot_dir .file_name() - .ok_or_else(|| StorageError::DbError("snapshot dir has no name".to_string()))? + .ok_or_else(|| StateMachineError::new("snapshot dir has no name"))? .to_string_lossy(); let checkpoint_dir = snapshot_dir.with_file_name(format!(".{name}.checkpoint")); if checkpoint_dir.exists() { @@ -339,36 +330,37 @@ impl RocksDBStateMachine { db: &DB, checkpoint_dir: &std::path::Path, snapshot_dir: &std::path::Path, - ) -> Result { - let to_err = |e: rocksdb::Error| StorageError::DbError(e.to_string()); + ) -> Result { rocksdb::checkpoint::Checkpoint::new(db) .and_then(|c| c.create_checkpoint(checkpoint_dir)) - .map_err(to_err)?; + .map_err(StateMachineError::new)?; // Open the frozen copy with every column family it contains. Exporting needs a writable // handle; background compaction is off so that nothing rewrites the files being exported. let mut opts = Options::default(); opts.set_disable_auto_compactions(true); opts.set_max_background_jobs(1); - let cf_names = DB::list_cf(&opts, checkpoint_dir).map_err(to_err)?; - let frozen = DB::open_cf(&opts, checkpoint_dir, &cf_names).map_err(to_err)?; - let checkpoint = rocksdb::checkpoint::Checkpoint::new(&frozen).map_err(to_err)?; + let cf_names = DB::list_cf(&opts, checkpoint_dir).map_err(StateMachineError::new)?; + let frozen = + DB::open_cf(&opts, checkpoint_dir, &cf_names).map_err(StateMachineError::new)?; + let checkpoint = + rocksdb::checkpoint::Checkpoint::new(&frozen).map_err(StateMachineError::new)?; let cf_sm = frozen .cf_handle(STATE_MACHINE_CF) - .ok_or_else(|| StorageError::DbError("SM CF not found".to_string()))?; + .ok_or_else(|| StateMachineError::new("SM CF not found"))?; let cf_sm_meta = frozen .cf_handle(STATE_MACHINE_META_CF) - .ok_or_else(|| StorageError::DbError("SM meta CF not found".to_string()))?; + .ok_or_else(|| StateMachineError::new("SM meta CF not found"))?; // export_column_family() creates only the final subdirectory (e.g. "sm"); the parent // (snapshot_dir) already exists. let sm_export = checkpoint .export_column_family(&cf_sm, snapshot_dir.join("sm")) - .map_err(to_err)?; + .map_err(StateMachineError::new)?; let sm_meta_export = checkpoint .export_column_family(&cf_sm_meta, snapshot_dir.join("sm_meta")) - .map_err(to_err)?; + .map_err(StateMachineError::new)?; Self::save_cf_export_metadata(&sm_export, &snapshot_dir.join("sm_metadata.bin"))?; Self::save_cf_export_metadata(&sm_meta_export, &snapshot_dir.join("sm_meta_metadata.bin"))?; // The position is read from the frozen copy, the same view the data was exported from. @@ -382,7 +374,7 @@ impl RocksDBStateMachine { fn save_cf_export_metadata( metadata: &ExportImportFilesMetaData, path: &std::path::Path, - ) -> Result<(), Error> { + ) -> Result<(), StateMachineError> { let files = metadata .get_files() .into_iter() @@ -405,8 +397,8 @@ impl RocksDBStateMachine { files, }; - let bytes = bincode::serialize(&meta).map_err(StorageError::BincodeError)?; - std::fs::write(path, bytes).map_err(|e| StorageError::DbError(e.to_string()))?; + let bytes = bincode::serialize(&meta).map_err(StateMachineError::new)?; + std::fs::write(path, bytes).map_err(StateMachineError::new)?; Ok(()) } @@ -415,14 +407,13 @@ impl RocksDBStateMachine { fn load_cf_export_metadata( path: &std::path::Path, actual_dir: &std::path::Path, - ) -> Result { - let bytes = std::fs::read(path).map_err(|e| StorageError::DbError(e.to_string()))?; - let meta: CfExportMeta = - bincode::deserialize(&bytes).map_err(StorageError::BincodeError)?; + ) -> Result { + let bytes = std::fs::read(path).map_err(StateMachineError::new)?; + let meta: CfExportMeta = bincode::deserialize(&bytes).map_err(StateMachineError::new)?; let dir_str = actual_dir .to_str() - .ok_or_else(|| StorageError::DbError("Snapshot path is not valid UTF-8".to_string()))? + .ok_or_else(|| StateMachineError::new("Snapshot path is not valid UTF-8"))? .to_string(); let live_files: Vec = meta @@ -445,20 +436,18 @@ impl RocksDBStateMachine { let mut export_metadata = ExportImportFilesMetaData::default(); export_metadata.set_db_comparator_name(&meta.db_comparator_name); - export_metadata - .set_files(&live_files) - .map_err(|e| StorageError::DbError(e.to_string()))?; + export_metadata.set_files(&live_files).map_err(StateMachineError::new)?; Ok(export_metadata) } - pub(crate) fn map_snapshot_join_error(e: tokio::task::JoinError) -> StorageError { + pub(crate) fn map_snapshot_join_error(e: tokio::task::JoinError) -> StateMachineError { let msg = if e.is_panic() { format!("snapshot blocking task panicked: {e}") } else { format!("snapshot blocking task was cancelled: {e}") }; - StorageError::DbError(msg) + StateMachineError::new(msg) } /// Scans all entries whose key starts with `prefix`. @@ -469,7 +458,7 @@ impl RocksDBStateMachine { pub fn scan_prefix( &self, prefix: &[u8], - ) -> Result { + ) -> Result { if prefix.is_empty() { let revision = self.last_applied_index.load(Ordering::SeqCst); return Ok(ScanResult { @@ -487,16 +476,16 @@ impl RocksDBStateMachine { let guard = self.db.load(); let Some(db) = guard.as_deref() else { - return Err(StorageError::NotServing("state machine stopped".into()).into()); + return Err(StateMachineError::new("state machine stopped")); }; let cf = db .cf_handle(STATE_MACHINE_CF) - .ok_or_else(|| StorageError::DbError("STATE_MACHINE_CF not found".into()))?; + .ok_or_else(|| StateMachineError::new("STATE_MACHINE_CF not found"))?; let iter = db.iterator_cf_opt(&cf, opts, IteratorMode::From(prefix, Direction::Forward)); let mut entries = Vec::new(); for item in iter { - let (k, v) = item.map_err(|e| StorageError::DbError(e.to_string()))?; + let (k, v) = item.map_err(StateMachineError::new)?; if !k.starts_with(prefix) { break; } @@ -516,9 +505,9 @@ impl RocksDBStateMachine { } /// Flushes the WAL and both state machine column families to disk. - pub fn flush(&self) -> Result<(), Error> { + pub fn flush(&self) -> Result<(), StateMachineError> { self.with_db(|db| { - db.flush_wal(true).map_err(|e| StorageError::DbError(e.to_string()))?; + db.flush_wal(true).map_err(StateMachineError::new)?; // db.flush() with no CF argument only flushes RocksDB's implicit // "default" column family, which this state machine never writes to — @@ -527,13 +516,13 @@ impl RocksDBStateMachine { // in create_snapshot()'s pre-export flush. let cf_sm = db .cf_handle(STATE_MACHINE_CF) - .ok_or_else(|| StorageError::DbError("State machine CF not found".to_string()))?; - let cf_sm_meta = db.cf_handle(STATE_MACHINE_META_CF).ok_or_else(|| { - StorageError::DbError("State machine meta CF not found".to_string()) - })?; + .ok_or_else(|| StateMachineError::new("State machine CF not found"))?; + let cf_sm_meta = db + .cf_handle(STATE_MACHINE_META_CF) + .ok_or_else(|| StateMachineError::new("State machine meta CF not found"))?; let flush_opts = rocksdb::FlushOptions::default(); db.flush_cfs_opt(&[&cf_sm, &cf_sm_meta], &flush_opts) - .map_err(|e| StorageError::DbError(e.to_string()))?; + .map_err(StateMachineError::new)?; Ok(()) }) // last_applied is already durable as of apply_chunk() — no separate @@ -584,7 +573,7 @@ impl StateMachine for RocksDBStateMachine { first_err.get_or_insert(e); } }; - record("flush", self.flush().map_err(StateMachineError::new)); + record("flush", self.flush()); if let Some(db) = self.db.load().as_deref() { db.cancel_all_background_work(true); } @@ -608,17 +597,14 @@ impl StateMachine for RocksDBStateMachine { } self.with_db(|db| { - let cf = db.cf_handle(STATE_MACHINE_CF).ok_or_else(|| { - Error::System(d_engine_core::SystemError::Storage(StorageError::DbError( - "State machine CF not found".to_string(), - ))) - })?; + let cf = db + .cf_handle(STATE_MACHINE_CF) + .ok_or_else(|| StateMachineError::new("State machine CF not found"))?; match db.get_cf(&cf, key_buffer).map_err(StateMachineError::new)? { Some(value) => Ok(Some(Bytes::copy_from_slice(&value))), None => Ok(None), } }) - .map_err(StateMachineError::new) } /// Thread-safe: called serially by the state machine worker @@ -781,7 +767,7 @@ impl StateMachine for RocksDBStateMachine { self.is_serving.store(true, Ordering::SeqCst); } - result.map_err(StateMachineError::new) + result } async fn generate_snapshot_data( @@ -796,15 +782,14 @@ impl StateMachine for RocksDBStateMachine { // run on tokio's blocking thread pool via spawn_blocking, not on an async worker thread. // This prevents snapshot disk I/O from starving the inbound event loop under concurrent // writes (#315). - let db = self.load_db().map_err(StateMachineError::new)?; // Arc: Send, safe to move into spawn_blocking + let db = self.load_db()?; // Arc: Send, safe to move into spawn_blocking let dir = new_snapshot_dir.clone(); - let position = tokio::task::spawn_blocking(move || -> Result { + let position = tokio::task::spawn_blocking(move || -> Result { std::fs::create_dir_all(&dir)?; Self::export_frozen_view(&db, &dir) }) .await - .map_err(|e| StateMachineError::new(Self::map_snapshot_join_error(e).to_string()))? - .map_err(StateMachineError::new)?; + .map_err(Self::map_snapshot_join_error)??; info!("Snapshot generated at {:?}", new_snapshot_dir); Ok(position) @@ -814,7 +799,7 @@ impl StateMachine for RocksDBStateMachine { &self, prefix: &[u8], ) -> Result { - self.scan_prefix(prefix).map_err(StateMachineError::new) + self.scan_prefix(prefix) } fn get_multi( @@ -827,11 +812,9 @@ impl StateMachine for RocksDBStateMachine { )); } self.with_db(|db| { - let cf = db.cf_handle(STATE_MACHINE_CF).ok_or_else(|| { - Error::System(d_engine_core::SystemError::Storage(StorageError::DbError( - "State machine CF not found".to_string(), - ))) - })?; + let cf = db + .cf_handle(STATE_MACHINE_CF) + .ok_or_else(|| StateMachineError::new("State machine CF not found"))?; // Single-key reads don't need cross-key point-in-time consistency, // so skip snapshot creation to avoid RocksDB's internal snapshot-list lock. @@ -839,7 +822,7 @@ impl StateMachine for RocksDBStateMachine { return db .get_cf(&cf, &keys[0]) .map(|v| vec![v.map(|b| Bytes::copy_from_slice(&b))]) - .map_err(|e| StorageError::DbError(e.to_string()).into()); + .map_err(StateMachineError::new); } // One snapshot covers the full batch: all keys read from the same point-in-time. @@ -849,11 +832,10 @@ impl StateMachine for RocksDBStateMachine { .map(|k| { snap.get_cf(&cf, k) .map(|v| v.map(|b| Bytes::copy_from_slice(&b))) - .map_err(|e| StorageError::DbError(e.to_string()).into()) + .map_err(StateMachineError::new) }) .collect() }) - .map_err(StateMachineError::new) } } impl Drop for RocksDBStateMachine { diff --git a/examples/sled-cluster/src/sled_state_machine.rs b/examples/sled-cluster/src/sled_state_machine.rs index 9757a443..13bd3935 100644 --- a/examples/sled-cluster/src/sled_state_machine.rs +++ b/examples/sled-cluster/src/sled_state_machine.rs @@ -17,7 +17,6 @@ use d_engine::BatchOp; use d_engine::Command; use d_engine::Result; use d_engine::ScanResult; -use d_engine::SnapshotError; use d_engine::StateMachine; use d_engine::StateMachineError; use d_engine::StorageError; @@ -96,7 +95,7 @@ impl StateMachine for SledStateMachine { // failed. The lock is released once no other handle to the db is alive. let saved = self.save_hard_state(); self.db.store(None); - saved.map_err(StateMachineError::new) + saved } fn last_applied(&self) -> LogId { @@ -110,12 +109,12 @@ impl StateMachine for SledStateMachine { &self, key_buffer: &[u8], ) -> std::result::Result, StateMachineError> { - match self.current_tree().map_err(StateMachineError::new)?.get(key_buffer) { + match self.current_tree()?.get(key_buffer) { Ok(Some(v)) => Ok(Some(Bytes::copy_from_slice(&v))), Ok(None) => Ok(None), Err(e) => { error!("state_machine get error: {}", e); - Err(StateMachineError::new(e.to_string())) + Err(StateMachineError::new(e)) } } } @@ -125,7 +124,7 @@ impl StateMachine for SledStateMachine { chunk: &[ApplyEntry], ) -> std::result::Result, StateMachineError> { trace!("Applying chunk: {} entries", chunk.len()); - let db = self.db().map_err(StateMachineError::new)?; + let db = self.db()?; let mut highest_log_id: Option = None; for entry in chunk { @@ -151,10 +150,10 @@ impl StateMachine for SledStateMachine { // the data. A compare-and-swap reads and writes inside the same transaction, so it sees // the entries applied before it in this chunk. sled may run the closure more than once on // a conflict, so it builds its results from scratch every time. - let data_tree = self.current_tree().map_err(StateMachineError::new)?; + let data_tree = self.current_tree()?; let meta_tree = db .open_tree(STATE_MACHINE_META_NAMESPACE) - .map_err(|e| StateMachineError::new(e.to_string()))?; + .map_err(StateMachineError::new)?; let results = (&data_tree, &meta_tree) .transaction(|(data, meta)| { @@ -228,7 +227,7 @@ impl StateMachine for SledStateMachine { }) .map_err(|e: sled::transaction::TransactionError| { error!("state_machine apply_chunk failed: {}", e); - StateMachineError::new(e.to_string()) + StateMachineError::new(e) })?; self.update_last_applied(position); @@ -249,10 +248,10 @@ impl StateMachine for SledStateMachine { // 2. Create a new state machine database instance let new_db = init_sled_state_machine_db(&new_snapshot_dir) - .map_err(|e| StateMachineError::new(e.to_string()))?; + .map_err(StateMachineError::new)?; - let exist_db_tree = self.current_tree().map_err(StateMachineError::new)?; - let new_state_machine_tree = new_tree(&new_db, STATE_MACHINE_TREE).map_err(StateMachineError::new)?; + let exist_db_tree = self.current_tree()?; + let new_state_machine_tree = new_tree(&new_db, STATE_MACHINE_TREE)?; let mut batch = sled::Batch::default(); let mut counter = 0; @@ -261,7 +260,7 @@ impl StateMachine for SledStateMachine { // as log indices. The SM already reflects state at last_included.index, so // all current entries are valid snapshot content. for item in exist_db_tree.iter() { - let (k, v) = item.map_err(|e| StateMachineError::new(e.to_string()))?; + let (k, v) = item.map_err(StateMachineError::new)?; batch.insert(k, v); counter += 1; @@ -269,17 +268,17 @@ impl StateMachine for SledStateMachine { if counter % 100 == 0 { new_state_machine_tree .apply_batch(batch) - .map_err(|e| StateMachineError::new(e.to_string()))?; + .map_err(StateMachineError::new)?; batch = sled::Batch::default(); } } new_state_machine_tree .apply_batch(batch) - .map_err(|e| StateMachineError::new(e.to_string()))?; + .map_err(StateMachineError::new)?; // Make sure flush into disk - new_db.flush().map_err(|e| StateMachineError::new(e.to_string()))?; + new_db.flush().map_err(StateMachineError::new)?; Ok(position) } @@ -346,7 +345,7 @@ impl StateMachine for SledStateMachine { debug!("Reopening state machine database at its permanent path"); let db = init_sled_state_machine_db(&self.path) - .map_err(|e| StateMachineError::new(e.to_string()))?; + .map_err(StateMachineError::new)?; self.db.store(Some(Arc::new(db))); // Best-effort: a leftover `*.pre-install-backup` is harmless (just disk space) @@ -361,7 +360,7 @@ impl StateMachine for SledStateMachine { ); self.update_last_applied(new_last_included); - self.save_hard_state().map_err(StateMachineError::new)?; + self.save_hard_state()?; Ok(()) } @@ -370,10 +369,10 @@ impl StateMachine for SledStateMachine { &self, prefix: &[u8], ) -> std::result::Result { - let tree = self.current_tree().map_err(StateMachineError::new)?; + let tree = self.current_tree()?; let mut entries = Vec::new(); for result in tree.scan_prefix(prefix) { - let (k, v) = result.map_err(|e| StateMachineError::new(e.to_string()))?; + let (k, v) = result.map_err(StateMachineError::new)?; entries.push((Bytes::copy_from_slice(&k), Bytes::copy_from_slice(&v))); } let revision = self.last_applied_index.load(Ordering::SeqCst); @@ -451,17 +450,17 @@ impl SledStateMachine { // Dynamically obtain the current Tree (lock-free cache optimization) #[inline] - fn current_tree(&self) -> Result { + fn current_tree(&self) -> std::result::Result { self.db()? .open_tree(STATE_MACHINE_TREE) - .map_err(|e| StorageError::DbError(e.to_string()).into()) + .map_err(StateMachineError::new) } /// The live db, or `NotServing` once `close()` has released it. - fn db(&self) -> Result> { + fn db(&self) -> std::result::Result, StateMachineError> { self.db .load_full() - .ok_or_else(|| StorageError::NotServing("state machine stopped".into()).into()) + .ok_or_else(|| StateMachineError::new("state machine stopped")) } pub(crate) fn update_last_applied( @@ -476,28 +475,28 @@ impl SledStateMachine { pub(crate) fn persist_last_applied( &self, last_applied: LogId, - ) -> Result<()> { + ) -> std::result::Result<(), StateMachineError> { debug!(%self.node_id, ?last_applied, "persist_last_applied"); let db = self.db()?; let tree = db .open_tree(STATE_MACHINE_META_NAMESPACE) - .map_err(|e| StorageError::DbError(e.to_string()))?; + .map_err(StateMachineError::new)?; tree.insert( STATE_MACHINE_META_KEY_LAST_APPLIED_INDEX, safe_kv(last_applied.index), ) - .map_err(|e| StorageError::DbError(e.to_string()))?; + .map_err(StateMachineError::new)?; tree.insert( STATE_MACHINE_META_KEY_LAST_APPLIED_TERM, safe_kv(last_applied.term), ) - .map_err(|e| StorageError::DbError(e.to_string()))?; + .map_err(StateMachineError::new)?; - tree.flush().map_err(|e| StorageError::DbError(e.to_string()))?; + tree.flush().map_err(StateMachineError::new)?; Ok(()) } - pub(crate) fn save_hard_state(&self) -> Result<()> { + pub(crate) fn save_hard_state(&self) -> std::result::Result<(), StateMachineError> { let last_applied = self.last_applied(); self.persist_last_applied(last_applied)?; @@ -505,9 +504,9 @@ impl SledStateMachine { Ok(()) } - pub(crate) fn flush(&self) -> Result<()> { + pub(crate) fn flush(&self) -> std::result::Result<(), StateMachineError> { let db = self.db()?; - let bytes = db.flush().map_err(|e| StorageError::DbError(e.to_string()))?; + let bytes = db.flush().map_err(StateMachineError::new)?; info!( "Successfully flushed State Machine, bytes flushed: {}", bytes @@ -541,9 +540,9 @@ fn copy_dir_recursive( fn new_tree( db: &sled::Db, key: &str, -) -> Result { +) -> std::result::Result { db.open_tree(key) - .map_err(|e| SnapshotError::OperationFailed(format!("{e:?}")).into()) + .map_err(StateMachineError::new) } pub fn init_sled_state_machine_db( From 2b7ad0cac61ea5e1880a20dc6db2c5cd3daa1354 Mon Sep 17 00:00:00 2001 From: Joshua Chi <112539+JoshuaChi@users.noreply.github.com> Date: Wed, 4 Nov 2026 23:34:57 +0800 Subject: [PATCH 19/19] refactor #400: StateMachine returns std::io::Result; remove StateMachineError and NotServing --- CHANGELOG.md | 2 +- MIGRATION_GUIDE.md | 15 +- d-engine-core/src/client/types.rs | 15 +- d-engine-core/src/client/types_test.rs | 19 +-- d-engine-core/src/errors.rs | 39 +---- d-engine-core/src/errors_test.rs | 45 +---- .../src/raft_role/follower_state_test.rs | 15 +- .../leader_state_test/client_read_test.rs | 21 +-- .../default_state_machine_handler_test.rs | 39 ++++- d-engine-core/src/storage/state_machine.rs | 56 ++---- .../src/storage/state_machine_test.rs | 28 ++- d-engine-server/README.md | 3 +- .../api/embedded_test/shutdown_order_test.rs | 13 +- d-engine-server/src/api/read_handle.rs | 7 +- d-engine-server/src/api/read_handle_test.rs | 26 ++- d-engine-server/src/lib.rs | 8 +- .../grpc/grpc_raft_service_fast_path_test.rs | 6 +- d-engine-server/src/read_actor_test.rs | 6 +- .../adaptors/file/file_state_machine.rs | 69 ++++---- .../adaptors/rocksdb/rocksdb_state_machine.rs | 160 ++++++++---------- .../tests/common/shared_state_machine.rs | 15 +- .../server_guide/customize-state-machine.md | 108 +++++------- d-engine/src/lib.rs | 2 - .../sled-cluster/src/sled_state_machine.rs | 92 +++++----- 24 files changed, 328 insertions(+), 481 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 004bb326..84578005 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -124,7 +124,7 @@ All notable changes to this project will be documented in this file. "MemFirst"` or `"DiskFirst"` are silently ignored, not an error — remove the field, `flush_policy` is the only persistence knob now. -- **⚠️ `StateMachine` trait slimmed from 17 methods to 8** (#400, #455, #469): `start`, `stop`, `is_running`, `len`, `is_empty`, `reset` and the snapshot-metadata methods are removed, `close_storage` is now `close`, methods return `StateMachineError`, and `generate_snapshot_data(dir)` returns the position of what it exported. The engine now owns snapshot checksums, records and install decisions. See [MIGRATION_GUIDE.md](./MIGRATION_GUIDE.md). +- **⚠️ `StateMachine` trait slimmed from 17 methods to 8** (#400, #455, #469): `start`, `stop`, `is_running`, `len`, `is_empty`, `reset` and the snapshot-metadata methods are removed, `close_storage` is now `close`, every method returns `std::io::Result` instead of `d_engine::Error` (as do the constructors of the built-in file and RocksDB state machines), and `generate_snapshot_data(dir)` returns the position of what it exported. The engine now owns snapshot checksums, records and install decisions. See [MIGRATION_GUIDE.md](./MIGRATION_GUIDE.md). - **⚠️ The engine takes the state machine by value** (#400): `start_custom`, `start_node` and `run_custom` take `SM` instead of `Arc`, and `EmbeddedEngine` and `EmbeddedClient` no longer have type parameters. diff --git a/MIGRATION_GUIDE.md b/MIGRATION_GUIDE.md index 134d3b72..1ece6c62 100644 --- a/MIGRATION_GUIDE.md +++ b/MIGRATION_GUIDE.md @@ -26,12 +26,22 @@ The trait went from 17 methods to 8, and the engine now owns snapshots. Go throu | `stop`, `close_storage` | Delete `stop`. Rename `close_storage` to `close`. | | `update_last_applied`, `persist_last_applied`, `update_last_snapshot_metadata`, `save_hard_state`, `flush`, `flush_async` | Delete. The engine never called them. | | `snapshot_metadata`, `persist_last_snapshot_metadata` | Delete. The engine records the snapshots it holds. | -| Methods that return `d_engine::Error` | Return `StateMachineError` (below). | +| Methods that return `d_engine::Error` | Return `std::io::Result` (see Errors below). | | `generate_snapshot_data(dir, last_included)` | Drop `last_included`. Return the position of the last entry the exported data contains. | | A snapshot checksum, or stale / duplicate / conflict checks in `apply_snapshot_from_file` | Delete. The engine does them and calls you only for a newer snapshot. | | `Arc` passed to the engine | Pass `MyStateMachine` by value. | -**Errors.** Report any failure with `StateMachineError::new("what went wrong")`; `?` converts an `io::Error`. A failed `apply_chunk` stops the node. A failed read fails only that read. +**Errors.** Every method returns `std::io::Result`. You no longer build `d_engine::Error` or `StorageError`. `?` works on an `io::Error` as is; convert any other error with `io::Error::other`: + +```rust +// before +self.db.put(k, v).map_err(|e| StorageError::DbError(e.to_string()))?; + +// after +self.db.put(k, v).map_err(io::Error::other)?; +``` + +The engine does not look at the error kind. A failed `apply_chunk` or `apply_snapshot_from_file` stops the node, which recovers from its replicas. A failed read fails only that read. **What your state machine must guarantee** @@ -53,6 +63,7 @@ EmbeddedEngine::start_custom(dir, storage, sm, None).await?; ``` - `EmbeddedEngine` and `EmbeddedClient` no longer have type parameters. +- `FileStateMachine::new` and `RocksDBStateMachine::new` return `std::io::Result` instead of `d_engine::Error`. Code that uses `?` keeps compiling; only code that matches on `d_engine::Error` variants from these constructors needs a change. - If you read your own storage directly (for example `DLmdb`), share the storage handles behind an `Arc` inside a `Clone` state machine; give the engine one clone and keep the other. - Embedded reads (`get_eventual`, lease reads) go through the ReadActor, as in standalone mode: one extra channel hop. - Existing snapshot archives have no `.meta` record, so a node reports no snapshot until it builds its next one. The snapshot directory may contain only regular files and directories. diff --git a/d-engine-core/src/client/types.rs b/d-engine-core/src/client/types.rs index d62a608d..780bf248 100644 --- a/d-engine-core/src/client/types.rs +++ b/d-engine-core/src/client/types.rs @@ -15,7 +15,6 @@ use bytes::Bytes; use serde::Deserialize; use serde::Serialize; use tracing::error; -use tracing::warn; // ReadConsistencyPolicy already exists as a native type in crate::config; re-export it // here so all client-facing types live in one module. @@ -181,14 +180,10 @@ impl TryFrom for ErrorCode { impl ErrorCode { /// The code a client sees when the state machine fails a read. /// - /// `DataCorruption` is reported to the client, not escalated to a node shutdown: the read - /// failed and the stored data may be damaged, and the client application decides what to do. + /// A state machine reports every failure as an `io::Error`, so a failed read is a storage I/O + /// error; anything else is uncategorized. pub fn for_read_failure(e: &Error) -> Self { match e { - _ if e.is_not_serving() => Self::RetryRequired, - Error::System(SystemError::Storage(StorageError::DataCorruption { .. })) => { - Self::DataCorruption - } Error::System(SystemError::Storage( StorageError::IoError(_) | StorageError::DbError(_), )) => Self::StorageIoError, @@ -297,11 +292,7 @@ impl ClientResponse { match result { Ok(entries) => Self::read_results(entries), Err(e) => { - if e.is_not_serving() { - warn!(error = ?e, "state machine cannot serve reads right now"); - } else { - error!(error = ?e, "state machine read failed"); - } + error!(error = ?e, "state machine read failed"); Self::client_error(ErrorCode::for_read_failure(&e)) } } diff --git a/d-engine-core/src/client/types_test.rs b/d-engine-core/src/client/types_test.rs index bebc4fb1..f72298e2 100644 --- a/d-engine-core/src/client/types_test.rs +++ b/d-engine-core/src/client/types_test.rs @@ -346,13 +346,8 @@ fn test_is_propose_failure_false_for_success() { // ── State machine read failures ────────────────────────────────────────────── -fn read_failure_cases() -> [(crate::Error, ErrorCode); 5] { +fn read_failure_cases() -> [(crate::Error, ErrorCode); 3] { [ - // Transient: the state machine is replacing its state, a retry will succeed. - ( - crate::StorageError::NotServing("restoring".into()).into(), - ErrorCode::RetryRequired, - ), ( crate::StorageError::DbError("disk failure".into()).into(), ErrorCode::StorageIoError, @@ -361,20 +356,12 @@ fn read_failure_cases() -> [(crate::Error, ErrorCode); 5] { crate::StorageError::IoError(std::io::Error::other("disk failure")).into(), ErrorCode::StorageIoError, ), - // The client is told the stored data may be damaged and decides what to do. - ( - crate::StorageError::DataCorruption { - location: "sst".into(), - } - .into(), - ErrorCode::DataCorruption, - ), (crate::Error::Fatal("boom".into()), ErrorCode::Uncategorized), ] } -/// Each failure class maps to one stable code, so clients can tell "retry", "storage problem", -/// "data may be damaged" and "internal bug" apart. +/// Each failure class maps to one stable code, so clients can tell a "storage problem" +/// from an "internal bug". #[test] fn test_for_read_failure_maps_each_failure_class_to_its_code() { for (error, expected) in read_failure_cases() { diff --git a/d-engine-core/src/errors.rs b/d-engine-core/src/errors.rs index 7a01a55f..89e6fd24 100644 --- a/d-engine-core/src/errors.rs +++ b/d-engine-core/src/errors.rs @@ -9,8 +9,6 @@ use std::time::Duration; use config::ConfigError; use tokio::task::JoinError; -use crate::StateMachineError; - /// Unified result type used throughout `StateMachine`/`StorageEngine`/`LogStore`/`MetaStore` /// trait signatures — equivalent to `std::result::Result`. pub type Result = std::result::Result; @@ -37,30 +35,17 @@ pub enum Error { impl Error { /// Returns true only for errors that require node shutdown. /// - /// Fatal: explicit `Fatal` marker, storage data corruption. + /// Fatal: explicit `Fatal` marker, snapshot boundary conflict. /// Non-fatal (warn + continue): network, snapshot, election, replication. pub fn is_fatal(&self) -> bool { matches!( self, Error::Fatal(_) - | Error::System(SystemError::Storage(StorageError::DataCorruption { .. })) | Error::Consensus(ConsensusError::Snapshot( SnapshotError::BoundaryConflict { .. } )) ) } - - /// Returns true if the state machine refused the operation because it cannot serve right - /// now: it is replacing its state (snapshot install) or it has been closed. - /// - /// Transient from the caller's point of view: route the request through another path, or - /// ask the client to retry. Not an error in the data. - pub fn is_not_serving(&self) -> bool { - matches!( - self, - Error::System(SystemError::Storage(StorageError::NotServing(_))) - ) - } } #[derive(Debug, thiserror::Error)] @@ -237,10 +222,6 @@ pub enum StorageError { // /// Snapshot creation/restoration failures // #[error("Snapshot operation failed: {0}")] // Snapshot(String), - /// Checksum validation failures - #[error("Data corruption detected at {location}")] - DataCorruption { location: String }, - /// Configuration storage failures #[error("Configuration storage error: {0}")] ConfigStorage(String), @@ -264,14 +245,6 @@ pub enum StorageError { /// ID allocation errors #[error(transparent)] IdAllocation(#[from] IdAllocationError), - - /// State machine not serving requests - /// - /// Returned when read operations are attempted during critical operations - /// such as snapshot restoration. This ensures reads never access inconsistent - /// or temporary state during database transitions. - #[error("State machine not serving: {0}")] - NotServing(String), } #[doc(hidden)] @@ -873,13 +846,3 @@ impl From for Error { StorageError::IoError(e).into() } } - -impl From for Error { - /// A state machine failure reaches the engine as a state machine error with its message; the - /// engine never needs to look inside it. - fn from(error: StateMachineError) -> Self { - Error::System(SystemError::Storage(StorageError::StateMachineError( - error.to_string(), - ))) - } -} diff --git a/d-engine-core/src/errors_test.rs b/d-engine-core/src/errors_test.rs index 88f3c2a4..d28afeba 100644 --- a/d-engine-core/src/errors_test.rs +++ b/d-engine-core/src/errors_test.rs @@ -72,15 +72,6 @@ fn test_network_error_signal_send_failed() { assert!(msg.contains("channel closed")); } -#[test] -fn test_storage_error_data_corruption() { - let err = StorageError::DataCorruption { - location: "raft_log_entry_42".to_string(), - }; - let msg = err.to_string(); - assert!(msg.contains("raft_log_entry_42")); -} - #[test] fn test_storage_error_path_error() { let io_err = std::io::Error::new(std::io::ErrorKind::NotFound, "file not found"); @@ -490,15 +481,6 @@ fn test_is_fatal_explicit_fatal_marker() { assert!(err.is_fatal()); } -#[test] -fn test_is_fatal_data_corruption() { - let err: Error = StorageError::DataCorruption { - location: "raft_log_entry_42".to_string(), - } - .into(); - assert!(err.is_fatal()); -} - #[test] fn test_is_fatal_snapshot_boundary_conflict() { // Same index, different term: the two sides disagree about what wrote this @@ -548,32 +530,7 @@ fn test_is_not_fatal_replication_quorum_not_reached() { #[test] fn test_is_not_fatal_storage_log_storage() { - // I/O-level log storage errors are not fatal (unlike DataCorruption) + // I/O-level log storage errors are not fatal (unlike an explicit `Fatal`) let err: Error = StorageError::LogStorage("write failed".to_string()).into(); assert!(!err.is_fatal()); } - -/// `NotServing` is the state machine saying "not right now" (it is replacing its state or has -/// been closed). Callers route around it or ask the client to retry, so it must not be confused -/// with a real failure of the data. -#[test] -fn test_is_not_serving_is_true_only_for_not_serving() { - let not_serving: Error = StorageError::NotServing("restoring".to_string()).into(); - assert!(not_serving.is_not_serving()); - - let failures: [Error; 4] = [ - StorageError::DbError("disk failure".to_string()).into(), - StorageError::IoError(std::io::Error::other("disk failure")).into(), - StorageError::DataCorruption { - location: "sst".to_string(), - } - .into(), - Error::Fatal("boom".to_string()), - ]; - for error in failures { - assert!( - !error.is_not_serving(), - "{error:?} is a failure, not 'not serving'" - ); - } -} diff --git a/d-engine-core/src/raft_role/follower_state_test.rs b/d-engine-core/src/raft_role/follower_state_test.rs index 04341068..a873b9c3 100644 --- a/d-engine-core/src/raft_role/follower_state_test.rs +++ b/d-engine-core/src/raft_role/follower_state_test.rs @@ -1890,17 +1890,10 @@ async fn test_handle_client_read_request_eventual_consistency_succeeds() { /// consistency. The client must get an error it can act on, not an empty "key not found" answer. #[tokio::test] async fn test_follower_eventual_read_reports_state_machine_failure_as_error() { - let cases: [(fn() -> Error, ErrorCode); 2] = [ - ( - || -> Error { crate::StorageError::DbError("disk failure".into()).into() }, - ErrorCode::StorageIoError, - ), - // Transient: the state machine is replacing its state, retrying will succeed. - ( - || -> Error { crate::StorageError::NotServing("restoring".into()).into() }, - ErrorCode::RetryRequired, - ), - ]; + let cases: [(fn() -> Error, ErrorCode); 1] = [( + || -> Error { crate::StorageError::DbError("disk failure".into()).into() }, + ErrorCode::StorageIoError, + )]; for (failure, expected) in cases { let (_graceful_tx, graceful_rx) = watch::channel(()); diff --git a/d-engine-core/src/raft_role/leader_state_test/client_read_test.rs b/d-engine-core/src/raft_role/leader_state_test/client_read_test.rs index 471f9dfc..0981b0b1 100644 --- a/d-engine-core/src/raft_role/leader_state_test/client_read_test.rs +++ b/d-engine-core/src/raft_role/leader_state_test/client_read_test.rs @@ -20,7 +20,6 @@ use crate::MockStateMachineHandler; use crate::RaftNodeConfig; use crate::ReadConsistencyPolicy; use crate::ScanResult; -use crate::StateMachineError; use crate::client::ClientReadRequest; use crate::client::ClientResponsePayload; use crate::client::ErrorCode; @@ -2303,7 +2302,7 @@ async fn test_scan_cmd_leader_propagates_sm_error() { let mut sm = mock_state_machine(); sm.expect_scan_prefix() - .returning(|_prefix| Err(StateMachineError::new("scan failed"))); + .returning(|_prefix| Err(std::io::Error::other("scan failed"))); let (_graceful_tx, graceful_rx) = watch::channel(()); let context = MockBuilder::new(graceful_rx) @@ -2325,18 +2324,16 @@ async fn test_scan_cmd_leader_propagates_sm_error() { assert_eq!(err.code(), Code::Internal); } -/// A lease read that fails in the state machine must reach the client as an error. The data -/// may be damaged, so the client gets the corruption code and decides what to do; it must not get -/// an empty result that looks like "key not found". +/// A lease read that fails in the state machine must reach the client as an error. A state +/// machine reports a failing disk as an `io::Error`, and the client gets the storage I/O code; it +/// must not get an empty result that looks like "key not found". #[tokio::test] async fn test_lease_read_reports_state_machine_failure_as_error() { let mut state_machine_handler = MockStateMachineHandler::new(); - state_machine_handler.expect_read_from_state_machine().times(1).returning(|_| { - Err(crate::StorageError::DataCorruption { - location: "sst".into(), - } - .into()) - }); + state_machine_handler + .expect_read_from_state_machine() + .times(1) + .returning(|_| Err(std::io::Error::other("disk failure").into())); let (_graceful_tx, graceful_rx) = watch::channel(()); let context = MockBuilder::new(graceful_rx) .with_db_path("/tmp/test_lease_read_reports_state_machine_failure_as_error") @@ -2363,6 +2360,6 @@ async fn test_lease_read_reports_state_machine_failure_as_error() { .expect("answering the client is not a node failure"); let response = resp_rx.recv().await.unwrap().unwrap(); - assert_eq!(response.error, ErrorCode::DataCorruption); + assert_eq!(response.error, ErrorCode::StorageIoError); assert!(response.result.is_none()); } diff --git a/d-engine-core/src/state_machine_handler/default_state_machine_handler_test.rs b/d-engine-core/src/state_machine_handler/default_state_machine_handler_test.rs index 7b1f5af6..812fe21d 100644 --- a/d-engine-core/src/state_machine_handler/default_state_machine_handler_test.rs +++ b/d-engine-core/src/state_machine_handler/default_state_machine_handler_test.rs @@ -43,7 +43,6 @@ use super::new_reader_writer_pair; use crate::MockSnapshotPolicy; use crate::MockStateMachine; use crate::MockTypeConfig; -use crate::StateMachineError; use crate::test_utils::create_test_compressed_snapshot; use crate::test_utils::snapshot_config; @@ -217,7 +216,7 @@ mod apply_chunk_test { if apply_chunk_error { state_machine .expect_apply_chunk() - .returning(|_| Err(StateMachineError::new("Test error"))); + .returning(|_| Err(std::io::Error::other("Test error"))); } else { state_machine.expect_apply_chunk().returning(|_| Ok(vec![])); } @@ -336,7 +335,7 @@ mod apply_chunk_test { let (watch_tx, mut watch_rx) = tokio::sync::broadcast::channel(16); let mut sm = MockStateMachine::new(); sm.expect_apply_chunk() - .returning(|_| Err(StateMachineError::new("batch failed"))); + .returning(|_| Err(std::io::Error::other("batch failed"))); let (handler, writer) = new_reader_writer_pair::( 1, @@ -1082,7 +1081,7 @@ mod create_snapshot_tests { fs::create_dir(&db_path).unwrap(); Ok(LogId { term: 1, index: 1 }) } else { - Err(StateMachineError::new("Concurrency failure")) + Err(std::io::Error::other("Concurrency failure")) } }); @@ -1210,7 +1209,7 @@ mod create_snapshot_tests { // Setup failing snapshot generation sm.expect_last_applied().returning(|| LogId { term: 1, index: 1 }); sm.expect_generate_snapshot_data() - .returning(|_| Err(StateMachineError::new("test failure"))); + .returning(|_| Err(std::io::Error::other("test failure"))); let mut config = snapshot_config(temp_path.to_path_buf()); config.retained_log_entries = 0; @@ -1280,7 +1279,7 @@ mod create_snapshot_tests { sm.expect_last_applied().returning(|| LogId { term: 1, index: 1 }); sm.expect_generate_snapshot_data() - .returning(|_| Err(StateMachineError::new("test error"))); + .returning(|_| Err(std::io::Error::other("test error"))); let config = snapshot_config(temp_path.clone()); let (handler, writer) = new_reader_writer_pair::( @@ -1509,7 +1508,7 @@ mod create_snapshot_tests { sm.expect_generate_snapshot_data().returning(|path| { // Simulate a partial write before the failure (e.g. checkpoint killed mid-copy). fs::create_dir_all(&path).unwrap(); - Err(StateMachineError::new("simulated crash")) + Err(std::io::Error::other("simulated crash")) }); let config = snapshot_config(snapshot_dir.clone()); @@ -1548,7 +1547,7 @@ mod create_snapshot_tests { // `create_dir_all`, otherwise this test can't distinguish "cleaned up" from // "never checked". if path.exists() { - return Err(StateMachineError::new("mkdir: File exists")); + return Err(std::io::Error::other("mkdir: File exists")); } fs::create_dir_all(&path).unwrap(); Ok(LogId { term: 1, index: 5 }) @@ -2923,7 +2922,7 @@ fn test_read_from_state_machine_returns_the_state_machine_error() { let mut state_machine = MockStateMachine::new(); state_machine .expect_get_multi() - .returning(|_| Err(StateMachineError::new("disk failure"))); + .returning(|_| Err(std::io::Error::other("disk failure"))); let handler = handler_over(state_machine, temp_dir.path()); let err = handler @@ -2933,6 +2932,28 @@ fn test_read_from_state_machine_returns_the_state_machine_error() { assert!(format!("{err:?}").contains("disk failure"), "got: {err:?}"); } +/// Business scenario: the disk under a node fails while a client reads. A state machine reports +/// that as a plain `io::Error`; the client must receive the "storage I/O error" code and no +/// payload, so it can tell a failing disk from a missing key. +#[test] +fn test_state_machine_io_error_reaches_client_as_storage_io_error() { + use crate::client::types::ClientResponse; + use crate::client::types::ErrorCode; + + let temp_dir = tempdir().unwrap(); + let mut state_machine = MockStateMachine::new(); + state_machine + .expect_get_multi() + .returning(|_| Err(std::io::Error::other("disk failure"))); + let handler = handler_over(state_machine, temp_dir.path()); + + let result = handler.read_from_state_machine(vec![bytes::Bytes::from_static(b"k")]); + let response = ClientResponse::from_read(result); + + assert_eq!(response.error, ErrorCode::StorageIoError); + assert!(response.result.is_none()); +} + /// Keys that do not exist are left out; the others keep the order they were asked in. All keys /// are read with one `get_multi` call, so they come from one consistent view. #[test] diff --git a/d-engine-core/src/storage/state_machine.rs b/d-engine-core/src/storage/state_machine.rs index 216edb42..9a2379ba 100644 --- a/d-engine-core/src/storage/state_machine.rs +++ b/d-engine-core/src/storage/state_machine.rs @@ -71,32 +71,6 @@ impl ApplyResult { } } -/// A failure reported by a [`StateMachine`]. The message says what went wrong. -/// -/// Return it for any failure: a full disk, an I/O error, data that failed a checksum. The engine -/// does not need to know which: when applying entries or installing a snapshot fails, the node -/// stops and recovers from its replicas; when a read fails, only that read fails. -/// -/// Build one from a message, `StateMachineError::new("disk full")`, or from any error type that -/// implements [`std::error::Error`]; an [`std::io::Error`] converts with `?`. The error chain is -/// kept for the log. -#[derive(Debug, thiserror::Error)] -#[error("state machine error: {0}")] -pub struct StateMachineError(#[source] Box); - -impl StateMachineError { - /// Wraps a message or an error. - pub fn new(error: impl Into>) -> Self { - Self(error.into()) - } -} - -impl From for StateMachineError { - fn from(error: std::io::Error) -> Self { - Self::new(error) - } -} - /// State machine trait for Raft consensus /// /// # Thread Safety Requirements @@ -123,10 +97,15 @@ impl From for StateMachineError { /// /// # Errors /// -/// Every method reports a failure with [`StateMachineError`]. A failure in `apply_chunk` stops the -/// node, which then recovers from its replicas: an entry that was committed cannot be skipped, so -/// the node must not go on without it. A failed `get`, `get_multi` or `scan_prefix` fails only that -/// read. +/// Every method reports a failure with a plain [`std::io::Error`]. The engine does not look at its +/// kind: the message is for the log. Build one from a message, `io::Error::other("disk full")`, or +/// wrap the error of the storage library you use, `.map_err(io::Error::other)`; the original error +/// stays attached as its source. +/// +/// What a failure does depends on the method. A failure in `apply_chunk` or +/// `apply_snapshot_from_file` stops the node, which then recovers from its replicas: an entry that +/// was committed cannot be skipped, so the node must not go on without it. A failed `get`, +/// `get_multi` or `scan_prefix` fails only that read. /// /// # Reads while a snapshot is installed /// @@ -157,7 +136,7 @@ pub trait StateMachine: Send + Sync + 'static { /// `last_applied` has to be persisted together with the data it describes. /// /// Default is a no-op, for implementations with nothing to persist or release. - fn close(&self) -> Result<(), StateMachineError> { + fn close(&self) -> std::io::Result<()> { Ok(()) } @@ -168,7 +147,7 @@ pub trait StateMachine: Send + Sync + 'static { fn get( &self, key_buffer: &[u8], - ) -> Result, StateMachineError>; + ) -> std::io::Result>; /// Retrieves multiple values by key from the state machine in a single call. /// @@ -199,7 +178,7 @@ pub trait StateMachine: Send + Sync + 'static { fn get_multi( &self, keys: &[Bytes], - ) -> Result>, StateMachineError> { + ) -> std::io::Result>> { keys.iter().map(|k| self.get(k)).collect() } @@ -214,7 +193,7 @@ pub trait StateMachine: Send + Sync + 'static { async fn apply_chunk( &self, chunk: &[ApplyEntry], - ) -> Result, StateMachineError>; + ) -> std::io::Result>; /// Gets the last applied log index and term. /// Sync operation as it reads from atomic variables. @@ -236,7 +215,7 @@ pub trait StateMachine: Send + Sync + 'static { &self, metadata: &SnapshotMetadata, snapshot_path: std::path::PathBuf, - ) -> Result<(), StateMachineError>; + ) -> std::io::Result<()>; /// Exports the state machine's current state into `new_snapshot_dir` and returns the position /// of the last applied entry that the exported state contains. @@ -281,7 +260,7 @@ pub trait StateMachine: Send + Sync + 'static { async fn generate_snapshot_data( &self, new_snapshot_dir: std::path::PathBuf, - ) -> Result; + ) -> std::io::Result; /// Returns all `(key, value)` pairs whose key starts with `prefix`, plus /// the `last_applied_index` at the time of the scan. @@ -294,9 +273,10 @@ pub trait StateMachine: Send + Sync + 'static { fn scan_prefix( &self, prefix: &[u8], - ) -> Result { + ) -> std::io::Result { let _ = prefix; - Err(StateMachineError::new( + Err(std::io::Error::new( + std::io::ErrorKind::Unsupported, "scan_prefix not supported by this state machine", )) } diff --git a/d-engine-core/src/storage/state_machine_test.rs b/d-engine-core/src/storage/state_machine_test.rs index ff5ebc7a..94136d39 100644 --- a/d-engine-core/src/storage/state_machine_test.rs +++ b/d-engine-core/src/storage/state_machine_test.rs @@ -18,7 +18,7 @@ use crate::storage::StateMachine; /// /// This suite provides comprehensive tests that can be used to validate /// any StateMachine implementation. Developers should implement the -/// `StateMachineBuilder` trait and then call `run_all_state_machine_tests` +/// `StateMachineBuilder` trait and then call `StateMachineTestSuite::run_all_tests` /// with their builder. pub struct StateMachineTestSuite; @@ -1748,7 +1748,6 @@ mod default_scan_prefix_tests { use crate::ApplyEntry; use crate::storage::state_machine::ApplyResult; use crate::storage::state_machine::StateMachine; - use crate::storage::state_machine::StateMachineError; struct MinimalSm { running: AtomicBool, @@ -1756,7 +1755,7 @@ mod default_scan_prefix_tests { #[async_trait] impl StateMachine for MinimalSm { - fn close(&self) -> Result<(), StateMachineError> { + fn close(&self) -> std::io::Result<()> { self.running.store(false, Ordering::SeqCst); Ok(()) } @@ -1764,14 +1763,14 @@ mod default_scan_prefix_tests { fn get( &self, _key_buffer: &[u8], - ) -> Result, StateMachineError> { + ) -> std::io::Result> { Ok(None) } async fn apply_chunk( &self, _chunk: &[ApplyEntry], - ) -> Result, StateMachineError> { + ) -> std::io::Result> { Ok(vec![]) } @@ -1783,14 +1782,14 @@ mod default_scan_prefix_tests { &self, _metadata: &SnapshotMetadata, _snapshot_path: std::path::PathBuf, - ) -> Result<(), StateMachineError> { + ) -> std::io::Result<()> { Ok(()) } async fn generate_snapshot_data( &self, _new_snapshot_dir: std::path::PathBuf, - ) -> Result { + ) -> std::io::Result { Ok(LogId::default()) } @@ -1813,8 +1812,8 @@ mod default_scan_prefix_tests { result.is_err(), "default scan_prefix must return Err; got Ok unexpectedly" ); - // The error type is StateMachineError — check the Debug representation which - // includes the nested message (Display only shows the outermost variant). + // The error type is io::Error — check the Debug representation, which + // includes the message. let err_debug = format!("{:?}", result.unwrap_err()); assert!( err_debug.contains("scan_prefix not supported"), @@ -1851,7 +1850,6 @@ mod install_reader_suite_tests { use crate::Command; use crate::Error; use crate::storage::StateMachine; - use crate::storage::StateMachineError; use crate::storage::state_machine::ApplyResult; /// What survives a restart: shared by every state machine the builder creates. @@ -1903,14 +1901,14 @@ mod install_reader_suite_tests { fn get( &self, key: &[u8], - ) -> Result, StateMachineError> { + ) -> std::io::Result> { Ok(self.durable.data.read().unwrap().get(key).cloned()) } async fn apply_chunk( &self, chunk: &[ApplyEntry], - ) -> Result, StateMachineError> { + ) -> std::io::Result> { // Data and position change under one lock, like a database transaction. let mut data = self.durable.data.write().unwrap(); for entry in chunk { @@ -1938,7 +1936,7 @@ mod install_reader_suite_tests { } } - fn close(&self) -> Result<(), StateMachineError> { + fn close(&self) -> std::io::Result<()> { if self.position_on_close_only { *self.durable.applied.write().unwrap() = *self.pending_position.read().unwrap(); } @@ -1948,7 +1946,7 @@ mod install_reader_suite_tests { async fn generate_snapshot_data( &self, new_snapshot_dir: PathBuf, - ) -> Result { + ) -> std::io::Result { std::fs::create_dir_all(&new_snapshot_dir)?; // Right: copy the data and read the position under the same lock. let (lines, mut position) = { @@ -1983,7 +1981,7 @@ mod install_reader_suite_tests { &self, _metadata: &SnapshotMetadata, snapshot_path: PathBuf, - ) -> Result<(), StateMachineError> { + ) -> std::io::Result<()> { let text = std::fs::read_to_string(snapshot_path.join("data.txt"))?; let new_data: HashMap = text .lines() diff --git a/d-engine-server/README.md b/d-engine-server/README.md index 608a01db..13fee908 100644 --- a/d-engine-server/README.md +++ b/d-engine-server/README.md @@ -135,7 +135,7 @@ Simple file-based storage for development and small deployments. use d_engine_server::{FileStorageEngine, FileStateMachine}; let storage = Arc::new(FileStorageEngine::new("./logs")?); -let sm = Arc::new(FileStateMachine::new("./sm").await?); +let sm = FileStateMachine::new("./sm").await?; ``` ### RocksDB Storage (Production) @@ -152,7 +152,6 @@ use d_engine_server::RocksDBUnifiedEngine; let (storage, sm) = RocksDBUnifiedEngine::open("./data/db")?; let storage = Arc::new(storage); -let sm = Arc::new(sm); ``` --- diff --git a/d-engine-server/src/api/embedded_test/shutdown_order_test.rs b/d-engine-server/src/api/embedded_test/shutdown_order_test.rs index 7b482b33..bfbef789 100644 --- a/d-engine-server/src/api/embedded_test/shutdown_order_test.rs +++ b/d-engine-server/src/api/embedded_test/shutdown_order_test.rs @@ -17,7 +17,6 @@ use d_engine_core::ApplyEntry; use d_engine_core::ApplyResult; use d_engine_core::ScanResult; use d_engine_core::StateMachine; -use d_engine_core::StateMachineError; use d_engine_proto::common::LogId; use d_engine_proto::server::storage::SnapshotMetadata; @@ -70,7 +69,7 @@ impl StateMachine for SlowApplyProbe { async fn apply_chunk( &self, chunk: &[ApplyEntry], - ) -> Result, StateMachineError> { + ) -> std::io::Result> { self.flags.apply_started.store(true, Ordering::SeqCst); let closed_at_start = self.flags.storage_closed.load(Ordering::SeqCst); tokio::time::sleep(self.apply_delay).await; @@ -81,7 +80,7 @@ impl StateMachine for SlowApplyProbe { result } - fn close(&self) -> Result<(), StateMachineError> { + fn close(&self) -> std::io::Result<()> { self.flags.storage_closed.store(true, Ordering::SeqCst); std::thread::sleep(self.close_delay); self.inner.close() @@ -91,7 +90,7 @@ impl StateMachine for SlowApplyProbe { fn get( &self, key_buffer: &[u8], - ) -> Result, StateMachineError> { + ) -> std::io::Result> { self.note_read(); self.inner.get(key_buffer) } @@ -102,19 +101,19 @@ impl StateMachine for SlowApplyProbe { &self, metadata: &SnapshotMetadata, snapshot_path: std::path::PathBuf, - ) -> Result<(), StateMachineError> { + ) -> std::io::Result<()> { self.inner.apply_snapshot_from_file(metadata, snapshot_path).await } async fn generate_snapshot_data( &self, new_snapshot_dir: std::path::PathBuf, - ) -> Result { + ) -> std::io::Result { self.inner.generate_snapshot_data(new_snapshot_dir).await } fn scan_prefix( &self, prefix: &[u8], - ) -> Result { + ) -> std::io::Result { self.note_read(); self.inner.scan_prefix(prefix) } diff --git a/d-engine-server/src/api/read_handle.rs b/d-engine-server/src/api/read_handle.rs index cb821ed4..e66775d6 100644 --- a/d-engine-server/src/api/read_handle.rs +++ b/d-engine-server/src/api/read_handle.rs @@ -128,12 +128,7 @@ pub(crate) fn read_failure(e: &Error) -> ClientApiError { error!(error = ?e, "state machine read failed"); metrics::counter!("server.read.errors").increment(1); - ClientApiError::Business { - code: ErrorCode::Uncategorized, - message: "Read failed due to an internal error. Please retry or contact the administrator." - .to_string(), - required_action: None, - } + ErrorCode::for_read_failure(e).into() } // ── ReadHandle ────────────────────────────────────────────────────── diff --git a/d-engine-server/src/api/read_handle_test.rs b/d-engine-server/src/api/read_handle_test.rs index d77ca1f8..b154609f 100644 --- a/d-engine-server/src/api/read_handle_test.rs +++ b/d-engine-server/src/api/read_handle_test.rs @@ -7,7 +7,7 @@ /// /// Proof technique (same as embedded_client_fast_path_test.rs): /// cmd_rx is dropped so any fallback to cmd_tx returns a Network/channel-closed error. -/// A Business error proves SmError was returned directly. +/// A Storage error proves SmError was returned directly. /// An Ok(value) with cmd_rx dropped proves the fast path was taken. #[cfg(test)] mod read_handle_tests { @@ -65,15 +65,13 @@ mod read_handle_tests { fn sm_unavailable() -> MockStateMachine { let mut sm = MockStateMachine::new(); - sm.expect_get_multi() - .returning(|_| Err(d_engine_core::StateMachineError::new("stopped"))); + sm.expect_get_multi().returning(|_| Err(std::io::Error::other("stopped"))); sm } fn sm_with_error() -> MockStateMachine { let mut sm = MockStateMachine::new(); - sm.expect_get_multi() - .returning(|_| Err(d_engine_core::StateMachineError::new("disk failure"))); + sm.expect_get_multi().returning(|_| Err(std::io::Error::other("disk failure"))); sm } @@ -137,7 +135,7 @@ mod read_handle_tests { let (rh, handle) = make_handle(sm_unavailable(), valid_lease(), true).await; let result = rh.get(b"k", ReadConsistencyPolicy::EventualConsistency, 1, TIMEOUT).await; assert!( - matches!(&result, Err(e) if e.code() == ErrorCode::Uncategorized), + matches!(&result, Err(e) if e.code() == ErrorCode::StorageIoError), "the state machine's error must reach the caller, got: {:?}", result ); @@ -199,7 +197,7 @@ mod read_handle_tests { let (rh, handle) = make_handle(sm_unavailable(), valid_lease(), true).await; let result = rh.get(b"k", ReadConsistencyPolicy::LeaseRead, 1, TIMEOUT).await; assert!( - matches!(&result, Err(e) if e.code() == ErrorCode::Uncategorized), + matches!(&result, Err(e) if e.code() == ErrorCode::StorageIoError), "the state machine's error must reach the caller, got: {:?}", result ); @@ -209,16 +207,16 @@ mod read_handle_tests { // ── SmError — direct return, no cmd_tx fallback ─────────────────────────── - /// sm.get() returns Err → SmError → Business error returned DIRECTLY. + /// sm.get() returns Err → SmError → Storage error returned DIRECTLY. /// cmd_rx dropped: if fallback were taken, error would be Network. - /// Business error proves SmError was returned without touching cmd_tx. + /// Storage error proves SmError was returned without touching cmd_tx. #[tokio::test] async fn test_sm_error_returned_directly_no_cmd_tx_fallback() { let (rh, handle) = make_handle(sm_with_error(), valid_lease(), true).await; let result = rh.get(b"k", ReadConsistencyPolicy::EventualConsistency, 1, TIMEOUT).await; assert!( - matches!(result, Err(ClientApiError::Business { .. })), - "SmError must produce Business error (direct), not Network (fallback), got: {:?}", + matches!(result, Err(ClientApiError::Storage { .. })), + "SmError must produce Storage error (direct), not Network (fallback), got: {:?}", result ); drop(rh); @@ -232,14 +230,14 @@ mod read_handle_tests { let (rh, handle) = make_handle(sm_with_error(), valid_lease(), true).await; let result = rh.get(b"k", ReadConsistencyPolicy::EventualConsistency, 1, TIMEOUT).await; match result { - Err(ClientApiError::Business { code, message, .. }) => { - assert_eq!(code, ErrorCode::Uncategorized); + Err(ClientApiError::Storage { code, message }) => { + assert_eq!(code, ErrorCode::StorageIoError); assert!( !message.contains("disk failure"), "engine detail leaked to the client: {message}" ); } - other => panic!("expected a Business error, got: {other:?}"), + other => panic!("expected a Storage error, got: {other:?}"), } drop(rh); handle.await.unwrap(); diff --git a/d-engine-server/src/lib.rs b/d-engine-server/src/lib.rs index 372335c0..afb6fc97 100644 --- a/d-engine-server/src/lib.rs +++ b/d-engine-server/src/lib.rs @@ -168,12 +168,12 @@ pub use d_engine_core::ScanResult; pub use d_engine_core::ScanResults; /// Snapshot-specific error type (internal to the engine). pub use d_engine_core::SnapshotError; -/// Storage trait for implementing custom storage backends +/// Your application's data, kept identical on every node. /// -/// Implement this trait to create your own storage engine. +/// d-engine hands each committed write to `apply_chunk`, in the same order on every node, and +/// answers reads through `get`. Implement this trait to store the data in your own database +/// instead of the built-in file or RocksDB state machines. pub use d_engine_core::StateMachine; -/// The error a [`StateMachine`] returns for any failure. -pub use d_engine_core::StateMachineError; /// Storage trait for implementing custom storage backends /// /// Implement this trait to create your own storage engine. diff --git a/d-engine-server/src/network/grpc/grpc_raft_service_fast_path_test.rs b/d-engine-server/src/network/grpc/grpc_raft_service_fast_path_test.rs index 19ebc695..919635b1 100644 --- a/d-engine-server/src/network/grpc/grpc_raft_service_fast_path_test.rs +++ b/d-engine-server/src/network/grpc/grpc_raft_service_fast_path_test.rs @@ -56,15 +56,13 @@ mod grpc_fast_path_tests { fn sm_unavailable() -> MockStateMachine { let mut sm = MockStateMachine::new(); - sm.expect_get_multi() - .returning(|_| Err(d_engine_core::StateMachineError::new("stopped"))); + sm.expect_get_multi().returning(|_| Err(std::io::Error::other("stopped"))); sm } fn sm_with_error() -> MockStateMachine { let mut sm = MockStateMachine::new(); - sm.expect_get_multi() - .returning(|_| Err(d_engine_core::StateMachineError::new("disk failure"))); + sm.expect_get_multi().returning(|_| Err(std::io::Error::other("disk failure"))); sm } diff --git a/d-engine-server/src/read_actor_test.rs b/d-engine-server/src/read_actor_test.rs index 6202377f..962fb79d 100644 --- a/d-engine-server/src/read_actor_test.rs +++ b/d-engine-server/src/read_actor_test.rs @@ -18,8 +18,7 @@ fn make_sm_running() -> MockStateMachine { fn make_sm_unavailable() -> MockStateMachine { let mut sm = MockStateMachine::new(); - sm.expect_get_multi() - .returning(|_| Err(d_engine_core::StateMachineError::new("stopped"))); + sm.expect_get_multi().returning(|_| Err(std::io::Error::other("stopped"))); sm } @@ -285,8 +284,7 @@ async fn test_read_actor_eventual_multi_key_uses_get_multi() { #[tokio::test] async fn test_read_actor_sm_get_multi_error_returns_sm_error() { let mut sm = make_sm_running(); - sm.expect_get_multi() - .returning(|_| Err(d_engine_core::StateMachineError::new("disk failure"))); + sm.expect_get_multi().returning(|_| Err(std::io::Error::other("disk failure"))); let (tx, rx) = mpsc::channel(8); let handle = tokio::spawn(run_read_actor(rx, valid_lease(), Arc::new(sm), 64)); diff --git a/d-engine-server/src/storage/adaptors/file/file_state_machine.rs b/d-engine-server/src/storage/adaptors/file/file_state_machine.rs index e4459d75..53c76ace 100644 --- a/d-engine-server/src/storage/adaptors/file/file_state_machine.rs +++ b/d-engine-server/src/storage/adaptors/file/file_state_machine.rs @@ -54,6 +54,7 @@ use std::collections::HashMap; use std::collections::HashSet; +use std::io; use std::io::Write; use std::path::PathBuf; use std::sync::Mutex; @@ -66,11 +67,8 @@ use bytes::Bytes; use d_engine_core::ApplyEntry; use d_engine_core::ApplyResult; use d_engine_core::Command; -use d_engine_core::Error; use d_engine_core::ScanResult; use d_engine_core::StateMachine; -use d_engine_core::StateMachineError; -use d_engine_core::StorageError; use d_engine_proto::common::LogId; use d_engine_proto::server::storage::SnapshotMetadata; use parking_lot::RwLock; @@ -237,8 +235,9 @@ impl FileStateMachine { /// * `data_dir` - Directory where data files will be stored /// /// # Returns - /// Result containing the initialized FileStateMachine - pub async fn new(data_dir: PathBuf) -> Result { + /// The initialized FileStateMachine, or an error if the files cannot be read or the WAL is + /// corrupt + pub async fn new(data_dir: PathBuf) -> io::Result { // Ensure data directory exists fs::create_dir_all(&data_dir).await?; @@ -259,7 +258,7 @@ impl FileStateMachine { } /// Loads state machine data from disk files - async fn load_from_disk(&self) -> Result<(), Error> { + async fn load_from_disk(&self) -> io::Result<()> { // Load last applied index and term from metadata file self.load_metadata().await?; @@ -274,7 +273,7 @@ impl FileStateMachine { } /// Loads metadata from disk - async fn load_metadata(&self) -> Result<(), Error> { + async fn load_metadata(&self) -> io::Result<()> { let metadata_path = self.data_dir.join("metadata.bin"); if !metadata_path.exists() { return Ok(()); @@ -302,7 +301,7 @@ impl FileStateMachine { } /// Loads key-value data from disk - async fn load_data(&self) -> Result<(), Error> { + async fn load_data(&self) -> io::Result<()> { let data_path = self.data_dir.join("state.data"); if !data_path.exists() { return Ok(()); @@ -397,7 +396,7 @@ impl FileStateMachine { } /// Replays write-ahead log for crash recovery - async fn replay_wal(&self) -> Result<(), Error> { + async fn replay_wal(&self) -> io::Result<()> { let wal_path = self.data_dir.join("wal.log"); if !wal_path.exists() { debug!("No WAL file found, skipping replay"); @@ -435,10 +434,12 @@ impl FileStateMachine { pos += 8; // Read operation code (1 byte) - let op_code = - WalOpCode::from_u8(buffer[pos]).ok_or_else(|| StorageError::DataCorruption { - location: format!("WAL opcode {} at byte offset {}", buffer[pos], pos), - })?; + let op_code = WalOpCode::from_u8(buffer[pos]).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("WAL opcode {} at byte offset {}", buffer[pos], pos), + ) + })?; pos += 1; // Check if we have enough bytes for key length @@ -570,7 +571,7 @@ impl FileStateMachine { } /// Persists key-value data to disk - fn persist_data(&self) -> Result<(), StateMachineError> { + fn persist_data(&self) -> io::Result<()> { // Collect data first to minimize lock time let data_copy: HashMap = { let data = self.data.read(); @@ -597,7 +598,7 @@ impl FileStateMachine { } /// Persists key-value data to disk - async fn persist_data_async(&self) -> Result<(), StateMachineError> { + async fn persist_data_async(&self) -> io::Result<()> { // Collect data under minimal lock scope let data_copy: HashMap = { let data = self.data.read(); @@ -643,12 +644,12 @@ impl FileStateMachine { pub(crate) fn persist_last_applied( &self, last_applied: LogId, - ) -> Result<(), StateMachineError> { + ) -> io::Result<()> { self.update_last_applied(last_applied); self.persist_metadata() } - pub(crate) fn save_hard_state(&self) -> Result<(), StateMachineError> { + pub(crate) fn save_hard_state(&self) -> io::Result<()> { let last_applied = self.last_applied(); self.persist_last_applied(last_applied)?; @@ -656,7 +657,7 @@ impl FileStateMachine { Ok(()) } - pub(crate) fn flush(&self) -> Result<(), StateMachineError> { + pub(crate) fn flush(&self) -> io::Result<()> { self.persist_data()?; self.persist_metadata()?; // self.clear_wal()?; @@ -664,7 +665,7 @@ impl FileStateMachine { } /// Persists metadata to disk - fn persist_metadata(&self) -> Result<(), StateMachineError> { + fn persist_metadata(&self) -> io::Result<()> { let metadata_path = self.data_dir.join("metadata.bin"); let mut file = std::fs::OpenOptions::new() .write(true) @@ -682,7 +683,7 @@ impl FileStateMachine { Ok(()) } - async fn persist_metadata_async(&self) -> Result<(), StateMachineError> { + async fn persist_metadata_async(&self) -> io::Result<()> { let metadata_path = self.data_dir.join("metadata.bin"); let mut file = OpenOptions::new() .write(true) @@ -703,7 +704,7 @@ impl FileStateMachine { /// Clears the write-ahead log (called after successful persistence) #[allow(unused)] - fn clear_wal(&self) -> Result<(), StateMachineError> { + fn clear_wal(&self) -> io::Result<()> { let wal_path = self.data_dir.join("wal.log"); let mut file = std::fs::OpenOptions::new() .write(true) @@ -717,7 +718,7 @@ impl FileStateMachine { } /// Clears the write-ahead log (called after successful persistence) - async fn clear_wal_async(&self) -> Result<(), StateMachineError> { + async fn clear_wal_async(&self) -> io::Result<()> { let wal_path = self.data_dir.join("wal.log"); let mut file = OpenOptions::new() .write(true) @@ -752,7 +753,7 @@ impl FileStateMachine { /// Checkpoint: snapshot full data + metadata to disk, then clear WAL. /// /// WAL is the primary crash-safety path; checkpoint bounds recovery replay time. - pub(crate) async fn checkpoint(&self) -> Result<(), StateMachineError> { + pub(crate) async fn checkpoint(&self) -> io::Result<()> { self.persist_data_async().await?; self.persist_metadata_async().await?; self.clear_wal_async().await?; @@ -768,11 +769,11 @@ impl FileStateMachine { /// Reads are served only while running. After `close()` the state machine may still be /// reachable through a handle the developer shares, and it must not answer from state that /// is no longer maintained. - fn ensure_serving(&self) -> Result<(), StateMachineError> { + fn ensure_serving(&self) -> io::Result<()> { if self.running.load(Ordering::Acquire) { Ok(()) } else { - Err(StateMachineError::new("state machine stopped")) + Err(io::Error::other("state machine stopped")) } } @@ -796,7 +797,7 @@ impl FileStateMachine { &self, entries: &[ApplyEntry], cas_outcomes: &[bool], - ) -> Result<(), Error> { + ) -> io::Result<()> { assert_eq!( entries.len(), cas_outcomes.len(), @@ -836,7 +837,7 @@ impl Drop for FileStateMachine { #[async_trait] impl StateMachine for FileStateMachine { - fn close(&self) -> Result<(), StateMachineError> { + fn close(&self) -> io::Result<()> { if !self.running.swap(false, Ordering::SeqCst) { return Ok(()); // already closed } @@ -849,7 +850,7 @@ impl StateMachine for FileStateMachine { fn get( &self, key_buffer: &[u8], - ) -> Result, StateMachineError> { + ) -> io::Result> { self.ensure_serving()?; let data = self.data.read(); Ok(data.get(key_buffer).map(|(value, _)| value.clone())) @@ -859,7 +860,7 @@ impl StateMachine for FileStateMachine { async fn apply_chunk( &self, chunk: &[ApplyEntry], - ) -> Result, StateMachineError> { + ) -> io::Result> { self.ensure_serving()?; let chunk_len = chunk.len(); let mut results = Vec::with_capacity(chunk_len); @@ -1066,10 +1067,10 @@ impl StateMachine for FileStateMachine { &self, metadata: &SnapshotMetadata, snapshot_dir: std::path::PathBuf, - ) -> Result<(), StateMachineError> { + ) -> io::Result<()> { let incoming = metadata .last_included - .ok_or_else(|| StateMachineError::new("Missing last_included in snapshot metadata"))?; + .ok_or_else(|| io::Error::other("Missing last_included in snapshot metadata"))?; info!("Applying snapshot from file: {:?}", snapshot_dir); @@ -1182,7 +1183,7 @@ impl StateMachine for FileStateMachine { async fn generate_snapshot_data( &self, new_snapshot_dir: std::path::PathBuf, - ) -> Result { + ) -> io::Result { // Create snapshot directory fs::create_dir_all(&new_snapshot_dir).await?; @@ -1223,7 +1224,7 @@ impl StateMachine for FileStateMachine { fn scan_prefix( &self, prefix: &[u8], - ) -> Result { + ) -> io::Result { self.ensure_serving()?; let data = self.data.read(); let entries: Vec<(Bytes, Bytes)> = data @@ -1238,7 +1239,7 @@ impl StateMachine for FileStateMachine { fn get_multi( &self, keys: &[Bytes], - ) -> Result>, StateMachineError> { + ) -> io::Result>> { self.ensure_serving()?; // Hold read lock for the full batch so all keys come from the same consistent view. // apply_chunk holds the write lock; parking_lot RwLock ensures they don't interleave. diff --git a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs index 95316fbd..afdf61cc 100644 --- a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs +++ b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs @@ -1,3 +1,4 @@ +use std::io; use std::path::Path; use std::sync::Arc; use std::sync::atomic::AtomicBool; @@ -10,11 +11,8 @@ use bytes::Bytes; use d_engine_core::ApplyEntry; use d_engine_core::ApplyResult; use d_engine_core::Command; -use d_engine_core::Error; use d_engine_core::ScanResult; use d_engine_core::StateMachine; -use d_engine_core::StateMachineError; -use d_engine_core::StorageError; use d_engine_proto::common::LogId; use d_engine_proto::server::storage::SnapshotMetadata; use rocksdb::Cache; @@ -112,7 +110,7 @@ impl RocksDBStateMachine { /// Opens (or creates) a dedicated RocksDB instance for state machine storage. /// /// Used when `unified_db = false` (default): each engine owns its own DB instance. - pub fn new>(path: P) -> Result { + pub fn new>(path: P) -> io::Result { let db_opts = super::base_db_options(); let cache = Cache::new_lru_cache(128 * 1024 * 1024); @@ -121,7 +119,7 @@ impl RocksDBStateMachine { ColumnFamilyDescriptor::new(STATE_MACHINE_META_CF, super::meta_cf_options(&cache)); let db = DB::open_cf_descriptors(&db_opts, path, vec![sm_cf, sm_meta_cf]) - .map_err(|e| StorageError::DbError(e.to_string()))?; + .map_err(io::Error::other)?; let db_arc = Arc::new(db); let (last_applied_index, last_applied_term) = Self::load_state_machine_metadata(&db_arc)?; @@ -138,7 +136,7 @@ impl RocksDBStateMachine { /// /// Called by `RocksDBUnifiedEngine::open()`. The DB must already have /// `STATE_MACHINE_CF` and `STATE_MACHINE_META_CF` column families open. - pub(super) fn from_shared_db(db: Arc) -> Result { + pub(super) fn from_shared_db(db: Arc) -> io::Result { let (last_applied_index, last_applied_term) = Self::load_state_machine_metadata(&db)?; Ok(Self { db: ArcSwapOption::new(Some(db)), @@ -160,7 +158,7 @@ impl RocksDBStateMachine { // ========== Private helper methods ========== - /// Run `f` with a borrowed `&DB`, returning `NotServing` if `close_db()` was called. + /// Run `f` with a borrowed `&DB`, returning an error if `close()` has released the DB. /// /// Uses `ArcSwapOption::load()` — a seqlock read with no Arc clone. The Guard never /// escapes this call, so the hot-path cost is identical to the pre-refactor code. @@ -176,14 +174,14 @@ impl RocksDBStateMachine { fn with_db( &self, f: F, - ) -> Result + ) -> io::Result where - F: FnOnce(&DB) -> Result, + F: FnOnce(&DB) -> io::Result, { let guard = self.db.load(); // Guard — no Arc clone match guard.as_deref() { Some(db) => f(db), - None => Err(StateMachineError::new("state machine stopped")), + None => Err(io::Error::other("state machine stopped")), } } @@ -192,25 +190,23 @@ impl RocksDBStateMachine { /// Costs 2 atomic ops (Arc increment + future decrement). Only use when the handle /// must be moved into a closure or across an await point. For all other cases prefer /// `with_db()` (zero Arc clone) or the inline Guard pattern. - fn load_db(&self) -> Result, StateMachineError> { - self.db - .load_full() - .ok_or_else(|| StateMachineError::new("state machine stopped")) + fn load_db(&self) -> io::Result> { + self.db.load_full().ok_or_else(|| io::Error::other("state machine stopped")) } - fn load_state_machine_metadata(db: &DB) -> Result<(u64, u64), StateMachineError> { + fn load_state_machine_metadata(db: &DB) -> io::Result<(u64, u64)> { let cf = db .cf_handle(STATE_MACHINE_META_CF) - .ok_or_else(|| StateMachineError::new("State machine meta CF not found"))?; + .ok_or_else(|| io::Error::other("State machine meta CF not found"))?; - let index = match db.get_cf(&cf, LAST_APPLIED_INDEX_KEY).map_err(StateMachineError::new)? { + let index = match db.get_cf(&cf, LAST_APPLIED_INDEX_KEY).map_err(io::Error::other)? { Some(bytes) if bytes.len() == 8 => u64::from_be_bytes([ bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], ]), _ => 0, }; - let term = match db.get_cf(&cf, LAST_APPLIED_TERM_KEY).map_err(StateMachineError::new)? { + let term = match db.get_cf(&cf, LAST_APPLIED_TERM_KEY).map_err(io::Error::other)? { Some(bytes) if bytes.len() == 8 => u64::from_be_bytes([ bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], ]), @@ -229,7 +225,7 @@ impl RocksDBStateMachine { &self, last_included: LogId, snapshot_dir: &std::path::Path, - ) -> Result<(), StateMachineError> { + ) -> io::Result<()> { self.with_db(|db| Self::restore_from_cf_export(db, snapshot_dir))?; info!("Snapshot restore complete"); @@ -248,7 +244,7 @@ impl RocksDBStateMachine { fn restore_from_cf_export( db: &DB, snapshot_dir: &std::path::Path, - ) -> Result<(), StateMachineError> { + ) -> io::Result<()> { let sm_meta = Self::load_cf_export_metadata( &snapshot_dir.join("sm_metadata.bin"), &snapshot_dir.join("sm"), @@ -266,20 +262,20 @@ impl RocksDBStateMachine { // will be absent on restart. RocksDB does not support CF rename, so true atomicity // is not achievable here. Recovery: the node will restart with missing CFs, return // errors on reads, and wait for the leader to re-send the snapshot (issue #308). - db.drop_cf(STATE_MACHINE_CF).map_err(StateMachineError::new)?; - db.drop_cf(STATE_MACHINE_META_CF).map_err(StateMachineError::new)?; + db.drop_cf(STATE_MACHINE_CF).map_err(io::Error::other)?; + db.drop_cf(STATE_MACHINE_META_CF).map_err(io::Error::other)?; // create_column_family_with_import requires at least one SST file. // A CF with no files (e.g. sm_meta before its first flush) must use create_cf instead. if sm_meta.get_files().is_empty() { - db.create_cf(STATE_MACHINE_CF, &cf_opts).map_err(StateMachineError::new)?; + db.create_cf(STATE_MACHINE_CF, &cf_opts).map_err(io::Error::other)?; } else { db.create_column_family_with_import(&cf_opts, STATE_MACHINE_CF, &import_opts, &sm_meta) - .map_err(StateMachineError::new)?; + .map_err(io::Error::other)?; } if sm_meta_meta.get_files().is_empty() { - db.create_cf(STATE_MACHINE_META_CF, &cf_opts).map_err(StateMachineError::new)?; + db.create_cf(STATE_MACHINE_META_CF, &cf_opts).map_err(io::Error::other)?; } else { db.create_column_family_with_import( &cf_opts, @@ -287,7 +283,7 @@ impl RocksDBStateMachine { &import_opts, &sm_meta_meta, ) - .map_err(StateMachineError::new)?; + .map_err(io::Error::other)?; } Ok(()) @@ -310,10 +306,10 @@ impl RocksDBStateMachine { fn export_frozen_view( db: &DB, snapshot_dir: &std::path::Path, - ) -> Result { + ) -> io::Result { let name = snapshot_dir .file_name() - .ok_or_else(|| StateMachineError::new("snapshot dir has no name"))? + .ok_or_else(|| io::Error::other("snapshot dir has no name"))? .to_string_lossy(); let checkpoint_dir = snapshot_dir.with_file_name(format!(".{name}.checkpoint")); if checkpoint_dir.exists() { @@ -330,37 +326,35 @@ impl RocksDBStateMachine { db: &DB, checkpoint_dir: &std::path::Path, snapshot_dir: &std::path::Path, - ) -> Result { + ) -> io::Result { rocksdb::checkpoint::Checkpoint::new(db) .and_then(|c| c.create_checkpoint(checkpoint_dir)) - .map_err(StateMachineError::new)?; + .map_err(io::Error::other)?; // Open the frozen copy with every column family it contains. Exporting needs a writable // handle; background compaction is off so that nothing rewrites the files being exported. let mut opts = Options::default(); opts.set_disable_auto_compactions(true); opts.set_max_background_jobs(1); - let cf_names = DB::list_cf(&opts, checkpoint_dir).map_err(StateMachineError::new)?; - let frozen = - DB::open_cf(&opts, checkpoint_dir, &cf_names).map_err(StateMachineError::new)?; - let checkpoint = - rocksdb::checkpoint::Checkpoint::new(&frozen).map_err(StateMachineError::new)?; + let cf_names = DB::list_cf(&opts, checkpoint_dir).map_err(io::Error::other)?; + let frozen = DB::open_cf(&opts, checkpoint_dir, &cf_names).map_err(io::Error::other)?; + let checkpoint = rocksdb::checkpoint::Checkpoint::new(&frozen).map_err(io::Error::other)?; let cf_sm = frozen .cf_handle(STATE_MACHINE_CF) - .ok_or_else(|| StateMachineError::new("SM CF not found"))?; + .ok_or_else(|| io::Error::other("SM CF not found"))?; let cf_sm_meta = frozen .cf_handle(STATE_MACHINE_META_CF) - .ok_or_else(|| StateMachineError::new("SM meta CF not found"))?; + .ok_or_else(|| io::Error::other("SM meta CF not found"))?; // export_column_family() creates only the final subdirectory (e.g. "sm"); the parent // (snapshot_dir) already exists. let sm_export = checkpoint .export_column_family(&cf_sm, snapshot_dir.join("sm")) - .map_err(StateMachineError::new)?; + .map_err(io::Error::other)?; let sm_meta_export = checkpoint .export_column_family(&cf_sm_meta, snapshot_dir.join("sm_meta")) - .map_err(StateMachineError::new)?; + .map_err(io::Error::other)?; Self::save_cf_export_metadata(&sm_export, &snapshot_dir.join("sm_metadata.bin"))?; Self::save_cf_export_metadata(&sm_meta_export, &snapshot_dir.join("sm_meta_metadata.bin"))?; // The position is read from the frozen copy, the same view the data was exported from. @@ -374,7 +368,7 @@ impl RocksDBStateMachine { fn save_cf_export_metadata( metadata: &ExportImportFilesMetaData, path: &std::path::Path, - ) -> Result<(), StateMachineError> { + ) -> io::Result<()> { let files = metadata .get_files() .into_iter() @@ -397,8 +391,8 @@ impl RocksDBStateMachine { files, }; - let bytes = bincode::serialize(&meta).map_err(StateMachineError::new)?; - std::fs::write(path, bytes).map_err(StateMachineError::new)?; + let bytes = bincode::serialize(&meta).map_err(io::Error::other)?; + std::fs::write(path, bytes)?; Ok(()) } @@ -407,13 +401,13 @@ impl RocksDBStateMachine { fn load_cf_export_metadata( path: &std::path::Path, actual_dir: &std::path::Path, - ) -> Result { - let bytes = std::fs::read(path).map_err(StateMachineError::new)?; - let meta: CfExportMeta = bincode::deserialize(&bytes).map_err(StateMachineError::new)?; + ) -> io::Result { + let bytes = std::fs::read(path)?; + let meta: CfExportMeta = bincode::deserialize(&bytes).map_err(io::Error::other)?; let dir_str = actual_dir .to_str() - .ok_or_else(|| StateMachineError::new("Snapshot path is not valid UTF-8"))? + .ok_or_else(|| io::Error::other("Snapshot path is not valid UTF-8"))? .to_string(); let live_files: Vec = meta @@ -436,18 +430,18 @@ impl RocksDBStateMachine { let mut export_metadata = ExportImportFilesMetaData::default(); export_metadata.set_db_comparator_name(&meta.db_comparator_name); - export_metadata.set_files(&live_files).map_err(StateMachineError::new)?; + export_metadata.set_files(&live_files).map_err(io::Error::other)?; Ok(export_metadata) } - pub(crate) fn map_snapshot_join_error(e: tokio::task::JoinError) -> StateMachineError { + pub(crate) fn map_snapshot_join_error(e: tokio::task::JoinError) -> io::Error { let msg = if e.is_panic() { format!("snapshot blocking task panicked: {e}") } else { format!("snapshot blocking task was cancelled: {e}") }; - StateMachineError::new(msg) + io::Error::other(msg) } /// Scans all entries whose key starts with `prefix`. @@ -458,7 +452,7 @@ impl RocksDBStateMachine { pub fn scan_prefix( &self, prefix: &[u8], - ) -> Result { + ) -> io::Result { if prefix.is_empty() { let revision = self.last_applied_index.load(Ordering::SeqCst); return Ok(ScanResult { @@ -476,16 +470,16 @@ impl RocksDBStateMachine { let guard = self.db.load(); let Some(db) = guard.as_deref() else { - return Err(StateMachineError::new("state machine stopped")); + return Err(io::Error::other("state machine stopped")); }; let cf = db .cf_handle(STATE_MACHINE_CF) - .ok_or_else(|| StateMachineError::new("STATE_MACHINE_CF not found"))?; + .ok_or_else(|| io::Error::other("STATE_MACHINE_CF not found"))?; let iter = db.iterator_cf_opt(&cf, opts, IteratorMode::From(prefix, Direction::Forward)); let mut entries = Vec::new(); for item in iter { - let (k, v) = item.map_err(StateMachineError::new)?; + let (k, v) = item.map_err(io::Error::other)?; if !k.starts_with(prefix) { break; } @@ -505,9 +499,9 @@ impl RocksDBStateMachine { } /// Flushes the WAL and both state machine column families to disk. - pub fn flush(&self) -> Result<(), StateMachineError> { + pub fn flush(&self) -> io::Result<()> { self.with_db(|db| { - db.flush_wal(true).map_err(StateMachineError::new)?; + db.flush_wal(true).map_err(io::Error::other)?; // db.flush() with no CF argument only flushes RocksDB's implicit // "default" column family, which this state machine never writes to — @@ -516,13 +510,13 @@ impl RocksDBStateMachine { // in create_snapshot()'s pre-export flush. let cf_sm = db .cf_handle(STATE_MACHINE_CF) - .ok_or_else(|| StateMachineError::new("State machine CF not found"))?; + .ok_or_else(|| io::Error::other("State machine CF not found"))?; let cf_sm_meta = db .cf_handle(STATE_MACHINE_META_CF) - .ok_or_else(|| StateMachineError::new("State machine meta CF not found"))?; + .ok_or_else(|| io::Error::other("State machine meta CF not found"))?; let flush_opts = rocksdb::FlushOptions::default(); db.flush_cfs_opt(&[&cf_sm, &cf_sm_meta], &flush_opts) - .map_err(StateMachineError::new)?; + .map_err(io::Error::other)?; Ok(()) }) // last_applied is already durable as of apply_chunk() — no separate @@ -557,7 +551,7 @@ impl RocksDBStateMachine { impl StateMachine for RocksDBStateMachine { /// Sequence: stop serving → flush → save hard state → cancel background /// work → drop the DB handle (releasing the LOCK). A second call finds no DB and does nothing. - fn close(&self) -> Result<(), StateMachineError> { + fn close(&self) -> io::Result<()> { if self.db.load().is_none() { return Ok(()); } @@ -566,8 +560,8 @@ impl StateMachine for RocksDBStateMachine { // Every step runs even if an earlier one failed, so the LOCK is always released; // the first error is returned. - let mut first_err: Option = None; - let mut record = |step: &str, result: Result<(), StateMachineError>| { + let mut first_err: Option = None; + let mut record = |step: &str, result: io::Result<()>| { if let Err(e) = result { error!("close: {step} failed: {e:?}"); first_err.get_or_insert(e); @@ -586,21 +580,19 @@ impl StateMachine for RocksDBStateMachine { fn get( &self, key_buffer: &[u8], - ) -> Result, StateMachineError> { + ) -> io::Result> { // Guard against reads during snapshot restoration // During apply_snapshot_from_file(), is_serving is set to false while the database // is being replaced. This prevents reads from accessing temporary or inconsistent state. if !self.is_serving.load(Ordering::SeqCst) { - return Err(StateMachineError::new( - "State machine is restoring from snapshot", - )); + return Err(io::Error::other("State machine is restoring from snapshot")); } self.with_db(|db| { let cf = db .cf_handle(STATE_MACHINE_CF) - .ok_or_else(|| StateMachineError::new("State machine CF not found"))?; - match db.get_cf(&cf, key_buffer).map_err(StateMachineError::new)? { + .ok_or_else(|| io::Error::other("State machine CF not found"))?; + match db.get_cf(&cf, key_buffer).map_err(io::Error::other)? { Some(value) => Ok(Some(Bytes::copy_from_slice(&value))), None => Ok(None), } @@ -612,20 +604,20 @@ impl StateMachine for RocksDBStateMachine { async fn apply_chunk( &self, chunk: &[ApplyEntry], - ) -> Result, StateMachineError> { + ) -> io::Result> { // Inline guard: db and cf must share a lifetime across the entire function body // (batch borrows cf, CAS reads borrow db). Using with_db() closure would require // moving the whole async function body into a sync closure — use Guard directly. let guard = self.db.load(); let Some(db) = guard.as_deref() else { - return Err(StateMachineError::new("state machine stopped")); + return Err(io::Error::other("state machine stopped")); }; let cf = db .cf_handle(STATE_MACHINE_CF) - .ok_or_else(|| StateMachineError::new("State machine CF not found"))?; + .ok_or_else(|| io::Error::other("State machine CF not found"))?; let meta_cf = db .cf_handle(STATE_MACHINE_META_CF) - .ok_or_else(|| StateMachineError::new("State machine meta CF not found"))?; + .ok_or_else(|| io::Error::other("State machine meta CF not found"))?; let mut batch = WriteBatchWithIndex::new(0, true); let mut highest_index_entry: Option = None; @@ -669,7 +661,7 @@ impl StateMachine for RocksDBStateMachine { // are visible, preventing stale-read linearizability violations. let current_value = batch .get_from_batch_and_db_cf(db, &cf, key, &ReadOptions::default()) - .map_err(|e| StateMachineError::new(format!("CAS read failed: {e}")))?; + .map_err(|e| io::Error::other(format!("CAS read failed: {e}")))?; let cas_success = match (current_value, expected) { (Some(current), Some(exp)) => current == exp.as_ref(), @@ -724,7 +716,7 @@ impl StateMachine for RocksDBStateMachine { let _t0 = std::time::Instant::now(); SM_WRITE_OPTS .with(|opts| db.write_wbwi_opt(&batch, opts)) - .map_err(StateMachineError::new)?; + .map_err(io::Error::other)?; if let Some(highest) = highest_index_entry { self.update_last_applied(highest); @@ -744,10 +736,10 @@ impl StateMachine for RocksDBStateMachine { &self, metadata: &SnapshotMetadata, snapshot_dir: std::path::PathBuf, - ) -> Result<(), StateMachineError> { + ) -> io::Result<()> { let incoming = metadata .last_included - .ok_or_else(|| StateMachineError::new("Missing last_included in snapshot metadata"))?; + .ok_or_else(|| io::Error::other("Missing last_included in snapshot metadata"))?; info!("Applying snapshot from: {:?}", snapshot_dir); @@ -773,7 +765,7 @@ impl StateMachine for RocksDBStateMachine { async fn generate_snapshot_data( &self, new_snapshot_dir: std::path::PathBuf, - ) -> Result { + ) -> io::Result { // Export only SM CFs: compact SST-only export, no Raft log data. // export_column_family() creates only the final subdirectory (e.g. "sm"), so the parent // (new_snapshot_dir) must already exist before calling it. @@ -784,7 +776,7 @@ impl StateMachine for RocksDBStateMachine { // writes (#315). let db = self.load_db()?; // Arc: Send, safe to move into spawn_blocking let dir = new_snapshot_dir.clone(); - let position = tokio::task::spawn_blocking(move || -> Result { + let position = tokio::task::spawn_blocking(move || -> io::Result { std::fs::create_dir_all(&dir)?; Self::export_frozen_view(&db, &dir) }) @@ -798,23 +790,21 @@ impl StateMachine for RocksDBStateMachine { fn scan_prefix( &self, prefix: &[u8], - ) -> Result { + ) -> io::Result { self.scan_prefix(prefix) } fn get_multi( &self, keys: &[Bytes], - ) -> Result>, StateMachineError> { + ) -> io::Result>> { if !self.is_serving.load(Ordering::SeqCst) { - return Err(StateMachineError::new( - "State machine is restoring from snapshot", - )); + return Err(io::Error::other("State machine is restoring from snapshot")); } self.with_db(|db| { let cf = db .cf_handle(STATE_MACHINE_CF) - .ok_or_else(|| StateMachineError::new("State machine CF not found"))?; + .ok_or_else(|| io::Error::other("State machine CF not found"))?; // Single-key reads don't need cross-key point-in-time consistency, // so skip snapshot creation to avoid RocksDB's internal snapshot-list lock. @@ -822,7 +812,7 @@ impl StateMachine for RocksDBStateMachine { return db .get_cf(&cf, &keys[0]) .map(|v| vec![v.map(|b| Bytes::copy_from_slice(&b))]) - .map_err(StateMachineError::new); + .map_err(io::Error::other); } // One snapshot covers the full batch: all keys read from the same point-in-time. @@ -832,7 +822,7 @@ impl StateMachine for RocksDBStateMachine { .map(|k| { snap.get_cf(&cf, k) .map(|v| v.map(|b| Bytes::copy_from_slice(&b))) - .map_err(StateMachineError::new) + .map_err(io::Error::other) }) .collect() }) diff --git a/d-engine-server/tests/common/shared_state_machine.rs b/d-engine-server/tests/common/shared_state_machine.rs index 16126436..0539d860 100644 --- a/d-engine-server/tests/common/shared_state_machine.rs +++ b/d-engine-server/tests/common/shared_state_machine.rs @@ -17,7 +17,6 @@ use d_engine_core::ApplyEntry; use d_engine_core::ApplyResult; use d_engine_core::ScanResult; use d_engine_core::StateMachine; -use d_engine_core::StateMachineError; use d_engine_proto::common::LogId; use d_engine_proto::server::storage::SnapshotMetadata; @@ -33,25 +32,25 @@ impl SharedStateMachine { #[async_trait] impl StateMachine for SharedStateMachine { - fn close(&self) -> Result<(), StateMachineError> { + fn close(&self) -> std::io::Result<()> { self.0.close() } fn get( &self, key_buffer: &[u8], - ) -> Result, StateMachineError> { + ) -> std::io::Result> { self.0.get(key_buffer) } fn get_multi( &self, keys: &[Bytes], - ) -> Result>, StateMachineError> { + ) -> std::io::Result>> { self.0.get_multi(keys) } async fn apply_chunk( &self, chunk: &[ApplyEntry], - ) -> Result, StateMachineError> { + ) -> std::io::Result> { self.0.apply_chunk(chunk).await } fn last_applied(&self) -> LogId { @@ -61,19 +60,19 @@ impl StateMachine for SharedStateMachine { &self, metadata: &SnapshotMetadata, snapshot_path: std::path::PathBuf, - ) -> Result<(), StateMachineError> { + ) -> std::io::Result<()> { self.0.apply_snapshot_from_file(metadata, snapshot_path).await } async fn generate_snapshot_data( &self, new_snapshot_dir: std::path::PathBuf, - ) -> Result { + ) -> std::io::Result { self.0.generate_snapshot_data(new_snapshot_dir).await } fn scan_prefix( &self, prefix: &[u8], - ) -> Result { + ) -> std::io::Result { self.0.scan_prefix(prefix) } } diff --git a/d-engine/src/docs/server_guide/customize-state-machine.md b/d-engine/src/docs/server_guide/customize-state-machine.md index 7236d599..b8ba0f6b 100644 --- a/d-engine/src/docs/server_guide/customize-state-machine.md +++ b/d-engine/src/docs/server_guide/customize-state-machine.md @@ -12,14 +12,16 @@ d-engine supports **pluggable state machines** through the `StateMachine` trait. ## 1. Implement the Trait ```rust,ignore -use d_engine::{ApplyEntry, ApplyResult, Command, LogId, SnapshotMetadata, StateMachine, StateMachineError}; +use std::io; + use async_trait::async_trait; +use bytes::Bytes; +use d_engine::{ApplyEntry, ApplyResult, Command, LogId, StateMachine}; struct CustomStateMachine { - // Your storage backend + // Your storage backend. Its calls return `io::Result`. If your storage library has its own + // error type, convert it where you call it: `.map_err(io::Error::other)?`. backend: Arc, - last_applied: AtomicU64, - snapshot_meta: Mutex> } #[async_trait] @@ -37,18 +39,18 @@ impl StateMachine for CustomStateMachine { // If you share your storage handle with the application (a cloneable state machine), the // state machine is not dropped when the engine stops, so do not rely on Drop alone. // - // fn close(&self) -> Result<(), StateMachineError> { + // fn close(&self) -> io::Result<()> { // let flushed = self.backend.flush(); // self.backend.close(); // release lock file / DB handle // flushed // } - fn get(&self, key_buffer: &[u8]) -> Result>, StateMachineError> { + fn get(&self, key_buffer: &[u8]) -> io::Result> { // Retrieve value by key self.backend.get(key_buffer) } - async fn apply_chunk(&self, chunk: &[ApplyEntry]) -> Result, StateMachineError> { + async fn apply_chunk(&self, chunk: &[ApplyEntry]) -> io::Result> { // The framework pre-decodes proto bytes — match on Command directly, no prost needed. let mut results = Vec::with_capacity(chunk.len()); for entry in chunk { @@ -92,8 +94,8 @@ impl StateMachine for CustomStateMachine { Ok(results) } - - // Implement all required methods... + // The other required methods are `last_applied`, `apply_snapshot_from_file` and + // `generate_snapshot_data`; see the table in section 3. } ``` @@ -108,7 +110,9 @@ impl StateMachine for CustomStateMachine { - **Snapshot Install**: The engine decides whether a snapshot is installed. `apply_snapshot_from_file()` is called only for a snapshot newer than `last_applied()`, and the engine checks afterwards that `last_applied()` equals the snapshot boundary - **Restore On Open**: The engine never asks you to restore state. When your state machine is opened, `last_applied()` must return the last entry whose effects are durable (persist it together with the data it describes). You store nothing about snapshots: the engine records the snapshots it holds itself - **Concurrency Control**: Use appropriate locking for state mutations -- **Resource Cleanup**: Implement `Drop` to ensure proper flush on shutdown +- **Errors**: Every method returns `std::io::Result`. Report a failure with `io::Error::other("disk full")`, or convert your storage library's error with `.map_err(io::Error::other)?`; the original error stays attached for the log. The engine does not look at the error kind. A failed `apply_chunk()` or `apply_snapshot_from_file()` stops the node, which then recovers from its replicas, because a committed entry cannot be skipped. A failed `get()`, `get_multi()` or `scan_prefix()` fails only that read +- **Reads During A Snapshot Install**: A read can run while `apply_snapshot_from_file()` replaces the state. It must see the old state, the new state, or fail; never a mix of the two +- **Resource Cleanup**: Implement `close()` to flush and release locks and handles on shutdown (see above). Do not rely on `Drop` alone ## 3. StateMachine API Reference @@ -116,6 +120,8 @@ impl StateMachine for CustomStateMachine { | ---------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | ----------- | | `close()` | Last step at shutdown: persist what must survive a restart and release exclusive OS resources (e.g. DB lock file), even if persisting fails; returns the first error. May be called more than once. Nothing is called after it starts. Default `Ok(())`. | Sync | Medium | | `get()` | Read value by key | Sync | High | +| `get_multi()` | Read several keys from one consistent state. Default calls `get()` per key; override it if your storage can read a batch from one snapshot | Sync | Medium | +| `scan_prefix()` | Return every key with the given prefix. Default returns an error; override it to support prefix scans (e.g. watch) | Sync | Optional | | `apply_chunk()` | Apply committed entries | Async | Critical | | `last_applied()` | Get last applied index | Sync | High | | `apply_snapshot_from_file()` | Replace state with snapshot | Async | Critical | @@ -125,81 +131,51 @@ impl StateMachine for CustomStateMachine { ### 4.1 Functional Tests -Use the built-in test suite to validate correctness: +Implement `StateMachineBuilder` once, then run the built-in suite against your state machine. It checks the contract described above, including that a restart finds the last applied entry together with its data: ```rust,ignore -use d_engine::{StateMachine, Error}; -use tempfile::TempDir; +use d_engine::state_machine_test::{StateMachineBuilder, StateMachineTestSuite}; +use d_engine::{Error, StateMachine}; -struct TestStateMachineBuilder { - temp_dir: TempDir, -} +struct MyStateMachineBuilder { /* e.g. a temporary directory */ } -impl TestStateMachineBuilder { - async fn build(&self) -> Result, Error> { - let path = self.temp_dir.path().join("sm_test"); - let sm = CustomStateMachine::new(path).await?; - Ok(Arc::new(sm)) +#[async_trait] +impl StateMachineBuilder for MyStateMachineBuilder { + async fn build(&self) -> Result, Error> { + // A state machine on new, empty storage (a new directory on every call) + Ok(Arc::new(CustomStateMachine::new().await?)) } -} -#[tokio::test] -async fn test_custom_state_machine() -> Result<(), Error> { - let builder = TestStateMachineBuilder::new(); - let sm = builder.build().await?; - - // Test apply_chunk with sample entries - let entries = vec![create_test_entry(1, 1)]; - sm.apply_chunk(&entries).await?; - assert_eq!(sm.last_applied().index, 1); + async fn reopen(&self) -> Result, Error> { + // The storage the last build() used, with whatever it holds now (a restart) + Ok(Arc::new(CustomStateMachine::open_existing().await?)) + } - // Test snapshot operations - let temp_dir = TempDir::new()?; - let position = sm.generate_snapshot_data(temp_dir.path().to_path_buf()).await?; - assert_eq!(position.index, 1); + async fn cleanup(&self) -> Result<(), Error> { + Ok(()) + } - sm.close()?; - Ok(()) + // Optional: override `build_from_crash_image()` to also test recovery after a crash. + // It returns `None` by default, and that test is skipped. } +#[tokio::test] +async fn test_custom_state_machine() -> Result<(), Error> { + StateMachineTestSuite::run_all_tests(MyStateMachineBuilder::new()).await +} ``` ### 4.2 Performance Tests -Use `run_performance_tests()` to detect performance regressions: +Use `run_performance_tests()` with the same builder to detect performance regressions: ```rust,ignore -use d_engine::state_machine_test::{StateMachineBuilder, StateMachineTestSuite}; - #[tokio::test] #[ignore] // Run with: cargo test -- --ignored async fn test_performance() -> Result<(), Error> { - struct MyStateMachineBuilder { /* ... */ } - - #[async_trait] - impl StateMachineBuilder for MyStateMachineBuilder { - async fn build(&self) -> Result, Error> { - // A state machine on new, empty storage (a new directory on every call) - Ok(Arc::new(MyStateMachine::new().await?)) - } - - async fn reopen(&self) -> Result, Error> { - // The storage the last build() used, with whatever it holds now (a restart) - Ok(Arc::new(MyStateMachine::open_existing().await?)) - } - - async fn cleanup(&self) -> Result<(), Error> { - Ok(()) - } - } - - let builder = MyStateMachineBuilder::new(); - // Runs smoke test (100 entries < 1s) and scalability test (O(N) complexity) - StateMachineTestSuite::run_performance_tests(builder).await?; - Ok(()) + StateMachineTestSuite::run_performance_tests(MyStateMachineBuilder::new()).await } - ``` **Performance Thresholds:** @@ -232,8 +208,8 @@ Running a standalone gRPC server instead of embedding? Use `StandaloneEngine::ru See complete implementations in: - `examples/three-nodes-standalone/src/main.rs` - Production cluster with RocksDB -- `examples/sled-cluster/src/main.rs` - Sled state machine -- `src/storage/adaptors/` - Built-in implementations +- `examples/sled-cluster/src/sled_state_machine.rs` - Sled state machine +- `d-engine-server/src/storage/adaptors/` - Built-in file and RocksDB implementations Enable RocksDB feature in your `Cargo.toml`: diff --git a/d-engine/src/lib.rs b/d-engine/src/lib.rs index 499f3933..8f2d4b53 100644 --- a/d-engine/src/lib.rs +++ b/d-engine/src/lib.rs @@ -47,8 +47,6 @@ pub mod prelude { #[cfg(feature = "server")] pub use d_engine_server::StateMachine; #[cfg(feature = "server")] - pub use d_engine_server::StateMachineError; - #[cfg(feature = "server")] pub use d_engine_server::StorageEngine; } diff --git a/examples/sled-cluster/src/sled_state_machine.rs b/examples/sled-cluster/src/sled_state_machine.rs index 13bd3935..77671a87 100644 --- a/examples/sled-cluster/src/sled_state_machine.rs +++ b/examples/sled-cluster/src/sled_state_machine.rs @@ -15,11 +15,8 @@ use d_engine::ApplyEntry; use d_engine::ApplyResult; use d_engine::BatchOp; use d_engine::Command; -use d_engine::Result; use d_engine::ScanResult; use d_engine::StateMachine; -use d_engine::StateMachineError; -use d_engine::StorageError; use d_engine::common::LogId; use d_engine::server_storage::SnapshotMetadata; use sled::Transactional; @@ -86,7 +83,7 @@ impl Drop for SledStateMachine { #[async_trait] impl StateMachine for SledStateMachine { - fn close(&self) -> std::result::Result<(), StateMachineError> { + fn close(&self) -> std::io::Result<()> { if self.db.load().is_none() { return Ok(()); } @@ -108,13 +105,13 @@ impl StateMachine for SledStateMachine { fn get( &self, key_buffer: &[u8], - ) -> std::result::Result, StateMachineError> { + ) -> std::io::Result> { match self.current_tree()?.get(key_buffer) { Ok(Some(v)) => Ok(Some(Bytes::copy_from_slice(&v))), Ok(None) => Ok(None), Err(e) => { error!("state_machine get error: {}", e); - Err(StateMachineError::new(e)) + Err(std::io::Error::other(e)) } } } @@ -122,7 +119,7 @@ impl StateMachine for SledStateMachine { async fn apply_chunk( &self, chunk: &[ApplyEntry], - ) -> std::result::Result, StateMachineError> { + ) -> std::io::Result> { trace!("Applying chunk: {} entries", chunk.len()); let db = self.db()?; @@ -153,7 +150,7 @@ impl StateMachine for SledStateMachine { let data_tree = self.current_tree()?; let meta_tree = db .open_tree(STATE_MACHINE_META_NAMESPACE) - .map_err(StateMachineError::new)?; + .map_err(std::io::Error::other)?; let results = (&data_tree, &meta_tree) .transaction(|(data, meta)| { @@ -227,7 +224,7 @@ impl StateMachine for SledStateMachine { }) .map_err(|e: sled::transaction::TransactionError| { error!("state_machine apply_chunk failed: {}", e); - StateMachineError::new(e) + std::io::Error::other(e) })?; self.update_last_applied(position); @@ -238,7 +235,7 @@ impl StateMachine for SledStateMachine { async fn generate_snapshot_data( &self, new_snapshot_dir: PathBuf, - ) -> std::result::Result { + ) -> std::io::Result { // Known limits of this example: the tree is iterated while `apply_chunk` may run, so the // copy is not a point-in-time view, and the position below comes from memory, not from the // exported data. Reading it before the copy keeps it at or behind the data. A state machine @@ -248,7 +245,7 @@ impl StateMachine for SledStateMachine { // 2. Create a new state machine database instance let new_db = init_sled_state_machine_db(&new_snapshot_dir) - .map_err(StateMachineError::new)?; + .map_err(std::io::Error::other)?; let exist_db_tree = self.current_tree()?; let new_state_machine_tree = new_tree(&new_db, STATE_MACHINE_TREE)?; @@ -260,7 +257,7 @@ impl StateMachine for SledStateMachine { // as log indices. The SM already reflects state at last_included.index, so // all current entries are valid snapshot content. for item in exist_db_tree.iter() { - let (k, v) = item.map_err(StateMachineError::new)?; + let (k, v) = item.map_err(std::io::Error::other)?; batch.insert(k, v); counter += 1; @@ -268,17 +265,17 @@ impl StateMachine for SledStateMachine { if counter % 100 == 0 { new_state_machine_tree .apply_batch(batch) - .map_err(StateMachineError::new)?; + .map_err(std::io::Error::other)?; batch = sled::Batch::default(); } } new_state_machine_tree .apply_batch(batch) - .map_err(StateMachineError::new)?; + .map_err(std::io::Error::other)?; // Make sure flush into disk - new_db.flush().map_err(StateMachineError::new)?; + new_db.flush().map_err(std::io::Error::other)?; Ok(position) } @@ -287,14 +284,14 @@ impl StateMachine for SledStateMachine { &self, metadata: &SnapshotMetadata, decompressed_snapshot_path: PathBuf, - ) -> std::result::Result<(), StateMachineError> { + ) -> std::io::Result<()> { // `StateMachineWorker` never runs this concurrently with `apply_chunk`, a snapshot // export or another install (#436). // The engine only calls this for a snapshot newer than `last_applied()`. let new_last_included = metadata .last_included - .ok_or_else(|| StateMachineError::new("Missing last_included in snapshot metadata"))?; + .ok_or_else(|| std::io::Error::other("Missing last_included in snapshot metadata"))?; // `decompressed_snapshot_path` is a temporary directory owned by the caller // (see `PreparedSnapshot`), not this state machine's own `self.path`. Opening a @@ -340,12 +337,12 @@ impl StateMachine for SledStateMachine { Ok(()) }) .await - .map_err(|e| StateMachineError::new(format!("snapshot install task panicked: {e}")))? - .map_err(|e| StateMachineError::new(format!("failed to move snapshot into place: {e}")))?; + .map_err(|e| std::io::Error::other(format!("snapshot install task panicked: {e}")))? + .map_err(|e| std::io::Error::other(format!("failed to move snapshot into place: {e}")))?; debug!("Reopening state machine database at its permanent path"); let db = init_sled_state_machine_db(&self.path) - .map_err(StateMachineError::new)?; + .map_err(std::io::Error::other)?; self.db.store(Some(Arc::new(db))); // Best-effort: a leftover `*.pre-install-backup` is harmless (just disk space) @@ -368,11 +365,11 @@ impl StateMachine for SledStateMachine { fn scan_prefix( &self, prefix: &[u8], - ) -> std::result::Result { + ) -> std::io::Result { let tree = self.current_tree()?; let mut entries = Vec::new(); for result in tree.scan_prefix(prefix) { - let (k, v) = result.map_err(StateMachineError::new)?; + let (k, v) = result.map_err(std::io::Error::other)?; entries.push((Bytes::copy_from_slice(&k), Bytes::copy_from_slice(&v))); } let revision = self.last_applied_index.load(Ordering::SeqCst); @@ -384,7 +381,7 @@ impl SledStateMachine { pub fn new + std::fmt::Debug>( path: P, node_id: u32, - ) -> Result { + ) -> std::io::Result { let path = path.as_ref().to_path_buf(); // Recovery from a crash mid-install (see `apply_snapshot_from_file`): if the @@ -399,17 +396,16 @@ impl SledStateMachine { ?backup_dir, "Recovering from an interrupted snapshot install: restoring pre-install backup" ); - std::fs::rename(&backup_dir, &db_dir) - .map_err(|e| StorageError::DbError(e.to_string()))?; + std::fs::rename(&backup_dir, &db_dir)?; } let db = Arc::new( - init_sled_state_machine_db(&path).map_err(|e| StorageError::DbError(e.to_string()))?, + init_sled_state_machine_db(&path).map_err(std::io::Error::other)?, ); let state_machine_meta_tree = db .open_tree(STATE_MACHINE_META_NAMESPACE) - .map_err(|e| StorageError::DbError(e.to_string()))?; + .map_err(std::io::Error::other)?; let (last_applied_index, last_applied_term) = Self::load_state_machine_metadata(&state_machine_meta_tree)?; @@ -432,35 +428,37 @@ impl SledStateMachine { Ok(sm) } - fn load_state_machine_metadata(tree: &sled::Tree) -> Result<(u64, u64)> { + fn load_state_machine_metadata(tree: &sled::Tree) -> std::io::Result<(u64, u64)> { let index = tree .get(STATE_MACHINE_META_KEY_LAST_APPLIED_INDEX) - .map_err(|e| StorageError::DbError(e.to_string()))? + .map_err(std::io::Error::other)? .map(safe_vk_ivec) - .unwrap_or(Ok(0))?; + .unwrap_or(Ok(0)) + .map_err(std::io::Error::other)?; let term = tree .get(STATE_MACHINE_META_KEY_LAST_APPLIED_TERM) - .map_err(|e| StorageError::DbError(e.to_string()))? + .map_err(std::io::Error::other)? .map(safe_vk_ivec) - .unwrap_or(Ok(0))?; + .unwrap_or(Ok(0)) + .map_err(std::io::Error::other)?; Ok((index, term)) } // Dynamically obtain the current Tree (lock-free cache optimization) #[inline] - fn current_tree(&self) -> std::result::Result { + fn current_tree(&self) -> std::io::Result { self.db()? .open_tree(STATE_MACHINE_TREE) - .map_err(StateMachineError::new) + .map_err(std::io::Error::other) } - /// The live db, or `NotServing` once `close()` has released it. - fn db(&self) -> std::result::Result, StateMachineError> { + /// The live db, or an error once `close()` has released it. + fn db(&self) -> std::io::Result> { self.db .load_full() - .ok_or_else(|| StateMachineError::new("state machine stopped")) + .ok_or_else(|| std::io::Error::other("state machine stopped")) } pub(crate) fn update_last_applied( @@ -475,28 +473,28 @@ impl SledStateMachine { pub(crate) fn persist_last_applied( &self, last_applied: LogId, - ) -> std::result::Result<(), StateMachineError> { + ) -> std::io::Result<()> { debug!(%self.node_id, ?last_applied, "persist_last_applied"); let db = self.db()?; let tree = db .open_tree(STATE_MACHINE_META_NAMESPACE) - .map_err(StateMachineError::new)?; + .map_err(std::io::Error::other)?; tree.insert( STATE_MACHINE_META_KEY_LAST_APPLIED_INDEX, safe_kv(last_applied.index), ) - .map_err(StateMachineError::new)?; + .map_err(std::io::Error::other)?; tree.insert( STATE_MACHINE_META_KEY_LAST_APPLIED_TERM, safe_kv(last_applied.term), ) - .map_err(StateMachineError::new)?; + .map_err(std::io::Error::other)?; - tree.flush().map_err(StateMachineError::new)?; + tree.flush().map_err(std::io::Error::other)?; Ok(()) } - pub(crate) fn save_hard_state(&self) -> std::result::Result<(), StateMachineError> { + pub(crate) fn save_hard_state(&self) -> std::io::Result<()> { let last_applied = self.last_applied(); self.persist_last_applied(last_applied)?; @@ -504,9 +502,9 @@ impl SledStateMachine { Ok(()) } - pub(crate) fn flush(&self) -> std::result::Result<(), StateMachineError> { + pub(crate) fn flush(&self) -> std::io::Result<()> { let db = self.db()?; - let bytes = db.flush().map_err(StateMachineError::new)?; + let bytes = db.flush().map_err(std::io::Error::other)?; info!( "Successfully flushed State Machine, bytes flushed: {}", bytes @@ -540,9 +538,9 @@ fn copy_dir_recursive( fn new_tree( db: &sled::Db, key: &str, -) -> std::result::Result { +) -> std::io::Result { db.open_tree(key) - .map_err(StateMachineError::new) + .map_err(std::io::Error::other) } pub fn init_sled_state_machine_db(