Fix: faster failover after an unplanned leader loss

examples/raft-kv-memstore-singlethreaded/src/store.rs

@@ -48,25 +48,6 @@ impl Request {
     }
 }
 
-#[cfg(test)]
-mod tests {
-    use std::marker::PhantomData;
-
-    use crate::store::Request;
-
-    #[test]
-    fn test_serde() {
-        let a = Request::Set {
-            key: "foo".to_string(),
-            value: "bar".to_string(),
-            _p: PhantomData,
-        };
-
-        let b = serde_json::to_string(&a).unwrap();
-        println!("{}", b);
-    }
-}
-
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct Response {
     pub value: Option<String>,
@@ -374,3 +355,22 @@ impl RaftLogStorage<TypeConfig> for Rc<LogStore> {
         self.clone()
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use std::marker::PhantomData;
+
+    use crate::store::Request;
+
+    #[test]
+    fn test_serde() {
+        let a = Request::Set {
+            key: "foo".to_string(),
+            value: "bar".to_string(),
+            _p: PhantomData,
+        };
+
+        let b = serde_json::to_string(&a).unwrap();
+        println!("{}", b);
+    }
+}

examples/raft-kv-rocksdb/src/lib.rs

@@ -84,7 +84,8 @@ where
     // Create a configuration for the raft instance.
     let config = Config {
         heartbeat_interval: 250,
-        election_timeout_min: 299,
+        election_timeout_min: 800,
+        election_timeout_max: 1_200,
         ..Default::default()
     };
 

openraft/src/config/config.rs

@@ -117,6 +117,10 @@ pub struct Config {
     pub cluster_name: String,
 
     /// The minimum election timeout in milliseconds
+    ///
+    /// It must exceed three halves of `heartbeat_interval`, the interval at which a leader sends
+    /// heartbeats, and should be at least twice that so one late heartbeat does not start an
+    /// election. It is also the follower's leader lease.
     #[clap(long, default_value = "150")]
     pub election_timeout_min: u64,
 
@@ -128,6 +132,17 @@ pub struct Config {
     #[clap(long, default_value = "50")]
     pub heartbeat_interval: u64,
 
+    /// The deadline of one AppendEntries RPC in milliseconds.
+    ///
+    /// It bounds every AppendEntries RPC a leader sends: log replication, heartbeats and the
+    /// heartbeats that confirm leadership for a linearizable read. Heartbeats are still sent every
+    /// `heartbeat_interval`, so a follower may take longer than one heartbeat interval to answer
+    /// without its request being abandoned and resent. It must be positive.
+    ///
+    /// When unset, the deadline is `heartbeat_interval`.
+    #[clap(long)]
+    pub append_entries_timeout: Option<u64>,
+
     /// The timeout for sending then installing the last snapshot segment,
     /// in millisecond. It is also used as the timeout for sending a non-last segment, if
     /// `send_snapshot_timeout` is 0.
@@ -243,19 +258,46 @@ pub struct Config {
            default_missing_value = "true"
     )]
     pub enable_elect: bool,
+
+    /// Whether a follower runs a Pre-Vote round before it increments its term and starts a real
+    /// election.
+    ///
+    /// When enabled, a voter whose election timer fires asks the other voters whether they *would*
+    /// grant it a vote at `term + 1`, without persisting a vote or changing its term. Only when a
+    /// quorum would grant does it start the real election. A voter that cannot currently win, for
+    /// example one that is cut off from the others, restarted, or behind on the log, therefore
+    /// does not raise its term and depose a healthy leader once it can be reached again.
+    ///
+    /// Pre-Vote uses the separate
+    /// [`RaftNetwork::pre_vote`](`crate::network::RaftNetwork::pre_vote`) RPC. A voter that the
+    /// network does not know to answer it, including every voter of a network that does not
+    /// implement it, makes the campaign run the classic election instead; a transport error is
+    /// never counted as a grant.
+    ///
+    /// `None`, the default, is treated as disabled.
+    // clap 4 requires `num_args = 0..=1`, or it complains about missing arg error
+    // https://github.com/clap-rs/clap/discussions/4374
+    #[clap(long,
+           action = clap::ArgAction::Set,
+           num_args = 0..=1,
+           default_missing_value = "true"
+    )]
+    pub enable_pre_vote: Option<bool>,
 }
 
 /// Updatable config for a raft runtime.
 pub(crate) struct RuntimeConfig {
     pub(crate) enable_heartbeat: AtomicBool,
     pub(crate) enable_elect: AtomicBool,
+    pub(crate) enable_pre_vote: AtomicBool,
 }
 
 impl RuntimeConfig {
     pub(crate) fn new(config: &Config) -> Self {
         Self {
             enable_heartbeat: AtomicBool::from(config.enable_heartbeat),
             enable_elect: AtomicBool::from(config.enable_elect),
+            enable_pre_vote: AtomicBool::from(config.get_enable_pre_vote()),
         }
     }
 }
@@ -272,6 +314,21 @@ impl Config {
         RT::thread_rng().gen_range(self.election_timeout_min..self.election_timeout_max)
     }
 
+    /// Whether a voter runs a Pre-Vote round before it starts a real election.
+    ///
+    /// `None` is treated as disabled.
+    pub(crate) fn get_enable_pre_vote(&self) -> bool {
+        self.enable_pre_vote.unwrap_or(false)
+    }
+
+    /// Get the deadline of one AppendEntries RPC.
+    ///
+    /// It is [`append_entries_timeout`](Self::append_entries_timeout) when set, otherwise
+    /// [`heartbeat_interval`](Self::heartbeat_interval).
+    pub fn append_entries_timeout(&self) -> Duration {
+        Duration::from_millis(self.append_entries_timeout.unwrap_or(self.heartbeat_interval))
+    }
+
     /// Get the timeout for sending and installing the last snapshot segment.
     pub fn install_snapshot_timeout(&self) -> Duration {
         Duration::from_millis(self.install_snapshot_timeout)
@@ -311,7 +368,10 @@ impl Config {
             });
         }
 
-        if self.election_timeout_min <= self.heartbeat_interval {
+        // A leader sends heartbeats on its engine tick, every three halves of the heartbeat
+        // interval. A follower campaigns once its sampled election timeout expires, so the
+        // smallest timeout has to outlast that tick.
+        if self.election_timeout_min <= self.heartbeat_interval.saturating_mul(3) / 2 {
             return Err(ConfigError::ElectionTimeoutLTHeartBeat {
                 election_timeout_min: self.election_timeout_min,
                 heartbeat_interval: self.heartbeat_interval,

openraft/src/config/config_test.rs

@@ -98,6 +98,67 @@ fn test_build() -> anyhow::Result<()> {
     Ok(())
 }
 
+#[test]
+fn test_election_timeout_min_must_outlast_the_heartbeat_tick() {
+    // Heartbeats go out every three halves of the heartbeat interval: 150 ms here.
+    let config = Config {
+        election_timeout_min: 150,
+        election_timeout_max: 300,
+        heartbeat_interval: 100,
+        ..Default::default()
+    };
+    assert_eq!(
+        config.validate().unwrap_err(),
+        ConfigError::ElectionTimeoutLTHeartBeat {
+            election_timeout_min: 150,
+            heartbeat_interval: 100
+        }
+    );
+
+    let config = Config {
+        election_timeout_min: 151,
+        election_timeout_max: 300,
+        heartbeat_interval: 100,
+        ..Default::default()
+    };
+    assert!(config.validate().is_ok());
+}
+
+#[test]
+fn test_config_append_entries_timeout() -> anyhow::Result<()> {
+    let config = Config::build(&["foo", "--heartbeat-interval=40"])?;
+    assert_eq!(None, config.append_entries_timeout);
+    assert_eq!(
+        Duration::from_millis(40),
+        config.append_entries_timeout(),
+        "by default an AppendEntries RPC is bounded by the heartbeat interval"
+    );
+
+    let config = Config::build(&["foo", "--heartbeat-interval=40", "--append-entries-timeout=900"])?;
+    assert_eq!(Some(900), config.append_entries_timeout);
+    assert_eq!(Duration::from_millis(900), config.append_entries_timeout());
+    assert_eq!(40, config.heartbeat_interval);
+
+    Ok(())
+}
+
+#[test]
+fn test_config_enable_pre_vote() -> anyhow::Result<()> {
+    let config = Config::build(&["foo"])?;
+    assert_eq!(None, config.enable_pre_vote);
+
+    let config = Config::build(&["foo", "--enable-pre-vote"])?;
+    assert_eq!(Some(true), config.enable_pre_vote);
+
+    let config = Config::build(&["foo", "--enable-pre-vote=true"])?;
+    assert_eq!(Some(true), config.enable_pre_vote);
+
+    let config = Config::build(&["foo", "--enable-pre-vote=false"])?;
+    assert_eq!(Some(false), config.enable_pre_vote);
+
+    Ok(())
+}
+
 #[test]
 fn test_config_snapshot_policy() -> anyhow::Result<()> {
     let config = Config::build(&["foo", "--snapshot-policy=never"])?;

openraft/src/config/error.rs

@@ -14,7 +14,9 @@ pub enum ConfigError {
     #[error("max_payload_entries must be > 0")]
     MaxPayloadIs0,
 
-    #[error("election_timeout_min({election_timeout_min}) must be > heartbeat_interval({heartbeat_interval})")]
+    #[error(
+        "election_timeout_min({election_timeout_min}) must be > heartbeat_interval({heartbeat_interval}) * 3 / 2, the interval at which a leader sends heartbeats"
+    )]
     ElectionTimeoutLTHeartBeat {
         election_timeout_min: u64,
         heartbeat_interval: u64,

openraft/src/core/notify.rs

@@ -1,5 +1,6 @@
 use crate::core::sm;
 use crate::error::Fatal;
+use crate::raft::PreVoteReply;
 use crate::raft::VoteResponse;
 use crate::replication;
 use crate::MessageSummary;
@@ -20,6 +21,14 @@ where C: RaftTypeConfig
         sender_vote: Vote<C::NodeId>,
     },
 
+    PreVoteResponse {
+        target: C::NodeId,
+        reply: PreVoteReply<C::NodeId>,
+
+        /// The Pre-Vote round that sent the request.
+        round: u64,
+    },
+
     /// Seen a higher `vote`.
     HigherVote {
         /// The ID of the target node from which the new term was observed.
@@ -73,6 +82,9 @@ where C: RaftTypeConfig
             } => {
                 format!("VoteResponse: from: {}: {}, res-vote: {}", target, resp.summary(), vote)
             }
+            Self::PreVoteResponse { target, reply, round } => {
+                format!("PreVoteResponse: from: {}: {}, round: {}", target, reply, round)
+            }
             Self::HigherVote {
                 ref target,
                 higher: ref new_vote,