diff --git a/operator/chart/README.md b/operator/chart/README.md index b3b338b7..51bd8934 100644 --- a/operator/chart/README.md +++ b/operator/chart/README.md @@ -181,9 +181,24 @@ The members talk over mutual TLS on port 9300, each reached through a Service of A version change replaces one member at a time, followers first and the leader last. The next is replaced only once every member is ready, knows the leader and holds an entry of its log: a member with an empty log is no voter, so replacing another before then could leave the quorum without a majority. Under `mode: Drain` the leader is replaced once its drain ends. A `PodDisruptionBudget` lets a node drain take one member at a time. Deleting the resource deletes the members' volumes. +The members prefer one node each, so a lost node takes one member, not the majority. This places new volumes only: a member whose node-local volume already shares a node with another member stays there until its volume is replaced. A member whose volume can only be on a node that is gone, as with `local-path` and other node-local storage, waits `Pending`. Two minutes later the operator deletes its claim and its pod, and the `StatefulSet` makes both again on another node. It does this one member at a time, and only while the others follow a leader and have caught up, because the new member starts with an empty log and does not vote until it catches up. Until then `Ready` is `False`, reason `MemberLost`. The same happens when a node holding a member is drained and stays unschedulable for two minutes. The member loses nothing it needs: the quorum replicates only who leads, and everything else is in PostgreSQL. + +With the majority lost (two of three members, or three of five), nothing is replaced, because there is no leader. The members' disks then hold nothing but terms and votes, so you can start the quorum over. Delete every member's claim, then their pods: + +```bash +kubectl -n delete pvc -l app.kubernetes.io/instance= --wait=false +``` + +```bash +kubectl -n delete pod -l app.kubernetes.io/instance= +``` + +The new quorum counts terms from the start again, so a fault the old one left is removed when its deadline passes, not straight away. + What it doesn't do yet: - It doesn't scale a quorum: `replicas` is set when the resource is made. +- It doesn't recover a quorum that lost its majority (see above). - No PostgreSQL Secret for the `INTEGRATION_CDC` test, and no `rbac.extraRules`: the Kates chart's `rbac.cdcSecretNamespaces` and `rbac.extraRules` have no counterpart yet. - It doesn't create PostgreSQL. A changed Secret is not noticed: delete the pod to pick it up. - Kafka over TLS with a truststore file isn't expressible yet. `spec.env` takes any other setting of the container, except the names `database`, `kafka` and `apiKeySecretRef` set. diff --git a/operator/chart/templates/_helpers.tpl b/operator/chart/templates/_helpers.tpl index c9cb2520..3b0363d8 100644 --- a/operator/chart/templates/_helpers.tpl +++ b/operator/chart/templates/_helpers.tpl @@ -68,6 +68,10 @@ quorum it labels the leader's pod, which the Service selects. */}} - apiGroups: [""] resources: ["pods"] verbs: ["get", "delete", "patch"] +{{- /* A quorum member whose volume's node is gone gets a new claim: the StatefulSet makes it. */}} +- apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "delete"] {{- /* kates-api-key: the one key Secret it reads, to set /api/drain. */}} - apiGroups: [""] resources: ["secrets"] diff --git a/operator/crates/kates-operator/src/reconcile/kates.rs b/operator/crates/kates-operator/src/reconcile/kates.rs index ecd9f999..e3056d42 100644 --- a/operator/crates/kates-operator/src/reconcile/kates.rs +++ b/operator/crates/kates-operator/src/reconcile/kates.rs @@ -13,7 +13,7 @@ use std::time::Duration; use jiff::Timestamp; use k8s_openapi::api::apps::v1::StatefulSet; -use k8s_openapi::api::core::v1::{Pod, Secret, Service, ServiceAccount}; +use k8s_openapi::api::core::v1::{PersistentVolumeClaim, Pod, Secret, Service, ServiceAccount}; use k8s_openapi::api::policy::v1::PodDisruptionBudget; use k8s_openapi::api::rbac::v1::ClusterRoleBinding; use k8s_openapi::apimachinery::pkg::apis::meta::v1::OwnerReference; @@ -448,6 +448,37 @@ async fn roll_quorum( now, ); }; + // A member given a new volume: its pod waits on the old claim until it goes, then the + // StatefulSet makes both again. + for member in members.iter().filter(|member| member.orphaned) { + if let Some(found) = pods.get(&member.pod) { + replace(ctx, &namespace, found).await?; + } + } + match quorum::lost(&members, now) { + quorum::Next::Done => {} + quorum::Next::Wait(why) => { + let conditions = &mut after.conditions; + status::set( + conditions, + "Ready", + "False", + "MemberLost", + &why, + generation, + now, + ); + return Ok((drain, None)); + } + quorum::Next::Replace(pod) => { + tracing::warn!(%pod, "its volume's node is gone: a new volume, then a new pod"); + forget(ctx, &namespace, &quorum::claim(&pod)).await?; + if let Some(found) = pods.get(&pod) { + replace(ctx, &namespace, found).await?; + } + return Ok((drain, Some(pod))); + } + } match quorum::next(&members) { quorum::Next::Done => Ok((drain, None)), quorum::Next::Wait(why) => { @@ -656,6 +687,16 @@ async fn replace(ctx: &Context, namespace: &str, pod: &Pod) -> Result<(), kube:: } } +/// Deletes a member's claim, and with it the term and vote it kept. +async fn forget(ctx: &Context, namespace: &str, claim: &str) -> Result<(), kube::Error> { + let claims = Api::::namespaced(ctx.client.clone(), namespace); + match claims.delete(claim, &DeleteParams::default()).await { + Ok(_) => Ok(()), + Err(kube::Error::Api(response)) if response.code == 404 => Ok(()), + Err(error) => Err(error), + } +} + #[allow(clippy::needless_pass_by_value)] pub fn error_policy(kates: Arc, error: &Error, ctx: Arc) -> Action { ctx.metrics.reconcile_failed(KIND, error.name()); @@ -874,6 +915,12 @@ fn with_quorum(objects: &mut Objects, kates: &Kates) { claim["spec"]["storageClassName"] = json!(class); } objects.set["spec"]["volumeClaimTemplates"] = json!([claim]); + // One member a node where there are nodes enough: two on one node die with it, and so does + // the majority. Preferred, so a one-node cluster still runs a quorum. + let selector = objects.set["spec"]["selector"]["matchLabels"].clone(); + objects.set["spec"]["template"]["spec"]["affinity"] = json!({"podAntiAffinity": { + "preferredDuringSchedulingIgnoredDuringExecution": [{"weight": 100, "podAffinityTerm": { + "topologyKey": "kubernetes.io/hostname", "labelSelector": {"matchLabels": selector}}}]}}); // Together: a member that starts after the others elected has an empty log, and is no voter // until it catches up. objects.set["spec"]["podManagementPolicy"] = json!("Parallel"); @@ -1331,6 +1378,13 @@ mod tests { set["volumeClaimTemplates"][0]["spec"]["resources"]["requests"]["storage"], "2Gi" ); + let spread = &set["template"]["spec"]["affinity"]["podAntiAffinity"]["preferredDuringSchedulingIgnoredDuringExecution"] + [0]["podAffinityTerm"]; + assert_eq!(spread["topologyKey"], "kubernetes.io/hostname"); + assert_eq!( + spread["labelSelector"]["matchLabels"], + set["selector"]["matchLabels"] + ); assert_eq!( objects.service["spec"]["selector"][quorum::ROLE], quorum::LEADER diff --git a/operator/crates/kates-operator/src/reconcile/quorum.rs b/operator/crates/kates-operator/src/reconcile/quorum.rs index c6d26627..e5c9d58a 100644 --- a/operator/crates/kates-operator/src/reconcile/quorum.rs +++ b/operator/crates/kates-operator/src/reconcile/quorum.rs @@ -9,7 +9,7 @@ use std::collections::BTreeMap; use jiff::{SignedDuration, Timestamp}; use k8s_openapi::ByteString; -use k8s_openapi::api::core::v1::{Pod, Secret}; +use k8s_openapi::api::core::v1::{PersistentVolumeClaim, Pod, Secret}; use kates_client::QuorumView; use kates_crds::Kates; use kube::api::{Api, Patch, PatchParams}; @@ -40,6 +40,11 @@ const CA_LIFE: SignedDuration = SignedDuration::from_hours(24 * 3650); const MEMBER_LIFE: SignedDuration = SignedDuration::from_hours(24 * 365); /// Re-issued at two thirds of its life, so a failing renewal has months to be noticed. const RENEW_AFTER: SignedDuration = SignedDuration::from_hours(24 * 243); +/// How long a member waits for its volume's node before it gets a new volume: past a node's +/// restart, so a member is moved only when its node is gone. +pub const LOST_AFTER: SignedDuration = SignedDuration::from_mins(2); +/// The claim the `StatefulSet` makes for a member's term and vote. +const CLAIM: &str = "quorum"; #[must_use] pub fn members(name: &str, replicas: i32) -> Vec { @@ -297,6 +302,17 @@ pub struct Member { /// It runs an old revision, or started before its current certificate was issued. pub stale: bool, pub view: Option, + /// Since when its pod waits for the one node its volume can be on, which is gone or refuses it. + pub lost_since: Option, + /// Its pod waits, and its claim is gone or going: the pod is deleted so the `StatefulSet` + /// makes both again. + pub orphaned: bool, +} + +/// The claim that holds a member's term and vote. +#[must_use] +pub fn claim(pod: &str) -> String { + format!("{CLAIM}-{pod}") } /// The member that says it leads, in the highest term any member reports. @@ -347,6 +363,64 @@ pub fn next(members: &[Member]) -> Next { .map_or(Next::Done, |member| Next::Replace(member.pod.clone())) } +/// A member whose volume's node has been gone for [`LOST_AFTER`] gets a new volume, one at a time +/// and only while every other member that is not lost too is ready and follows a leader. Its log +/// is then empty, so it is no voter until it catches up: the quorum keeps its majority only if +/// the others have one without it. With no leader, nothing is replaced. +#[must_use] +pub fn lost(members: &[Member], now: Timestamp) -> Next { + let Some(member) = members.iter().find(|member| { + member + .lost_since + .is_some_and(|since| now.duration_since(since) >= LOST_AFTER) + }) else { + return Next::Done; + }; + let Some(leader) = leader(members) else { + return Next::Wait(format!( + "{}'s volume is on a node that is gone, and the quorum has no leader to replace it under", + member.pod + )); + }; + let unsettled = members.iter().find(|other| { + let caught_up = other.view.as_ref().is_some_and(|view| { + view.leader_id.as_deref() == Some(leader) && view.applied_index >= 0 + }); + other.lost_since.is_none() && (!other.ready || !caught_up) + }); + if let Some(other) = unsettled { + return Next::Wait(format!( + "{}'s volume is on a node that is gone; it is replaced once {} is ready and caught up", + member.pod, other.pod + )); + } + Next::Replace(member.pod.clone()) +} + +/// The scheduler found no node for the pod because its volume can only be on another. +fn volume_elsewhere(message: &str) -> bool { + message.contains("didn't match PersistentVolume's node affinity") + || message.contains("volume node affinity conflict") +} + +/// When the scheduler last found no node for this pending pod because of its volume. +fn lost_since(pod: &Pod) -> Option { + let status = pod.status.as_ref()?; + if status.phase.as_deref() != Some("Pending") { + return None; + } + // shortcut: matched on the scheduler's VolumeBinding reason text, in its wording before and + // since Kubernetes 1.26; read the volume's node instead if the operator ever reads nodes. + status.conditions.iter().flatten().find_map(|c| { + (c.type_ == "PodScheduled" + && c.status == "False" + && c.message.as_deref().is_some_and(volume_elsewhere)) + .then_some(c.last_transition_time.as_ref()) + .flatten() + .and_then(|at| at.0.to_string().parse().ok()) + }) +} + /// Reads each member: its pod and its public `/api/health`. /// /// # Errors @@ -360,6 +434,7 @@ pub async fn observe( ) -> Result<(Vec, BTreeMap), Error> { let name = kates.name_any(); let pods = Api::::namespaced(ctx.client.clone(), namespace); + let claims = Api::::namespaced(ctx.client.clone(), namespace); let mut members = Vec::new(); let mut found = BTreeMap::new(); for pod_name in self::members(&name, kates.spec.replicas) { @@ -369,6 +444,8 @@ pub async fn observe( ready: false, stale: false, view: None, + lost_since: None, + orphaned: false, }); continue; }; @@ -395,11 +472,23 @@ pub async fn observe( Ok(client) => client.quorum().await.ok(), Err(_) => None, }; + let pending = pod + .status + .as_ref() + .is_some_and(|status| status.phase.as_deref() == Some("Pending")); + let orphaned = pending + && pod.metadata.deletion_timestamp.is_none() + && claims + .get_opt(&claim(&pod_name)) + .await? + .is_none_or(|claim| claim.metadata.deletion_timestamp.is_some()); members.push(Member { pod: pod_name.clone(), ready, stale, view, + lost_since: lost_since(&pod), + orphaned, }); found.insert(pod_name, pod); } @@ -447,9 +536,70 @@ mod tests { term: 2, applied_index: 4, }), + lost_since: None, + orphaned: false, + } + } + + fn gone(pod: &str, at: Timestamp) -> Member { + Member { + ready: false, + view: None, + lost_since: Some(at), + ..member(pod, false, "", false) } } + #[test] + fn a_member_whose_node_is_gone_is_replaced_only_under_a_settled_leader() { + let now: Timestamp = "2026-11-15T12:00:00Z".parse().unwrap(); + let long_ago = now - LOST_AFTER; + let healthy = [ + member("main-0", false, "main-0", true), + member("main-1", false, "main-0", false), + ]; + let with = |lost: Vec| [healthy.to_vec(), lost].concat(); + + assert_eq!(lost(&healthy, now), Next::Done); + // Not yet: its node may only be restarting. + assert_eq!(lost(&with(vec![gone("main-2", now)]), now), Next::Done); + assert_eq!( + lost(&with(vec![gone("main-2", long_ago)]), now), + Next::Replace("main-2".to_owned()) + ); + // Two of five lost: one at a time, each other lost member left out of the check. + let five = with(vec![ + member("main-2", false, "main-0", false), + gone("main-3", long_ago), + gone("main-4", long_ago), + ]); + assert_eq!(lost(&five, now), Next::Replace("main-3".to_owned())); + // Another member is catching up: wait. + let mut catching_up = with(vec![gone("main-2", long_ago)]); + catching_up[1].view.as_mut().unwrap().applied_index = -1; + assert!(matches!(lost(&catching_up, now), Next::Wait(why) if why.contains("main-1"))); + // Two of three lost: no leader, nothing done. + let quorum_lost = [ + member("main-0", false, "", false), + gone("main-1", long_ago), + gone("main-2", long_ago), + ]; + assert!(matches!(lost(&quorum_lost, now), Next::Wait(why) if why.contains("no leader"))); + } + + #[test] + fn a_volume_on_another_node_is_read_from_the_schedulers_message_old_and_new() { + assert!(volume_elsewhere( + "0/3 nodes are available: 1 node(s) had untolerated taint(s), 2 node(s) didn't match PersistentVolume's node affinity." + )); + assert!(volume_elsewhere( + "0/2 nodes are available: 2 node(s) had volume node affinity conflict." + )); + assert!(!volume_elsewhere( + "0/3 nodes are available: 3 Insufficient cpu." + )); + } + #[test] fn peers_name_every_member_by_its_dns_name() { assert_eq!( diff --git a/operator/e2e-node-loss.sh b/operator/e2e-node-loss.sh new file mode 100755 index 00000000..7c36bee8 --- /dev/null +++ b/operator/e2e-node-loss.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# A quorum member whose node is gone gets a new volume on another node, and the quorum is whole +# again, with no one acting. Deletes a Kind worker node, so it needs a Kind cluster of three +# nodes or more, not CI's one; the node is started again at the end. +# +# KUBECONFIG= operator/e2e-node-loss.sh +# +# On a Ready Kates resource with replicas 3, as operator/e2e-quorum.sh makes, under an operator +# built from this tree. +set -uo pipefail + +ns=${1:?namespace} +name=${2:?Kates resource} +k=(kubectl -n "$ns") +failed=0 + +pass() { echo " ok $1"; } +fail() { echo " FAIL $1"; failed=1; } +node_of() { "${k[@]}" get pod "$1" -o jsonpath='{.spec.nodeName}' 2>/dev/null && echo; } +claim_uid() { "${k[@]}" get pvc "quorum-$1" -o jsonpath='{.metadata.uid}' 2>/dev/null; } +leader() { "${k[@]}" get kates "$name" -o jsonpath='{.status.leader}'; } +ready() { "${k[@]}" get kates "$name" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}'; } + +echo "== three members, one a node" +"${k[@]}" wait --for=condition=Ready "kates/$name" --timeout=5m >/dev/null || { fail "not Ready"; exit 1; } +members=("$name-0" "$name-1" "$name-2") +nodes=$(for pod in "${members[@]}"; do node_of "$pod"; done | sort -u | wc -l | tr -d ' ') +# Two on the lost node would take the majority, and nothing would be replaced: by design. +[ "$nodes" = 3 ] && pass "spread over three nodes" || { fail "on $nodes nodes: give one a new volume first"; exit 1; } + +# A follower on a worker: the control plane's node holds the API server. +lead=$(leader) +victim= +for pod in "${members[@]}"; do + node=$(node_of "$pod") + [ "$pod" != "$lead" ] && [[ "$(kubectl get node "$node" -o jsonpath='{.metadata.labels}')" != *control-plane* ]] \ + && victim=$pod && break +done +[ -n "$victim" ] || { fail "no follower on a worker node"; exit 1; } +node=$(node_of "$victim") +container=$(kubectl get node "$node" -o jsonpath='{.spec.providerID}' | sed 's|.*/||') +before=$(claim_uid "$victim") +trap 'docker start "$container" >/dev/null 2>&1' EXIT + +echo "== $node, holding $victim and its volume, is gone" +start=$(date +%s) +docker stop "$container" >/dev/null +kubectl delete node "$node" --wait=false >/dev/null +moved= +for _ in $(seq 1 240); do + now=$(claim_uid "$victim") + [ -n "$now" ] && [ "$now" != "$before" ] && [ -n "$(node_of "$victim")" ] && [ "$(node_of "$victim")" != "$node" ] \ + && [ "$("${k[@]}" get pod "$victim" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}')" = True ] \ + && moved=1 && break + sleep 2 +done +[ -n "$moved" ] && pass "$victim runs on $(node_of "$victim") with a new volume, $(( $(date +%s) - start )) s after the loss" \ + || fail "$victim was not replaced within 8 minutes ($("${k[@]}" get pod "$victim" -o jsonpath='{.status.phase}'))" +check_lead=$(leader) +[ "$check_lead" = "$lead" ] && pass "$lead led throughout" || fail "the leader changed from $lead to '$check_lead'" +for _ in $(seq 1 60); do [ "$(ready)" = True ] && break; sleep 2; done +[ "$(ready)" = True ] && pass "Ready with three members" \ + || fail "not Ready: $("${k[@]}" get kates "$name" -o jsonpath='{.status.conditions[?(@.type=="Ready")].message}')" +# Ready before caught up: the pod's readiness does not wait for the log. +for _ in $(seq 1 30); do + applied=$(kubectl get --raw "/api/v1/namespaces/$ns/services/$victim:8080/proxy/api/health" | jq '.quorum.appliedIndex') + [ "${applied:--1}" -ge 0 ] && break + sleep 2 +done +[ "${applied:--1}" -ge 0 ] && pass "$victim caught up (applied index $applied)" || fail "$victim has not caught up: '$applied'" + +echo "== $node comes back" +docker start "$container" >/dev/null +trap - EXIT +for _ in $(seq 1 60); do kubectl get node "$node" >/dev/null 2>&1 && break; sleep 2; done +kubectl wait --for=condition=Ready "node/$node" --timeout=3m >/dev/null && pass "$node registered again" \ + || fail "$node did not come back" + +exit $failed