From c23a6aedd68d661a6afa459480a8afc30f90ea03 Mon Sep 17 00:00:00 2001 From: Said Boudjelda Date: Mon, 16 Nov 2026 19:47:53 +0100 Subject: [PATCH] fix(cli,operator): a release deploy runs its own backend, and an old one fails fast On kind, kates deploy ran a local kates:native-local whenever Docker had one, even from a release's published charts. One built before the quorum never answers with a quorum, so the operator reported NoLeader and deploy waited out its 10 minutes. - A release's published charts run the published backend on kind too; a checkout deploy still prefers a local build. - The client tells a backend older than the quorum (its health has no quorum at all) from one whose quorum is off or not started yet. - A member of the current spec on such a backend sets Ready=False, reason NoQuorum, naming the image, and stops the roll: it never leads or catches up, so the roll would wait on it forever or move the majority onto it. NoLeader stays for a quorum that is still electing. - Under a leader, the roll replaces a stale follower that runs no quorum (crashing, or older than it) first: it holds no vote, and waiting for it to catch up stalled every rollback from such an image. - Deploy polls the Ready condition, stops at once on NoQuorum, and counts only a condition of the resource's current generation. --- cli/cmd/deploy_kates_operator.go | 44 +++++++--- cli/cmd/deploy_kates_operator_test.go | 40 +++++++-- cli/cmd/deploy_localimage.go | 10 ++- cli/cmd/deploy_localimage_test.go | 15 ++++ operator/chart/README.md | 2 +- operator/crates/kates-client/src/lib.rs | 33 ++++++- .../kates-operator/src/reconcile/kates.rs | 36 +++++++- .../kates-operator/src/reconcile/quorum.rs | 87 +++++++++++++++++++ 8 files changed, 242 insertions(+), 25 deletions(-) diff --git a/cli/cmd/deploy_kates_operator.go b/cli/cmd/deploy_kates_operator.go index 1497a15c1..9b2bbc251 100644 --- a/cli/cmd/deploy_kates_operator.go +++ b/cli/cmd/deploy_kates_operator.go @@ -224,21 +224,41 @@ func (dc *deployContext) installKatesResource(ctx context.Context) error { return runExecStdinFn(ctx, "kubectl", []string{"apply", "--server-side", "--field-manager", "kates-deploy", "-f", "-"}, manifest) } +// How long, and how often, deploy reads the Kates resource's Ready condition. +var katesReadyTimeout, katesReadyPoll = 10 * time.Minute, 5 * time.Second + // waitKatesResource waits for the operator to report the backend Ready: every -// member up and, with a quorum, a leader. +// member up and, with a quorum, a leader. It stops at once on NoQuorum, which +// only another image ends, rather than waiting out the timeout. func waitKatesResource(ctx context.Context, appNS string) error { - err := runExecFn(ctx, "kubectl", "wait", "--for=condition=Ready", "kates.kates.io/"+katesResourceName, - "-n", appNS, "--timeout=10m") - if err == nil { - return nil - } - // The operator says why in the condition; a bare timeout says nothing. - out, readErr := runExecOutputFn(ctx, "kubectl", "get", "kates.kates.io", katesResourceName, "-n", appNS, - "-o", `jsonpath={.status.conditions[?(@.type=="Ready")].reason}: {.status.conditions[?(@.type=="Ready")].message}`) - if why := strings.TrimSpace(string(out)); readErr == nil && why != "" && why != ":" { - return fmt.Errorf("the Kates resource %s/%s is not Ready (%s): %w", appNS, katesResourceName, why, err) + const ready = `{.status.conditions[?(@.type=="Ready")]` + deadline := time.Now().Add(katesReadyTimeout) + why := "" + for { + out, err := runExecOutputFn(ctx, "kubectl", "get", "kates.kates.io", katesResourceName, "-n", appNS, "-o", + "jsonpath={.metadata.generation}|"+ready+".observedGeneration}|"+ready+".status}|"+ready+".reason}|"+ready+".message}") + // A condition from before the spec this deploy applied says nothing about it. + if fields := strings.SplitN(strings.TrimSpace(string(out)), "|", 5); err == nil && len(fields) == 5 && fields[0] == fields[1] { + if fields[2] == "True" { + return nil + } + why = fields[3] + ": " + fields[4] + if fields[3] == "NoQuorum" { + return fmt.Errorf("the Kates resource %s/%s cannot become Ready (%s)", appNS, katesResourceName, why) + } + } + if time.Now().After(deadline) { + if why == "" { + return fmt.Errorf("the Kates resource %s/%s is not Ready after %s", appNS, katesResourceName, katesReadyTimeout) + } + return fmt.Errorf("the Kates resource %s/%s is not Ready after %s (%s)", appNS, katesResourceName, katesReadyTimeout, why) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(katesReadyPoll): + } } - return err } // katesImage is the image the backend runs, as the chart path picks it: a diff --git a/cli/cmd/deploy_kates_operator_test.go b/cli/cmd/deploy_kates_operator_test.go index 05d02ac5c..7ee932541 100644 --- a/cli/cmd/deploy_kates_operator_test.go +++ b/cli/cmd/deploy_kates_operator_test.go @@ -3,12 +3,12 @@ package cmd import ( "context" "encoding/base64" - "errors" "os" "path/filepath" "slices" "strings" "testing" + "time" "gopkg.in/yaml.v3" ) @@ -230,13 +230,43 @@ func TestTheOperatorChartIsCheckedAgainstItsOwnSigner(t *testing.T) { } } +// katesReadyAnswers stubs the Ready reads, one answer per poll, the last one +// repeated, and counts them. +func katesReadyAnswers(t *testing.T, timeout time.Duration, answers ...string) *int { + t.Helper() + origTimeout, origPoll := katesReadyTimeout, katesReadyPoll + t.Cleanup(func() { katesReadyTimeout, katesReadyPoll = origTimeout, origPoll }) + katesReadyTimeout, katesReadyPoll = timeout, time.Millisecond + reads := 0 + operatorModeCluster(t, nil) + runExecOutputFn = func(context.Context, string, ...string) ([]byte, error) { + answer := answers[min(reads, len(answers)-1)] + reads++ + return []byte(answer), nil + } + return &reads +} + func TestANotReadyKatesResourceSaysWhy(t *testing.T) { - operatorModeCluster(t, map[string]string{"jsonpath={.status.conditions": "NoLeader: the quorum has no leader"}) - origExec := runExecFn - t.Cleanup(func() { runExecFn = origExec }) - runExecFn = func(context.Context, string, ...string) error { return errors.New("timed out") } + katesReadyAnswers(t, 20*time.Millisecond, "2|2|False|NoLeader|the quorum has no leader") err := waitKatesResource(context.Background(), "kates") if err == nil || !strings.Contains(err.Error(), "NoLeader: the quorum has no leader") { t.Fatalf("error = %v", err) } } + +func TestAKatesResourceWithoutAQuorumStopsTheWaitAtOnce(t *testing.T) { + reads := katesReadyAnswers(t, time.Hour, "2|2|False|NoQuorum|kates-0 answers without a quorum: kates:native-local is older than the quorum") + err := waitKatesResource(context.Background(), "kates") + if err == nil || !strings.Contains(err.Error(), "kates:native-local is older than the quorum") || *reads != 1 { + t.Fatalf("error = %v after %d reads", err, *reads) + } +} + +func TestOnlyTheCurrentGenerationsConditionCounts(t *testing.T) { + // Ready, and NoQuorum, from before the spec this deploy applied: neither ends the wait. + reads := katesReadyAnswers(t, time.Hour, "3|2|True||", "3|2|False|NoQuorum|old image", "3|3|True||") + if err := waitKatesResource(context.Background(), "kates"); err != nil || *reads != 3 { + t.Fatalf("error = %v after %d reads", err, *reads) + } +} diff --git a/cli/cmd/deploy_localimage.go b/cli/cmd/deploy_localimage.go index 40e224794..7a747836f 100644 --- a/cli/cmd/deploy_localimage.go +++ b/cli/cmd/deploy_localimage.go @@ -19,7 +19,8 @@ import ( // So a local deploy runs the native image built from the working tree, which // the chart already describes: values-native-local.yaml pins kates:native-local // with pullPolicy Never, sized for a laptop. This applies ONLY to kind (see -// isKind in deploy.go); every other cluster runs the published native image. +// isKind in deploy.go) deploying a checkout's charts; every other cluster, and +// a release's published charts, run the published native image. const ( // localNativeImage is the tag `make kates-image-native-local` builds and // loads onto the kind node, and the one values-native-local.yaml names. @@ -75,6 +76,13 @@ func kindBackendImageArgs(ctx context.Context, kubeContext string) ([]string, er dl.Printf(" - Backend image: published native image, from %s (KATES_IMAGE_REGISTRY)\n", mirror) return []string{"-f", chartRootPath(publishedNativeValues)}, nil } + // A release's charts and operator expect that release's backend: a local + // build is whatever the working tree was when it was made, and one older + // than the quorum leaves the operator's members with no leader, ever. + if chartsPublished { + dl.Println(" - Backend image: published native image, the one this release's charts name") + return []string{"-f", chartRootPath(publishedNativeValues)}, nil + } for _, image := range localNativeCandidates { if !localImageOnHost(ctx, image) { diff --git a/cli/cmd/deploy_localimage_test.go b/cli/cmd/deploy_localimage_test.go index 0cc8ee3a9..060bb9f64 100644 --- a/cli/cmd/deploy_localimage_test.go +++ b/cli/cmd/deploy_localimage_test.go @@ -175,6 +175,21 @@ func TestLocalImageArgs(t *testing.T) { } } +// A release's published charts run the release's backend, even with a local +// build on the host: one made before the quorum never elects a leader. +func TestKindBackendImageArgsUsesThePublishedImageWithPublishedCharts(t *testing.T) { + loaded, setOutput := installKindExecSpies(t) + setOutput(hostHasImage(localNativeImage)) + orig := chartsPublished + t.Cleanup(func() { chartsPublished = orig }) + chartsPublished = true + + got, err := kindBackendImageArgs(context.Background(), "kind-panda") + if err != nil || strings.Join(got, " ") != "-f "+chartRootPath(publishedNativeValues) || len(*loaded) != 0 { + t.Fatalf("args %v, %v, loads %v; want the published native image and no load", got, err, *loaded) + } +} + // Behind a mirror the local build is not used: global.imageRegistry would // rename it, and the nodes hold it under its bare name only. func TestKindBackendImageArgsUsesThePublishedImageBehindAMirror(t *testing.T) { diff --git a/operator/chart/README.md b/operator/chart/README.md index d95de0e65..c9353d099 100644 --- a/operator/chart/README.md +++ b/operator/chart/README.md @@ -99,7 +99,7 @@ spec: # storageClassName: fast ``` -With `replicas: 3` the members elect a leader among themselves, and keep one with a member gone; `5` keeps one with two gone. Only the leader takes writes: the others answer them 503 with `Retry-After` and serve reads. The `Kates` Service sends traffic to the leader alone, through the label `kates.io/quorum-role=leader`, which the operator moves to a new leader within about five seconds. `kubectl get kates` shows the leader. When the leader dies, a new one is elected within seconds, and before it takes a write it fails the runs the dead leader had live and removes the faults it had left. +With `replicas: 3` the members elect a leader among themselves, and keep one with a member gone; `5` keeps one with two gone. Only the leader takes writes: the others answer them 503 with `Retry-After` and serve reads. The `Kates` Service sends traffic to the leader alone, through the label `kates.io/quorum-role=leader`, which the operator moves to a new leader within about five seconds. `kubectl get kates` shows the leader. When the leader dies, a new one is elected within seconds, and before it takes a write it fails the runs the dead leader had live and removes the faults it had left. Until a leader is elected `Ready` is `False`, reason `NoLeader`. A member whose image is older than the quorum never leads or catches up, which no election or wait fixes: `Ready` is `False`, reason `NoQuorum`, naming the image, and a roll onto that image stops there. Rolling back replaces such a member, or one that crashes, before any other, since it holds no vote. The members talk over mutual TLS on port 9300, each reached through a Service of its own name, `-0` and so on. A Service's address stays the same when its pod restarts, so a member's peers reach it again within seconds instead of trying the old pod's address. The operator makes a CA for the quorum and a certificate for each member, valid a year, and renews each at two-thirds of its life by replacing that member's pod. They are kept in two Secrets, named `kates-quorum-ca` (never mounted) and `kates-quorum-tls`: the chart lets the operator read and change only Secrets of those names, so a namespace holds one `Kates` with a quorum, and a second gets `Ready=False`, reason `QuorumSecretTaken`. diff --git a/operator/crates/kates-client/src/lib.rs b/operator/crates/kates-client/src/lib.rs index 0ddbd9053..a1838681b 100644 --- a/operator/crates/kates-client/src/lib.rs +++ b/operator/crates/kates-client/src/lib.rs @@ -116,6 +116,9 @@ pub struct QuorumView { /// The last entry it applied; -1 with an empty log, when it is no voter. #[serde(default = "no_entry")] pub applied_index: i64, + /// Its health has no `quorum` at all: the backend is older than the quorum. + #[serde(skip)] + pub predates: bool, } fn no_entry() -> i64 { @@ -168,21 +171,23 @@ impl Client { } /// The member's view of its quorum, from the public `GET /api/health`: no key is sent. A - /// backend with no quorum, or one older than it, says `enabled: false`. + /// backend with no quorum says `enabled: false`; one older than the quorum is also `predates`. /// /// # Errors /// See [`Error`]. pub async fn quorum(&self) -> Result { #[derive(Deserialize)] struct Health { - #[serde(default)] - quorum: QuorumView, + quorum: Option, } let none = SecretString::from(String::new()); let health: Health = self .json(Method::GET, "/api/health", "/api/health", &none, None, None) .await?; - Ok(health.quorum) + Ok(health.quorum.unwrap_or(QuorumView { + predates: true, + ..QuorumView::default() + })) } /// `GET /api/drain`. @@ -396,6 +401,26 @@ mod tests { client(&server).drain(&key()).await.unwrap_err() } + #[tokio::test] + async fn only_a_health_with_no_quorum_at_all_predates_it() { + let health = |body: serde_json::Value| async move { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/api/health")) + .respond_with(ResponseTemplate::new(200).set_body_json(body)) + .mount(&server) + .await; + client(&server).quorum().await.unwrap() + }; + let old = health(json!({"status": "UP"})).await; + assert!(old.predates && !old.enabled); + // Configured with no peers, or not started yet: no quorum, but not an old backend. + let off = health(json!({"status": "UP", "quorum": {"enabled": false}})).await; + assert!(!off.predates && !off.enabled); + let on = health(json!({"quorum": {"enabled": true, "leader": true, "term": 3}})).await; + assert!(!on.predates && on.leader && on.term == 3); + } + #[tokio::test] async fn set_drain_puts_the_switch_and_reads_the_state_back() { let server = MockServer::start().await; diff --git a/operator/crates/kates-operator/src/reconcile/kates.rs b/operator/crates/kates-operator/src/reconcile/kates.rs index 8b5054d97..4dd345bea 100644 --- a/operator/crates/kates-operator/src/reconcile/kates.rs +++ b/operator/crates/kates-operator/src/reconcile/kates.rs @@ -459,9 +459,8 @@ async fn roll_quorum( // lost() replaces one member at a time. quorum::Next::Done | quorum::Next::ReplaceAll(_) => {} quorum::Next::Wait(why) => { - let conditions = &mut after.conditions; status::set( - conditions, + &mut after.conditions, "Ready", "False", "MemberLost", @@ -480,6 +479,9 @@ async fn roll_quorum( return Ok((drain, Some(pod))); } } + if no_quorum(kates, &members, after, now) { + return Ok((drain, None)); + } match quorum::next(&members) { quorum::Next::Done => Ok((drain, None)), quorum::Next::Wait(why) => { @@ -507,6 +509,36 @@ async fn roll_quorum( } } +/// `Ready=False`, reason `NoQuorum`, when a member runs a backend older than the quorum, and the +/// roll stops: that member never catches up, so the roll would wait on it forever, or move the +/// majority onto its image. +fn no_quorum( + kates: &Kates, + members: &[quorum::Member], + after: &mut KatesStatus, + now: Timestamp, +) -> bool { + let Some(pod) = quorum::without_quorum(members) else { + return false; + }; + let message = format!( + "{pod} runs {}:{}, which is older than the quorum, so it can never lead or catch up; run an image that has it", + kates.spec.image.as_deref().unwrap_or(DEFAULT_IMAGE), + kates.spec.version + ); + let generation = kates.metadata.generation; + status::set( + &mut after.conditions, + "Ready", + "False", + "NoQuorum", + &message, + generation, + now, + ); + true +} + /// The revision a pod must run, once the `StatefulSet` has seen its latest spec. fn update_revision(set: &StatefulSet) -> Option { let observed = set.status.clone().unwrap_or_default(); diff --git a/operator/crates/kates-operator/src/reconcile/quorum.rs b/operator/crates/kates-operator/src/reconcile/quorum.rs index 60aa8312b..f1ce07ec8 100644 --- a/operator/crates/kates-operator/src/reconcile/quorum.rs +++ b/operator/crates/kates-operator/src/reconcile/quorum.rs @@ -325,6 +325,16 @@ pub fn leader(members: &[Member]) -> Option<&str> { .map(|member| member.pod.as_str()) } +/// A member that runs this spec on a backend older than the quorum. It never leads, votes or +/// catches up, so no election, and no roll that waits on it, ends; only another image does. +#[must_use] +pub fn without_quorum(members: &[Member]) -> Option<&str> { + members + .iter() + .find(|member| !member.stale && member.view.as_ref().is_some_and(|view| view.predates)) + .map(|member| member.pod.as_str()) +} + /// What the roll does next. #[derive(Debug, PartialEq, Eq)] pub enum Next { @@ -359,6 +369,14 @@ pub fn next(members: &[Member]) -> Next { let Some(leader) = leader(members) else { return Next::Wait("the quorum has no leader".to_owned()); }; + // A stale follower that runs no quorum, down or older than it, holds no vote: replacing it + // costs the majority nothing, while waiting for it to catch up would wait forever. + if let Some(member) = members + .iter() + .find(|member| member.stale && member.pod != leader && !may_run_quorum(member)) + { + return Next::Replace(member.pod.clone()); + } let unsettled = members.iter().find(|member| { let caught_up = member.view.as_ref().is_some_and(|view| { view.leader_id.as_deref() == Some(leader) && view.applied_index >= 0 @@ -548,6 +566,7 @@ mod tests { leader_id: Some(leader_id.to_owned()), term: 2, applied_index: 4, + predates: false, }), lost_since: None, orphaned: false, @@ -600,6 +619,74 @@ mod tests { assert!(matches!(lost(&quorum_lost, now), Next::Wait(why) if why.contains("no leader"))); } + #[test] + fn a_stale_follower_that_runs_no_quorum_is_replaced_first_under_a_leader() { + let old = Member { + view: Some(QuorumView { + predates: true, + ..QuorumView::default() + }), + ..member("main-0", true, "", false) + }; + let crashing = Member { + ready: false, + view: None, + ..member("main-1", true, "", false) + }; + let leader = member("main-2", false, "main-2", true); + // Rolled back from an image older than the quorum: it never catches up, so it goes. + let rollback = [ + old.clone(), + member("main-1", false, "main-2", false), + leader.clone(), + ]; + assert_eq!(next(&rollback), Next::Replace("main-0".to_owned())); + // Before a stale voter: replacing that one would leave no majority. + let mixed = [ + member("main-0", true, "main-2", false), + crashing, + leader.clone(), + ]; + assert_eq!(next(&mixed), Next::Replace("main-1".to_owned())); + // With no leader, nothing is replaced. + let leaderless = [old, member("main-1", false, "", false)]; + assert!(matches!(next(&leaderless), Next::Wait(why) if why.contains("no leader"))); + } + + #[test] + fn only_a_current_member_older_than_the_quorum_is_without_one() { + let no_quorum = |pod: &str, stale: bool| Member { + view: Some(QuorumView { + predates: true, + ..QuorumView::default() + }), + ..member(pod, stale, "", false) + }; + // Still electing: every member has a quorum, none leads yet. + let electing = [ + member("main-0", false, "", false), + member("main-1", false, "", false), + ]; + assert_eq!(without_quorum(&electing), None); + // A new backend whose quorum has not started: it will. + let starting = Member { + view: Some(QuorumView::default()), + ..member("main-0", false, "", false) + }; + assert_eq!(without_quorum(&[starting]), None); + // Not answering is not the same as answering without a quorum. + assert_eq!(without_quorum(&[gone("main-0", Timestamp::now())]), None); + // An old member the roll is about to replace. + assert_eq!(without_quorum(&[no_quorum("main-0", true)]), None); + assert_eq!( + without_quorum(&[ + member("main-0", false, "", false), + no_quorum("main-1", false) + ]), + Some("main-1") + ); + } + #[test] fn a_volume_on_another_node_is_read_from_the_schedulers_message_old_and_new() { assert!(volume_elsewhere(