Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 32 additions & 12 deletions cli/cmd/deploy_kates_operator.go
Original file line number Diff line number Diff line change
Expand Up @@ -224,21 +224,41 @@ func (dc *deployContext) installKatesResource(ctx context.Context) error {
return runExecStdinFn(ctx, "kubectl", []string{"apply", "--server-side", "--field-manager", "kates-deploy", "-f", "-"}, manifest)
}

// How long, and how often, deploy reads the Kates resource's Ready condition.
var katesReadyTimeout, katesReadyPoll = 10 * time.Minute, 5 * time.Second

// waitKatesResource waits for the operator to report the backend Ready: every
// member up and, with a quorum, a leader.
// member up and, with a quorum, a leader. It stops at once on NoQuorum, which
// only another image ends, rather than waiting out the timeout.
func waitKatesResource(ctx context.Context, appNS string) error {
err := runExecFn(ctx, "kubectl", "wait", "--for=condition=Ready", "kates.kates.io/"+katesResourceName,
"-n", appNS, "--timeout=10m")
if err == nil {
return nil
}
// The operator says why in the condition; a bare timeout says nothing.
out, readErr := runExecOutputFn(ctx, "kubectl", "get", "kates.kates.io", katesResourceName, "-n", appNS,
"-o", `jsonpath={.status.conditions[?(@.type=="Ready")].reason}: {.status.conditions[?(@.type=="Ready")].message}`)
if why := strings.TrimSpace(string(out)); readErr == nil && why != "" && why != ":" {
return fmt.Errorf("the Kates resource %s/%s is not Ready (%s): %w", appNS, katesResourceName, why, err)
const ready = `{.status.conditions[?(@.type=="Ready")]`
deadline := time.Now().Add(katesReadyTimeout)
why := ""
for {
out, err := runExecOutputFn(ctx, "kubectl", "get", "kates.kates.io", katesResourceName, "-n", appNS, "-o",
"jsonpath={.metadata.generation}|"+ready+".observedGeneration}|"+ready+".status}|"+ready+".reason}|"+ready+".message}")
// A condition from before the spec this deploy applied says nothing about it.
if fields := strings.SplitN(strings.TrimSpace(string(out)), "|", 5); err == nil && len(fields) == 5 && fields[0] == fields[1] {
if fields[2] == "True" {
return nil
}
why = fields[3] + ": " + fields[4]
if fields[3] == "NoQuorum" {
return fmt.Errorf("the Kates resource %s/%s cannot become Ready (%s)", appNS, katesResourceName, why)
}
}
if time.Now().After(deadline) {
if why == "" {
return fmt.Errorf("the Kates resource %s/%s is not Ready after %s", appNS, katesResourceName, katesReadyTimeout)
}
return fmt.Errorf("the Kates resource %s/%s is not Ready after %s (%s)", appNS, katesResourceName, katesReadyTimeout, why)
}
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(katesReadyPoll):
}
}
return err
}

// katesImage is the image the backend runs, as the chart path picks it: a
Expand Down
40 changes: 35 additions & 5 deletions cli/cmd/deploy_kates_operator_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,12 @@ package cmd
import (
"context"
"encoding/base64"
"errors"
"os"
"path/filepath"
"slices"
"strings"
"testing"
"time"

"gopkg.in/yaml.v3"
)
Expand Down Expand Up @@ -230,13 +230,43 @@ func TestTheOperatorChartIsCheckedAgainstItsOwnSigner(t *testing.T) {
}
}

// katesReadyAnswers stubs the Ready reads, one answer per poll, the last one
// repeated, and counts them.
func katesReadyAnswers(t *testing.T, timeout time.Duration, answers ...string) *int {
t.Helper()
origTimeout, origPoll := katesReadyTimeout, katesReadyPoll
t.Cleanup(func() { katesReadyTimeout, katesReadyPoll = origTimeout, origPoll })
katesReadyTimeout, katesReadyPoll = timeout, time.Millisecond
reads := 0
operatorModeCluster(t, nil)
runExecOutputFn = func(context.Context, string, ...string) ([]byte, error) {
answer := answers[min(reads, len(answers)-1)]
reads++
return []byte(answer), nil
}
return &reads
}

func TestANotReadyKatesResourceSaysWhy(t *testing.T) {
operatorModeCluster(t, map[string]string{"jsonpath={.status.conditions": "NoLeader: the quorum has no leader"})
origExec := runExecFn
t.Cleanup(func() { runExecFn = origExec })
runExecFn = func(context.Context, string, ...string) error { return errors.New("timed out") }
katesReadyAnswers(t, 20*time.Millisecond, "2|2|False|NoLeader|the quorum has no leader")
err := waitKatesResource(context.Background(), "kates")
if err == nil || !strings.Contains(err.Error(), "NoLeader: the quorum has no leader") {
t.Fatalf("error = %v", err)
}
}

func TestAKatesResourceWithoutAQuorumStopsTheWaitAtOnce(t *testing.T) {
reads := katesReadyAnswers(t, time.Hour, "2|2|False|NoQuorum|kates-0 answers without a quorum: kates:native-local is older than the quorum")
err := waitKatesResource(context.Background(), "kates")
if err == nil || !strings.Contains(err.Error(), "kates:native-local is older than the quorum") || *reads != 1 {
t.Fatalf("error = %v after %d reads", err, *reads)
}
}

func TestOnlyTheCurrentGenerationsConditionCounts(t *testing.T) {
// Ready, and NoQuorum, from before the spec this deploy applied: neither ends the wait.
reads := katesReadyAnswers(t, time.Hour, "3|2|True||", "3|2|False|NoQuorum|old image", "3|3|True||")
if err := waitKatesResource(context.Background(), "kates"); err != nil || *reads != 3 {
t.Fatalf("error = %v after %d reads", err, *reads)
}
}
10 changes: 9 additions & 1 deletion cli/cmd/deploy_localimage.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ import (
// So a local deploy runs the native image built from the working tree, which
// the chart already describes: values-native-local.yaml pins kates:native-local
// with pullPolicy Never, sized for a laptop. This applies ONLY to kind (see
// isKind in deploy.go); every other cluster runs the published native image.
// isKind in deploy.go) deploying a checkout's charts; every other cluster, and
// a release's published charts, run the published native image.
const (
// localNativeImage is the tag `make kates-image-native-local` builds and
// loads onto the kind node, and the one values-native-local.yaml names.
Expand Down Expand Up @@ -75,6 +76,13 @@ func kindBackendImageArgs(ctx context.Context, kubeContext string) ([]string, er
dl.Printf(" - Backend image: published native image, from %s (KATES_IMAGE_REGISTRY)\n", mirror)
return []string{"-f", chartRootPath(publishedNativeValues)}, nil
}
// A release's charts and operator expect that release's backend: a local
// build is whatever the working tree was when it was made, and one older
// than the quorum leaves the operator's members with no leader, ever.
if chartsPublished {
dl.Println(" - Backend image: published native image, the one this release's charts name")
return []string{"-f", chartRootPath(publishedNativeValues)}, nil
}

for _, image := range localNativeCandidates {
if !localImageOnHost(ctx, image) {
Expand Down
15 changes: 15 additions & 0 deletions cli/cmd/deploy_localimage_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,21 @@ func TestLocalImageArgs(t *testing.T) {
}
}

// A release's published charts run the release's backend, even with a local
// build on the host: one made before the quorum never elects a leader.
func TestKindBackendImageArgsUsesThePublishedImageWithPublishedCharts(t *testing.T) {
loaded, setOutput := installKindExecSpies(t)
setOutput(hostHasImage(localNativeImage))
orig := chartsPublished
t.Cleanup(func() { chartsPublished = orig })
chartsPublished = true

got, err := kindBackendImageArgs(context.Background(), "kind-panda")
if err != nil || strings.Join(got, " ") != "-f "+chartRootPath(publishedNativeValues) || len(*loaded) != 0 {
t.Fatalf("args %v, %v, loads %v; want the published native image and no load", got, err, *loaded)
}
}

// Behind a mirror the local build is not used: global.imageRegistry would
// rename it, and the nodes hold it under its bare name only.
func TestKindBackendImageArgsUsesThePublishedImageBehindAMirror(t *testing.T) {
Expand Down
2 changes: 1 addition & 1 deletion operator/chart/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ spec:
# storageClassName: fast
```

With `replicas: 3` the members elect a leader among themselves, and keep one with a member gone; `5` keeps one with two gone. Only the leader takes writes: the others answer them 503 with `Retry-After` and serve reads. The `Kates` Service sends traffic to the leader alone, through the label `kates.io/quorum-role=leader`, which the operator moves to a new leader within about five seconds. `kubectl get kates` shows the leader. When the leader dies, a new one is elected within seconds, and before it takes a write it fails the runs the dead leader had live and removes the faults it had left.
With `replicas: 3` the members elect a leader among themselves, and keep one with a member gone; `5` keeps one with two gone. Only the leader takes writes: the others answer them 503 with `Retry-After` and serve reads. The `Kates` Service sends traffic to the leader alone, through the label `kates.io/quorum-role=leader`, which the operator moves to a new leader within about five seconds. `kubectl get kates` shows the leader. When the leader dies, a new one is elected within seconds, and before it takes a write it fails the runs the dead leader had live and removes the faults it had left. Until a leader is elected `Ready` is `False`, reason `NoLeader`. A member whose image is older than the quorum never leads or catches up, which no election or wait fixes: `Ready` is `False`, reason `NoQuorum`, naming the image, and a roll onto that image stops there. Rolling back replaces such a member, or one that crashes, before any other, since it holds no vote.

The members talk over mutual TLS on port 9300, each reached through a Service of its own name, `<resource>-0` and so on. A Service's address stays the same when its pod restarts, so a member's peers reach it again within seconds instead of trying the old pod's address. The operator makes a CA for the quorum and a certificate for each member, valid a year, and renews each at two-thirds of its life by replacing that member's pod. They are kept in two Secrets, named `kates-quorum-ca` (never mounted) and `kates-quorum-tls`: the chart lets the operator read and change only Secrets of those names, so a namespace holds one `Kates` with a quorum, and a second gets `Ready=False`, reason `QuorumSecretTaken`.

Expand Down
33 changes: 29 additions & 4 deletions operator/crates/kates-client/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,9 @@ pub struct QuorumView {
/// The last entry it applied; -1 with an empty log, when it is no voter.
#[serde(default = "no_entry")]
pub applied_index: i64,
/// Its health has no `quorum` at all: the backend is older than the quorum.
#[serde(skip)]
pub predates: bool,
}

fn no_entry() -> i64 {
Expand Down Expand Up @@ -168,21 +171,23 @@ impl Client {
}

/// The member's view of its quorum, from the public `GET /api/health`: no key is sent. A
/// backend with no quorum, or one older than it, says `enabled: false`.
/// backend with no quorum says `enabled: false`; one older than the quorum is also `predates`.
///
/// # Errors
/// See [`Error`].
pub async fn quorum(&self) -> Result<QuorumView, Error> {
#[derive(Deserialize)]
struct Health {
#[serde(default)]
quorum: QuorumView,
quorum: Option<QuorumView>,
}
let none = SecretString::from(String::new());
let health: Health = self
.json(Method::GET, "/api/health", "/api/health", &none, None, None)
.await?;
Ok(health.quorum)
Ok(health.quorum.unwrap_or(QuorumView {
predates: true,
..QuorumView::default()
}))
}

/// `GET /api/drain`.
Expand Down Expand Up @@ -396,6 +401,26 @@ mod tests {
client(&server).drain(&key()).await.unwrap_err()
}

#[tokio::test]
async fn only_a_health_with_no_quorum_at_all_predates_it() {
let health = |body: serde_json::Value| async move {
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/api/health"))
.respond_with(ResponseTemplate::new(200).set_body_json(body))
.mount(&server)
.await;
client(&server).quorum().await.unwrap()
};
let old = health(json!({"status": "UP"})).await;
assert!(old.predates && !old.enabled);
// Configured with no peers, or not started yet: no quorum, but not an old backend.
let off = health(json!({"status": "UP", "quorum": {"enabled": false}})).await;
assert!(!off.predates && !off.enabled);
let on = health(json!({"quorum": {"enabled": true, "leader": true, "term": 3}})).await;
assert!(!on.predates && on.leader && on.term == 3);
}

#[tokio::test]
async fn set_drain_puts_the_switch_and_reads_the_state_back() {
let server = MockServer::start().await;
Expand Down
36 changes: 34 additions & 2 deletions operator/crates/kates-operator/src/reconcile/kates.rs
Original file line number Diff line number Diff line change
Expand Up @@ -459,9 +459,8 @@ async fn roll_quorum(
// lost() replaces one member at a time.
quorum::Next::Done | quorum::Next::ReplaceAll(_) => {}
quorum::Next::Wait(why) => {
let conditions = &mut after.conditions;
status::set(
conditions,
&mut after.conditions,
"Ready",
"False",
"MemberLost",
Expand All @@ -480,6 +479,9 @@ async fn roll_quorum(
return Ok((drain, Some(pod)));
}
}
if no_quorum(kates, &members, after, now) {
return Ok((drain, None));
}
match quorum::next(&members) {
quorum::Next::Done => Ok((drain, None)),
quorum::Next::Wait(why) => {
Expand Down Expand Up @@ -507,6 +509,36 @@ async fn roll_quorum(
}
}

/// `Ready=False`, reason `NoQuorum`, when a member runs a backend older than the quorum, and the
/// roll stops: that member never catches up, so the roll would wait on it forever, or move the
/// majority onto its image.
fn no_quorum(
kates: &Kates,
members: &[quorum::Member],
after: &mut KatesStatus,
now: Timestamp,
) -> bool {
let Some(pod) = quorum::without_quorum(members) else {
return false;
};
let message = format!(
"{pod} runs {}:{}, which is older than the quorum, so it can never lead or catch up; run an image that has it",
kates.spec.image.as_deref().unwrap_or(DEFAULT_IMAGE),
kates.spec.version
);
let generation = kates.metadata.generation;
status::set(
&mut after.conditions,
"Ready",
"False",
"NoQuorum",
&message,
generation,
now,
);
true
}

/// The revision a pod must run, once the `StatefulSet` has seen its latest spec.
fn update_revision(set: &StatefulSet) -> Option<String> {
let observed = set.status.clone().unwrap_or_default();
Expand Down
Loading
Loading