feat(keeper): hand leadership over before restarting the leader

docs/guides/configuration.mdx

@@ -70,6 +70,19 @@ spec:
         storage: 5Gi
 ```
 
+### Leadership handover on restart {#keeper-leadership-handover}
+
+When a Keeper pod stops, whether the operator is rolling it out or Kubernetes is evicting it,
+its container runs a `preStop` hook that asks the server to hand Raft leadership to another
+replica through the HTTP commands API (`GET /api/v1/commands?command=ydld` on the
+`http_control` port). Restarting the leader then costs a leadership transfer of a few dozen
+milliseconds instead of an election, so ClickHouse does not see a leaderless ensemble.
+
+Keeper serves the commands API from version 26.3. On older servers the request returns 404,
+Kubernetes records a `FailedPreStopHook` event on the pod, and the container stops as before.
+A server that does not answer on the `http_control` port at all holds the hook open for up to
+`terminationGracePeriodSeconds` before the container is stopped.
+
 ## Storage configuration {#storage-configuration}
 
 Configure persistent storage with `dataVolumeClaimSpec`, a standard Kubernetes

internal/controller/keeper/constants.go

@@ -32,6 +32,9 @@ const (
 	ContainerName          = "clickhouse-keeper"
 	DefaultRevisionHistory = 10
 	MaximalAffinityWeight  = 100
+
+	// FLWYieldLeadership is the four letter word that asks a leader to hand leadership to another replica.
+	FLWYieldLeadership = "ydld"
 )
 
 var breakingStatefulSetVersion, _ = semver.Parse("0.0.1")

internal/controller/keeper/templates.go

@@ -525,6 +525,23 @@ func templateContainer(cr *v1.KeeperCluster) (corev1.Container, error) {
 		SecurityContext:          controller.DefaultContainerSecurityContext(),
 	}
 
+	// Whoever stops the pod, the operator during a rollout or Kubernetes during an eviction, the
+	// container first asks the server to hand Raft leadership to a peer, so a leader restart costs a
+	// transfer instead of an election. Sent over http_control so it works without a shell in the
+	// image; a follower or a lone replica treats the command as a no-op. Keeper serves the commands
+	// API from 26.3; on older servers the request 404s, the kubelet records FailedPreStopHook and
+	// stops the container as before. The hook is deliberately not gated on Status.Version: that
+	// value comes from running replicas, so a gate would roll every new cluster once it reports in.
+	container.Lifecycle = &corev1.Lifecycle{
+		PreStop: &corev1.LifecycleHandler{
+			HTTPGet: &corev1.HTTPGetAction{
+				Path:   "/api/v1/commands?command=" + FLWYieldLeadership,
+				Port:   intstr.FromInt32(PortHTTPControl),
+				Scheme: corev1.URISchemeHTTP,
+			},
+		},
+	}
+
 	if !cr.Spec.Settings.TLS.Enabled || !cr.Spec.Settings.TLS.Required {
 		container.Ports = append(container.Ports, corev1.ContainerPort{
 			Protocol:      corev1.ProtocolTCP,

internal/controller/keeper/templates_test.go

@@ -393,3 +393,34 @@ var _ = Describe("TopologySpreadConstraints", func() {
 		Expect(tsc.LabelSelector.MatchLabels).To(HaveKeyWithValue(controllerutil.LabelAppKey, cr.SpecificName()))
 	})
 })
+
+var _ = Describe("PreStopLeadershipHandover", func() {
+	lifecycleFor := func(version string) *corev1.Lifecycle {
+		cr := &v1.KeeperCluster{
+			Name:   "test",
+			Spec:   v1.KeeperClusterSpec{Replicas: new(int32(3))},
+			Status: v1.KeeperClusterStatus{Version: version},
+		}
+
+		container, err := templateContainer(cr)
+		Expect(err).NotTo(HaveOccurred())
+
+		return container.Lifecycle
+	}
+
+	It("asks the server to yield leadership through the HTTP commands API before it stops", func() {
+		lifecycle := lifecycleFor("26.7.5.10")
+		Expect(lifecycle).NotTo(BeNil())
+		Expect(lifecycle.PreStop).NotTo(BeNil())
+		Expect(lifecycle.PreStop.HTTPGet).NotTo(BeNil())
+		Expect(lifecycle.PreStop.HTTPGet.Path).To(Equal("/api/v1/commands?command=ydld"))
+		Expect(lifecycle.PreStop.HTTPGet.Port.IntValue()).To(Equal(int(PortHTTPControl)))
+		Expect(lifecycle.PreStop.HTTPGet.Scheme).To(Equal(corev1.URISchemeHTTP))
+	})
+
+	It("does not depend on the observed version, which only arrives once replicas are running", func() {
+		// A template that changed with Status.Version would roll every freshly created cluster.
+		Expect(lifecycleFor("")).To(Equal(lifecycleFor("25.8.32.4")))
+		Expect(lifecycleFor("")).To(Equal(lifecycleFor("26.7.5.10")))
+	})
+})