diff --git a/docs/guides/configuration.mdx b/docs/guides/configuration.mdx index 38ee9849f..f2371ed2d 100644 --- a/docs/guides/configuration.mdx +++ b/docs/guides/configuration.mdx @@ -70,6 +70,19 @@ spec: storage: 5Gi ``` +### Leadership handover on restart {#keeper-leadership-handover} + +When a Keeper pod stops, whether the operator is rolling it out or Kubernetes is evicting it, +its container runs a `preStop` hook that asks the server to hand Raft leadership to another +replica through the HTTP commands API (`GET /api/v1/commands?command=ydld` on the +`http_control` port). Restarting the leader then costs a leadership transfer of a few dozen +milliseconds instead of an election, so ClickHouse does not see a leaderless ensemble. + +Keeper serves the commands API from version 26.3. On older servers the request returns 404, +Kubernetes records a `FailedPreStopHook` event on the pod, and the container stops as before. +A server that does not answer on the `http_control` port at all holds the hook open for up to +`terminationGracePeriodSeconds` before the container is stopped. + ## Storage configuration {#storage-configuration} Configure persistent storage with `dataVolumeClaimSpec`, a standard Kubernetes diff --git a/internal/controller/keeper/constants.go b/internal/controller/keeper/constants.go index 2f1e435ae..3c357412c 100644 --- a/internal/controller/keeper/constants.go +++ b/internal/controller/keeper/constants.go @@ -32,6 +32,9 @@ const ( ContainerName = "clickhouse-keeper" DefaultRevisionHistory = 10 MaximalAffinityWeight = 100 + + // FLWYieldLeadership is the four letter word that asks a leader to hand leadership to another replica. + FLWYieldLeadership = "ydld" ) var breakingStatefulSetVersion, _ = semver.Parse("0.0.1") diff --git a/internal/controller/keeper/templates.go b/internal/controller/keeper/templates.go index 111fe0e84..0660d14c3 100644 --- a/internal/controller/keeper/templates.go +++ b/internal/controller/keeper/templates.go @@ -525,6 +525,23 @@ func templateContainer(cr *v1.KeeperCluster) (corev1.Container, error) { SecurityContext: controller.DefaultContainerSecurityContext(), } + // Whoever stops the pod, the operator during a rollout or Kubernetes during an eviction, the + // container first asks the server to hand Raft leadership to a peer, so a leader restart costs a + // transfer instead of an election. Sent over http_control so it works without a shell in the + // image; a follower or a lone replica treats the command as a no-op. Keeper serves the commands + // API from 26.3; on older servers the request 404s, the kubelet records FailedPreStopHook and + // stops the container as before. The hook is deliberately not gated on Status.Version: that + // value comes from running replicas, so a gate would roll every new cluster once it reports in. + container.Lifecycle = &corev1.Lifecycle{ + PreStop: &corev1.LifecycleHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/api/v1/commands?command=" + FLWYieldLeadership, + Port: intstr.FromInt32(PortHTTPControl), + Scheme: corev1.URISchemeHTTP, + }, + }, + } + if !cr.Spec.Settings.TLS.Enabled || !cr.Spec.Settings.TLS.Required { container.Ports = append(container.Ports, corev1.ContainerPort{ Protocol: corev1.ProtocolTCP, diff --git a/internal/controller/keeper/templates_test.go b/internal/controller/keeper/templates_test.go index 688e4da42..da9b66a10 100644 --- a/internal/controller/keeper/templates_test.go +++ b/internal/controller/keeper/templates_test.go @@ -393,3 +393,34 @@ var _ = Describe("TopologySpreadConstraints", func() { Expect(tsc.LabelSelector.MatchLabels).To(HaveKeyWithValue(controllerutil.LabelAppKey, cr.SpecificName())) }) }) + +var _ = Describe("PreStopLeadershipHandover", func() { + lifecycleFor := func(version string) *corev1.Lifecycle { + cr := &v1.KeeperCluster{ + Name: "test", + Spec: v1.KeeperClusterSpec{Replicas: new(int32(3))}, + Status: v1.KeeperClusterStatus{Version: version}, + } + + container, err := templateContainer(cr) + Expect(err).NotTo(HaveOccurred()) + + return container.Lifecycle + } + + It("asks the server to yield leadership through the HTTP commands API before it stops", func() { + lifecycle := lifecycleFor("26.7.5.10") + Expect(lifecycle).NotTo(BeNil()) + Expect(lifecycle.PreStop).NotTo(BeNil()) + Expect(lifecycle.PreStop.HTTPGet).NotTo(BeNil()) + Expect(lifecycle.PreStop.HTTPGet.Path).To(Equal("/api/v1/commands?command=ydld")) + Expect(lifecycle.PreStop.HTTPGet.Port.IntValue()).To(Equal(int(PortHTTPControl))) + Expect(lifecycle.PreStop.HTTPGet.Scheme).To(Equal(corev1.URISchemeHTTP)) + }) + + It("does not depend on the observed version, which only arrives once replicas are running", func() { + // A template that changed with Status.Version would roll every freshly created cluster. + Expect(lifecycleFor("")).To(Equal(lifecycleFor("25.8.32.4"))) + Expect(lifecycleFor("")).To(Equal(lifecycleFor("26.7.5.10"))) + }) +})