diff --git a/api/v1alpha1/common.go b/api/v1alpha1/common.go index f06583ad8..99e526cdd 100644 --- a/api/v1alpha1/common.go +++ b/api/v1alpha1/common.go @@ -352,6 +352,11 @@ type ContainerTemplateSpec struct { // containers time to come up. The operator sets no startup probe by default. // +optional StartupProbe *corev1.Probe `json:"startupProbe,omitempty"` + + // Lifecycle defines actions the kubelet triggers on container lifecycle events (PostStart/PreStop). + // A non-nil Lifecycle fully replaces operator defaults; the user owns the entire struct. + // +optional + Lifecycle *corev1.Lifecycle `json:"lifecycle,omitempty"` } // ClusterTLSSpec defines cluster TLS configuration. diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index dd08e6a50..c6f8cec08 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -368,6 +368,11 @@ func (in *ContainerTemplateSpec) DeepCopyInto(out *ContainerTemplateSpec) { *out = new(v1.Probe) (*in).DeepCopyInto(*out) } + if in.Lifecycle != nil { + in, out := &in.Lifecycle, &out.Lifecycle + *out = new(v1.Lifecycle) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContainerTemplateSpec. diff --git a/config/crd/bases/clickhouse.com_clickhouseclusters.yaml b/config/crd/bases/clickhouse.com_clickhouseclusters.yaml index 7dc64e400..5fc8a2f41 100644 --- a/config/crd/bases/clickhouse.com_clickhouseclusters.yaml +++ b/config/crd/bases/clickhouse.com_clickhouseclusters.yaml @@ -535,6 +535,247 @@ spec: - Never - IfNotPresent type: string + lifecycle: + description: |- + Lifecycle defines actions the kubelet triggers on container lifecycle events (PostStart/PreStop). + A non-nil Lifecycle fully replaces operator defaults; the user owns the entire struct. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command to execute in the + container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to + perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + protocol: + description: |- + Protocol selects the wire protocol for the probe connection. + Nil defaults to HTTP/1.1. + type: string + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration that the container + should sleep. + properties: + seconds: + description: Seconds is the number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command to execute in the + container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to + perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + protocol: + description: |- + Protocol selects the wire protocol for the probe connection. + Nil defaults to HTTP/1.1. + type: string + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration that the container + should sleep. + properties: + seconds: + description: Seconds is the number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + stopSignal: + description: |- + StopSignal defines which signal will be sent to a container when it is being stopped. + If not specified, the default is defined by the container runtime in use. + StopSignal can only be set for Pods with a non-empty .spec.os.name + type: string + type: object livenessProbe: description: LivenessProbe overrides the operator's default liveness probe. diff --git a/config/crd/bases/clickhouse.com_keeperclusters.yaml b/config/crd/bases/clickhouse.com_keeperclusters.yaml index 94c7b4a37..9274e529f 100644 --- a/config/crd/bases/clickhouse.com_keeperclusters.yaml +++ b/config/crd/bases/clickhouse.com_keeperclusters.yaml @@ -265,6 +265,247 @@ spec: - Never - IfNotPresent type: string + lifecycle: + description: |- + Lifecycle defines actions the kubelet triggers on container lifecycle events (PostStart/PreStop). + A non-nil Lifecycle fully replaces operator defaults; the user owns the entire struct. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command to execute in the + container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to + perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + protocol: + description: |- + Protocol selects the wire protocol for the probe connection. + Nil defaults to HTTP/1.1. + type: string + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration that the container + should sleep. + properties: + seconds: + description: Seconds is the number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command to execute in the + container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to + perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + protocol: + description: |- + Protocol selects the wire protocol for the probe connection. + Nil defaults to HTTP/1.1. + type: string + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration that the container + should sleep. + properties: + seconds: + description: Seconds is the number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + stopSignal: + description: |- + StopSignal defines which signal will be sent to a container when it is being stopped. + If not specified, the default is defined by the container runtime in use. + StopSignal can only be set for Pods with a non-empty .spec.os.name + type: string + type: object livenessProbe: description: LivenessProbe overrides the operator's default liveness probe. diff --git a/dist/chart/templates/crd/clickhouseclusters.clickhouse.com.yaml b/dist/chart/templates/crd/clickhouseclusters.clickhouse.com.yaml index 7e672dfe6..8016a0234 100644 --- a/dist/chart/templates/crd/clickhouseclusters.clickhouse.com.yaml +++ b/dist/chart/templates/crd/clickhouseclusters.clickhouse.com.yaml @@ -538,6 +538,247 @@ spec: - Never - IfNotPresent type: string + lifecycle: + description: |- + Lifecycle defines actions the kubelet triggers on container lifecycle events (PostStart/PreStop). + A non-nil Lifecycle fully replaces operator defaults; the user owns the entire struct. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command to execute in the + container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to + perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + protocol: + description: |- + Protocol selects the wire protocol for the probe connection. + Nil defaults to HTTP/1.1. + type: string + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration that the container + should sleep. + properties: + seconds: + description: Seconds is the number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command to execute in the + container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to + perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + protocol: + description: |- + Protocol selects the wire protocol for the probe connection. + Nil defaults to HTTP/1.1. + type: string + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration that the container + should sleep. + properties: + seconds: + description: Seconds is the number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + stopSignal: + description: |- + StopSignal defines which signal will be sent to a container when it is being stopped. + If not specified, the default is defined by the container runtime in use. + StopSignal can only be set for Pods with a non-empty .spec.os.name + type: string + type: object livenessProbe: description: LivenessProbe overrides the operator's default liveness probe. diff --git a/dist/chart/templates/crd/keeperclusters.clickhouse.com.yaml b/dist/chart/templates/crd/keeperclusters.clickhouse.com.yaml index f9644bbf7..2bff78df9 100644 --- a/dist/chart/templates/crd/keeperclusters.clickhouse.com.yaml +++ b/dist/chart/templates/crd/keeperclusters.clickhouse.com.yaml @@ -268,6 +268,247 @@ spec: - Never - IfNotPresent type: string + lifecycle: + description: |- + Lifecycle defines actions the kubelet triggers on container lifecycle events (PostStart/PreStop). + A non-nil Lifecycle fully replaces operator defaults; the user owns the entire struct. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command to execute in the + container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to + perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + protocol: + description: |- + Protocol selects the wire protocol for the probe connection. + Nil defaults to HTTP/1.1. + type: string + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration that the container + should sleep. + properties: + seconds: + description: Seconds is the number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command to execute in the + container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to + perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + protocol: + description: |- + Protocol selects the wire protocol for the probe connection. + Nil defaults to HTTP/1.1. + type: string + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration that the container + should sleep. + properties: + seconds: + description: Seconds is the number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + stopSignal: + description: |- + StopSignal defines which signal will be sent to a container when it is being stopped. + If not specified, the default is defined by the container runtime in use. + StopSignal can only be set for Pods with a non-empty .spec.os.name + type: string + type: object livenessProbe: description: LivenessProbe overrides the operator's default liveness probe. diff --git a/docs/guides/configuration.mdx b/docs/guides/configuration.mdx index 283986e17..09155db74 100644 --- a/docs/guides/configuration.mdx +++ b/docs/guides/configuration.mdx @@ -70,6 +70,31 @@ spec: storage: 5Gi ``` +### Leadership handover on termination {#keeper-leadership-handover} + +The Keeper container has a default `preStop` hook that asks the replica to hand off Raft +leadership before it stops, and waits for the handover to complete before letting the +container terminate. This runs on every graceful stop of the container, not just +operator-initiated rollouts: a descheduler eviction, a node drain, or an autoscaler +action all trigger the same hook, since it is a kubelet-level `preStop` handler rather +than reconcile-loop logic. + +Without this, restarting the current leader leaves the ensemble without a leader until a +new election completes, which can take several seconds under load and shows up to +ClickHouse as Keeper connection loss and elevated query latency. The hook asks the +replica to yield leadership (Keeper's `ydld` four-letter-word command, sent over the +`http_control` port) and then polls the replica's status until it is no longer the +leader, rather than firing the request and immediately letting the container stop — a +fire-and-forget request can still lose the race with `SIGTERM` under write load, so +waiting is what keeps the leaderless window bounded. + +The hook needs no configuration and degrades gracefully: on Keeper versions older than +26.3, which don't yet serve the leadership commands used here, the request is a no-op and +the container stops as it would without the hook. It runs a shell script, so it requires +`bash` in the image; if you run a distroless or scratch Keeper image, override or remove +it via `containerTemplate.lifecycle` — see +[API Reference](/products/kubernetes-operator/reference/api-reference#containertemplatespec). + ## Storage configuration {#storage-configuration} Configure persistent storage with `dataVolumeClaimSpec`, a standard Kubernetes diff --git a/docs/reference/api-reference.mdx b/docs/reference/api-reference.mdx index 9af86fd67..30ac43a35 100644 --- a/docs/reference/api-reference.mdx +++ b/docs/reference/api-reference.mdx @@ -203,6 +203,7 @@ ContainerTemplateSpec describes the container configuration overrides for the cl | `livenessProbe` | [Probe](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#probe-v1-core) | LivenessProbe overrides the operator's default liveness probe. | false | | | `readinessProbe` | [Probe](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#probe-v1-core) | ReadinessProbe overrides the operator's default readiness probe. | false | | | `startupProbe` | [Probe](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#probe-v1-core) | StartupProbe holds back the liveness and readiness probes until it succeeds, giving slow-starting
containers time to come up. The operator sets no startup probe by default. | false | | +| `lifecycle` | [Lifecycle](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#lifecycle-v1-core) | Lifecycle defines actions the kubelet triggers on container lifecycle events (PostStart/PreStop).
A non-nil Lifecycle fully replaces operator defaults; the user owns the entire struct. | false | | Appears in: - [ClickHouseClusterSpec](#clickhouseclusterspec) diff --git a/docs/styles/config/vocabularies/ClickHouse/accept.txt b/docs/styles/config/vocabularies/ClickHouse/accept.txt index 44b6c86ff..7d355976d 100644 --- a/docs/styles/config/vocabularies/ClickHouse/accept.txt +++ b/docs/styles/config/vocabularies/ClickHouse/accept.txt @@ -45,3 +45,7 @@ maxSkew minDomains [Rr]esyncs? [Rr]eloadable +kubelet +distroless +descheduler +rollouts diff --git a/internal/controller/keeper/commands.go b/internal/controller/keeper/commands.go index 680096d19..a2283a68c 100644 --- a/internal/controller/keeper/commands.go +++ b/internal/controller/keeper/commands.go @@ -15,6 +15,9 @@ import ( const ( FLWCommand = "mntr" + // FLWYieldLeadership is the Keeper four-letter-word command that asks a leader to hand + // Raft leadership to a peer, issued via the HTTP commands API on PortHTTPControl. + FLWYieldLeadership = "ydld" ModeLeader = "leader" ModeFollower = "follower" diff --git a/internal/controller/keeper/lifecycle.go b/internal/controller/keeper/lifecycle.go new file mode 100644 index 000000000..46835543c --- /dev/null +++ b/internal/controller/keeper/lifecycle.go @@ -0,0 +1,45 @@ +package keeper + +import "fmt" + +// preStopPollAttempts and preStopPollInterval bound how long the PreStop hook waits for +// the outgoing leader to hand off Raft leadership, and for a new leader to be elected, +// before the kubelet is allowed to send SIGTERM. 10 * 0.5s = 5s, comfortably inside the +// pod's default 30s terminationGracePeriodSeconds. This cap only matters when http_control has stopped +// answering entirely (a hung server): a healthy handover finishes in well under a +// second even under heavy write load, and a server that answers with a non-leader role +// or an error exits the loop immediately. +const ( + preStopPollAttempts = 10 + preStopPollInterval = "0.5" // seconds, passed to `sleep` +) + +// buildPreStopScript returns the PreStop hook script run inside the Keeper container. +// It asks the current replica to yield Raft leadership (the FLWYieldLeadership four +// letter word, sent over the HTTP commands API), then polls the same http_control +// port's /ready endpoint until the replica's role is no longer "leader" and the quorum +// reports a leader ("hasLeader":true), capped at preStopPollAttempts * preStopPollInterval. +// Waiting for the new leader (rather than only for this replica to step down) keeps the +// yielding replica alive to vote in the election. Only "hasLeader":false keeps the loop +// waiting, so versions whose /ready lacks the field fall back to the role check alone. +// It talks to http_control specifically (not the client port) because that port stays +// plain HTTP even when TLS is required for client connections; see templateContainer's +// readinessProbe for the same reasoning. +// +// Requires /bin/bash: /dev/tcp/... is a bash builtin, not POSIX sh, and the image ships +// no curl. On an image without bash, the exec fails immediately and the kubelet records +// a FailedPreStopHook event; termination proceeds without hanging. +// +// On Keeper versions without the commands API (pre-26.3), the yield-leadership request +// 404s harmlessly (swallowed below) and the script falls straight into the /ready poll, +// which exists on every supported version, so behavior degrades to "wait once, then +// proceed" rather than failing, and no cr.Status.Version gate is needed here. A gate +// would also be wrong for another reason: status.version is only populated +// asynchronously from mntr on running replicas, so a template that depends on it would +// change after pod startup and roll every freshly created cluster once. +func buildPreStopScript() string { + return fmt.Sprintf(`h() { exec 3<>/dev/tcp/127.0.0.1/%[1]d; printf 'GET %%s HTTP/1.0\r\n\r\n' "$1" >&3; cat <&3; exec 3<&-; } +h '/api/v1/commands?command=%[2]s' >/dev/null 2>&1 +for i in $(seq 1 %[3]d); do h /ready 2>/dev/null | grep -Eq '"role":"leader"|"hasLeader":false' || exit 0; sleep %[4]s; done +`, PortHTTPControl, FLWYieldLeadership, preStopPollAttempts, preStopPollInterval) +} diff --git a/internal/controller/keeper/templates.go b/internal/controller/keeper/templates.go index 57c55d392..7056d9c3c 100644 --- a/internal/controller/keeper/templates.go +++ b/internal/controller/keeper/templates.go @@ -525,6 +525,13 @@ func templateContainer(cr *v1.KeeperCluster) (corev1.Container, error) { TerminationMessagePath: corev1.TerminationMessagePathDefault, TerminationMessagePolicy: corev1.TerminationMessageReadFile, SecurityContext: controller.DefaultContainerSecurityContext(), + Lifecycle: &corev1.Lifecycle{ + PreStop: &corev1.LifecycleHandler{ + Exec: &corev1.ExecAction{ + Command: []string{"/bin/bash", "-c", buildPreStopScript()}, + }, + }, + }, } if !cr.Spec.Settings.TLS.Enabled || !cr.Spec.Settings.TLS.Required { diff --git a/internal/controller/keeper/templates_test.go b/internal/controller/keeper/templates_test.go index d57e7b465..2df99f629 100644 --- a/internal/controller/keeper/templates_test.go +++ b/internal/controller/keeper/templates_test.go @@ -410,3 +410,78 @@ var _ = Describe("TopologySpreadConstraints", func() { Expect(tsc.LabelSelector.MatchLabels).To(HaveKeyWithValue(controllerutil.LabelAppKey, cr.SpecificName())) }) }) + +var _ = Describe("PreStopLeadershipHandover", func() { + It("asks the server to yield leadership then waits for it to lose the role before stopping", func() { + cr := &v1.KeeperCluster{ + Name: "test", + Spec: v1.KeeperClusterSpec{Replicas: new(int32(3))}, + } + + container, err := templateContainer(cr) + Expect(err).NotTo(HaveOccurred()) + + Expect(container.Lifecycle).NotTo(BeNil()) + Expect(container.Lifecycle.PreStop).NotTo(BeNil()) + Expect(container.Lifecycle.PreStop.Exec).NotTo(BeNil()) + Expect(container.Lifecycle.PreStop.Exec.Command).To(Equal([]string{"/bin/bash", "-c", buildPreStopScript()})) + + script := buildPreStopScript() + Expect(script).To(ContainSubstring("/dev/tcp/127.0.0.1/9123")) + Expect(script).To(ContainSubstring("command=ydld")) + Expect(script).To(ContainSubstring(`"role":"leader"`)) + Expect(script).To(ContainSubstring(`"hasLeader":false`)) + + // Existing probes must be unaffected by the new default. + Expect(container.LivenessProbe).NotTo(BeNil()) + Expect(container.ReadinessProbe).NotTo(BeNil()) + }) + + It("stays plain HTTP on the http_control port even when TLS is required for client connections", func() { + cr := &v1.KeeperCluster{ + Name: "test", + Spec: v1.KeeperClusterSpec{ + Replicas: new(int32(3)), + Settings: v1.KeeperSettings{ + TLS: v1.ClusterTLSSpec{Enabled: true, Required: true}, + }, + }, + } + + container, err := templateContainer(cr) + Expect(err).NotTo(HaveOccurred()) + Expect(container.Lifecycle.PreStop.Exec.Command).To(Equal([]string{"/bin/bash", "-c", buildPreStopScript()})) + }) + + It("does not vary with the observed version, which arrives asynchronously after pod startup", func() { + lifecycleFor := func(version string) *corev1.Lifecycle { + cr := &v1.KeeperCluster{ + Name: "test", + Spec: v1.KeeperClusterSpec{Replicas: new(int32(3))}, + Status: v1.KeeperClusterStatus{Version: version}, + } + + container, err := templateContainer(cr) + Expect(err).NotTo(HaveOccurred()) + + return container.Lifecycle + } + + // A template that changed with Status.Version would roll every freshly created cluster + // once it started reporting in. + empty := lifecycleFor("") + Expect(lifecycleFor("25.8.32.4")).To(Equal(empty)) + Expect(lifecycleFor("26.7.5.10")).To(Equal(empty)) + }) + + It("is harmless on a single-replica cluster since the script takes no CR-derived input", func() { + cr := &v1.KeeperCluster{ + Name: "test", + Spec: v1.KeeperClusterSpec{Replicas: new(int32(1))}, + } + + container, err := templateContainer(cr) + Expect(err).NotTo(HaveOccurred()) + Expect(container.Lifecycle.PreStop.Exec.Command).To(Equal([]string{"/bin/bash", "-c", buildPreStopScript()})) + }) +}) diff --git a/internal/controller/overrides.go b/internal/controller/overrides.go index dc838f642..f745488d4 100644 --- a/internal/controller/overrides.go +++ b/internal/controller/overrides.go @@ -163,6 +163,10 @@ func ApplyContainerTemplateOverrides(container *corev1.Container, t *v1.Containe base.StartupProbe = nil } + if t.Lifecycle != nil { + base.Lifecycle = nil + } + patchContainer := corev1.Container{ Name: base.Name, Image: t.Image.String(), @@ -174,6 +178,7 @@ func ApplyContainerTemplateOverrides(container *corev1.Container, t *v1.Containe LivenessProbe: t.LivenessProbe, ReadinessProbe: t.ReadinessProbe, StartupProbe: t.StartupProbe, + Lifecycle: t.Lifecycle, } mergedContainer, err := patchResource(base, patchContainer, containerSchema) diff --git a/internal/controller/overrides_test.go b/internal/controller/overrides_test.go index 8ff4467e8..cb3bab5cf 100644 --- a/internal/controller/overrides_test.go +++ b/internal/controller/overrides_test.go @@ -324,6 +324,48 @@ var _ = Describe("ApplyContainerTemplateOverrides", func() { Expect(container.StartupProbe).To(BeNil()) }) }) + + Describe("Lifecycle", func() { + It("should preserve operator Lifecycle when user Lifecycle is nil", func() { + container, err := ApplyContainerTemplateOverrides( + &corev1.Container{ + Name: "server", + Lifecycle: &corev1.Lifecycle{ + PreStop: &corev1.LifecycleHandler{Exec: &corev1.ExecAction{Command: []string{"sleep", "1"}}}, + }, + }, + &v1.ContainerTemplateSpec{}, + ) + + Expect(err).ToNot(HaveOccurred()) + Expect(container.Lifecycle).NotTo(BeNil()) + Expect(container.Lifecycle.PreStop).NotTo(BeNil()) + Expect(container.Lifecycle.PreStop.Exec.Command).To(Equal([]string{"sleep", "1"})) + }) + + It("should fully replace operator Lifecycle when user provides one", func() { + container, err := ApplyContainerTemplateOverrides( + &corev1.Container{ + Name: "server", + Lifecycle: &corev1.Lifecycle{ + PreStop: &corev1.LifecycleHandler{Exec: &corev1.ExecAction{Command: []string{"sleep", "1"}}}, + }, + }, + &v1.ContainerTemplateSpec{ + Lifecycle: &corev1.Lifecycle{ + PostStart: &corev1.LifecycleHandler{Exec: &corev1.ExecAction{Command: []string{"echo", "hi"}}}, + }, + }, + ) + + Expect(err).ToNot(HaveOccurred()) + Expect(container.Lifecycle).NotTo(BeNil()) + Expect(container.Lifecycle.PreStop).To(BeNil(), + "operator PreStop must NOT survive: a user-provided Lifecycle fully replaces operator defaults") + Expect(container.Lifecycle.PostStart).NotTo(BeNil()) + Expect(container.Lifecycle.PostStart.Exec.Command).To(Equal([]string{"echo", "hi"})) + }) + }) }) var _ = Describe("ApplyPodTemplateOverrides", func() { diff --git a/test/deploy/deploy_test.go b/test/deploy/deploy_test.go index a50673f26..33fb95014 100644 --- a/test/deploy/deploy_test.go +++ b/test/deploy/deploy_test.go @@ -427,9 +427,14 @@ var _ = Describe("Operator upgrade", Ordered, ContinueOnFailure, Label("upgrade" env.WaitDeploymentAvailable(ctx, namespace, deploymentName, 3*time.Minute) By("updating keeper and verifying it stays writable", func() { - Expect(k8sClient.Get(ctx, keeperCR.NamespacedName(), &keeperCR)).To(Succeed()) - keeperCR.Spec.Annotations = map[string]string{"e2e.clickhouse.com/upgrade": "reconciled"} - Expect(k8sClient.Update(ctx, &keeperCR)).To(Succeed()) + // The freshly upgraded operator may be writing status onto this object in the same + // instant, bumping its resourceVersion between our Get and Update, so retry through + // the resulting conflict rather than fetching only once. + Eventually(func(g Gomega) { + g.Expect(k8sClient.Get(ctx, keeperCR.NamespacedName(), &keeperCR)).To(Succeed()) + keeperCR.Spec.Annotations = map[string]string{"e2e.clickhouse.com/upgrade": "reconciled"} + g.Expect(k8sClient.Update(ctx, &keeperCR)).To(Succeed()) + }, "30s", "2s").Should(Succeed()) Eventually(func(g Gomega) { var cluster v1.KeeperCluster g.Expect(k8sClient.Get(ctx, keeperCR.NamespacedName(), &cluster)).To(Succeed()) @@ -453,9 +458,13 @@ var _ = Describe("Operator upgrade", Ordered, ContinueOnFailure, Label("upgrade" }) By("updating clickhouse and verifying its health", func() { - Expect(k8sClient.Get(ctx, chCR.NamespacedName(), &chCR)).To(Succeed()) - chCR.Spec.Annotations = map[string]string{"e2e.clickhouse.com/upgrade": "reconciled"} - Expect(k8sClient.Update(ctx, &chCR)).To(Succeed()) + // Same resourceVersion race as the keeper update above: retry through a concurrent + // status write from the operator instead of fetching only once. + Eventually(func(g Gomega) { + g.Expect(k8sClient.Get(ctx, chCR.NamespacedName(), &chCR)).To(Succeed()) + chCR.Spec.Annotations = map[string]string{"e2e.clickhouse.com/upgrade": "reconciled"} + g.Expect(k8sClient.Update(ctx, &chCR)).To(Succeed()) + }, "30s", "2s").Should(Succeed()) Eventually(func(g Gomega) { var cluster v1.ClickHouseCluster g.Expect(k8sClient.Get(ctx, chCR.NamespacedName(), &cluster)).To(Succeed())