From 78dddccc8116d4718f8a963ea5a0155f1a3816c9 Mon Sep 17 00:00:00 2001 From: Francesco Pantano Date: Mon, 9 Feb 2026 12:55:04 +0100 Subject: [PATCH 1/4] Probe configuration with override support Add ProbeOverrides interface and CreateProbeSet() function from lib-common for unified probe management across Cinder services. Enable probe customization through CRD overrides and remove code duplication. Updates all services (API, Scheduler, Volume, Backup) to use the new pattern with proper scheme handling and consistent defaults. In addition, webhook validation for probes have been introduced. Signed-off-by: Francesco Pantano --- .../cinder.openstack.org_cinderapis.yaml | 87 ++++ .../cinder.openstack.org_cinderbackups.yaml | 92 ++++ api/bases/cinder.openstack.org_cinders.yaml | 460 ++++++++++++++++++ ...cinder.openstack.org_cinderschedulers.yaml | 92 ++++ .../cinder.openstack.org_cindervolumes.yaml | 92 ++++ api/v1beta1/cinder_webhook.go | 89 ++++ api/v1beta1/cinderapi_types.go | 3 + api/v1beta1/cinderbackup_types.go | 11 + api/v1beta1/cinderscheduler_types.go | 11 + api/v1beta1/cindervolume_types.go | 12 + api/v1beta1/zz_generated.deepcopy.go | 52 ++ .../cinder.openstack.org_cinderapis.yaml | 87 ++++ .../cinder.openstack.org_cinderbackups.yaml | 92 ++++ .../bases/cinder.openstack.org_cinders.yaml | 460 ++++++++++++++++++ ...cinder.openstack.org_cinderschedulers.yaml | 92 ++++ .../cinder.openstack.org_cindervolumes.yaml | 92 ++++ internal/cinder/const.go | 24 + internal/cinderapi/statefuleset.go | 42 +- internal/cinderbackup/statefulset.go | 44 +- internal/cinderscheduler/statefulset.go | 43 +- internal/cindervolume/statefulset.go | 42 +- .../controller/cinderbackup_controller.go | 11 +- .../controller/cinderscheduler_controller.go | 11 +- .../controller/cindervolume_controller.go | 11 +- test/functional/base_test.go | 11 + test/functional/cinder_controller_test.go | 108 ++++ 26 files changed, 2071 insertions(+), 100 deletions(-) diff --git a/api/bases/cinder.openstack.org_cinderapis.yaml b/api/bases/cinder.openstack.org_cinderapis.yaml index 104c6d12..8d1137ff 100644 --- a/api/bases/cinder.openstack.org_cinderapis.yaml +++ b/api/bases/cinder.openstack.org_cinderapis.yaml @@ -1224,6 +1224,93 @@ spec: description: Override, provides the ability to override the generated manifest of several child resources. properties: + probes: + description: Override probes and other common fields in the StatefulSet + properties: + livenessProbes: + description: |- + Override configuration for the StatefulSet like Probes and other tunable + fields + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + readinessProbes: + description: |- + ProbeConf - the configuration for liveness and readiness probes + LivenessPath - Endpoint path for the liveness probe + ReadinessPath - Endpoint path for the readiness probe + InitialDelaySeconds - Number of seconds after the container starts before liveness/readiness probes are initiated + TimeoutSeconds - Number of seconds after which the probe times out + PeriodSeconds - How often (in seconds) to perform the probe + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + startupProbes: + description: |- + ProbeConf - the configuration for liveness and readiness probes + LivenessPath - Endpoint path for the liveness probe + ReadinessPath - Endpoint path for the readiness probe + InitialDelaySeconds - Number of seconds after the container starts before liveness/readiness probes are initiated + TimeoutSeconds - Number of seconds after which the probe times out + PeriodSeconds - How often (in seconds) to perform the probe + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + type: object service: additionalProperties: description: |- diff --git a/api/bases/cinder.openstack.org_cinderbackups.yaml b/api/bases/cinder.openstack.org_cinderbackups.yaml index 3198f681..45313818 100644 --- a/api/bases/cinder.openstack.org_cinderbackups.yaml +++ b/api/bases/cinder.openstack.org_cinderbackups.yaml @@ -1220,6 +1220,98 @@ spec: notificationsURLSecret: description: Secret containing Notifications transport URL type: string + override: + description: Override, provides the ability to override the generated + manifest of several child resources. + properties: + probes: + description: Override probes and other common fields in the StatefulSet + properties: + livenessProbes: + description: |- + Override configuration for the StatefulSet like Probes and other tunable + fields + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + readinessProbes: + description: |- + ProbeConf - the configuration for liveness and readiness probes + LivenessPath - Endpoint path for the liveness probe + ReadinessPath - Endpoint path for the readiness probe + InitialDelaySeconds - Number of seconds after the container starts before liveness/readiness probes are initiated + TimeoutSeconds - Number of seconds after which the probe times out + PeriodSeconds - How often (in seconds) to perform the probe + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + startupProbes: + description: |- + ProbeConf - the configuration for liveness and readiness probes + LivenessPath - Endpoint path for the liveness probe + ReadinessPath - Endpoint path for the readiness probe + InitialDelaySeconds - Number of seconds after the container starts before liveness/readiness probes are initiated + TimeoutSeconds - Number of seconds after which the probe times out + PeriodSeconds - How often (in seconds) to perform the probe + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + type: object + type: object passwordSelectors: default: service: CinderPassword diff --git a/api/bases/cinder.openstack.org_cinders.yaml b/api/bases/cinder.openstack.org_cinders.yaml index b752efab..53a9d786 100644 --- a/api/bases/cinder.openstack.org_cinders.yaml +++ b/api/bases/cinder.openstack.org_cinders.yaml @@ -100,6 +100,94 @@ spec: description: Override, provides the ability to override the generated manifest of several child resources. properties: + probes: + description: Override probes and other common fields in the + StatefulSet + properties: + livenessProbes: + description: |- + Override configuration for the StatefulSet like Probes and other tunable + fields + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + readinessProbes: + description: |- + ProbeConf - the configuration for liveness and readiness probes + LivenessPath - Endpoint path for the liveness probe + ReadinessPath - Endpoint path for the readiness probe + InitialDelaySeconds - Number of seconds after the container starts before liveness/readiness probes are initiated + TimeoutSeconds - Number of seconds after which the probe times out + PeriodSeconds - How often (in seconds) to perform the probe + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + startupProbes: + description: |- + ProbeConf - the configuration for liveness and readiness probes + LivenessPath - Endpoint path for the liveness probe + ReadinessPath - Endpoint path for the readiness probe + InitialDelaySeconds - Number of seconds after the container starts before liveness/readiness probes are initiated + TimeoutSeconds - Number of seconds after which the probe times out + PeriodSeconds - How often (in seconds) to perform the probe + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + type: object service: additionalProperties: description: |- @@ -408,6 +496,99 @@ spec: NodeSelector to target subset of worker nodes running this service. Setting here overrides any global NodeSelector settings within the Cinder CR. type: object + override: + description: Override, provides the ability to override the generated + manifest of several child resources. + properties: + probes: + description: Override probes and other common fields in the + StatefulSet + properties: + livenessProbes: + description: |- + Override configuration for the StatefulSet like Probes and other tunable + fields + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + readinessProbes: + description: |- + ProbeConf - the configuration for liveness and readiness probes + LivenessPath - Endpoint path for the liveness probe + ReadinessPath - Endpoint path for the readiness probe + InitialDelaySeconds - Number of seconds after the container starts before liveness/readiness probes are initiated + TimeoutSeconds - Number of seconds after which the probe times out + PeriodSeconds - How often (in seconds) to perform the probe + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + startupProbes: + description: |- + ProbeConf - the configuration for liveness and readiness probes + LivenessPath - Endpoint path for the liveness probe + ReadinessPath - Endpoint path for the readiness probe + InitialDelaySeconds - Number of seconds after the container starts before liveness/readiness probes are initiated + TimeoutSeconds - Number of seconds after which the probe times out + PeriodSeconds - How often (in seconds) to perform the probe + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + type: object + type: object replicas: default: 1 description: Replicas - Cinder Backup Replicas @@ -532,6 +713,99 @@ spec: NodeSelector to target subset of worker nodes running this service. Setting here overrides any global NodeSelector settings within the Cinder CR. type: object + override: + description: Override, provides the ability to override the + generated manifest of several child resources. + properties: + probes: + description: Override probes and other common fields in + the StatefulSet + properties: + livenessProbes: + description: |- + Override configuration for the StatefulSet like Probes and other tunable + fields + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + readinessProbes: + description: |- + ProbeConf - the configuration for liveness and readiness probes + LivenessPath - Endpoint path for the liveness probe + ReadinessPath - Endpoint path for the readiness probe + InitialDelaySeconds - Number of seconds after the container starts before liveness/readiness probes are initiated + TimeoutSeconds - Number of seconds after which the probe times out + PeriodSeconds - How often (in seconds) to perform the probe + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + startupProbes: + description: |- + ProbeConf - the configuration for liveness and readiness probes + LivenessPath - Endpoint path for the liveness probe + ReadinessPath - Endpoint path for the readiness probe + InitialDelaySeconds - Number of seconds after the container starts before liveness/readiness probes are initiated + TimeoutSeconds - Number of seconds after which the probe times out + PeriodSeconds - How often (in seconds) to perform the probe + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + type: object + type: object replicas: default: 1 description: Replicas - Cinder Backup Replicas @@ -658,6 +932,99 @@ spec: NodeSelector to target subset of worker nodes running this service. Setting here overrides any global NodeSelector settings within the Cinder CR. type: object + override: + description: Override, provides the ability to override the generated + manifest of several child resources. + properties: + probes: + description: Override probes and other common fields in the + StatefulSet + properties: + livenessProbes: + description: |- + Override configuration for the StatefulSet like Probes and other tunable + fields + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + readinessProbes: + description: |- + ProbeConf - the configuration for liveness and readiness probes + LivenessPath - Endpoint path for the liveness probe + ReadinessPath - Endpoint path for the readiness probe + InitialDelaySeconds - Number of seconds after the container starts before liveness/readiness probes are initiated + TimeoutSeconds - Number of seconds after which the probe times out + PeriodSeconds - How often (in seconds) to perform the probe + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + startupProbes: + description: |- + ProbeConf - the configuration for liveness and readiness probes + LivenessPath - Endpoint path for the liveness probe + ReadinessPath - Endpoint path for the readiness probe + InitialDelaySeconds - Number of seconds after the container starts before liveness/readiness probes are initiated + TimeoutSeconds - Number of seconds after which the probe times out + PeriodSeconds - How often (in seconds) to perform the probe + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + type: object + type: object replicas: default: 1 description: Replicas - Cinder Scheduler Replicas @@ -782,6 +1149,99 @@ spec: NodeSelector to target subset of worker nodes running this service. Setting here overrides any global NodeSelector settings within the Cinder CR. type: object + override: + description: Override, provides the ability to override the + generated manifest of several child resources. + properties: + probes: + description: Override probes and other common fields in + the StatefulSet + properties: + livenessProbes: + description: |- + Override configuration for the StatefulSet like Probes and other tunable + fields + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + readinessProbes: + description: |- + ProbeConf - the configuration for liveness and readiness probes + LivenessPath - Endpoint path for the liveness probe + ReadinessPath - Endpoint path for the readiness probe + InitialDelaySeconds - Number of seconds after the container starts before liveness/readiness probes are initiated + TimeoutSeconds - Number of seconds after which the probe times out + PeriodSeconds - How often (in seconds) to perform the probe + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + startupProbes: + description: |- + ProbeConf - the configuration for liveness and readiness probes + LivenessPath - Endpoint path for the liveness probe + ReadinessPath - Endpoint path for the readiness probe + InitialDelaySeconds - Number of seconds after the container starts before liveness/readiness probes are initiated + TimeoutSeconds - Number of seconds after which the probe times out + PeriodSeconds - How often (in seconds) to perform the probe + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + type: object + type: object replicas: default: 1 description: Replicas - Cinder Volume Replicas diff --git a/api/bases/cinder.openstack.org_cinderschedulers.yaml b/api/bases/cinder.openstack.org_cinderschedulers.yaml index 5f1b37ba..c93d0b68 100644 --- a/api/bases/cinder.openstack.org_cinderschedulers.yaml +++ b/api/bases/cinder.openstack.org_cinderschedulers.yaml @@ -1220,6 +1220,98 @@ spec: notificationsURLSecret: description: Secret containing Notifications transport URL type: string + override: + description: Override, provides the ability to override the generated + manifest of several child resources. + properties: + probes: + description: Override probes and other common fields in the StatefulSet + properties: + livenessProbes: + description: |- + Override configuration for the StatefulSet like Probes and other tunable + fields + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + readinessProbes: + description: |- + ProbeConf - the configuration for liveness and readiness probes + LivenessPath - Endpoint path for the liveness probe + ReadinessPath - Endpoint path for the readiness probe + InitialDelaySeconds - Number of seconds after the container starts before liveness/readiness probes are initiated + TimeoutSeconds - Number of seconds after which the probe times out + PeriodSeconds - How often (in seconds) to perform the probe + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + startupProbes: + description: |- + ProbeConf - the configuration for liveness and readiness probes + LivenessPath - Endpoint path for the liveness probe + ReadinessPath - Endpoint path for the readiness probe + InitialDelaySeconds - Number of seconds after the container starts before liveness/readiness probes are initiated + TimeoutSeconds - Number of seconds after which the probe times out + PeriodSeconds - How often (in seconds) to perform the probe + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + type: object + type: object passwordSelectors: default: service: CinderPassword diff --git a/api/bases/cinder.openstack.org_cindervolumes.yaml b/api/bases/cinder.openstack.org_cindervolumes.yaml index 79ee5ded..80590174 100644 --- a/api/bases/cinder.openstack.org_cindervolumes.yaml +++ b/api/bases/cinder.openstack.org_cindervolumes.yaml @@ -1220,6 +1220,98 @@ spec: notificationsURLSecret: description: Secret containing Notifications transport URL type: string + override: + description: Override, provides the ability to override the generated + manifest of several child resources. + properties: + probes: + description: Override probes and other common fields in the StatefulSet + properties: + livenessProbes: + description: |- + Override configuration for the StatefulSet like Probes and other tunable + fields + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + readinessProbes: + description: |- + ProbeConf - the configuration for liveness and readiness probes + LivenessPath - Endpoint path for the liveness probe + ReadinessPath - Endpoint path for the readiness probe + InitialDelaySeconds - Number of seconds after the container starts before liveness/readiness probes are initiated + TimeoutSeconds - Number of seconds after which the probe times out + PeriodSeconds - How often (in seconds) to perform the probe + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + startupProbes: + description: |- + ProbeConf - the configuration for liveness and readiness probes + LivenessPath - Endpoint path for the liveness probe + ReadinessPath - Endpoint path for the readiness probe + InitialDelaySeconds - Number of seconds after the container starts before liveness/readiness probes are initiated + TimeoutSeconds - Number of seconds after which the probe times out + PeriodSeconds - How often (in seconds) to perform the probe + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + type: object + type: object passwordSelectors: default: service: CinderPassword diff --git a/api/v1beta1/cinder_webhook.go b/api/v1beta1/cinder_webhook.go index e8784099..6684a167 100644 --- a/api/v1beta1/cinder_webhook.go +++ b/api/v1beta1/cinder_webhook.go @@ -263,6 +263,10 @@ func (spec *CinderSpec) ValidateCreate( allErrs = append(allErrs, bkpErrs...) allWarns = append(allWarns, bkpWarns...) + // Probes validation + probeErrs := spec.ValidateProbes(basePath) + allErrs = append(allErrs, probeErrs...) + return allWarns, allErrs } @@ -292,6 +296,10 @@ func (spec *CinderSpecCore) ValidateCreate( allErrs = append(allErrs, bkpErrs...) allWarns = append(allWarns, bkpWarns...) + // Probes validation + probeErrs := spec.ValidateProbes(basePath) + allErrs = append(allErrs, probeErrs...) + return allWarns, allErrs } @@ -370,6 +378,10 @@ func (spec *CinderSpec) ValidateUpdate( allErrs = append(allErrs, bkpErrs...) allWarns = append(allWarns, bkpWarns...) + // Probes validation + probeErrs := spec.ValidateProbes(basePath) + allErrs = append(allErrs, probeErrs...) + return allWarns, allErrs } @@ -401,6 +413,10 @@ func (spec *CinderSpecCore) ValidateUpdate( allErrs = append(allErrs, bkpErrs...) allWarns = append(allWarns, bkpWarns...) + // Probes validation + probeErrs := spec.ValidateProbes(basePath) + allErrs = append(allErrs, probeErrs...) + return allWarns, allErrs } @@ -594,3 +610,76 @@ func (spec *CinderSpecCore) ValidateCinderBackup(basePath *field.Path) ([]string } return allWarns, allErrs } + +// ValidateProbes - +// NOTE: Remove this after the API spec refactoring +func (spec *CinderSpec) ValidateProbes(basePath *field.Path) field.ErrorList { + var allErrs field.ErrorList + + // CinderAPI probes validation + apiPath := basePath.Child("cinderAPI").Child("override") + apiProbeErrs := spec.CinderAPI.Override.Probes.ValidateProbes( + apiPath.Child("probes")) + allErrs = append(allErrs, apiProbeErrs...) + + // CinderScheduler probes validation + schedPath := basePath.Child("cinderScheduler").Child("override") + schedProbeErrs := spec.CinderScheduler.Override.Probes.ValidateProbes( + schedPath.Child("probes")) + allErrs = append(allErrs, schedProbeErrs...) + + // CinderVolumes probes validation + volPath := basePath.Child("cinderVolumes") + for name, vol := range spec.CinderVolumes { + volProbeErrs := vol.Override.Probes.ValidateProbes( + volPath.Child(name).Child("override")) + allErrs = append(allErrs, volProbeErrs...) + } + + // CinderBackups probes validation + bkpPath := basePath.Child("cinderBackups") + if spec.CinderBackups != nil { + for name, bkp := range *spec.CinderBackups { + bkpProbeErrs := bkp.Override.Probes.ValidateProbes( + bkpPath.Child(name).Child("override")) + allErrs = append(allErrs, bkpProbeErrs...) + } + } + return allErrs +} + +// ValidateProbes - +func (spec *CinderSpecCore) ValidateProbes(basePath *field.Path) field.ErrorList { + var allErrs field.ErrorList + + // CinderAPI probes validation + apiPath := basePath.Child("cinderAPI").Child("override") + apiProbeErrs := spec.CinderAPI.Override.Probes.ValidateProbes( + apiPath.Child("probes")) + allErrs = append(allErrs, apiProbeErrs...) + + // CinderScheduler probes validation + schedPath := basePath.Child("cinderScheduler").Child("override") + schedProbeErrs := spec.CinderScheduler.Override.Probes.ValidateProbes( + schedPath.Child("probes")) + allErrs = append(allErrs, schedProbeErrs...) + + // CinderVolumes probes validation + volPath := basePath.Child("cinderVolumes") + for name, vol := range spec.CinderVolumes { + volProbeErrs := vol.Override.Probes.ValidateProbes( + volPath.Child(name).Child("override")) + allErrs = append(allErrs, volProbeErrs...) + } + + // CinderBackups probes validation + bkpPath := basePath.Child("cinderBackups") + if spec.CinderBackups != nil { + for name, bkp := range *spec.CinderBackups { + bkpProbeErrs := bkp.Override.Probes.ValidateProbes( + bkpPath.Child(name).Child("override")) + allErrs = append(allErrs, bkpProbeErrs...) + } + } + return allErrs +} diff --git a/api/v1beta1/cinderapi_types.go b/api/v1beta1/cinderapi_types.go index 079a7461..1abcfea9 100644 --- a/api/v1beta1/cinderapi_types.go +++ b/api/v1beta1/cinderapi_types.go @@ -21,6 +21,7 @@ import ( condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" "github.com/openstack-k8s-operators/lib-common/modules/common/service" "github.com/openstack-k8s-operators/lib-common/modules/common/tls" + "github.com/openstack-k8s-operators/lib-common/modules/common/probes" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -59,6 +60,8 @@ type APIOverrideSpec struct { // Override configuration for the Service created to serve traffic to the cluster. // The key must be the endpoint type (public, internal) Service map[service.Endpoint]service.RoutedOverrideSpec `json:"service,omitempty"` + // Override probes and other common fields in the StatefulSet + Probes probes.OverrideSpec `json:"probes,omitempty"` } // AuthSpec defines authentication parameters diff --git a/api/v1beta1/cinderbackup_types.go b/api/v1beta1/cinderbackup_types.go index e873240c..9042f89f 100644 --- a/api/v1beta1/cinderbackup_types.go +++ b/api/v1beta1/cinderbackup_types.go @@ -21,8 +21,15 @@ import ( condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" "github.com/openstack-k8s-operators/lib-common/modules/common/tls" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/openstack-k8s-operators/lib-common/modules/common/probes" ) +// CinderBackupOverrideSpec to override the generated manifest of several child resources. +type BackupOverrideSpec struct { + // Override probes and other common fields in the StatefulSet + Probes probes.OverrideSpec `json:"probes,omitempty"` +} + // CinderBackupTemplate defines the input parameters for the Cinder Backup service type CinderBackupTemplateCore struct { // Common input parameters for the Cinder Backup service @@ -33,6 +40,10 @@ type CinderBackupTemplateCore struct { // +kubebuilder:validation:Minimum=0 // Replicas - Cinder Backup Replicas Replicas *int32 `json:"replicas"` + + // +kubebuilder:validation:Optional + // Override, provides the ability to override the generated manifest of several child resources. + Override BackupOverrideSpec `json:"override,omitempty"` } // CinderBackupTemplate defines the input parameters for the Cinder Backup service diff --git a/api/v1beta1/cinderscheduler_types.go b/api/v1beta1/cinderscheduler_types.go index 51d5954a..21367cf2 100644 --- a/api/v1beta1/cinderscheduler_types.go +++ b/api/v1beta1/cinderscheduler_types.go @@ -20,9 +20,16 @@ import ( topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" "github.com/openstack-k8s-operators/lib-common/modules/common/tls" + "github.com/openstack-k8s-operators/lib-common/modules/common/probes" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +// SchedulerOverrideSpec to override the generated manifest of several child resources. +type SchedulerOverrideSpec struct { + // Override probes and other common fields in the StatefulSet + Probes probes.OverrideSpec `json:"probes,omitempty"` +} + // CinderSchedulerTemplate defines the input parameters for the Cinder Scheduler service type CinderSchedulerTemplateCore struct { // Common input parameters for the Cinder Scheduler service @@ -33,6 +40,10 @@ type CinderSchedulerTemplateCore struct { // +kubebuilder:validation:Minimum=0 // Replicas - Cinder Scheduler Replicas Replicas *int32 `json:"replicas"` + + // +kubebuilder:validation:Optional + // Override, provides the ability to override the generated manifest of several child resources. + Override SchedulerOverrideSpec `json:"override,omitempty"` } // CinderSchedulerTemplate defines the input parameters for the Cinder Scheduler service diff --git a/api/v1beta1/cindervolume_types.go b/api/v1beta1/cindervolume_types.go index a5aea96c..3245604c 100644 --- a/api/v1beta1/cindervolume_types.go +++ b/api/v1beta1/cindervolume_types.go @@ -19,6 +19,7 @@ package v1beta1 import ( topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/probes" "github.com/openstack-k8s-operators/lib-common/modules/common/tls" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -28,6 +29,13 @@ const ( Backend = "backend" ) + +// VolumeOverrideSpec to override the generated manifest of several child resources. +type VolumeOverrideSpec struct { + // Override probes and other common fields in the StatefulSet + Probes probes.OverrideSpec `json:"probes,omitempty"` +} + // CinderVolumeTemplate defines the input parameters for the Cinder Volume service type CinderVolumeTemplateCore struct { // Common input parameters for the Cinder Volume service @@ -39,6 +47,10 @@ type CinderVolumeTemplateCore struct { // +kubebuilder:validation:Maximum=1 // Replicas - Cinder Volume Replicas Replicas *int32 `json:"replicas"` + + // +kubebuilder:validation:Optional + // Override, provides the ability to override the generated manifest of several child resources. + Override VolumeOverrideSpec `json:"override,omitempty"` } // CinderVolumeTemplate defines the input parameters for the Cinder Volume service diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 0914910a..f24d3695 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -39,6 +39,7 @@ func (in *APIOverrideSpec) DeepCopyInto(out *APIOverrideSpec) { (*out)[key] = *val.DeepCopy() } } + in.Probes.DeepCopyInto(&out.Probes) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new APIOverrideSpec. @@ -66,6 +67,22 @@ func (in *AuthSpec) DeepCopy() *AuthSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BackupOverrideSpec) DeepCopyInto(out *BackupOverrideSpec) { + *out = *in + in.Probes.DeepCopyInto(&out.Probes) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupOverrideSpec. +func (in *BackupOverrideSpec) DeepCopy() *BackupOverrideSpec { + if in == nil { + return nil + } + out := new(BackupOverrideSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Cinder) DeepCopyInto(out *Cinder) { *out = *in @@ -459,6 +476,7 @@ func (in *CinderBackupTemplateCore) DeepCopyInto(out *CinderBackupTemplateCore) *out = new(int32) **out = **in } + in.Override.DeepCopyInto(&out.Override) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CinderBackupTemplateCore. @@ -704,6 +722,7 @@ func (in *CinderSchedulerTemplateCore) DeepCopyInto(out *CinderSchedulerTemplate *out = new(int32) **out = **in } + in.Override.DeepCopyInto(&out.Override) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CinderSchedulerTemplateCore. @@ -1137,6 +1156,7 @@ func (in *CinderVolumeTemplateCore) DeepCopyInto(out *CinderVolumeTemplateCore) *out = new(int32) **out = **in } + in.Override.DeepCopyInto(&out.Override) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CinderVolumeTemplateCore. @@ -1178,3 +1198,35 @@ func (in *PasswordSelector) DeepCopy() *PasswordSelector { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SchedulerOverrideSpec) DeepCopyInto(out *SchedulerOverrideSpec) { + *out = *in + in.Probes.DeepCopyInto(&out.Probes) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SchedulerOverrideSpec. +func (in *SchedulerOverrideSpec) DeepCopy() *SchedulerOverrideSpec { + if in == nil { + return nil + } + out := new(SchedulerOverrideSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VolumeOverrideSpec) DeepCopyInto(out *VolumeOverrideSpec) { + *out = *in + in.Probes.DeepCopyInto(&out.Probes) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeOverrideSpec. +func (in *VolumeOverrideSpec) DeepCopy() *VolumeOverrideSpec { + if in == nil { + return nil + } + out := new(VolumeOverrideSpec) + in.DeepCopyInto(out) + return out +} diff --git a/config/crd/bases/cinder.openstack.org_cinderapis.yaml b/config/crd/bases/cinder.openstack.org_cinderapis.yaml index 104c6d12..8d1137ff 100644 --- a/config/crd/bases/cinder.openstack.org_cinderapis.yaml +++ b/config/crd/bases/cinder.openstack.org_cinderapis.yaml @@ -1224,6 +1224,93 @@ spec: description: Override, provides the ability to override the generated manifest of several child resources. properties: + probes: + description: Override probes and other common fields in the StatefulSet + properties: + livenessProbes: + description: |- + Override configuration for the StatefulSet like Probes and other tunable + fields + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + readinessProbes: + description: |- + ProbeConf - the configuration for liveness and readiness probes + LivenessPath - Endpoint path for the liveness probe + ReadinessPath - Endpoint path for the readiness probe + InitialDelaySeconds - Number of seconds after the container starts before liveness/readiness probes are initiated + TimeoutSeconds - Number of seconds after which the probe times out + PeriodSeconds - How often (in seconds) to perform the probe + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + startupProbes: + description: |- + ProbeConf - the configuration for liveness and readiness probes + LivenessPath - Endpoint path for the liveness probe + ReadinessPath - Endpoint path for the readiness probe + InitialDelaySeconds - Number of seconds after the container starts before liveness/readiness probes are initiated + TimeoutSeconds - Number of seconds after which the probe times out + PeriodSeconds - How often (in seconds) to perform the probe + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + type: object service: additionalProperties: description: |- diff --git a/config/crd/bases/cinder.openstack.org_cinderbackups.yaml b/config/crd/bases/cinder.openstack.org_cinderbackups.yaml index 3198f681..45313818 100644 --- a/config/crd/bases/cinder.openstack.org_cinderbackups.yaml +++ b/config/crd/bases/cinder.openstack.org_cinderbackups.yaml @@ -1220,6 +1220,98 @@ spec: notificationsURLSecret: description: Secret containing Notifications transport URL type: string + override: + description: Override, provides the ability to override the generated + manifest of several child resources. + properties: + probes: + description: Override probes and other common fields in the StatefulSet + properties: + livenessProbes: + description: |- + Override configuration for the StatefulSet like Probes and other tunable + fields + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + readinessProbes: + description: |- + ProbeConf - the configuration for liveness and readiness probes + LivenessPath - Endpoint path for the liveness probe + ReadinessPath - Endpoint path for the readiness probe + InitialDelaySeconds - Number of seconds after the container starts before liveness/readiness probes are initiated + TimeoutSeconds - Number of seconds after which the probe times out + PeriodSeconds - How often (in seconds) to perform the probe + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + startupProbes: + description: |- + ProbeConf - the configuration for liveness and readiness probes + LivenessPath - Endpoint path for the liveness probe + ReadinessPath - Endpoint path for the readiness probe + InitialDelaySeconds - Number of seconds after the container starts before liveness/readiness probes are initiated + TimeoutSeconds - Number of seconds after which the probe times out + PeriodSeconds - How often (in seconds) to perform the probe + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + type: object + type: object passwordSelectors: default: service: CinderPassword diff --git a/config/crd/bases/cinder.openstack.org_cinders.yaml b/config/crd/bases/cinder.openstack.org_cinders.yaml index b752efab..53a9d786 100644 --- a/config/crd/bases/cinder.openstack.org_cinders.yaml +++ b/config/crd/bases/cinder.openstack.org_cinders.yaml @@ -100,6 +100,94 @@ spec: description: Override, provides the ability to override the generated manifest of several child resources. properties: + probes: + description: Override probes and other common fields in the + StatefulSet + properties: + livenessProbes: + description: |- + Override configuration for the StatefulSet like Probes and other tunable + fields + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + readinessProbes: + description: |- + ProbeConf - the configuration for liveness and readiness probes + LivenessPath - Endpoint path for the liveness probe + ReadinessPath - Endpoint path for the readiness probe + InitialDelaySeconds - Number of seconds after the container starts before liveness/readiness probes are initiated + TimeoutSeconds - Number of seconds after which the probe times out + PeriodSeconds - How often (in seconds) to perform the probe + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + startupProbes: + description: |- + ProbeConf - the configuration for liveness and readiness probes + LivenessPath - Endpoint path for the liveness probe + ReadinessPath - Endpoint path for the readiness probe + InitialDelaySeconds - Number of seconds after the container starts before liveness/readiness probes are initiated + TimeoutSeconds - Number of seconds after which the probe times out + PeriodSeconds - How often (in seconds) to perform the probe + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + type: object service: additionalProperties: description: |- @@ -408,6 +496,99 @@ spec: NodeSelector to target subset of worker nodes running this service. Setting here overrides any global NodeSelector settings within the Cinder CR. type: object + override: + description: Override, provides the ability to override the generated + manifest of several child resources. + properties: + probes: + description: Override probes and other common fields in the + StatefulSet + properties: + livenessProbes: + description: |- + Override configuration for the StatefulSet like Probes and other tunable + fields + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + readinessProbes: + description: |- + ProbeConf - the configuration for liveness and readiness probes + LivenessPath - Endpoint path for the liveness probe + ReadinessPath - Endpoint path for the readiness probe + InitialDelaySeconds - Number of seconds after the container starts before liveness/readiness probes are initiated + TimeoutSeconds - Number of seconds after which the probe times out + PeriodSeconds - How often (in seconds) to perform the probe + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + startupProbes: + description: |- + ProbeConf - the configuration for liveness and readiness probes + LivenessPath - Endpoint path for the liveness probe + ReadinessPath - Endpoint path for the readiness probe + InitialDelaySeconds - Number of seconds after the container starts before liveness/readiness probes are initiated + TimeoutSeconds - Number of seconds after which the probe times out + PeriodSeconds - How often (in seconds) to perform the probe + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + type: object + type: object replicas: default: 1 description: Replicas - Cinder Backup Replicas @@ -532,6 +713,99 @@ spec: NodeSelector to target subset of worker nodes running this service. Setting here overrides any global NodeSelector settings within the Cinder CR. type: object + override: + description: Override, provides the ability to override the + generated manifest of several child resources. + properties: + probes: + description: Override probes and other common fields in + the StatefulSet + properties: + livenessProbes: + description: |- + Override configuration for the StatefulSet like Probes and other tunable + fields + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + readinessProbes: + description: |- + ProbeConf - the configuration for liveness and readiness probes + LivenessPath - Endpoint path for the liveness probe + ReadinessPath - Endpoint path for the readiness probe + InitialDelaySeconds - Number of seconds after the container starts before liveness/readiness probes are initiated + TimeoutSeconds - Number of seconds after which the probe times out + PeriodSeconds - How often (in seconds) to perform the probe + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + startupProbes: + description: |- + ProbeConf - the configuration for liveness and readiness probes + LivenessPath - Endpoint path for the liveness probe + ReadinessPath - Endpoint path for the readiness probe + InitialDelaySeconds - Number of seconds after the container starts before liveness/readiness probes are initiated + TimeoutSeconds - Number of seconds after which the probe times out + PeriodSeconds - How often (in seconds) to perform the probe + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + type: object + type: object replicas: default: 1 description: Replicas - Cinder Backup Replicas @@ -658,6 +932,99 @@ spec: NodeSelector to target subset of worker nodes running this service. Setting here overrides any global NodeSelector settings within the Cinder CR. type: object + override: + description: Override, provides the ability to override the generated + manifest of several child resources. + properties: + probes: + description: Override probes and other common fields in the + StatefulSet + properties: + livenessProbes: + description: |- + Override configuration for the StatefulSet like Probes and other tunable + fields + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + readinessProbes: + description: |- + ProbeConf - the configuration for liveness and readiness probes + LivenessPath - Endpoint path for the liveness probe + ReadinessPath - Endpoint path for the readiness probe + InitialDelaySeconds - Number of seconds after the container starts before liveness/readiness probes are initiated + TimeoutSeconds - Number of seconds after which the probe times out + PeriodSeconds - How often (in seconds) to perform the probe + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + startupProbes: + description: |- + ProbeConf - the configuration for liveness and readiness probes + LivenessPath - Endpoint path for the liveness probe + ReadinessPath - Endpoint path for the readiness probe + InitialDelaySeconds - Number of seconds after the container starts before liveness/readiness probes are initiated + TimeoutSeconds - Number of seconds after which the probe times out + PeriodSeconds - How often (in seconds) to perform the probe + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + type: object + type: object replicas: default: 1 description: Replicas - Cinder Scheduler Replicas @@ -782,6 +1149,99 @@ spec: NodeSelector to target subset of worker nodes running this service. Setting here overrides any global NodeSelector settings within the Cinder CR. type: object + override: + description: Override, provides the ability to override the + generated manifest of several child resources. + properties: + probes: + description: Override probes and other common fields in + the StatefulSet + properties: + livenessProbes: + description: |- + Override configuration for the StatefulSet like Probes and other tunable + fields + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + readinessProbes: + description: |- + ProbeConf - the configuration for liveness and readiness probes + LivenessPath - Endpoint path for the liveness probe + ReadinessPath - Endpoint path for the readiness probe + InitialDelaySeconds - Number of seconds after the container starts before liveness/readiness probes are initiated + TimeoutSeconds - Number of seconds after which the probe times out + PeriodSeconds - How often (in seconds) to perform the probe + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + startupProbes: + description: |- + ProbeConf - the configuration for liveness and readiness probes + LivenessPath - Endpoint path for the liveness probe + ReadinessPath - Endpoint path for the readiness probe + InitialDelaySeconds - Number of seconds after the container starts before liveness/readiness probes are initiated + TimeoutSeconds - Number of seconds after which the probe times out + PeriodSeconds - How often (in seconds) to perform the probe + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + type: object + type: object replicas: default: 1 description: Replicas - Cinder Volume Replicas diff --git a/config/crd/bases/cinder.openstack.org_cinderschedulers.yaml b/config/crd/bases/cinder.openstack.org_cinderschedulers.yaml index 5f1b37ba..c93d0b68 100644 --- a/config/crd/bases/cinder.openstack.org_cinderschedulers.yaml +++ b/config/crd/bases/cinder.openstack.org_cinderschedulers.yaml @@ -1220,6 +1220,98 @@ spec: notificationsURLSecret: description: Secret containing Notifications transport URL type: string + override: + description: Override, provides the ability to override the generated + manifest of several child resources. + properties: + probes: + description: Override probes and other common fields in the StatefulSet + properties: + livenessProbes: + description: |- + Override configuration for the StatefulSet like Probes and other tunable + fields + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + readinessProbes: + description: |- + ProbeConf - the configuration for liveness and readiness probes + LivenessPath - Endpoint path for the liveness probe + ReadinessPath - Endpoint path for the readiness probe + InitialDelaySeconds - Number of seconds after the container starts before liveness/readiness probes are initiated + TimeoutSeconds - Number of seconds after which the probe times out + PeriodSeconds - How often (in seconds) to perform the probe + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + startupProbes: + description: |- + ProbeConf - the configuration for liveness and readiness probes + LivenessPath - Endpoint path for the liveness probe + ReadinessPath - Endpoint path for the readiness probe + InitialDelaySeconds - Number of seconds after the container starts before liveness/readiness probes are initiated + TimeoutSeconds - Number of seconds after which the probe times out + PeriodSeconds - How often (in seconds) to perform the probe + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + type: object + type: object passwordSelectors: default: service: CinderPassword diff --git a/config/crd/bases/cinder.openstack.org_cindervolumes.yaml b/config/crd/bases/cinder.openstack.org_cindervolumes.yaml index 79ee5ded..80590174 100644 --- a/config/crd/bases/cinder.openstack.org_cindervolumes.yaml +++ b/config/crd/bases/cinder.openstack.org_cindervolumes.yaml @@ -1220,6 +1220,98 @@ spec: notificationsURLSecret: description: Secret containing Notifications transport URL type: string + override: + description: Override, provides the ability to override the generated + manifest of several child resources. + properties: + probes: + description: Override probes and other common fields in the StatefulSet + properties: + livenessProbes: + description: |- + Override configuration for the StatefulSet like Probes and other tunable + fields + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + readinessProbes: + description: |- + ProbeConf - the configuration for liveness and readiness probes + LivenessPath - Endpoint path for the liveness probe + ReadinessPath - Endpoint path for the readiness probe + InitialDelaySeconds - Number of seconds after the container starts before liveness/readiness probes are initiated + TimeoutSeconds - Number of seconds after which the probe times out + PeriodSeconds - How often (in seconds) to perform the probe + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + startupProbes: + description: |- + ProbeConf - the configuration for liveness and readiness probes + LivenessPath - Endpoint path for the liveness probe + ReadinessPath - Endpoint path for the readiness probe + InitialDelaySeconds - Number of seconds after the container starts before liveness/readiness probes are initiated + TimeoutSeconds - Number of seconds after which the probe times out + PeriodSeconds - How often (in seconds) to perform the probe + properties: + failureThreshold: + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + format: int32 + minimum: 0 + type: integer + path: + pattern: ^(/.*)?$ + type: string + periodSeconds: + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + format: int32 + minimum: 1 + type: integer + type: object + type: object + type: object passwordSelectors: default: service: CinderPassword diff --git a/internal/cinder/const.go b/internal/cinder/const.go index 0471dc04..949fd1ca 100644 --- a/internal/cinder/const.go +++ b/internal/cinder/const.go @@ -21,6 +21,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" + "github.com/openstack-k8s-operators/lib-common/modules/common/probes" "github.com/openstack-k8s-operators/lib-common/modules/storage" ) @@ -103,3 +104,26 @@ var CinderBackupPropagation = []storage.PropagationType{Cinder, CinderBackup} // It allows the CinderVolume pods to mount volumes destined to Cinder and CinderVolume // ServiceTypes var CinderVolumePropagation = []storage.PropagationType{Cinder, CinderVolume} + +// DefaultProbeConf - Default values applied to Cinder StatefulSets when no +// overrides are provided +var DefaultProbeConf = probes.OverrideSpec{ + LivenessProbes: &probes.ProbeConf{ + Path: "/healthcheck", + TimeoutSeconds: 10, + PeriodSeconds: 6, + InitialDelaySeconds: 10, + }, + ReadinessProbes: &probes.ProbeConf{ + Path: "/healthcheck", + TimeoutSeconds: 10, + PeriodSeconds: 10, + InitialDelaySeconds: 10, + }, + StartupProbes: &probes.ProbeConf{ + TimeoutSeconds: 10, + FailureThreshold: 12, + PeriodSeconds: 10, + InitialDelaySeconds: 10, + }, +} diff --git a/internal/cinderapi/statefuleset.go b/internal/cinderapi/statefuleset.go index e2717946..4e0d37c6 100644 --- a/internal/cinderapi/statefuleset.go +++ b/internal/cinderapi/statefuleset.go @@ -21,13 +21,13 @@ import ( memcachedv1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1" topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" "github.com/openstack-k8s-operators/lib-common/modules/common/env" + "github.com/openstack-k8s-operators/lib-common/modules/common/probes" "github.com/openstack-k8s-operators/lib-common/modules/common/service" "github.com/openstack-k8s-operators/lib-common/modules/common/tls" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" ) const ( @@ -47,34 +47,28 @@ func StatefulSet( runAsUser := int64(0) cinderUser := int64(cinderv1beta1.CinderUserID) - livenessProbe := &corev1.Probe{ - // TODO might need tuning - TimeoutSeconds: 5, - PeriodSeconds: 3, - InitialDelaySeconds: 5, + scheme := corev1.URISchemeHTTP + if instance.Spec.TLS.API.Enabled(service.EndpointPublic) { + scheme = corev1.URISchemeHTTPS } - readinessProbe := &corev1.Probe{ - // TODO might need tuning - TimeoutSeconds: 5, - PeriodSeconds: 5, - InitialDelaySeconds: 5, + + // Note that by default we create probes with the same URIScheme and port + probes, err := probes.CreateProbeSet( + int32(cinder.CinderPublicPort), + &scheme, + instance.Spec.Override.Probes, + cinder.DefaultProbeConf, + ) + + // Could not process probes config + if err != nil { + return nil, err } args := []string{"-c", ServiceCommand} // // https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/ // - livenessProbe.HTTPGet = &corev1.HTTPGetAction{ - Path: "/healthcheck", - Port: intstr.IntOrString{Type: intstr.Int, IntVal: int32(cinder.CinderPublicPort)}, - } - readinessProbe.HTTPGet = livenessProbe.HTTPGet - - if instance.Spec.TLS.API.Enabled(service.EndpointPublic) { - livenessProbe.HTTPGet.Scheme = corev1.URISchemeHTTPS - readinessProbe.HTTPGet.Scheme = corev1.URISchemeHTTPS - } - // create Volume and VolumeMounts volumes := GetVolumes( cinder.GetOwningCinderName(instance), @@ -172,8 +166,8 @@ func StatefulSet( Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), VolumeMounts: volumeMounts, Resources: instance.Spec.Resources, - ReadinessProbe: readinessProbe, - LivenessProbe: livenessProbe, + ReadinessProbe: probes.Readiness, + LivenessProbe: probes.Liveness, }, }, Volumes: volumes, diff --git a/internal/cinderbackup/statefulset.go b/internal/cinderbackup/statefulset.go index 85977129..3f969104 100644 --- a/internal/cinderbackup/statefulset.go +++ b/internal/cinderbackup/statefulset.go @@ -22,10 +22,10 @@ import ( topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" "github.com/openstack-k8s-operators/lib-common/modules/common/env" + "github.com/openstack-k8s-operators/lib-common/modules/common/probes" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" ) const ( @@ -41,34 +41,28 @@ func StatefulSet( annotations map[string]string, topology *topologyv1.Topology, memcached *memcachedv1.Memcached, -) *appsv1.StatefulSet { +) (*appsv1.StatefulSet, error) { trueVar := true cinderUser := int64(cinderv1.CinderUserID) cinderGroup := int64(cinderv1.CinderGroupID) - // TODO until we determine how to properly query for these - livenessProbe := &corev1.Probe{ - // TODO might need tuning - TimeoutSeconds: 5, - PeriodSeconds: 3, - InitialDelaySeconds: 3, - } - - startupProbe := &corev1.Probe{ - TimeoutSeconds: 5, - FailureThreshold: 12, - PeriodSeconds: 5, - InitialDelaySeconds: 5, + // Both scheme and port are set according to the healthcheck.py script + scheme := corev1.URISchemeHTTP + probesPort := int32(8080) + + probes, err := probes.CreateProbeSet( + probesPort, + &scheme, + instance.Spec.Override.Probes, + cinder.DefaultProbeConf, + ) + // Could not process probes config + if err != nil { + return nil, err } args := []string{"-c", ServiceCommand} - var probeCommand []string - // Use the HTTP probe now that we have a simple server running - livenessProbe.HTTPGet = &corev1.HTTPGetAction{ - Port: intstr.FromInt(8080), - } - startupProbe.HTTPGet = livenessProbe.HTTPGet - probeCommand = []string{ + probeCommand := []string{ "/usr/local/bin/container-scripts/healthcheck.py", "backup", "/etc/cinder/cinder.conf.d", @@ -141,8 +135,8 @@ func StatefulSet( Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), VolumeMounts: volumeMounts, Resources: instance.Spec.Resources, - LivenessProbe: livenessProbe, - StartupProbe: startupProbe, + LivenessProbe: probes.Liveness, + StartupProbe: probes.Startup, }, { Name: "probe", @@ -174,5 +168,5 @@ func StatefulSet( statefulset.Spec.Template.Spec.Affinity = cinder.GetPodAffinity(ComponentName) } - return statefulset + return statefulset, nil } diff --git a/internal/cinderscheduler/statefulset.go b/internal/cinderscheduler/statefulset.go index 57541174..29f2908b 100644 --- a/internal/cinderscheduler/statefulset.go +++ b/internal/cinderscheduler/statefulset.go @@ -22,10 +22,10 @@ import ( topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" "github.com/openstack-k8s-operators/lib-common/modules/common/env" + "github.com/openstack-k8s-operators/lib-common/modules/common/probes" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" ) const ( @@ -41,32 +41,27 @@ func StatefulSet( annotations map[string]string, topology *topologyv1.Topology, memcached *memcachedv1.Memcached, -) *appsv1.StatefulSet { +) (*appsv1.StatefulSet, error) { cinderUser := int64(cinderv1.CinderUserID) cinderGroup := int64(cinderv1.CinderGroupID) - // TODO until we determine how to properly query for these - livenessProbe := &corev1.Probe{ - // TODO might need tuning - TimeoutSeconds: 5, - PeriodSeconds: 3, - InitialDelaySeconds: 3, - } - - startupProbe := &corev1.Probe{ - TimeoutSeconds: 5, - FailureThreshold: 12, - PeriodSeconds: 5, - InitialDelaySeconds: 5, + // Both scheme and port are set according to the healthcheck.py script + scheme := corev1.URISchemeHTTP + probesPort := int32(8080) + + probes, err := probes.CreateProbeSet( + probesPort, + &scheme, + instance.Spec.Override.Probes, + cinder.DefaultProbeConf, + ) + // Could not process probes config + if err != nil { + return nil, err } args := []string{"-c", ServiceCommand} - var probeCommand []string - livenessProbe.HTTPGet = &corev1.HTTPGetAction{ - Port: intstr.FromInt(8080), - } - startupProbe.HTTPGet = livenessProbe.HTTPGet - probeCommand = []string{ + probeCommand := []string{ "/usr/local/bin/container-scripts/healthcheck.py", "scheduler", "/etc/cinder/cinder.conf.d", @@ -126,8 +121,8 @@ func StatefulSet( Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), VolumeMounts: volumeMounts, Resources: instance.Spec.Resources, - LivenessProbe: livenessProbe, - StartupProbe: startupProbe, + LivenessProbe: probes.Liveness, + StartupProbe: probes.Startup, }, { Name: "probe", @@ -159,5 +154,5 @@ func StatefulSet( statefulset.Spec.Template.Spec.Affinity = cinder.GetPodAffinity(ComponentName) } - return statefulset + return statefulset, nil } diff --git a/internal/cindervolume/statefulset.go b/internal/cindervolume/statefulset.go index 3ccda648..51104eca 100644 --- a/internal/cindervolume/statefulset.go +++ b/internal/cindervolume/statefulset.go @@ -21,11 +21,11 @@ import ( memcachedv1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1" topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" "github.com/openstack-k8s-operators/lib-common/modules/common/env" + "github.com/openstack-k8s-operators/lib-common/modules/common/probes" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" ) const ( @@ -42,34 +42,28 @@ func StatefulSet( usesLVM bool, topology *topologyv1.Topology, memcached *memcachedv1.Memcached, -) *appsv1.StatefulSet { +) (*appsv1.StatefulSet, error) { trueVar := true cinderUser := int64(cinderv1.CinderUserID) cinderGroup := int64(cinderv1.CinderGroupID) - // TODO until we determine how to properly query for these - livenessProbe := &corev1.Probe{ - // TODO might need tuning - TimeoutSeconds: 5, - PeriodSeconds: 3, - InitialDelaySeconds: 3, - } + // Both scheme and port are set according to the healthcheck.py script + scheme := corev1.URISchemeHTTP + probesPort := int32(8080) - startupProbe := &corev1.Probe{ - TimeoutSeconds: 5, - FailureThreshold: 12, - PeriodSeconds: 5, - InitialDelaySeconds: 5, + probes, err := probes.CreateProbeSet( + probesPort, + &scheme, + instance.Spec.Override.Probes, + cinder.DefaultProbeConf, + ) + // Could not process probes config + if err != nil { + return nil, err } args := []string{"-c", ServiceCommand} - var probeCommand []string - // Use the HTTP probe now that we have a simple server running - livenessProbe.HTTPGet = &corev1.HTTPGetAction{ - Port: intstr.FromInt(8080), - } - startupProbe.HTTPGet = livenessProbe.HTTPGet - probeCommand = []string{ + probeCommand := []string{ "/usr/local/bin/container-scripts/healthcheck.py", "volume", "/etc/cinder/cinder.conf.d", @@ -148,8 +142,8 @@ func StatefulSet( Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), VolumeMounts: volumeMounts, Resources: instance.Spec.Resources, - LivenessProbe: livenessProbe, - StartupProbe: startupProbe, + LivenessProbe: probes.Liveness, + StartupProbe: probes.Startup, }, { Name: "probe", @@ -181,5 +175,5 @@ func StatefulSet( statefulset.Spec.Template.Spec.Affinity = cinder.GetPodAffinity(ComponentName) } - return statefulset + return statefulset, nil } diff --git a/internal/controller/cinderbackup_controller.go b/internal/controller/cinderbackup_controller.go index 62df43f2..798d02d4 100644 --- a/internal/controller/cinderbackup_controller.go +++ b/internal/controller/cinderbackup_controller.go @@ -590,7 +590,16 @@ func (r *CinderBackupReconciler) reconcileNormal(ctx context.Context, instance * } // Deploy a statefulset - ssDef := cinderbackup.StatefulSet(instance, inputHash, serviceLabels, serviceAnnotations, topology, memcached) + ssDef, err := cinderbackup.StatefulSet(instance, inputHash, serviceLabels, serviceAnnotations, topology, memcached) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.DeploymentReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } ss := statefulset.NewStatefulSet(ssDef, cinder.ShortDuration) var ssData appsv1.StatefulSet diff --git a/internal/controller/cinderscheduler_controller.go b/internal/controller/cinderscheduler_controller.go index c429c41a..b3dac321 100644 --- a/internal/controller/cinderscheduler_controller.go +++ b/internal/controller/cinderscheduler_controller.go @@ -588,7 +588,16 @@ func (r *CinderSchedulerReconciler) reconcileNormal(ctx context.Context, instanc } // Deploy a statefulset - ssDef := cinderscheduler.StatefulSet(instance, inputHash, serviceLabels, serviceAnnotations, topology, memcached) + ssDef, err := cinderscheduler.StatefulSet(instance, inputHash, serviceLabels, serviceAnnotations, topology, memcached) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.DeploymentReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } ss := statefulset.NewStatefulSet(ssDef, cinder.ShortDuration) var ssData appsv1.StatefulSet diff --git a/internal/controller/cindervolume_controller.go b/internal/controller/cindervolume_controller.go index 2392a685..4b7115ac 100644 --- a/internal/controller/cindervolume_controller.go +++ b/internal/controller/cindervolume_controller.go @@ -592,7 +592,16 @@ func (r *CinderVolumeReconciler) reconcileNormal(ctx context.Context, instance * } // Deploy a statefulset - ssDef := cindervolume.StatefulSet(instance, inputHash, serviceLabels, serviceAnnotations, usesLVM, topology, memcached) + ssDef, err := cindervolume.StatefulSet(instance, inputHash, serviceLabels, serviceAnnotations, usesLVM, topology, memcached) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.DeploymentReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } ss := statefulset.NewStatefulSet(ssDef, cinder.ShortDuration) var ssData appsv1.StatefulSet diff --git a/test/functional/base_test.go b/test/functional/base_test.go index 1e4dbdc8..3f630635 100644 --- a/test/functional/base_test.go +++ b/test/functional/base_test.go @@ -453,3 +453,14 @@ func CreateCinderInvalidSecret(namespace string, name string) *corev1.Secret { }, ) } + +// GetProbeConfOverrides returns a set of parameters to override the default +// probes values +func GetProbeConfOverrides() map[string]any { + return map[string]any{ + "path": "/healthcheck", + "initialDelaySeconds": int32(20), + "timeoutSeconds": int32(30), + "periodSeconds": int32(10), + } +} diff --git a/test/functional/cinder_controller_test.go b/test/functional/cinder_controller_test.go index 0a3a5302..4699922c 100644 --- a/test/functional/cinder_controller_test.go +++ b/test/functional/cinder_controller_test.go @@ -2064,6 +2064,114 @@ var _ = Describe("Cinder Webhook", func() { }, timeout, interval).Should(Succeed()) }) }) + + When("Cinder CR instance is built with custom probes", func() { + stsOverride := GetProbeConfOverrides() + BeforeEach(func() { + spec := GetDefaultCinderSpec() + + spec["cinderAPI"] = map[string]any{ + "override": map[string]any{ + "probes": map[string]any{ + "livenessProbes": stsOverride, + "readinessProbes": stsOverride, + }, + }, + } + spec["cinderScheduler"] = map[string]any{ + "override": map[string]any{ + "probes": map[string]any{ + "livenessProbes": stsOverride, + // StartupProbes are set to the default values as no + // overrides are provided + }, + }, + } + spec["cinderVolumes"] = map[string]any{ + "volume1": map[string]any{ + "override": map[string]any{ + "probes": map[string]any{ + "livenessProbes": stsOverride, + // StartupProbes are set to the default values as no + // overrides are provided + }, + }, + }, + } + DeferCleanup(th.DeleteInstance, CreateCinder(cinderTest.Instance, spec)) + DeferCleanup(k8sClient.Delete, ctx, CreateCinderMessageBusSecret(cinderTest.Instance.Namespace, cinderTest.RabbitmqSecretName)) + DeferCleanup( + mariadb.DeleteDBService, + mariadb.CreateDBService( + cinderTest.Instance.Namespace, + GetCinder(cinderName).Spec.DatabaseInstance, + corev1.ServiceSpec{ + Ports: []corev1.ServicePort{{Port: 3306}}, + }, + ), + ) + infra.SimulateTransportURLReady(cinderTest.CinderTransportURL) + DeferCleanup(infra.DeleteMemcached, infra.CreateMemcached(namespace, cinderTest.MemcachedInstance, memcachedv1.MemcachedSpec{})) + infra.SimulateMemcachedReady(cinderTest.CinderMemcached) + DeferCleanup(keystone.DeleteKeystoneAPI, keystone.CreateKeystoneAPI(cinderTest.Instance.Namespace)) + mariadb.SimulateMariaDBAccountCompleted(cinderTest.Database) + mariadb.SimulateMariaDBDatabaseCompleted(cinderTest.Database) + th.SimulateJobSuccess(cinderTest.CinderDBSync) + keystone.SimulateKeystoneServiceReady(cinderTest.CinderKeystoneService) + keystone.SimulateKeystoneEndpointReady(cinderTest.CinderKeystoneEndpoint) + th.SimulateStatefulSetReplicaReady(cinderTest.CinderAPI) + th.SimulateStatefulSetReplicaReady(cinderTest.CinderScheduler) + }) + It("Check the resulting probes generated in the statefulSet", func() { + th.SimulateStatefulSetReplicaReady(cinderTest.CinderAPI) + th.SimulateStatefulSetReplicaReady(cinderTest.CinderScheduler) + keystone.SimulateKeystoneEndpointReady(cinderTest.CinderKeystoneEndpoint) + // Check CinderAPI + Eventually(func(g Gomega) { + apiContainer := th.GetStatefulSet(cinderTest.CinderAPI).Spec.Template.Spec.Containers[1] + g.Expect(apiContainer.LivenessProbe).ToNot(BeNil()) + g.Expect(apiContainer.LivenessProbe.HTTPGet.Path).To(Equal(stsOverride["path"])) + g.Expect(apiContainer.LivenessProbe.InitialDelaySeconds).To(Equal(stsOverride["initialDelaySeconds"])) + g.Expect(apiContainer.LivenessProbe.TimeoutSeconds).To(Equal(stsOverride["timeoutSeconds"])) + g.Expect(apiContainer.LivenessProbe.PeriodSeconds).To(Equal(stsOverride["periodSeconds"])) + g.Expect(apiContainer.StartupProbe).To(BeNil()) + g.Expect(apiContainer.LivenessProbe).ToNot(BeNil()) + g.Expect(apiContainer.ReadinessProbe.InitialDelaySeconds).To(Equal(stsOverride["initialDelaySeconds"])) + g.Expect(apiContainer.ReadinessProbe.TimeoutSeconds).To(Equal(stsOverride["timeoutSeconds"])) + g.Expect(apiContainer.ReadinessProbe.PeriodSeconds).To(Equal(stsOverride["periodSeconds"])) + }, timeout, interval).Should(Succeed()) + // Check CinderScheduler + Eventually(func(g Gomega) { + schedContainer := th.GetStatefulSet(cinderTest.CinderScheduler).Spec.Template.Spec.Containers[0] + g.Expect(schedContainer.LivenessProbe).ToNot(BeNil()) + g.Expect(schedContainer.LivenessProbe.HTTPGet.Path).To(Equal("/healthcheck")) + g.Expect(schedContainer.LivenessProbe.InitialDelaySeconds).To(Equal(int32(20))) + g.Expect(schedContainer.LivenessProbe.TimeoutSeconds).To(Equal(int32(30))) + g.Expect(schedContainer.LivenessProbe.PeriodSeconds).To(Equal(int32(10))) + g.Expect(schedContainer.ReadinessProbe).To(BeNil()) + // We do not override StartupProbes and we apply the defaults defined in the cinder-operator + g.Expect(schedContainer.StartupProbe.InitialDelaySeconds).To(Equal(int32(cinder.DefaultProbeConf.StartupProbes.InitialDelaySeconds))) + g.Expect(schedContainer.StartupProbe.TimeoutSeconds).To(Equal(int32(cinder.DefaultProbeConf.StartupProbes.TimeoutSeconds))) + g.Expect(schedContainer.StartupProbe.PeriodSeconds).To(Equal(int32(cinder.DefaultProbeConf.StartupProbes.PeriodSeconds))) + g.Expect(schedContainer.StartupProbe.FailureThreshold).To(Equal(int32(cinder.DefaultProbeConf.StartupProbes.FailureThreshold))) + }, timeout, interval).Should(Succeed()) + // Check CinderVolume + Eventually(func(g Gomega) { + volContainer := th.GetStatefulSet(cinderTest.CinderVolumes[0]).Spec.Template.Spec.Containers[0] + g.Expect(volContainer.LivenessProbe).ToNot(BeNil()) + g.Expect(volContainer.LivenessProbe.HTTPGet.Path).To(Equal("/healthcheck")) + g.Expect(volContainer.LivenessProbe.InitialDelaySeconds).To(Equal(int32(20))) + g.Expect(volContainer.LivenessProbe.TimeoutSeconds).To(Equal(int32(30))) + g.Expect(volContainer.LivenessProbe.PeriodSeconds).To(Equal(int32(10))) + g.Expect(volContainer.ReadinessProbe).To(BeNil()) + // We do not override StartupProbes and we apply the defaults defined in the cinder-operator + g.Expect(volContainer.StartupProbe.InitialDelaySeconds).To(Equal(int32(cinder.DefaultProbeConf.StartupProbes.InitialDelaySeconds))) + g.Expect(volContainer.StartupProbe.TimeoutSeconds).To(Equal(int32(cinder.DefaultProbeConf.StartupProbes.TimeoutSeconds))) + g.Expect(volContainer.StartupProbe.PeriodSeconds).To(Equal(int32(cinder.DefaultProbeConf.StartupProbes.PeriodSeconds))) + g.Expect(volContainer.StartupProbe.FailureThreshold).To(Equal(int32(cinder.DefaultProbeConf.StartupProbes.FailureThreshold))) + }, timeout, interval).Should(Succeed()) + }) + }) }) var _ = Describe("Cinder with RabbitMQ custom vhost and user", func() { From 9c460056868fb4402588819b3948b57fe83f3eb5 Mon Sep 17 00:00:00 2001 From: Francesco Pantano Date: Fri, 20 Mar 2026 15:07:46 +0100 Subject: [PATCH 2/4] Make probe timeouts dependent on APITimeout configuration Replace static probe timeouts with dynamic scaling based on APITimeout parameter. Creates separate probe configurations for API services (HTTP endpoints) and RPC workers (internal services) with appropriate scaling factors. API services use full APITimeout scaling while RPC workers get proportional timeouts, preventing premature pod kills during high load scenarios. Signed-off-by: Francesco Pantano --- .../cinder.openstack.org_cinderapis.yaml | 5 ++ api/v1beta1/cinderapi_types.go | 6 ++ .../cinder.openstack.org_cinderapis.yaml | 5 ++ internal/cinder/const.go | 28 ++------ internal/cinder/funcs.go | 68 +++++++++++++++++++ internal/cinderapi/statefuleset.go | 3 +- internal/cinderbackup/statefulset.go | 2 +- internal/cinderscheduler/statefulset.go | 2 +- internal/cindervolume/statefulset.go | 2 +- internal/controller/cinder_controller.go | 1 + internal/controller/cinderapi_controller.go | 2 +- test/functional/cinder_controller_test.go | 16 ++--- .../common/assert_sample_deployment.yaml | 4 +- 13 files changed, 105 insertions(+), 39 deletions(-) diff --git a/api/bases/cinder.openstack.org_cinderapis.yaml b/api/bases/cinder.openstack.org_cinderapis.yaml index 8d1137ff..eb46f7dc 100644 --- a/api/bases/cinder.openstack.org_cinderapis.yaml +++ b/api/bases/cinder.openstack.org_cinderapis.yaml @@ -52,6 +52,11 @@ spec: spec: description: CinderAPISpec defines the desired state of CinderAPI properties: + apiTimeout: + default: 60 + description: APITimeout for HAProxy, Apache, and rpc_response_timeout + minimum: 10 + type: integer containerImage: description: ContainerImage - Cinder Container Image URL (will be set to environmental default if empty) diff --git a/api/v1beta1/cinderapi_types.go b/api/v1beta1/cinderapi_types.go index 1abcfea9..4bc07215 100644 --- a/api/v1beta1/cinderapi_types.go +++ b/api/v1beta1/cinderapi_types.go @@ -103,6 +103,12 @@ type CinderAPISpec struct { // +kubebuilder:default=memcached // Memcached instance name. MemcachedInstance *string `json:"memcachedInstance"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default=60 + // +kubebuilder:validation:Minimum=10 + // APITimeout for HAProxy, Apache, and rpc_response_timeout + APITimeout int `json:"apiTimeout"` } // CinderAPIStatus defines the observed state of CinderAPI diff --git a/config/crd/bases/cinder.openstack.org_cinderapis.yaml b/config/crd/bases/cinder.openstack.org_cinderapis.yaml index 8d1137ff..eb46f7dc 100644 --- a/config/crd/bases/cinder.openstack.org_cinderapis.yaml +++ b/config/crd/bases/cinder.openstack.org_cinderapis.yaml @@ -52,6 +52,11 @@ spec: spec: description: CinderAPISpec defines the desired state of CinderAPI properties: + apiTimeout: + default: 60 + description: APITimeout for HAProxy, Apache, and rpc_response_timeout + minimum: 10 + type: integer containerImage: description: ContainerImage - Cinder Container Image URL (will be set to environmental default if empty) diff --git a/internal/cinder/const.go b/internal/cinder/const.go index 949fd1ca..10480dd4 100644 --- a/internal/cinder/const.go +++ b/internal/cinder/const.go @@ -21,7 +21,6 @@ import ( ctrl "sigs.k8s.io/controller-runtime" - "github.com/openstack-k8s-operators/lib-common/modules/common/probes" "github.com/openstack-k8s-operators/lib-common/modules/storage" ) @@ -72,6 +71,10 @@ const ( // Cinder is the global ServiceType that refers to all the components deployed // by the cinder operator Cinder storage.PropagationType = "Cinder" + // CinderServiceDownTime - Maximum time since last check-in for a service + // to be considered up. This is based on: + // https://opendev.org/openstack/cinder/src/branch/master/cinder/common/config.py#L121 + CinderServiceDownTime = 60 // ShortDuration is a short duration for quick retries ShortDuration = time.Duration(5) * time.Second @@ -104,26 +107,3 @@ var CinderBackupPropagation = []storage.PropagationType{Cinder, CinderBackup} // It allows the CinderVolume pods to mount volumes destined to Cinder and CinderVolume // ServiceTypes var CinderVolumePropagation = []storage.PropagationType{Cinder, CinderVolume} - -// DefaultProbeConf - Default values applied to Cinder StatefulSets when no -// overrides are provided -var DefaultProbeConf = probes.OverrideSpec{ - LivenessProbes: &probes.ProbeConf{ - Path: "/healthcheck", - TimeoutSeconds: 10, - PeriodSeconds: 6, - InitialDelaySeconds: 10, - }, - ReadinessProbes: &probes.ProbeConf{ - Path: "/healthcheck", - TimeoutSeconds: 10, - PeriodSeconds: 10, - InitialDelaySeconds: 10, - }, - StartupProbes: &probes.ProbeConf{ - TimeoutSeconds: 10, - FailureThreshold: 12, - PeriodSeconds: 10, - InitialDelaySeconds: 10, - }, -} diff --git a/internal/cinder/funcs.go b/internal/cinder/funcs.go index 4e86c067..0ed78f92 100644 --- a/internal/cinder/funcs.go +++ b/internal/cinder/funcs.go @@ -3,8 +3,10 @@ package cinder import ( common "github.com/openstack-k8s-operators/lib-common/modules/common" "github.com/openstack-k8s-operators/lib-common/modules/common/affinity" + "github.com/openstack-k8s-operators/lib-common/modules/common/probes" corev1 "k8s.io/api/core/v1" + "math" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -47,3 +49,69 @@ func GetPodAffinity(componentName string) *corev1.Affinity { corev1.LabelHostname, ) } + +// GetDefaultProbesAPI - +func GetDefaultProbesAPI(apiTimeout int) probes.OverrideSpec { + const failureCount = 3 + period := int32(math.Floor(float64(apiTimeout) / float64(failureCount))) + // For startup probes, use shorter period for faster startup detection + startupPeriod := int32(math.Max(5, float64(period)/2)) + + // Default values applied to CinderAPI StatefulSets when no + // overrides are provided + return probes.OverrideSpec{ + LivenessProbes: &probes.ProbeConf{ + Path: "/healthcheck", + TimeoutSeconds: 5, + PeriodSeconds: period, + InitialDelaySeconds: 5, + }, + ReadinessProbes: &probes.ProbeConf{ + Path: "/healthcheck", + TimeoutSeconds: 5, + PeriodSeconds: period, + InitialDelaySeconds: 5, + }, + StartupProbes: &probes.ProbeConf{ + TimeoutSeconds: 5, + PeriodSeconds: startupPeriod, + InitialDelaySeconds: 5, + FailureThreshold: 12, + }, + } +} + +// GetDefaultProbesRPCWorker returns default probe configuration for RPC worker +// processes (cinder-volume, cinder-scheduler, cinder-backup). These processes +// expose no HTTP healthcheck endpoint, so Path is intentionally omitted. +// A dedicated serviceDownTime field is not currently exposed in the operator's +// API, and we rely on the default provided by cinder to compute the default +// values. +// This is an established value, and dedicated probes tuning can be performed +// via the dedicated interface. +// https://opendev.org/openstack/cinder/src/branch/master/cinder/common/config.py#L121 +func GetDefaultProbesRPCWorker(serviceDownTime int) probes.OverrideSpec { + const failureCount = 3 + period := int32(math.Floor(float64(serviceDownTime) / float64(failureCount))) + // For startup probes, use shorter period for faster startup detection + startupPeriod := int32(math.Max(5, float64(period)/2)) + + return probes.OverrideSpec{ + LivenessProbes: &probes.ProbeConf{ + TimeoutSeconds: 5, + PeriodSeconds: period, + InitialDelaySeconds: 15, + }, + ReadinessProbes: &probes.ProbeConf{ + TimeoutSeconds: 5, + PeriodSeconds: period, + InitialDelaySeconds: 15, + }, + StartupProbes: &probes.ProbeConf{ + TimeoutSeconds: 5, + PeriodSeconds: startupPeriod, + InitialDelaySeconds: period, + FailureThreshold: 12, + }, + } +} diff --git a/internal/cinderapi/statefuleset.go b/internal/cinderapi/statefuleset.go index 4e0d37c6..9e370477 100644 --- a/internal/cinderapi/statefuleset.go +++ b/internal/cinderapi/statefuleset.go @@ -43,6 +43,7 @@ func StatefulSet( annotations map[string]string, topology *topologyv1.Topology, memcached *memcachedv1.Memcached, + timeout int, ) (*appsv1.StatefulSet, error) { runAsUser := int64(0) cinderUser := int64(cinderv1beta1.CinderUserID) @@ -57,7 +58,7 @@ func StatefulSet( int32(cinder.CinderPublicPort), &scheme, instance.Spec.Override.Probes, - cinder.DefaultProbeConf, + cinder.GetDefaultProbesAPI(timeout), ) // Could not process probes config diff --git a/internal/cinderbackup/statefulset.go b/internal/cinderbackup/statefulset.go index 3f969104..6525ebcb 100644 --- a/internal/cinderbackup/statefulset.go +++ b/internal/cinderbackup/statefulset.go @@ -54,7 +54,7 @@ func StatefulSet( probesPort, &scheme, instance.Spec.Override.Probes, - cinder.DefaultProbeConf, + cinder.GetDefaultProbesRPCWorker(cinder.CinderServiceDownTime), ) // Could not process probes config if err != nil { diff --git a/internal/cinderscheduler/statefulset.go b/internal/cinderscheduler/statefulset.go index 29f2908b..1c846155 100644 --- a/internal/cinderscheduler/statefulset.go +++ b/internal/cinderscheduler/statefulset.go @@ -53,7 +53,7 @@ func StatefulSet( probesPort, &scheme, instance.Spec.Override.Probes, - cinder.DefaultProbeConf, + cinder.GetDefaultProbesRPCWorker(cinder.CinderServiceDownTime), ) // Could not process probes config if err != nil { diff --git a/internal/cindervolume/statefulset.go b/internal/cindervolume/statefulset.go index 51104eca..e09c525c 100644 --- a/internal/cindervolume/statefulset.go +++ b/internal/cindervolume/statefulset.go @@ -55,7 +55,7 @@ func StatefulSet( probesPort, &scheme, instance.Spec.Override.Probes, - cinder.DefaultProbeConf, + cinder.GetDefaultProbesRPCWorker(cinder.CinderServiceDownTime), ) // Could not process probes config if err != nil { diff --git a/internal/controller/cinder_controller.go b/internal/controller/cinder_controller.go index 8c7b30f3..f02d1270 100644 --- a/internal/controller/cinder_controller.go +++ b/internal/controller/cinder_controller.go @@ -1273,6 +1273,7 @@ func (r *CinderReconciler) apiDeploymentCreateOrUpdate(ctx context.Context, inst TransportURLSecret: instance.Status.TransportURLSecret, ServiceAccount: instance.RbacResourceName(), MemcachedInstance: &instance.Spec.MemcachedInstance, + APITimeout: instance.Spec.APITimeout, } if cinderAPISpec.NodeSelector == nil { diff --git a/internal/controller/cinderapi_controller.go b/internal/controller/cinderapi_controller.go index 46fc8b6b..560e1587 100644 --- a/internal/controller/cinderapi_controller.go +++ b/internal/controller/cinderapi_controller.go @@ -935,7 +935,7 @@ func (r *CinderAPIReconciler) reconcileNormal(ctx context.Context, instance *cin } // Deploy a statefulset - ssDef, err := cinderapi.StatefulSet(instance, inputHash, serviceLabels, serviceAnnotations, topology, memcached) + ssDef, err := cinderapi.StatefulSet(instance, inputHash, serviceLabels, serviceAnnotations, topology, memcached, instance.Spec.APITimeout) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.DeploymentReadyCondition, diff --git a/test/functional/cinder_controller_test.go b/test/functional/cinder_controller_test.go index 4699922c..5985f775 100644 --- a/test/functional/cinder_controller_test.go +++ b/test/functional/cinder_controller_test.go @@ -2150,10 +2150,10 @@ var _ = Describe("Cinder Webhook", func() { g.Expect(schedContainer.LivenessProbe.PeriodSeconds).To(Equal(int32(10))) g.Expect(schedContainer.ReadinessProbe).To(BeNil()) // We do not override StartupProbes and we apply the defaults defined in the cinder-operator - g.Expect(schedContainer.StartupProbe.InitialDelaySeconds).To(Equal(int32(cinder.DefaultProbeConf.StartupProbes.InitialDelaySeconds))) - g.Expect(schedContainer.StartupProbe.TimeoutSeconds).To(Equal(int32(cinder.DefaultProbeConf.StartupProbes.TimeoutSeconds))) - g.Expect(schedContainer.StartupProbe.PeriodSeconds).To(Equal(int32(cinder.DefaultProbeConf.StartupProbes.PeriodSeconds))) - g.Expect(schedContainer.StartupProbe.FailureThreshold).To(Equal(int32(cinder.DefaultProbeConf.StartupProbes.FailureThreshold))) + g.Expect(schedContainer.StartupProbe.InitialDelaySeconds).To(Equal(int32(cinder.GetDefaultProbesRPCWorker(60).StartupProbes.InitialDelaySeconds))) + g.Expect(schedContainer.StartupProbe.TimeoutSeconds).To(Equal(int32(cinder.GetDefaultProbesRPCWorker(60).StartupProbes.TimeoutSeconds))) + g.Expect(schedContainer.StartupProbe.PeriodSeconds).To(Equal(int32(cinder.GetDefaultProbesRPCWorker(60).StartupProbes.PeriodSeconds))) + g.Expect(schedContainer.StartupProbe.FailureThreshold).To(Equal(int32(cinder.GetDefaultProbesRPCWorker(60).StartupProbes.FailureThreshold))) }, timeout, interval).Should(Succeed()) // Check CinderVolume Eventually(func(g Gomega) { @@ -2165,10 +2165,10 @@ var _ = Describe("Cinder Webhook", func() { g.Expect(volContainer.LivenessProbe.PeriodSeconds).To(Equal(int32(10))) g.Expect(volContainer.ReadinessProbe).To(BeNil()) // We do not override StartupProbes and we apply the defaults defined in the cinder-operator - g.Expect(volContainer.StartupProbe.InitialDelaySeconds).To(Equal(int32(cinder.DefaultProbeConf.StartupProbes.InitialDelaySeconds))) - g.Expect(volContainer.StartupProbe.TimeoutSeconds).To(Equal(int32(cinder.DefaultProbeConf.StartupProbes.TimeoutSeconds))) - g.Expect(volContainer.StartupProbe.PeriodSeconds).To(Equal(int32(cinder.DefaultProbeConf.StartupProbes.PeriodSeconds))) - g.Expect(volContainer.StartupProbe.FailureThreshold).To(Equal(int32(cinder.DefaultProbeConf.StartupProbes.FailureThreshold))) + g.Expect(volContainer.StartupProbe.InitialDelaySeconds).To(Equal(int32(cinder.GetDefaultProbesRPCWorker(60).StartupProbes.InitialDelaySeconds))) + g.Expect(volContainer.StartupProbe.TimeoutSeconds).To(Equal(int32(cinder.GetDefaultProbesRPCWorker(60).StartupProbes.TimeoutSeconds))) + g.Expect(volContainer.StartupProbe.PeriodSeconds).To(Equal(int32(cinder.GetDefaultProbesRPCWorker(60).StartupProbes.PeriodSeconds))) + g.Expect(volContainer.StartupProbe.FailureThreshold).To(Equal(int32(cinder.GetDefaultProbesRPCWorker(60).StartupProbes.FailureThreshold))) }, timeout, interval).Should(Succeed()) }) }) diff --git a/test/kuttl/common/assert_sample_deployment.yaml b/test/kuttl/common/assert_sample_deployment.yaml index c17d469f..97d9274b 100644 --- a/test/kuttl/common/assert_sample_deployment.yaml +++ b/test/kuttl/common/assert_sample_deployment.yaml @@ -89,7 +89,7 @@ spec: port: 8776 scheme: HTTP initialDelaySeconds: 5 - periodSeconds: 3 + periodSeconds: 20 successThreshold: 1 timeoutSeconds: 5 name: cinder-api @@ -100,7 +100,7 @@ spec: port: 8776 scheme: HTTP initialDelaySeconds: 5 - periodSeconds: 5 + periodSeconds: 20 successThreshold: 1 timeoutSeconds: 5 resources: {} From f870a944bbdebfaa4b60c4f04c9aafebd7debc23 Mon Sep 17 00:00:00 2001 From: Francesco Pantano Date: Mon, 20 Apr 2026 14:40:23 +0200 Subject: [PATCH 3/4] Document healthcheck design decisions This patch introduces a document where the design decisions related to the probes settings are described. Signed-off-by: Francesco Pantano --- docs/probes.md | 326 ++++++++++++++++++++++++ internal/cinderapi/statefuleset.go | 6 +- internal/cinderbackup/statefulset.go | 6 +- internal/cinderscheduler/statefulset.go | 6 +- internal/cindervolume/statefulset.go | 6 +- 5 files changed, 338 insertions(+), 12 deletions(-) create mode 100644 docs/probes.md diff --git a/docs/probes.md b/docs/probes.md new file mode 100644 index 00000000..597c62e6 --- /dev/null +++ b/docs/probes.md @@ -0,0 +1,326 @@ +# Probes Defaults Configuration + +The cinder-operator provides flexible health check configuration through +Kubernetes probes. This document explains the probe configuration, timing +calculations, and best practices. + +## Overview + +The operator supports three types of probes: +- **Liveness Probes** - Determines if a container needs to be restarted +- **Readiness Probes** - Determines if a container can receive traffic +- **Startup Probes** - Handles initial container startup before other probes begin + +Cinder has two distinct service categories, each with its own probe strategy: + +| Service | Liveness | Readiness | Startup | Health Check Method | +|---------------|----------|-----------|---------|-----------------------------| +| CinderAPI | Yes | Yes | No | HTTP `/healthcheck` endpoint| +| CinderScheduler | Yes | No | Yes | Sidecar `healthcheck.py` | +| CinderBackup | Yes | No | Yes | Sidecar `healthcheck.py` | +| CinderVolume | Yes | No | Yes | Sidecar `healthcheck.py` | + +**CinderAPI** exposes an HTTP endpoint and serves external traffic, so it uses +readiness probes to control traffic routing. Startup probes are not applied +because the API process initializes quickly behind HAProxy/Apache. + +**RPC worker processes** (scheduler, backup, volume) do not expose HTTP +endpoints. Instead, a dedicated sidecar container runs `healthcheck.py`, which +verifies the service state by checking the cinder service report timestamps +against the configuration files. These workers use startup probes to handle +potentially slow initialization (e.g., connecting to storage backends), but +do not need readiness probes since they do not receive direct traffic. + +## Default Probe Configuration + +### CinderAPI: Dynamic Timing Based on APITimeout + +The CinderAPI probe timings are dynamically calculated based on the `apiTimeout` +parameter to ensure alignment with HAProxy and Apache timeout settings: + +```go +period = floor(apiTimeout / failureCount) // failureCount = 3 +timeout = 5 // fixed value +startupPeriod = max(5, period / 2) // faster startup detection +``` + +### RPC Workers: Timing Based on ServiceDownTime + +The RPC worker probe timings (cinder-scheduler, cinder-backup, cinder-volume) +are derived from the upstream cinder `service_down_time` configuration value, +which defaults to 60 seconds. This value represents the maximum time since +the last service check-in before a service is considered down. + +A dedicated `serviceDownTime` field is not currently exposed in the operator's +API, and we rely on the +[default provided by cinder](https://opendev.org/openstack/cinder/src/branch/master/cinder/common/config.py#L121) +to compute the default probe timings. Further tuning can be performed via the +dedicated override interface. + +```go +period = floor(serviceDownTime / failureCount) // failureCount = 3 +timeout = 5 // fixed value +startupPeriod = max(5, period / 2) // faster startup detection +``` + +### Default Values (apiTimeout=60s, serviceDownTime=60s) + +#### CinderAPI + +| Probe Type | Timeout | Period | Initial Delay | Failure Threshold | Total Unhealthy Time | +|------------|---------|--------|---------------|-------------------|----------------------| +| Liveness | 5s | 20s | 5s | 3 (default) | 60s (3 x 20s) | +| Readiness | 5s | 20s | 5s | 3 (default) | 60s (3 x 20s) | + +#### RPC Workers (Scheduler, Backup, Volume) + +| Probe Type | Timeout | Period | Initial Delay | Failure Threshold | Total Unhealthy Time | +|------------|---------|--------|---------------|-------------------|----------------------| +| Liveness | 5s | 20s | 15s | 3 (default) | 60s (3 x 20s) | +| Startup | 5s | 10s | 20s | 12 | 120s (12 x 10s) | + +> **Note**: RPC workers use a longer `initialDelaySeconds` (15s for liveness, +> `period` for startup) compared to CinderAPI (5s), because RPC processes need +> additional time to establish AMQP connections and register with the message bus. + +### CinderAPI Timing Examples for Different APITimeout Values + +| apiTimeout | Period | Startup Period | Max Startup Time | Total Unhealthy Time | +|-----------|--------|----------------|------------------|----------------------| +| 30s | 10s | 5s (min) | 60s (1 min) | 30s (3 x 10s) | +| 60s | 20s | 10s | 120s (2 min) | 60s (3 x 20s) | +| 120s | 40s | 20s | 240s (4 min) | 120s (3 x 40s) | +| 300s | 100s | 50s | 600s (10 min) | 300s (3 x 100s) | + +> **Note**: Unlike the glance-operator, the cinder-operator does not cap the +> `startupPeriod` at 10 seconds. For large `apiTimeout` values, the startup +> detection window grows proportionally. Since CinderAPI does not currently +> apply startup probes in its StatefulSet, this only affects RPC workers when +> their defaults are manually overridden. The RPC worker timing remains +> constant since `serviceDownTime` is not exposed as a configurable parameter. + +## Design Decisions + +### 1. Fixed Timeout (5 seconds) + +Unlike some operators that compute timeout as a percentage of period, the +cinder-operator uses a fixed 5-second timeout for all probes. This provides a +consistent and predictable health check window regardless of the `apiTimeout` +value, and avoids unnecessarily long probe timeouts that could delay failure +detection. + +### 2. Period = APITimeout / FailureCount + +The probe period is derived by dividing `apiTimeout` (or `serviceDownTime`) by +the failure count (3). This ensures the total unhealthy time aligns exactly with +the configured timeout: + +`Total Unhealthy Time = periodSeconds x failureThreshold = (apiTimeout / 3) x 3 = apiTimeout` + +This means a pod is marked unhealthy at exactly the `apiTimeout` boundary, +providing a tight health checking window that detects failures in sync with +the HAProxy/Apache timeout configuration. + +### 3. Sidecar-Based Probes for RPC Workers + +RPC worker processes (cinder-volume, cinder-scheduler, cinder-backup) do not +expose HTTP endpoints. Instead, the operator deploys a sidecar container +running `healthcheck.py` that: + +1. Reads the cinder configuration from `/etc/cinder/cinder.conf.d` +2. Queries the cinder database for the service's last check-in timestamp +3. Compares against `service_down_time` to determine health +4. Exposes the result on port 8080 for Kubernetes probes to query + +This approach checks actual service health rather than merely verifying process +liveness, catching scenarios where the process is running but unable to +communicate with the message bus. + +### 4. No Readiness Probes for RPC Workers + +RPC workers communicate exclusively via the message bus (RabbitMQ) and never +receive direct HTTP traffic. Readiness probes control whether a pod receives +traffic from Kubernetes Services, which is not relevant for these workers. +Liveness probes ensure the process is restarted if it becomes unresponsive, +while startup probes handle the initialization window. + +### 5. No Startup Probes for CinderAPI + +CinderAPI starts behind HAProxy and Apache, which handle connection queuing +during initialization. The readiness probe already prevents traffic from +reaching the pod before it is ready to serve, making a separate startup probe +unnecessary. The liveness probe uses a short `initialDelaySeconds` (5s) since +the WSGI process initializes quickly. + +## Customizing Probes + +### CinderAPI CR Override + +You can override default probe settings for the CinderAPI service: + +```yaml +apiVersion: cinder.openstack.org/v1beta1 +kind: CinderAPI +metadata: + name: cinder-api +spec: + apiTimeout: 120 + override: + probes: + livenessProbes: + path: "/healthcheck" + initialDelaySeconds: 10 + timeoutSeconds: 30 + periodSeconds: 40 + failureThreshold: 5 + readinessProbes: + path: "/healthcheck" + initialDelaySeconds: 10 + timeoutSeconds: 30 + periodSeconds: 40 + failureThreshold: 3 +``` + +All probe overrides are configured through the top-level Cinder CR, which +propagates settings to the individual sub-services: + +```yaml +apiVersion: cinder.openstack.org/v1beta1 +kind: Cinder +metadata: + name: cinder +spec: + apiTimeout: 120 + cinderAPI: + override: + probes: + livenessProbes: + path: "/healthcheck" + timeoutSeconds: 30 + periodSeconds: 40 + cinderScheduler: + override: + probes: + livenessProbes: + timeoutSeconds: 10 + periodSeconds: 25 + startupProbes: + timeoutSeconds: 10 + periodSeconds: 5 + failureThreshold: 20 + cinderVolumes: + volume1: + override: + probes: + livenessProbes: + timeoutSeconds: 10 + periodSeconds: 25 + startupProbes: + timeoutSeconds: 10 + periodSeconds: 5 + failureThreshold: 20 + cinderBackups: + backup1: + override: + probes: + livenessProbes: + timeoutSeconds: 10 + periodSeconds: 25 +``` + +### Field-Level Overrides + +Users can customize individual probe parameters while other settings retain +their computed defaults. For example, to only increase the failure threshold +on a CinderVolume instance: + +```yaml +cinderVolumes: + ceph-backend: + override: + probes: + startupProbes: + failureThreshold: 24 +``` + +This doubles the startup detection window (from 120s to 240s) without changing +any other probe timing, which can be useful for backends that require additional +initialization time. + +## TLS Considerations + +When TLS is enabled for the CinderAPI endpoints: +- Probe scheme automatically switches to `HTTPS` +- No configuration changes needed + +The operator automatically configures the correct scheme based on the TLS +settings: + +```go +if instance.Spec.TLS.API.Enabled(service.EndpointPublic) { + scheme = corev1.URISchemeHTTPS +} +``` + +RPC workers are not affected by TLS settings since their probe sidecar +communicates over localhost on port 8080 using plain HTTP. + +## Best Practices + +### 1. Align apiTimeout with Expected Response Times + +Set `apiTimeout` based on your slowest expected Cinder API operation: +- Standard deployments: 60s (default) +- Large volume operations: 120-300s +- High-latency storage backends: 180-300s + +Changing `apiTimeout` automatically adjusts CinderAPI probe timings. RPC worker +probe timings are not affected by `apiTimeout`. + +### 2. Tune RPC Worker Probes for Slow Backends + +Storage backends with slow initialization (e.g., establishing iSCSI sessions, +connecting to remote Ceph clusters) may require adjusted startup probes: + +```yaml +cinderVolumes: + slow-backend: + override: + probes: + startupProbes: + failureThreshold: 24 # 240s max startup time + periodSeconds: 15 +``` + +### 3. Monitor Probe Failures + +Watch for probe failure patterns: + +```bash +# Check probe failures in pod events +oc describe pod cinder-volume-vol1-0 -n openstack + +# View probe timing in container spec +oc get pod cinder-api-0 -n openstack -o jsonpath='{.spec.containers[1].livenessProbe}' +``` + +### 4. Understand the Sidecar Probe Architecture + +For RPC workers, remember that the probes target the sidecar `probe` container, +not the main service container. If probe failures occur, check both containers: + +```bash +# Check main container logs +oc logs cinder-volume-vol1-0 -c cinder-volume -n openstack + +# Check probe sidecar logs +oc logs cinder-volume-vol1-0 -c probe -n openstack +``` + +## References + +- [Kubernetes Probe Documentation](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/) +- [lib-common Probes Module](https://github.com/openstack-k8s-operators/lib-common/tree/main/modules/common/probes) +- [Cinder ServiceDownTime Configuration](https://opendev.org/openstack/cinder/src/branch/master/cinder/common/config.py#L121) +- [CinderAPI CRD Reference](../api/v1beta1/cinderapi_types.go) +- [Probes Defaults Implementation](../internal/cinder/funcs.go) diff --git a/internal/cinderapi/statefuleset.go b/internal/cinderapi/statefuleset.go index 9e370477..62ae5854 100644 --- a/internal/cinderapi/statefuleset.go +++ b/internal/cinderapi/statefuleset.go @@ -54,7 +54,7 @@ func StatefulSet( } // Note that by default we create probes with the same URIScheme and port - probes, err := probes.CreateProbeSet( + apiProbes, err := probes.CreateProbeSet( int32(cinder.CinderPublicPort), &scheme, instance.Spec.Override.Probes, @@ -167,8 +167,8 @@ func StatefulSet( Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), VolumeMounts: volumeMounts, Resources: instance.Spec.Resources, - ReadinessProbe: probes.Readiness, - LivenessProbe: probes.Liveness, + ReadinessProbe: apiProbes.Readiness, + LivenessProbe: apiProbes.Liveness, }, }, Volumes: volumes, diff --git a/internal/cinderbackup/statefulset.go b/internal/cinderbackup/statefulset.go index 6525ebcb..a080f620 100644 --- a/internal/cinderbackup/statefulset.go +++ b/internal/cinderbackup/statefulset.go @@ -50,7 +50,7 @@ func StatefulSet( scheme := corev1.URISchemeHTTP probesPort := int32(8080) - probes, err := probes.CreateProbeSet( + backupProbes, err := probes.CreateProbeSet( probesPort, &scheme, instance.Spec.Override.Probes, @@ -135,8 +135,8 @@ func StatefulSet( Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), VolumeMounts: volumeMounts, Resources: instance.Spec.Resources, - LivenessProbe: probes.Liveness, - StartupProbe: probes.Startup, + LivenessProbe: backupProbes.Liveness, + StartupProbe: backupProbes.Startup, }, { Name: "probe", diff --git a/internal/cinderscheduler/statefulset.go b/internal/cinderscheduler/statefulset.go index 1c846155..8df8d5fa 100644 --- a/internal/cinderscheduler/statefulset.go +++ b/internal/cinderscheduler/statefulset.go @@ -49,7 +49,7 @@ func StatefulSet( scheme := corev1.URISchemeHTTP probesPort := int32(8080) - probes, err := probes.CreateProbeSet( + schedProbes, err := probes.CreateProbeSet( probesPort, &scheme, instance.Spec.Override.Probes, @@ -121,8 +121,8 @@ func StatefulSet( Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), VolumeMounts: volumeMounts, Resources: instance.Spec.Resources, - LivenessProbe: probes.Liveness, - StartupProbe: probes.Startup, + LivenessProbe: schedProbes.Liveness, + StartupProbe: schedProbes.Startup, }, { Name: "probe", diff --git a/internal/cindervolume/statefulset.go b/internal/cindervolume/statefulset.go index e09c525c..10259dd9 100644 --- a/internal/cindervolume/statefulset.go +++ b/internal/cindervolume/statefulset.go @@ -51,7 +51,7 @@ func StatefulSet( scheme := corev1.URISchemeHTTP probesPort := int32(8080) - probes, err := probes.CreateProbeSet( + volumeProbes, err := probes.CreateProbeSet( probesPort, &scheme, instance.Spec.Override.Probes, @@ -142,8 +142,8 @@ func StatefulSet( Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), VolumeMounts: volumeMounts, Resources: instance.Spec.Resources, - LivenessProbe: probes.Liveness, - StartupProbe: probes.Startup, + LivenessProbe: volumeProbes.Liveness, + StartupProbe: volumeProbes.Startup, }, { Name: "probe", From 30ff07a70444dfc05bf421c4a285fb10712d0bf2 Mon Sep 17 00:00:00 2001 From: Francesco Pantano Date: Tue, 5 May 2026 12:06:33 +0200 Subject: [PATCH 4/4] Enable cinder-backup testing in envTest This patch enables cinder-backup in envTests and extend the probes override test to this component. GetDefaultCinderSpec is now extended to return cinderBackup as part of the top-level CR and we can now test overrides for probes. Signed-off-by: Francesco Pantano --- api/v1beta1/cinderbackup_types.go | 2 +- test/functional/base_test.go | 13 +++++++++++++ test/functional/cinder_controller_test.go | 22 ++++++++++++++++++++++ test/functional/cinder_test_data.go | 5 +++++ test/functional/suite_test.go | 7 +++++++ 5 files changed, 48 insertions(+), 1 deletion(-) diff --git a/api/v1beta1/cinderbackup_types.go b/api/v1beta1/cinderbackup_types.go index 9042f89f..20dc0465 100644 --- a/api/v1beta1/cinderbackup_types.go +++ b/api/v1beta1/cinderbackup_types.go @@ -24,7 +24,7 @@ import ( "github.com/openstack-k8s-operators/lib-common/modules/common/probes" ) -// CinderBackupOverrideSpec to override the generated manifest of several child resources. +// BackupOverrideSpec to override the generated manifest of several child resources. type BackupOverrideSpec struct { // Override probes and other common fields in the StatefulSet Probes probes.OverrideSpec `json:"probes,omitempty"` diff --git a/test/functional/base_test.go b/test/functional/base_test.go index 3f630635..c09b8bc4 100644 --- a/test/functional/base_test.go +++ b/test/functional/base_test.go @@ -78,6 +78,7 @@ func GetDefaultCinderSpec() map[string]any { "cinderAPI": GetDefaultCinderAPISpec(), "cinderScheduler": GetDefaultCinderSchedulerSpec(), "cinderVolume": GetDefaultCinderVolumeSpec(), + "cinderBackup": GetDefaultCinderBackupSpec(), } } @@ -88,6 +89,7 @@ func GetTLSCinderSpec() map[string]any { "cinderAPI": GetTLSCinderAPISpec(), "cinderScheduler": GetDefaultCinderSchedulerSpec(), "cinderVolume": GetDefaultCinderVolumeSpec(), + "cinderBackup": GetDefaultCinderBackupSpec(), } } @@ -143,6 +145,17 @@ func GetDefaultCinderVolumeSpec() map[string]any { } } +func GetDefaultCinderBackupSpec() map[string]any { + return map[string]any{ + "secret": SecretName, + "replicas": 1, + "containerImage": cinderTest.ContainerImage, + "serviceAccount": cinderTest.CinderSA.Name, + "databaseHostname": cinderTest.DatabaseHostname, + "transportURLSecret": cinderTest.RabbitmqSecretName, + } +} + func GetCinder(name types.NamespacedName) *cinderv1.Cinder { instance := &cinderv1.Cinder{} Eventually(func(g Gomega) { diff --git a/test/functional/cinder_controller_test.go b/test/functional/cinder_controller_test.go index 5985f775..e1d19f28 100644 --- a/test/functional/cinder_controller_test.go +++ b/test/functional/cinder_controller_test.go @@ -2098,6 +2098,13 @@ var _ = Describe("Cinder Webhook", func() { }, }, } + spec["cinderBackup"] = map[string]any{ + "override": map[string]any{ + "probes": map[string]any{ + "livenessProbes": stsOverride, + }, + }, + } DeferCleanup(th.DeleteInstance, CreateCinder(cinderTest.Instance, spec)) DeferCleanup(k8sClient.Delete, ctx, CreateCinderMessageBusSecret(cinderTest.Instance.Namespace, cinderTest.RabbitmqSecretName)) DeferCleanup( @@ -2170,6 +2177,21 @@ var _ = Describe("Cinder Webhook", func() { g.Expect(volContainer.StartupProbe.PeriodSeconds).To(Equal(int32(cinder.GetDefaultProbesRPCWorker(60).StartupProbes.PeriodSeconds))) g.Expect(volContainer.StartupProbe.FailureThreshold).To(Equal(int32(cinder.GetDefaultProbesRPCWorker(60).StartupProbes.FailureThreshold))) }, timeout, interval).Should(Succeed()) + // Check CinderBackup + Eventually(func(g Gomega) { + bkpContainer := th.GetStatefulSet(cinderTest.CinderBackup).Spec.Template.Spec.Containers[0] + g.Expect(bkpContainer.LivenessProbe).ToNot(BeNil()) + g.Expect(bkpContainer.LivenessProbe.HTTPGet.Path).To(Equal("/healthcheck")) + g.Expect(bkpContainer.LivenessProbe.InitialDelaySeconds).To(Equal(int32(20))) + g.Expect(bkpContainer.LivenessProbe.TimeoutSeconds).To(Equal(int32(30))) + g.Expect(bkpContainer.LivenessProbe.PeriodSeconds).To(Equal(int32(10))) + g.Expect(bkpContainer.ReadinessProbe).To(BeNil()) + // We do not override StartupProbes and we apply the defaults defined in the cinder-operator + g.Expect(bkpContainer.StartupProbe.InitialDelaySeconds).To(Equal(int32(cinder.GetDefaultProbesRPCWorker(60).StartupProbes.InitialDelaySeconds))) + g.Expect(bkpContainer.StartupProbe.TimeoutSeconds).To(Equal(int32(cinder.GetDefaultProbesRPCWorker(60).StartupProbes.TimeoutSeconds))) + g.Expect(bkpContainer.StartupProbe.PeriodSeconds).To(Equal(int32(cinder.GetDefaultProbesRPCWorker(60).StartupProbes.PeriodSeconds))) + g.Expect(bkpContainer.StartupProbe.FailureThreshold).To(Equal(int32(cinder.GetDefaultProbesRPCWorker(60).StartupProbes.FailureThreshold))) + }, timeout, interval).Should(Succeed()) }) }) }) diff --git a/test/functional/cinder_test_data.go b/test/functional/cinder_test_data.go index 0b6f75a8..a372cd5c 100644 --- a/test/functional/cinder_test_data.go +++ b/test/functional/cinder_test_data.go @@ -73,6 +73,7 @@ type CinderTestData struct { Cinder types.NamespacedName CinderAPI types.NamespacedName CinderScheduler types.NamespacedName + CinderBackup types.NamespacedName CinderVolumes []types.NamespacedName InternalAPINAD types.NamespacedName ContainerImage string @@ -121,6 +122,10 @@ func GetCinderTestData(cinderName types.NamespacedName) CinderTestData { Name: fmt.Sprintf("%s-volume-volume2", cinderName.Name), }, }, + CinderBackup: types.NamespacedName{ + Namespace: cinderName.Namespace, + Name: fmt.Sprintf("%s-backup", cinderName.Name), + }, CinderRole: types.NamespacedName{ Namespace: cinderName.Namespace, Name: fmt.Sprintf("cinder-%s-role", cinderName.Name), diff --git a/test/functional/suite_test.go b/test/functional/suite_test.go index dd1b91f4..08f647cd 100644 --- a/test/functional/suite_test.go +++ b/test/functional/suite_test.go @@ -242,6 +242,13 @@ var _ = BeforeSuite(func() { }).SetupWithManager(context.Background(), k8sManager) Expect(err).ToNot(HaveOccurred()) + err = (&controllers.CinderBackupReconciler{ + Client: k8sManager.GetClient(), + Scheme: k8sManager.GetScheme(), + Kclient: kclient, + }).SetupWithManager(context.Background(), k8sManager) + Expect(err).ToNot(HaveOccurred()) + go func() { defer GinkgoRecover() err = k8sManager.Start(ctx)