diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e8ccadd..c32f6f7 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -13,7 +13,25 @@ concurrency: cancel-in-progress: true jobs: + resolve-server: + runs-on: ubuntu-latest + outputs: + revision: ${{ steps.resolve.outputs.revision }} + steps: + - id: resolve + env: + CANDIDATE_REF: ${{ github.head_ref || 'main' }} + run: | + remote=https://github.com/batchstream/sink.git + revision=$(git ls-remote --heads "$remote" "refs/heads/$CANDIDATE_REF" | awk '{print $1}') + if [[ -z "$revision" ]]; then + revision=$(git ls-remote --heads "$remote" refs/heads/main | awk '{print $1}') + fi + [[ "$revision" =~ ^[0-9a-f]{40}$ ]] + echo "revision=$revision" >> "$GITHUB_OUTPUT" + validate: + needs: resolve-server runs-on: ubuntu-latest timeout-minutes: 10 steps: @@ -38,10 +56,20 @@ jobs: helm template sink ./charts/sink --kube-version 1.30.0 \ -f examples/cluster-values.yaml -f examples/logging-values.yaml --set metrics.enabled=true \ | kubeconform -strict -summary -kubernetes-version 1.30.0 - - name: Check complete configurations with released Sink - run: python tests/check_runtime.py + - name: Check out matching Sink candidate + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: batchstream/sink + path: sink-candidate + ref: ${{ needs.resolve-server.outputs.revision }} + - name: Check generated files with candidate parser + run: | + go -C sink-candidate build -o "$RUNNER_TEMP/sink" ./cmd/sink + python tests/check_runtime.py --sink-binary "$RUNNER_TEMP/sink" + qualification: + needs: resolve-server name: Kubernetes qualification (${{ matrix.scenario }}) strategy: fail-fast: false @@ -65,8 +93,19 @@ jobs: run: | python -m pip install -r requirements-dev.txt go install sigs.k8s.io/kind@v0.33.0 + - name: Check out matching Sink candidate + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: batchstream/sink + path: sink-candidate + ref: ${{ needs.resolve-server.outputs.revision }} + - name: Build compatible rollout candidates + run: | + docker build --build-arg VERSION=chart-candidate-a -t sink-chart-candidate:a sink-candidate + docker build --build-arg VERSION=chart-candidate-b -t sink-chart-candidate:b sink-candidate - name: Qualify rollouts, Store lifecycle and KEDA with business traffic - run: make integration SCENARIO=${{ matrix.scenario }} + run: python tests/integration/run.py --scenario ${{ matrix.scenario }} --sink-image sink-chart-candidate:a --upgrade-image sink-chart-candidate:b + - name: Preserve qualification evidence if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 diff --git a/README.md b/README.md index 008baae..2bbb466 100644 --- a/README.md +++ b/README.md @@ -6,8 +6,8 @@ configuration with shared Store settings and managed operational defaults; external Secrets contain credential values only. One release deploys a Sink cluster: a shared Gateway and independently configured -Engine and Worker Deployments for each Store. Chart `0.7.0` generates runtime -configuration and references individual credential keys in external Secrets. It targets Sink **0.18.0**, SDK **0.8.0**, and +Engine and Worker Deployments for each Store. Chart `0.8.0` generates runtime +configuration and references individual credential keys in external Secrets. It targets Sink **0.19.0**, SDK **0.8.0**, and Kubernetes **1.30+** with native lifecycle sleep hooks enabled. - Stable Store names, generated headless discovery, staged activation and retirement. @@ -24,6 +24,18 @@ budget assumptions, not a zero-error guarantee under arbitrary DNS/backend failu ## Install +This configuration requires Sink 0.19.0. Until that release is published, build +its matching candidate and override `image.repository`, `image.tag`, and +`image.digest: ""`. Do not pair Chart 0.8 with a 0.18 or older image. + +Each Store gets one `store.yaml` ConfigMap shared by Engine and Worker. Both roles +load it using `--store-config`; their own `sink.yaml` holds only role tuning. +Use `gateway.runtime.forwarding` / `request`, `engineDefaults.runtime.execution` / +`batching` / `producer`, and `workerDefaults.runtime.execution` / `consumer`. +Consumer group belongs in `stores..worker.runtime.consumer.group_id`. +There is no service-wide request timeout; callers control request lifetime. + + Create the release namespace and provision `sink-mongo-v1` and `sink-archive-v1` Secrets there. Each example references a key named `uri` containing its complete MongoDB URI. Use your secret manager; never commit credentials to values files. @@ -127,21 +139,18 @@ Rendering tests are offline. `make integration` is an explicit Docker/Kind test; see [tests/integration](tests/integration/README.md). CI also checks Kubernetes schemas and complete example configurations against the pinned Sink image. -### Demand-based memory admission - -The pinned Sink 0.18.0 image supports demand-based memory admission through -`gateway.runtime.memory`, `engineDefaults.runtime.memory`, and -`workerDefaults.runtime.memory`, with per-Store overrides under each role. -The map supports `max_bytes`, `burst_percent` (1–99), and `wait_timeout`. -Leave it empty for server automatic sizing and its measured 10% reserve default. -Empty maps are omitted from generated YAML, which also permits explicit legacy -image overrides during staged upgrades. Keep these fields empty on Sink 0.16.0. - -[The memory KEDA overlay](examples/memory-keda-values.yaml) supplies managed-capacity -pressure and temporary-rejection signals with `metricType: Value`. Install KEDA, -configure Prometheus scraping and adjust job/namespace selectors to one Gateway -Deployment before enabling it. Engine selectors must also isolate the Store; -Workers should retain Kafka lag triggers. Missing metrics must remain a scaler -error rather than zero load. Upgrade Engines before Gateways for the private -framed-response protocol; roll back Gateways first. Applying the example changes -scaling configuration; plan and verify the rollout using the operational runbook. +### Process memory admission + +Sink 0.19.0 uses high/low watermarks through `gateway.runtime.memory`, +`engineDefaults.runtime.memory`, and `workerDefaults.runtime.memory`, with per-Store +overrides. Supported fields are `max_bytes`, `high_watermark_percent` (default 80), +and `low_watermark_percent` (default 70). Require `0 < low < high < 100`. +An empty map selects server defaults. Startup panics when estimated minimum +working memory does not fit below the high watermark; larger gRPC/Lua/Kafka +buffers may require larger Pod limits. See [server sizing](https://github.com/batchstream/sink/blob/main/docs/design/demand-based-admission.md). + +[The memory KEDA overlay](examples/memory-keda-values.yaml) uses observed memory +pressure and RPC rejection ratios with `metricType: Value`. Configure Prometheus +selectors for one role/Deployment/Store and retain Worker Kafka lag triggers. +Missing data remains a scaler error. Gateway and Engine must use matching private +protocol versions; incompatible versions require an isolated cutover. diff --git a/charts/sink/Chart.yaml b/charts/sink/Chart.yaml index a65f122..c53e7a1 100644 --- a/charts/sink/Chart.yaml +++ b/charts/sink/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: sink description: Store-isolated Sink clusters with bounded rollouts and independent autoscaling. type: application -version: 0.7.0 -appVersion: "0.18.0" +version: 0.8.0 +appVersion: "0.19.0" kubeVersion: ">=1.30.0-0" home: https://github.com/batchstream/sink sources: diff --git a/charts/sink/templates/_helpers.tpl b/charts/sink/templates/_helpers.tpl index 76368fa..cda96f6 100644 --- a/charts/sink/templates/_helpers.tpl +++ b/charts/sink/templates/_helpers.tpl @@ -59,11 +59,11 @@ helm.sh/chart: {{ printf "%s-%s" .root.Chart.Name .root.Chart.Version | quote }} {{- end -}} {{- define "sink.preStop" -}} {{- $minimum := 0 -}} -{{- if eq .role "engine" -}}{{- $minimum = add (include "sink.discoverySeconds" .root | int) .root.Values.requestTimeoutSeconds .root.Values.discovery.safetyMarginSeconds -}}{{- end -}} -{{- if eq .role "gateway" -}}{{- $minimum = add .root.Values.discovery.clientWithdrawalSeconds .root.Values.requestTimeoutSeconds .root.Values.discovery.safetyMarginSeconds -}}{{- end -}} +{{- if eq .role "engine" -}}{{- $minimum = add (include "sink.discoverySeconds" .root | int) .root.Values.discovery.safetyMarginSeconds -}}{{- end -}} +{{- if eq .role "gateway" -}}{{- $minimum = add .root.Values.discovery.clientWithdrawalSeconds .root.Values.discovery.safetyMarginSeconds -}}{{- end -}} {{- $seconds := $minimum -}} {{- if ne .settings.pod.preStopSeconds nil -}}{{- $seconds = int .settings.pod.preStopSeconds -}}{{- end -}} -{{- if lt (int $seconds) (int $minimum) -}}{{- fail (printf "%s/%s preStopSeconds must be >= %d (discovery + request + margin)" .role .store $minimum) -}}{{- end -}} +{{- if lt (int $seconds) (int $minimum) -}}{{- fail (printf "%s/%s preStopSeconds must be >= %d (discovery + margin)" .role .store $minimum) -}}{{- end -}} {{- $seconds -}} {{- end -}} {{- define "sink.grace" -}} @@ -88,10 +88,10 @@ helm.sh/chart: {{ printf "%s-%s" .root.Chart.Name .root.Chart.Version | quote }} {{- $routes = append $routes (dict "store" $store "target" (printf "dns:///%s.%s.svc.%s:8080" $name $.Release.Namespace $.Values.clusterDomain) "tls" (dict "insecure" true)) -}} {{- end -}} {{- end -}} -{{- $gateway := mergeOverwrite (deepCopy .Values.gateway.runtime.gateway) (dict "routes" $routes "dns_refresh_interval" (printf "%ds" (int .Values.discovery.dnsRefreshSeconds))) -}} -{{- $service := mergeOverwrite (deepCopy .Values.gateway.runtime.service) (dict "request" (dict "timeout" (printf "%ds" (int .Values.requestTimeoutSeconds)))) -}} +{{- $gateway := mergeOverwrite (deepCopy .Values.gateway.runtime.forwarding) (dict "routes" $routes "dns_refresh_interval" (printf "%ds" (int .Values.discovery.dnsRefreshSeconds))) -}} {{- $grpc := mergeOverwrite (deepCopy .Values.gateway.runtime.grpc) (dict "address" ":8080") -}} -{{- $config := (dict "mode" "gateway" "gateway" $gateway "service" $service "grpc" $grpc "health" (dict "address" ":8081") "prometheus" (dict "enabled" .Values.metrics.enabled "address" ":9090") "shutdown_timeout" (printf "%ds" (int .Values.gateway.pod.shutdownTimeoutSeconds))) -}} +{{- $config := (dict "mode" "gateway" "forwarding" $gateway "grpc" $grpc "health" (dict "address" ":8081") "prometheus" (dict "enabled" .Values.metrics.enabled "address" ":9090") "shutdown_timeout" (printf "%ds" (int .Values.gateway.pod.shutdownTimeoutSeconds))) -}} +{{- with .Values.gateway.runtime.request -}}{{- $_ := set $config "request" . -}}{{- end -}} {{- with .Values.gateway.runtime.memory -}}{{- $_ := set $config "memory" . -}}{{- end -}} {{- with .Values.gateway.runtime.logging -}}{{- $_ := set $config "logging" . -}}{{- end -}} {{- toYaml $config -}} diff --git a/charts/sink/templates/_store.tpl b/charts/sink/templates/_store.tpl index 2a2e365..6cc861a 100644 --- a/charts/sink/templates/_store.tpl +++ b/charts/sink/templates/_store.tpl @@ -16,9 +16,11 @@ {{- end -}} {{- define "sink.storeConfig" -}} -{{- $storage := dict "name" .store "driver" .storage.driver -}} +{{- $storage := dict "driver" .storage.driver -}} {{- if eq .storage.driver "mongodb" -}} -{{- $mongo := dict "metadata_field" (default "__sink" .storage.mongodb.metadataField) "max_concurrent_writes" (default 64 .storage.mongodb.maxConcurrentWrites) "max_concurrent_groups" (default 16 .storage.mongodb.maxConcurrentGroups) -}} +{{- $mongo := dict "metadata_field" (default "__sink" .storage.mongodb.metadataField) -}} +{{- $_ := set $mongo "max_concurrent_writes" (default 64 .storage.mongodb.maxConcurrentWrites) -}} +{{- $_ := set $mongo "max_concurrent_groups" (default 16 .storage.mongodb.maxConcurrentGroups) -}} {{- $_ := set $mongo "uri_file" "/etc/sink-secrets/mongodb-uri" -}} {{- $_ := set $storage "mongodb" $mongo -}} {{- else -}} @@ -31,15 +33,24 @@ {{- end -}} {{- $_ := set $storage "search" $search -}} {{- end -}} +{{- $config := dict "name" .store "storage" $storage -}} {{- if .kafka -}} {{- $kafka := deepCopy (default dict .kafka.runtime) -}} -{{- $managed := dict "enabled" true "brokers" .kafka.brokers "topic" (dict "name" .kafka.topic "partitions" .kafka.partitions "replication_factor" (default 3 .kafka.replicationFactor) "min_insync_replicas" (default 2 .kafka.minInSyncReplicas)) "consumer" (dict "group_id" .kafka.consumerGroup) -}} -{{- $_ := set $storage "kafka" (mergeOverwrite $kafka $managed) -}} -{{- end -}} -{{- $service := mergeOverwrite (deepCopy .settings.runtime.service) (dict "request" (dict "timeout" (printf "%ds" (int .root.Values.requestTimeoutSeconds)))) -}} -{{- $grpc := mergeOverwrite (deepCopy .settings.runtime.grpc) (dict "address" ":8080") -}} -{{- $config := (dict "mode" .role "storage" $storage "service" $service "grpc" $grpc "health" (dict "address" ":8081") "prometheus" (dict "enabled" .root.Values.metrics.enabled "address" ":9090") "shutdown_timeout" (printf "%ds" (int .settings.pod.shutdownTimeoutSeconds))) -}} -{{- with .settings.runtime.memory -}}{{- $_ := set $config "memory" . -}}{{- end -}} -{{- with .settings.runtime.logging -}}{{- $_ := set $config "logging" . -}}{{- end -}} +{{- $managed := dict "enabled" true "brokers" .kafka.brokers "partitions" .kafka.partitions "replication_factor" (default 3 .kafka.replicationFactor) "min_insync_replicas" (default 1 .kafka.minInSyncReplicas) "topic" (dict "name" .kafka.topic) -}} +{{- $_ := set $config "kafka" (mergeOverwrite $kafka $managed) -}} +{{- end -}} +{{- toYaml $config -}} +{{- end -}} + +{{- define "sink.roleConfig" -}} +{{- $config := deepCopy .settings.runtime -}} +{{- $_ := set $config "mode" .role -}} +{{- $_ := set $config "health" (dict "address" ":8081") -}} +{{- $_ := set $config "prometheus" (dict "enabled" .root.Values.metrics.enabled "address" ":9090") -}} +{{- $_ := set $config "shutdown_timeout" (printf "%ds" (int .settings.pod.shutdownTimeoutSeconds)) -}} +{{- if eq .role "engine" -}} +{{- $_ := set $config "grpc" (mergeOverwrite (default dict $config.grpc) (dict "address" ":8080")) -}} +{{- end -}} +{{- range $key, $value := $config -}}{{- if empty $value -}}{{- $_ := unset $config $key -}}{{- end -}}{{- end -}} {{- toYaml $config -}} {{- end -}} diff --git a/charts/sink/templates/autoscaling.yaml b/charts/sink/templates/autoscaling.yaml index 015b44a..e748978 100644 --- a/charts/sink/templates/autoscaling.yaml +++ b/charts/sink/templates/autoscaling.yaml @@ -77,7 +77,7 @@ spec: {{- end }} metadata: bootstrapServers: {{ join "," $c.kafka.brokers | quote }} - consumerGroup: {{ $c.kafka.consumerGroup | quote }} + consumerGroup: {{ $c.settings.runtime.consumer.group_id | quote }} topic: {{ $c.kafka.topic | quote }} lagThreshold: {{ default 1000 $c.kafka.lagThreshold | quote }} activationLagThreshold: "0" diff --git a/charts/sink/templates/config.yaml b/charts/sink/templates/config.yaml index a084a8d..ea41e7a 100644 --- a/charts/sink/templates/config.yaml +++ b/charts/sink/templates/config.yaml @@ -11,19 +11,22 @@ {{- $engine := mergeOverwrite (deepCopy $.Values.engineDefaults) (default dict $settings.engine) }} {{- $worker := mergeOverwrite (deepCopy $.Values.workerDefaults) (default dict $settings.worker) }} {{- $kafka := omit (default dict $settings.kafka) "lagThreshold" "runtime" }} +{{- if $kafka }}{{- $_ := set $kafka "consumerGroup" (default "" $worker.runtime.consumer.group_id) }}{{- end }} {{- $kafkaRuntime := default dict (default dict $settings.kafka).runtime }} {{- $deadLetter := default dict $kafkaRuntime.dead_letter }} -{{- with $deadLetter.topic }}{{- $_ := set $kafka "deadLetterTopic" . }}{{- end }} -{{- $engineConfig := include "sink.storeConfig" (dict "root" $ "role" "engine" "store" $store "settings" $engine "storage" $settings.storage "kafka" (default dict $settings.kafka)) }} +{{- with $deadLetter.name }}{{- $_ := set $kafka "deadLetterTopic" . }}{{- end }} +{{- $engineConfig := include "sink.roleConfig" (dict "root" $ "role" "engine" "store" $store "settings" $engine "storage" $settings.storage "kafka" (default dict $settings.kafka)) }} {{- $entry := dict "state" $settings.state "kafka" $kafka "workerEnabled" $worker.enabled "engineHash" (toJson (dict "engine" $engine "storage" $settings.storage "kafkaRuntime" (default dict $settings.kafka) "credentialRevision" (default "" $settings.credentialRevision) "runtime" $engineConfig "image" $.Values.image "discovery" $.Values.discovery) | sha256sum) }} {{- $_ := set $current $store $entry }} {{- if $kafka }} -{{- range $pair := list (dict "name" "topic" "value" $kafka.topic "seen" $topics) (dict "name" "deadLetterTopic" "value" (default (printf "%s.dlq" (trim $kafka.topic)) $deadLetter.topic) "seen" $topics) (dict "name" "consumerGroup" "value" $kafka.consumerGroup "seen" $groups) }} +{{- range $pair := list (dict "name" "topic" "value" $kafka.topic "seen" $topics) (dict "name" "deadLetterTopic" "value" (default (printf "%s.dlq" (trim $kafka.topic)) $deadLetter.name) "seen" $topics) (dict "name" "consumerGroup" "value" $kafka.consumerGroup "seen" $groups) }} {{- $identity := trim $pair.value }} +{{- if $identity }} {{- if hasKey $pair.seen $identity }}{{- fail (printf "Kafka %s must be unique per Store" $pair.name) }}{{- end }} {{- $_ := set $pair.seen $identity true }} {{- end }} {{- end }} +{{- end }} {{- if $hasInventory }} {{- $old := get $previous $store }} {{- if not $old }} @@ -102,6 +105,20 @@ metadata: {{- include "sink.labels" $c | nindent 4 }} data: sink.yaml: | - {{- include "sink.storeConfig" $c | nindent 4 }} + {{- include "sink.roleConfig" $c | nindent 4 }} +{{- end }} {{- end }} + +{{- range $store, $settings := .Values.stores }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "sink.componentName" (dict "root" $ "role" "store" "store" $store) }} + labels: + app.kubernetes.io/instance: {{ $.Release.Name | quote }} + app.kubernetes.io/managed-by: {{ $.Release.Service }} +data: + store.yaml: | + {{- include "sink.storeConfig" (dict "store" $store "storage" $settings.storage "kafka" (default dict $settings.kafka)) | nindent 4 }} {{- end }} diff --git a/charts/sink/templates/deployment.yaml b/charts/sink/templates/deployment.yaml index 067ebfc..f0b42be 100644 --- a/charts/sink/templates/deployment.yaml +++ b/charts/sink/templates/deployment.yaml @@ -36,7 +36,7 @@ spec: {{- if eq $c.role "gateway" }} sink.batchstream.io/config-checksum: {{ include "sink.gatewayConfig" $ | sha256sum | quote }} {{- else }} - sink.batchstream.io/config-checksum: {{ include "sink.storeConfig" $c | sha256sum | quote }} + sink.batchstream.io/config-checksum: {{ printf "%s\n%s" (include "sink.roleConfig" $c) (include "sink.storeConfig" $c) | sha256sum | quote }} sink.batchstream.io/credential-revision: {{ $c.credentialRevision | quote }} {{- end }} {{- with $s.pod.annotations }}{{ toYaml . | nindent 8 }}{{- end }} @@ -82,7 +82,7 @@ spec: - name: sink image: {{ if $image.digest }}{{ printf "%s@%s" $image.repository $image.digest | quote }}{{ else }}{{ printf "%s:%s" $image.repository $image.tag | quote }}{{ end }} imagePullPolicy: {{ $image.pullPolicy }} - args: ["--config", "/etc/sink/sink.yaml"] + args: ["--config", "/etc/sink/sink.yaml"{{ if ne $c.role "gateway" }}, "--store-config", "/etc/sink/store.yaml"{{ end }}] securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true @@ -146,8 +146,14 @@ spec: {{- end }} volumes: - name: config - configMap: - name: {{ $name }} + projected: + sources: + - configMap: + name: {{ $name }} + {{- if ne $c.role "gateway" }} + - configMap: + name: {{ include "sink.componentName" (dict "root" $ "role" "store" "store" $c.store) }} + {{- end }} {{- if ne $c.role "gateway" }} {{- $sources := include "sink.secretSources" $c | fromJsonArray }} {{- if $sources }} diff --git a/charts/sink/templates/validate.yaml b/charts/sink/templates/validate.yaml index 8e91949..0090eb0 100644 --- a/charts/sink/templates/validate.yaml +++ b/charts/sink/templates/validate.yaml @@ -16,7 +16,6 @@ {{- if and $s.disruptionBudget.enabled (ge (int $s.disruptionBudget.maxUnavailable) $minimum) -}}{{- fail "disruptionBudget.maxUnavailable must be less than minimum replicas" -}}{{- end -}} {{- $grace := include "sink.grace" $c | int -}} {{- if lt (int $s.pod.shutdownBudgetSeconds) (mul 4 $s.pod.shutdownTimeoutSeconds | int) -}}{{- fail "shutdownBudgetSeconds must cover at least four sequential shutdown timeouts" -}}{{- end -}} -{{- if and (eq $c.role "gateway") (lt (int $s.pod.shutdownTimeoutSeconds) (int $.Values.requestTimeoutSeconds)) -}}{{- fail "Gateway shutdownTimeoutSeconds must cover requestTimeoutSeconds" -}}{{- end -}} {{- if ne $a.mode "none" -}} {{- range $policy := $a.behavior.scaleDown.policies -}}{{- if lt (int $policy.periodSeconds) $grace -}}{{- fail (printf "%s/%s scaleDown policy periodSeconds must cover termination grace (%d)" $c.role $c.store $grace) -}}{{- end -}}{{- end -}} {{- end -}} @@ -30,7 +29,7 @@ {{- if or (lt (int $a.keda.fallback.replicas) $minimum) (gt (int $a.keda.fallback.replicas) (int $a.maxReplicas)) -}}{{- fail "KEDA fallback replicas must be within the scaling range" -}}{{- end -}} {{- end -}} {{- if eq $c.role "worker" -}} -{{- if not (and $c.kafka.brokers $c.kafka.topic $c.kafka.consumerGroup $c.kafka.partitions) -}}{{- fail (printf "Store %s Worker requires kafka brokers, topic, consumerGroup and partitions" $c.store) -}}{{- end -}} +{{- if not (and $c.kafka.brokers $c.kafka.topic $s.runtime.consumer.group_id $c.kafka.partitions) -}}{{- fail (printf "Store %s Worker requires kafka brokers, topic, partitions and worker.runtime.consumer.group_id" $c.store) -}}{{- end -}} {{- $max := int $s.replicaCount -}}{{- if ne $a.mode "none" -}}{{- $max = int $a.maxReplicas -}}{{- end -}} {{- if gt $max (int $c.kafka.partitions) -}}{{- fail (printf "Store %s Worker maximum replicas exceeds Kafka partitions" $c.store) -}}{{- end -}} {{- end -}} @@ -46,7 +45,7 @@ {{- range $key, $_ := $s.pod.labels -}}{{- if or (hasPrefix "app.kubernetes.io/" $key) (hasPrefix "sink.batchstream.io/" $key) (eq $key "helm.sh/chart") -}}{{- fail "pod.labels cannot override chart identity labels" -}}{{- end -}}{{- end -}} {{- range $key, $_ := $s.pod.annotations -}}{{- if hasPrefix "sink.batchstream.io/" $key -}}{{- fail "pod.annotations cannot override chart configuration annotations" -}}{{- end -}}{{- end -}} {{- end -}} -{{- if or (hasKey .Values.gateway.runtime.gateway "routes") (hasKey .Values.gateway.runtime.gateway "dns_refresh_interval") (hasKey .Values.gateway.runtime.grpc "address") (hasKey (default dict .Values.gateway.runtime.service.request) "timeout") -}}{{- fail "Gateway routes, DNS refresh, listeners and request timeout are chart-managed" -}}{{- end -}} +{{- if or (hasKey .Values.gateway.runtime.forwarding "routes") (hasKey .Values.gateway.runtime.forwarding "dns_refresh_interval") (hasKey .Values.gateway.runtime.grpc "address") -}}{{- fail "Gateway routes, DNS refresh, listeners are chart-managed" -}}{{- end -}} {{- range $store, $settings := .Values.stores -}} {{- if ne $settings.storage.driver "mongodb" -}} {{- $s := $settings.storage.search -}} @@ -59,14 +58,18 @@ {{- if and $s.apiKeySecretRef $basic -}}{{- fail "search apiKeySecretRef cannot be combined with basic authentication" -}}{{- end -}} {{- end -}} {{- with $settings.kafka -}} -{{- if gt (int (default 2 .minInSyncReplicas)) (int (default 3 .replicationFactor)) -}}{{- fail "Kafka minInSyncReplicas cannot exceed replicationFactor" -}}{{- end -}} +{{- if gt (int (default 1 .minInSyncReplicas)) (int (default 3 .replicationFactor)) -}}{{- fail "Kafka minInSyncReplicas cannot exceed replicationFactor" -}}{{- end -}} {{- $runtime := default dict .runtime -}} {{- $topic := default dict $runtime.topic -}} -{{- if or (hasKey $topic "name") (hasKey $topic "partitions") (hasKey $topic "replication_factor") (hasKey $topic "min_insync_replicas") (hasKey (default dict $runtime.consumer) "group_id") -}}{{- fail "Kafka identity and replication policy are chart-managed" -}}{{- end -}} +{{- if or (hasKey $topic "name") (hasKey $runtime "partitions") (hasKey $runtime "replication_factor") (hasKey $runtime "min_insync_replicas") -}}{{- fail "Kafka identity and replication policy are chart-managed" -}}{{- end -}} {{- end -}} {{- end -}} {{- range $component := include "sink.components" . | fromJsonArray -}} +{{- $memory := default dict $component.settings.runtime.memory -}} +{{- $high := int (default 80 $memory.high_watermark_percent) -}} +{{- $low := int (default 70 $memory.low_watermark_percent) -}} +{{- if ge $low $high -}}{{- fail "memory.low_watermark_percent must be lower than high_watermark_percent" -}}{{- end -}} {{- if ne $component.role "gateway" -}} -{{- if or (hasKey $component.settings.runtime.grpc "address") (hasKey (default dict $component.settings.runtime.service.request) "timeout") -}}{{- fail "Store listeners and request timeout are chart-managed" -}}{{- end -}} +{{- if hasKey (default dict $component.settings.runtime.grpc) "address" -}}{{- fail "Store listeners are chart-managed" -}}{{- end -}} {{- end -}} {{- end -}} diff --git a/charts/sink/values.schema.json b/charts/sink/values.schema.json index 8ff08fb..f7e74a4 100644 --- a/charts/sink/values.schema.json +++ b/charts/sink/values.schema.json @@ -94,11 +94,6 @@ } } }, - "requestTimeoutSeconds": { - "type": "integer", - "minimum": 1, - "maximum": 300 - }, "lifecycle": { "type": "object", "additionalProperties": false, @@ -215,20 +210,76 @@ "type": "object", "additionalProperties": false, "properties": { - "gateway": { - "type": "object" - }, - "service": { - "type": "object" - }, "grpc": { - "type": "object" + "type": "object", + "additionalProperties": false, + "properties": { + "address": { + "type": "string" + }, + "max_receive_message_bytes": { + "oneOf": [ + { + "type": "integer", + "minimum": 1024 + }, + { + "type": "string", + "pattern": "^[1-9][0-9]*(KiB|MiB|GiB)?$" + } + ] + }, + "max_send_message_bytes": { + "oneOf": [ + { + "type": "integer", + "minimum": 1024 + }, + { + "type": "string", + "pattern": "^[1-9][0-9]*(KiB|MiB|GiB)?$" + } + ] + } + } }, "memory": { "$ref": "#/definitions/memoryCapacity" }, "logging": { "$ref": "#/definitions/logging" + }, + "forwarding": { + "type": "object", + "additionalProperties": false, + "properties": { + "dns_refresh_interval": { + "type": "string", + "minLength": 1 + }, + "idle_timeout": { + "type": "string", + "minLength": 1 + }, + "max_connections": { + "type": "integer", + "minimum": 1 + }, + "max_fanout": { + "type": "integer", + "minimum": 1 + } + } + }, + "request": { + "type": "object", + "additionalProperties": false, + "properties": { + "max_operations": { + "type": "integer", + "minimum": 1 + } + } } } }, @@ -280,10 +331,6 @@ "type": "string", "minLength": 1 }, - "consumerGroup": { - "type": "string", - "minLength": 1 - }, "partitions": { "type": "integer", "minimum": 1 @@ -305,16 +352,40 @@ "additionalProperties": false, "properties": { "topic": { - "type": "object" - }, - "producer": { - "type": "object" - }, - "consumer": { - "type": "object" + "type": "object", + "additionalProperties": false, + "properties": { + "retention": { + "type": "string", + "minLength": 1 + } + } }, "dead_letter": { - "type": "object" + "type": "object", + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "retention": { + "type": "string", + "minLength": 1 + } + } + }, + "max_record_bytes": { + "oneOf": [ + { + "type": "integer", + "minimum": 1 + }, + { + "type": "string", + "pattern": "^[1-9][0-9]*(B|KiB|MiB|GiB)?$" + } + ] } } } @@ -322,7 +393,6 @@ "required": [ "brokers", "topic", - "consumerGroup", "partitions" ] }, @@ -772,17 +842,168 @@ "type": "object", "additionalProperties": false, "properties": { - "service": { - "type": "object" - }, "grpc": { - "type": "object" + "type": "object", + "additionalProperties": false, + "properties": { + "address": { + "type": "string" + }, + "max_receive_message_bytes": { + "oneOf": [ + { + "type": "integer", + "minimum": 1024 + }, + { + "type": "string", + "pattern": "^[1-9][0-9]*(KiB|MiB|GiB)?$" + } + ] + }, + "max_send_message_bytes": { + "oneOf": [ + { + "type": "integer", + "minimum": 1024 + }, + { + "type": "string", + "pattern": "^[1-9][0-9]*(KiB|MiB|GiB)?$" + } + ] + } + } }, "memory": { "$ref": "#/definitions/memoryCapacity" }, "logging": { "$ref": "#/definitions/logging" + }, + "execution": { + "type": "object", + "additionalProperties": false, + "properties": { + "merge": { + "type": "object", + "additionalProperties": false, + "properties": { + "max_attempts": { + "type": "integer", + "minimum": 1 + }, + "lua": { + "type": "object", + "additionalProperties": false, + "properties": { + "timeout": { + "type": "string", + "minLength": 1 + }, + "max_source_bytes": { + "oneOf": [ + { + "type": "integer", + "minimum": 1024 + }, + { + "type": "string", + "pattern": "^[1-9][0-9]*(KiB|MiB|GiB)?$" + } + ] + }, + "max_result_bytes": { + "oneOf": [ + { + "type": "integer", + "minimum": 1024 + }, + { + "type": "string", + "pattern": "^[1-9][0-9]*(KiB|MiB|GiB)?$" + } + ] + }, + "max_cached_programs": { + "type": "integer", + "minimum": 1 + }, + "max_instructions": { + "type": "integer", + "minimum": 1 + } + } + } + } + } + } + }, + "batching": { + "type": "object", + "additionalProperties": false, + "properties": { + "max_wait": { + "type": "string", + "minLength": 1 + }, + "max_operations": { + "type": "integer", + "minimum": 1 + }, + "max_bytes": { + "oneOf": [ + { + "type": "integer", + "minimum": 1024 + }, + { + "type": "string", + "pattern": "^[1-9][0-9]*(KiB|MiB|GiB)?$" + } + ] + }, + "queue": { + "type": "object", + "additionalProperties": false, + "properties": { + "max_operations": { + "type": "integer", + "minimum": 1 + }, + "max_bytes": { + "oneOf": [ + { + "type": "integer", + "minimum": 1024 + }, + { + "type": "string", + "pattern": "^[1-9][0-9]*(KiB|MiB|GiB)?$" + } + ] + } + } + } + } + }, + "producer": { + "type": "object", + "additionalProperties": false, + "properties": { + "max_buffered_bytes": { + "oneOf": [ + { + "type": "integer", + "minimum": 1024 + }, + { + "type": "string", + "pattern": "^[1-9][0-9]*(KiB|MiB|GiB)?$" + } + ] + } + } } } }, @@ -818,17 +1039,105 @@ "type": "object", "additionalProperties": false, "properties": { - "service": { - "type": "object" - }, - "grpc": { - "type": "object" - }, "memory": { "$ref": "#/definitions/memoryCapacity" }, "logging": { "$ref": "#/definitions/logging" + }, + "execution": { + "type": "object", + "additionalProperties": false, + "properties": { + "merge": { + "type": "object", + "additionalProperties": false, + "properties": { + "max_attempts": { + "type": "integer", + "minimum": 1 + }, + "lua": { + "type": "object", + "additionalProperties": false, + "properties": { + "timeout": { + "type": "string", + "minLength": 1 + }, + "max_source_bytes": { + "oneOf": [ + { + "type": "integer", + "minimum": 1024 + }, + { + "type": "string", + "pattern": "^[1-9][0-9]*(KiB|MiB|GiB)?$" + } + ] + }, + "max_result_bytes": { + "oneOf": [ + { + "type": "integer", + "minimum": 1024 + }, + { + "type": "string", + "pattern": "^[1-9][0-9]*(KiB|MiB|GiB)?$" + } + ] + }, + "max_cached_programs": { + "type": "integer", + "minimum": 1 + }, + "max_instructions": { + "type": "integer", + "minimum": 1 + } + } + } + } + } + } + }, + "consumer": { + "type": "object", + "additionalProperties": false, + "properties": { + "group_id": { + "type": "string", + "minLength": 1 + }, + "max_poll_records": { + "type": "integer", + "minimum": 1 + }, + "processing_timeout": { + "type": "string", + "minLength": 1 + }, + "retry": { + "type": "object", + "additionalProperties": false, + "properties": { + "max_attempts": { + "type": "integer", + "minimum": 1 + }, + "backoff": { + "type": "string", + "minLength": 1 + }, + "max_backoff": { + "type": "string", + "minLength": 1 + } + } + } + } } } }, @@ -1052,14 +1361,15 @@ } ] }, - "burst_percent": { + "high_watermark_percent": { "type": "integer", "minimum": 1, "maximum": 99 }, - "wait_timeout": { - "type": "string", - "pattern": "^[1-9][0-9]*(ms|s|m|h)$" + "low_watermark_percent": { + "type": "integer", + "minimum": 1, + "maximum": 99 } } }, @@ -1201,15 +1511,15 @@ }, "flush_interval": { "type": "string", - "pattern": "^\\+?([0-9]+(\\.[0-9]*)?|\\.[0-9]+)(ns|us|µs|μs|ms|s|m|h)(([0-9]+(\\.[0-9]*)?|\\.[0-9]+)(ns|us|µs|μs|ms|s|m|h))*$" + "pattern": "^\\+?([0-9]+(\\.[0-9]*)?|\\.[0-9]+)(ns|us|\u00b5s|\u03bcs|ms|s|m|h)(([0-9]+(\\.[0-9]*)?|\\.[0-9]+)(ns|us|\u00b5s|\u03bcs|ms|s|m|h))*$" }, "export_timeout": { "type": "string", - "pattern": "^\\+?([0-9]+(\\.[0-9]*)?|\\.[0-9]+)(ns|us|µs|μs|ms|s|m|h)(([0-9]+(\\.[0-9]*)?|\\.[0-9]+)(ns|us|µs|μs|ms|s|m|h))*$" + "pattern": "^\\+?([0-9]+(\\.[0-9]*)?|\\.[0-9]+)(ns|us|\u00b5s|\u03bcs|ms|s|m|h)(([0-9]+(\\.[0-9]*)?|\\.[0-9]+)(ns|us|\u00b5s|\u03bcs|ms|s|m|h))*$" }, "shutdown_timeout": { "type": "string", - "pattern": "^\\+?([0-9]+(\\.[0-9]*)?|\\.[0-9]+)(ns|us|µs|μs|ms|s|m|h)(([0-9]+(\\.[0-9]*)?|\\.[0-9]+)(ns|us|µs|μs|ms|s|m|h))*$" + "pattern": "^\\+?([0-9]+(\\.[0-9]*)?|\\.[0-9]+)(ns|us|\u00b5s|\u03bcs|ms|s|m|h)(([0-9]+(\\.[0-9]*)?|\\.[0-9]+)(ns|us|\u00b5s|\u03bcs|ms|s|m|h))*$" } } } diff --git a/charts/sink/values.yaml b/charts/sink/values.yaml index 94955f8..f06d073 100644 --- a/charts/sink/values.yaml +++ b/charts/sink/values.yaml @@ -3,10 +3,10 @@ nameOverride: "" fullnameOverride: "" clusterDomain: cluster.local image: - # Pin the tested multi-architecture release digest. + # Requires the role-config release; override with a candidate image for pre-release testing. repository: ghcr.io/batchstream/sink - tag: "0.18.0" - digest: sha256:9a5cef79198cbda78e6c4b596746079c7633233393e20c9f70928c078a6429e2 + tag: "0.19.0" + digest: "" pullPolicy: IfNotPresent imagePullSecrets: [] @@ -20,7 +20,6 @@ discovery: safetyMarginSeconds: 10 # Includes client DNS, load balancer, proxy and EndpointSlice propagation. clientWithdrawalSeconds: 60 -requestTimeoutSeconds: 30 lifecycle: # Live Helm upgrades use the inventory ConfigMap and Pod/Deployment metadata. @@ -53,9 +52,9 @@ gateway: runtime: # Additional runtime tuning; managed routing/listeners cannot be overridden. logging: {} # Sink 0.18+: level, components, console, labels, failure_body, max_body_bytes, otlp. - memory: {} # Omit to auto-detect capacity; new memory-admission Sink builds only. - gateway: {} - service: {} + memory: {} # Omit to detect the process ceiling; high/low watermarks default to 80%/70%. + forwarding: {} + request: {} grpc: {} pod: configRevision: "" @@ -117,7 +116,8 @@ gateway: engineDefaults: image: {} # Per-Store engine.image overrides these fields. replicaCount: 2 - runtime: {logging: {}, memory: {}, service: {}, grpc: {}} # Nonsecret per-role tuning only. + # Inherit Sink defaults: batching max_operations=32, max_wait=2ms; memory high/low watermarks=80%/70%. + runtime: {logging: {}, memory: {}, execution: {}, batching: {}, producer: {}, grpc: {}} # Per-Engine tuning. readiness: storage # storage | kafka | process; do not combine dependency restarts with liveness. pod: configRevision: "" # Bump to reload an in-place Secret change; prefer versioned Secret names. @@ -177,7 +177,7 @@ workerDefaults: enabled: false # Enable together with the Store's Kafka settings. replicaCount: 2 allowScaleToZero: false # Explicitly accept delayed application while scaled to zero. - runtime: {logging: {}, memory: {}, service: {}, grpc: {}} + runtime: {logging: {}, memory: {}, execution: {}, consumer: {}} pod: configRevision: "" annotations: {} @@ -243,11 +243,10 @@ workerDefaults: # mongodb: # uriSecretRef: {name: orders-mongo-v1, key: uri} # metadataField: __sink -# maxConcurrentWrites: 64 +# maxConcurrentWrites: 64 # Per process; both roles use this Store policy. # maxConcurrentGroups: 16 # credentialRevision: "" # Roll both roles after an in-place Secret update. -# engine: {runtime: {memory: {}, service: {}, grpc: {}}} # worker: {enabled: false} # Add kafka metadata to enable async delivery; defaults: replicationFactor=3, -# minInSyncReplicas=2. kafka.runtime carries additional nonsecret Kafka tuning. +# minInSyncReplicas=1. kafka.runtime carries additional nonsecret Kafka tuning. stores: {} diff --git a/docs/operations.md b/docs/operations.md index 7266e72..2c3d61b 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -2,17 +2,18 @@ ## Configuration and initial admission -Chart 0.4 uses a shared `stores..storage` object for each Engine/Worker +Chart 0.8 uses a shared `stores..storage` object for each Engine/Worker pair. **Maintain only values, not a separate Sink config YAML.** The chart generates -the complete configuration for every role. It requires Sink 0.16.0+. The previous complete configuration Secret interface +the complete configuration for every role. It requires Sink 0.19.0+. The previous complete configuration Secret interface has been removed. Store map keys remain stable Sink identities; changing a key creates a different Store. Only active Stores appear in the Gateway routes. Common operational values are an abstraction over Sink configuration: listener ports, time budgets, Store identity, Kafka policy and credentials are configured once and rendered consistently for each role. MongoDB tuning uses -`metadataField` (default `__sink`), `maxConcurrentWrites` (64), and -`maxConcurrentGroups` (16). Resource/memory limits, discovery convergence, rolling +`storage.mongodb.metadataField` (default `__sink`), `maxConcurrentWrites` (64), +and `maxConcurrentGroups` (16). Both roles reuse these settings; concurrency +limits apply independently within each process. Resource/memory limits, discovery convergence, rolling surge and graceful exit have the defaults described below. Advanced tuning remains inside values under `runtime`; it never requires a separately maintained config file. @@ -23,15 +24,28 @@ the headless discovery name `sink-sink-gateway-headless.sink.svc.cluster.local`. The latter is suitable for SDK client-side endpoint discovery. Renaming a release or namespace changes both names and is a cluster migration. -The chart generates ordinary runtime YAML in ConfigMaps. It manages role, Store -name, listener/metrics addresses, request and shutdown timeouts, and Kafka identity -from values. `engineDefaults.runtime`, `workerDefaults.runtime`, and their per-Store -overrides accept additional `service` and `grpc` tuning. `gateway.runtime` accepts -additional Gateway tuning. Managed fields cannot be overridden there. Kafka -brokers/topic/group/partitions and `replicationFactor` (default 3)/`minInSyncReplicas` -(default 2) are shared with both roles; `kafka.runtime` accepts other supported -topic/producer/consumer/dead_letter tuning. Validate tuning with the pinned Sink -before deployment. KEDA uses the same broker/topic/group metadata. +The chart renders one `store.yaml` ConfigMap per Store and one `sink.yaml` per +role. Engine and Worker project the same Store ConfigMap and load it through +`--store-config`. Component changes roll only the affected role; shared changes +roll both. Gateway mounts only its own configuration. + +`runtime.execution`, `runtime.batching`, and `runtime.producer` tune Engine; +`runtime.execution` and `runtime.consumer` tune Worker. Only Gateway accepts +`runtime.request.max_operations` and `runtime.forwarding`. Worker memory exposes +only `max_bytes`. Kafka brokers, Topic/DLQ and replication policy stay shared; +`kafka.runtime` accepts `max_record_bytes`, `topic.retention`, and +`dead_letter.name`/`dead_letter.retention`. Partitions, replication factor, minimum +ISR (default `1`), and record size apply to both Topics; generated Store YAML places +these shared settings directly under `kafka`. Worker group belongs to +`stores..worker.runtime.consumer.group_id`, which KEDA also uses. + +Empty `engineDefaults.runtime.batching` inherits Sink's measured starting values: +`max_operations: 32` and `max_wait: 2ms`. Memory watermarks default to 80% for rejection and 70% for recovery. +The Chart does not emit duplicate overrides for these runtime defaults. Tune a +Store through `stores..engine.runtime.batching`; the 32-operation batch +target does not change Gateway's 1,000-operation public request limit. See +[default selection](https://github.com/batchstream/sink/blob/main/docs/batching.md#default-selection) +for the benchmark evidence and workload limits. ## Store credentials @@ -132,16 +146,11 @@ those policies or silently expose a public LoadBalancer by default. ### Staged image upgrades -Chart 0.7 defaults to Sink 0.18.0. Existing 0.16 installations must explicitly -retain their current global `image.tag` and `image.digest` when moving to the new -chart, then advance role images in the sequence below. Once every role uses the -new release, the global default can replace those temporary role pins. Keep -`runtime.memory` empty on roles still using 0.16.0. - -When a release changes the private forwarding protocol, upgrade every Engine -before the Gateway. Kubernetes does not order Deployment rollouts inside one -Helm upgrade. Sink builds using `ForwardStream` cannot call a v0.16.0 Engine; -old Gateways can call the retained unary endpoint on new Engines. +Chart 0.8 requires the new role/Store schema and private forwarding version in +Sink 0.19. Deploy a separate matching cluster for an upgrade from 0.18 or older; +the former mixed-version forwarding path is not supported across this boundary. +The steps below apply only to builds with compatible configuration and forwarding +protocols. Qualification accepts explicit baseline and target candidate images. Keep the global `image` pinned to the current release. In the first values revision, set `engineDefaults.image.digest` to the new release digest and apply @@ -163,19 +172,24 @@ transition checks still apply independently. ### Drain timing +Business RPCs have no implicit deadline. DNS withdrawal waits before SIGTERM, +then `shutdownTimeoutSeconds` bounds graceful drain; an indefinitely waiting +request can be interrupted when that shutdown bound expires. Configure it for +operational needs and let clients choose request deadlines. + The defaults encode the measured failure mechanism from Sink's qualification: a Gateway captures Engine membership for an accepted batch, so removing an endpoint from DNS alone does not make it safe to close its listener. ``` Engine preStop >= endpoint publication + DNS cache + Gateway DNS refresh - + lookup/retry allowance + maximum remaining request + margin -Gateway preStop >= client/proxy/LB withdrawal + maximum remaining request + margin + + lookup/retry allowance + margin +Gateway preStop >= client/proxy/LB withdrawal + margin Pod termination grace >= preStop + whole-process shutdown budget + margin ``` -Default Engine: `5 + 30 + 10 + 10 + 30 + 10 = 95s` preStop, `255s` total grace. -Default Gateway: `60 + 30 + 10 = 100s` preStop, `260s` total grace. +Default Engine: `5 + 30 + 10 + 10 + 10 = 65s` preStop, `225s` total grace. +Default Gateway: `60 + 10 = 70s` preStop, `230s` total grace. Worker: no preStop wait, `160s` grace, allowing prompt consumer departure. Each uses a 150s process shutdown budget and 10s final margin. The chart rejects shorter overrides and a process budget smaller than four configured shutdown @@ -281,15 +295,17 @@ or milli-byte quantities are rejected. `GOMEMLIMIT` defaults to 80% of the conta memory limit, leaving headroom for non-Go memory. It is a soft Go runtime target, not an OOM guarantee. Do not override it through `env`; use `goMemoryLimitPercent`. -Application queues, execution bytes, publish buffers, Kafka fetch sizes and gRPC -message limits must fit the memory budget too. Set these in the supported `runtime.service`/`runtime.grpc` -configuration and qualify representative maximum documents/fanout. Increasing +Application queues, publish buffers, Kafka fetch sizes and gRPC +message limits must fit the memory budget too. Set these in the supported `runtime.execution`/`runtime.batching`/`runtime.grpc` +configuration and qualify representative maximum documents/fanout. Startup panics +when its minimum working-memory estimate exceeds the high-watermark allowance; +this is a sizing check, not an OOM guarantee. Increasing replicas cannot repair an overloaded database or Kafka partition. Budget peak resources as desired replicas **plus surge and terminating replicas**, multiplied by per-Pod requests, then add Kafka/backend/controllers and operational headroom. The prior fixed-resource benchmark is workload-specific, not a cluster sizing -promise. Compare achieved throughput, p95/p99, rejection/error rate, queue age, +promise. Compare achieved throughput, p65/p99, rejection/error rate, queue age, Kafka lag age and memory at a fixed total resource budget. Choose thresholds below sustained saturation with room for one replica's withdrawal. Observe backend pool and connection growth when scaling; more replicas may reduce useful throughput. diff --git a/examples/cluster-values.yaml b/examples/cluster-values.yaml index 04ea636..bfce2b2 100644 --- a/examples/cluster-values.yaml +++ b/examples/cluster-values.yaml @@ -21,6 +21,9 @@ stores: maxReplicas: 8 worker: enabled: true + runtime: + consumer: + group_id: mongo-workers-v1 pod: resources: requests: @@ -36,7 +39,6 @@ stores: brokers: - kafka.example.invalid:9092 topic: mongo-mutations-v1 - consumerGroup: mongo-workers-v1 partitions: 8 lagThreshold: 1000 # Same credential projection and backend settings for both roles. diff --git a/examples/memory-keda-values.yaml b/examples/memory-keda-values.yaml index efa34c9..1a2a8c2 100644 --- a/examples/memory-keda-values.yaml +++ b/examples/memory-keda-values.yaml @@ -1,11 +1,11 @@ -# Requires a Sink image with demand-based memory admission (after 0.16.0). +# Requires Sink 0.19.0+ with process memory watermark admission. # Overlay cluster-values.yaml. Update Prometheus namespace/job selectors to the -# actual Gateway scrape target. Upgrade Engines before Gateways. +# actual Gateway scrape target. Use matching Gateway/Engine private protocol versions. gateway: runtime: memory: - burst_percent: 10 - wait_timeout: 2s + high_watermark_percent: 80 + low_watermark_percent: 70 autoscaling: mode: keda minReplicas: 2 @@ -20,12 +20,11 @@ gateway: ignoreNullValues: "false" query: >- ( - (sum(sink_memory_used_bytes{namespace="sink",job="sink-gateway",role="gateway"}) - + sum(sink_memory_waiting_bytes{namespace="sink",job="sink-gateway",role="gateway"})) - / sum(sink_memory_capacity_bytes{namespace="sink",job="sink-gateway",role="gateway",pool="normal"}) + sum(sink_memory_used_bytes{namespace="sink",job="sink-gateway",role="gateway"}) + / sum(sink_memory_watermark_bytes{namespace="sink",job="sink-gateway",role="gateway",watermark="high"}) ) and on() (min(up{namespace="sink",job="sink-gateway"}) == 1) - and on() (count(sink_memory_capacity_bytes{namespace="sink",job="sink-gateway",role="gateway",pool="normal"}) + and on() (count(sink_memory_watermark_bytes{namespace="sink",job="sink-gateway",role="gateway",watermark="high"}) == count(up{namespace="sink",job="sink-gateway"})) - type: prometheus metricType: Value @@ -35,9 +34,9 @@ gateway: ignoreNullValues: "false" query: >- ( - sum(rate(sink_memory_rejected_total{namespace="sink",job="sink-gateway",role="gateway",phase="request",reason="busy"}[1m])) + sum(rate(sink_memory_rejected_total{namespace="sink",job="sink-gateway",role="gateway"}[1m])) / clamp_min( - sum(rate(sink_memory_admitted_total{namespace="sink",job="sink-gateway",role="gateway",phase="request"}[1m])) - + sum(rate(sink_memory_rejected_total{namespace="sink",job="sink-gateway",role="gateway",phase="request",reason="busy"}[1m])), 1) + sum(rate(sink_memory_admitted_total{namespace="sink",job="sink-gateway",role="gateway"}[1m])) + + sum(rate(sink_memory_rejected_total{namespace="sink",job="sink-gateway",role="gateway"}[1m])), 1) ) and on() (min(up{namespace="sink",job="sink-gateway"}) == 1) diff --git a/tests/check_runtime.py b/tests/check_runtime.py index 4a614cd..c96e61c 100644 --- a/tests/check_runtime.py +++ b/tests/check_runtime.py @@ -12,6 +12,7 @@ def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--sink-image", help="explicit candidate image; defaults to the chart's pinned release") + parser.add_argument("--sink-binary", type=Path, help="local candidate binary; no Docker required") args = parser.parse_args() defaults = yaml.safe_load((ROOT / "charts/sink/values.yaml").read_text())["image"] image = args.sink_image or (f"{defaults['repository']}@{defaults['digest']}" if defaults["digest"] else f"{defaults['repository']}:{defaults['tag']}") @@ -22,20 +23,31 @@ def main(): for example in files: command.extend(["-f", str(ROOT / "examples" / example)]) rendered = subprocess.check_output(command, text=True) - configs = [doc for doc in yaml.safe_load_all(rendered) - if doc and doc["kind"] == "ConfigMap" and "sink.yaml" in doc.get("data", {})] + documents = [doc for doc in yaml.safe_load_all(rendered) if doc] + configs = {doc["metadata"]["name"]: doc["data"] for doc in documents if doc["kind"] == "ConfigMap"} + deployments = [doc for doc in documents if doc["kind"] == "Deployment"] with tempfile.TemporaryDirectory(prefix="sink-chart-config-") as directory: root = Path(directory) root.chmod(0o755) # Public fixtures readable by the image's nonroot UID. for name, value in {"mongodb-uri": "mongodb://example.invalid:27017", "search-username": "fixture", "search-password": " fixture:$#quotes\"'\\\nunicode-雪 \n", "search-api-key": "fixture-key"}.items(): (root / name).write_text(value) - for config in configs: - filename = config["metadata"]["name"] + ".yaml" - (root / filename).write_text(config["data"]["sink.yaml"]) - subprocess.run(["docker", "run", "--rm", "--network", "none", "-v", f"{root}:/etc/sink:ro", - "-v", f"{root}:/etc/sink-secrets:ro", image, - "config", "check", "--config", f"/etc/sink/{filename}"], check=True) + for deployment in deployments: + pod = deployment["spec"]["template"]["spec"] + for source in pod["volumes"][0]["projected"]["sources"]: + for filename, content in configs[source["configMap"]["name"]].items(): + if args.sink_binary: + content = content.replace("/etc/sink-secrets/", str(root) + "/") + (root / filename).write_text(content) + arguments = pod["containers"][0]["args"] + if args.sink_binary: + arguments = [arg.replace("/etc/sink/", str(root) + "/") for arg in arguments] + command = [str(args.sink_binary.resolve()), "config", "check", *arguments] + else: + command = ["docker", "run", "--rm", "--network", "none", "-v", f"{root}:/etc/sink:ro", + "-v", f"{root}:/etc/sink-secrets:ro", image, "config", "check", *arguments] + subprocess.run(command, check=True) + if __name__ == "__main__": diff --git a/tests/integration/README.md b/tests/integration/README.md index 2b95609..892c174 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -14,10 +14,10 @@ owned cluster. Its CoreDNS TTL is set to 30 seconds to exercise the chart cache budget; KEDA CRDs are installed only there. No current/default Kubernetes context is used. The cluster and probe image are deleted in `finally`, including on test failures. Logs and JSON reconciliation evidence remain in `.reports/`. -The `all` and `scaling` scenarios require Sink 0.18+ and enable chart-generated +The `all` and `scaling` scenarios require Sink 0.19+ and enable chart-generated logging on all roles with an unreachable local OTLP receiver. Business traffic, -readiness, and bounded shutdown must still work. The mixed-version `upgrades` -scenario leaves logging empty for compatibility with its legacy images. +readiness, and bounded shutdown must still work. The `upgrades` scenario exercises separate compatible image references; it does +not assert compatibility with the retired 0.18 configuration or forwarding protocol. Interrupting the entire host/process can bypass cleanup; use `kind get clusters` and delete only the recorded test cluster if that happens. @@ -56,13 +56,15 @@ is still deleted on completion. Never point this option at a shared cluster. Use `--scenario scaling` to rerun only the Kafka/KEDA regression in a fresh cluster. Use `--sink-image repository:tag` to qualify an explicitly built local candidate. -Only the disposable test cluster loads that image. Default deployment/scaling runs use the chart's pinned release; the upgrades -scenario starts from the explicit legacy baseline below. +Only the disposable test cluster loads that image. Default deployment/scaling +runs require the chart's matching 0.19 release. Before release, pass a locally +built role-config candidate. `upgrades` requires an explicit baseline candidate. ## Mixed-version upgrades -CI also runs `make integration SCENARIO=upgrades` in a separate disposable Kind -cluster. The checked pair is Sink 0.16.0 → 0.18.0 → 0.16.0, covering the private +CI builds two differently version-stamped images from the matching candidate and +runs `--scenario upgrades --sink-image sink-chart-candidate:a --upgrade-image sink-chart-candidate:b` +in a separate disposable Kind cluster, covering the private unary-to-streaming forwarding transition. Engines upgrade first, then Gateways; rollback restores Gateways before Engines. A 0.16.0 Worker remains active to check Kafka compatibility with each Engine version. Every stage waits for old diff --git a/tests/integration/run.py b/tests/integration/run.py index 45d7f80..47befaf 100644 --- a/tests/integration/run.py +++ b/tests/integration/run.py @@ -15,8 +15,7 @@ import yaml ROOT = Path(__file__).resolve().parents[2] -BASELINE_IMAGE = "ghcr.io/batchstream/sink:0.16.0" -UPGRADE_IMAGE = "ghcr.io/batchstream/sink:0.18.0" +UPGRADE_IMAGE = "ghcr.io/batchstream/sink:0.19.0" NODE = "kindest/node:v1.35.8@sha256:07b2536e30b803ed61d1677a79df6115f798ce64c80f9e22f6ed45afd09323c0" @@ -37,11 +36,11 @@ def __init__(self, args, directory): self.helm = ["helm", "--kubeconfig", self.kubeconfig, "--kube-context", f"kind-{self.cluster}"] self.values = { "gateway": {"pod": {"resources": {"requests": {"cpu": "100m", "memory": "128Mi"}, "limits": {"memory": "512Mi"}}}}, - "engineDefaults": {"pod": {"resources": {"requests": {"cpu": "100m", "memory": "256Mi"}, "limits": {"memory": "512Mi"}}}}, + "engineDefaults": {"pod": {"resources": {"requests": {"cpu": "100m", "memory": "256Mi"}, "limits": {"memory": "1Gi"}}}}, "workerDefaults": {"pod": {"resources": {"requests": {"cpu": "100m", "memory": "256Mi"}, "limits": {"cpu": "200m", "memory": "512Mi"}}}}, "stores": {"mongo": {"state": "active", "storage": self.storage("mongo-v1"), "engine": {}, - "worker": {"enabled": True, "replicaCount": 0, "allowScaleToZero": True, "disruptionBudget": {"enabled": False}}, - "kafka": {"brokers": ["kafka:9092"], "topic": "mongo-mutations", "consumerGroup": "mongo-workers", "partitions": 4, "lagThreshold": 100, "replicationFactor": 1, "minInSyncReplicas": 1}}}, + "worker": {"runtime": {"consumer": {"group_id": "mongo-workers"}}, "enabled": True, "replicaCount": 0, "allowScaleToZero": True, "disruptionBudget": {"enabled": False}}, + "kafka": {"brokers": ["kafka:9092"], "topic": "mongo-mutations", "partitions": 4, "lagThreshold": 100, "replicationFactor": 1, "minInSyncReplicas": 1}}}, } if args.sink_image: self.values["image"] = {"repository": args.sink_image.rsplit(":", 1)[0], "tag": args.sink_image.rsplit(":", 1)[1], "digest": ""} @@ -321,7 +320,7 @@ def exercise_upgrades(self): old_image = self.get("deployment", "qual-sink-gateway")["spec"]["template"]["spec"]["containers"][0]["image"] new_image = self.args.upgrade_image if old_image == new_image: - raise RuntimeError("mixed-version qualification requires two distinct images") + raise RuntimeError("rollout qualification requires two distinct image references from compatible builds") repository, tag = new_image.rsplit(":", 1) target = {"repository": repository, "tag": tag, "digest": ""} self.values["stores"]["mongo"]["worker"]["replicaCount"] = 1 @@ -448,8 +447,7 @@ def main(): parser.add_argument("--kubeconfig", help="explicit owned Kind kubeconfig for development reuse; cluster is still deleted") args = parser.parse_args() if args.scenario == "upgrades" and not args.sink_image: - # Keep the previous chart's image as the baseline after advancing chart defaults. - args.sink_image = BASELINE_IMAGE + parser.error("upgrades requires --sink-image and a compatible --upgrade-image") for executable in ["kind", "kubectl", "helm", "docker", "go"]: if not shutil.which(executable): parser.error(f"required executable missing: {executable}") diff --git a/tests/test_chart.py b/tests/test_chart.py index dc0916f..f648ecd 100644 --- a/tests/test_chart.py +++ b/tests/test_chart.py @@ -12,8 +12,8 @@ ROOT = Path(__file__).resolve().parents[1] CHART = ROOT / "charts" / "sink" STORAGE = {"driver": "mongodb", "mongodb": {"uriSecretRef": {"name": "mongo-v1", "key": "uri"}}} -BASE = {"stores": {"mongo": {"storage": STORAGE, "state": "active"}}} -KAFKA = {"brokers": ["kafka:9092"], "topic": "mongo", "consumerGroup": "mongo-workers", "partitions": 8} +BASE = {"workerDefaults": {"runtime": {"consumer": {"group_id": "mongo-workers"}}}, "stores": {"mongo": {"storage": STORAGE, "state": "active"}}} +KAFKA = {"brokers": ["kafka:9092"], "topic": "mongo", "partitions": 8} def render(values, *options): @@ -50,6 +50,8 @@ def test_logging_inheritance_outputs_and_rollout(self): "flush_interval": "500ms", "export_timeout": "2s", "shutdown_timeout": "5s"}} for role in ["gateway", "engineDefaults", "workerDefaults"]: values[role] = {"runtime": {"logging": copy.deepcopy(logging)}} + if role == "workerDefaults": + values[role]["runtime"]["consumer"] = {"group_id": "mongo-workers"} engine_override = {"failure_body": False, "console": {"enabled": False}, "otlp": {"tls": {"enabled": False}}} values["stores"]["mongo"]["engine"] = {"runtime": {"logging": engine_override}} @@ -201,17 +203,19 @@ def test_memory_capacity_is_optional_and_configurable_per_role(self): default_config = yaml.safe_load(defaults["ConfigMap", "test-sink-gateway"]["data"]["sink.yaml"]) self.assertNotIn("memory", default_config) values = copy.deepcopy(BASE) - values["gateway"] = {"runtime": {"memory": {"burst_percent": 15}}} - values["engineDefaults"] = {"runtime": {"memory": {"max_bytes": "256MiB", "burst_percent": 10}}} - values["stores"]["mongo"]["engine"] = {"runtime": {"memory": {"wait_timeout": "500ms"}}} + values["gateway"] = {"runtime": {"memory": {"high_watermark_percent": 90}}} + values["engineDefaults"] = {"runtime": {"memory": {"max_bytes": "768MiB", "high_watermark_percent": 80}}} + values["stores"]["mongo"]["engine"] = {"runtime": {"memory": {"low_watermark_percent": 65}}} docs = self.manifests(values) gateway = yaml.safe_load(docs["ConfigMap", "test-sink-gateway"]["data"]["sink.yaml"]) engine = yaml.safe_load(docs["ConfigMap", "test-sink-mongo-engine"]["data"]["sink.yaml"]) - gateway_expected = {"burst_percent": 15} + gateway_expected = {"high_watermark_percent": 90} self.assertEqual(gateway["memory"], gateway_expected) - engine_expected = {"max_bytes": "256MiB", "burst_percent": 10, "wait_timeout": "500ms"} + engine_expected = {"max_bytes": "768MiB", "high_watermark_percent": 80, "low_watermark_percent": 65} self.assertEqual(engine["memory"], engine_expected) - values["gateway"]["runtime"]["memory"]["burst_percent"] = 100 + values["gateway"]["runtime"]["memory"]["high_watermark_percent"] = 100 + self.assertNotEqual(render(values).returncode, 0) + values["gateway"]["runtime"]["memory"]["high_watermark_percent"] = 70 self.assertNotEqual(render(values).returncode, 0) def test_empty_and_staged_cluster(self): @@ -223,12 +227,12 @@ def test_empty_and_staged_cluster(self): def test_discovery_shutdown_security_and_memory(self): docs = self.manifests() - for role, name, delay, grace in [("engine", "test-sink-mongo-engine", 95, 255), - ("gateway", "test-sink-gateway", 100, 260)]: + for role, name, delay, grace in [("engine", "test-sink-mongo-engine", 65, 225), + ("gateway", "test-sink-gateway", 70, 230)]: deployment = docs["Deployment", name] pod = deployment["spec"]["template"] container = pod["spec"]["containers"][0] - self.assertEqual(container["args"], ["--config", "/etc/sink/sink.yaml"]) + self.assertEqual(container["args"], ["--config", "/etc/sink/sink.yaml"] + (["--store-config", "/etc/sink/store.yaml"] if role == "engine" else [])) self.assertEqual(container["lifecycle"]["preStop"]["sleep"]["seconds"], delay) self.assertEqual(pod["spec"]["terminationGracePeriodSeconds"], grace) self.assertEqual(deployment["spec"]["strategy"]["rollingUpdate"], {"maxUnavailable": 0, "maxSurge": 1}) @@ -257,7 +261,7 @@ def test_routes_are_only_active_and_config_changes_roll_gateway(self): values["stores"].update({"archive": {"storage": STORAGE, "state": "staged"}, "old": {"storage": STORAGE, "state": "retiring"}}) first = self.manifests(values) config = yaml.safe_load(first["ConfigMap", "test-sink-gateway"]["data"]["sink.yaml"]) - self.assertEqual(config["gateway"]["routes"], [{"store": "mongo", "target": "dns:///test-sink-mongo-engine.sink.svc.cluster.local:8080", "tls": {"insecure": True}}]) + self.assertEqual(config["forwarding"]["routes"], [{"store": "mongo", "target": "dns:///test-sink-mongo-engine.sink.svc.cluster.local:8080", "tls": {"insecure": True}}]) values["stores"]["archive"]["state"] = "active" second = self.manifests(values) a = first["Deployment", "test-sink-gateway"]["spec"]["template"]["metadata"]["annotations"] @@ -273,7 +277,7 @@ def test_per_store_merge_does_not_leak(self): mongo = docs["Deployment", "test-sink-mongo-engine"]["spec"]["template"]["spec"] self.assertEqual(archive["containers"][0]["resources"]["limits"]["memory"], "2Gi") self.assertEqual(mongo["containers"][0]["resources"]["limits"]["memory"], "1Gi") - self.assertEqual(archive["volumes"][0]["configMap"], {"name": "test-sink-archive-engine"}) + self.assertEqual(archive["volumes"][0]["projected"]["sources"], [{"configMap": {"name": "test-sink-archive-engine"}}, {"configMap": {"name": "test-sink-archive-store"}}]) def test_shared_secret_projection_and_runtime(self): values = copy.deepcopy(BASE) @@ -290,14 +294,24 @@ def test_shared_secret_projection_and_runtime(self): "items": [{"key": "uri", "path": "mongodb-uri"}]}}]) config = yaml.safe_load(docs["ConfigMap", name]["data"]["sink.yaml"]) self.assertEqual(config["mode"], role) - self.assertEqual(config["storage"]["mongodb"]["uri_file"], "/etc/sink-secrets/mongodb-uri") - self.assertEqual(config["storage"]["mongodb"]["metadata_field"], "__sink") - self.assertEqual(config["storage"]["mongodb"]["max_concurrent_writes"], 64) - self.assertEqual(config["storage"]["mongodb"]["max_concurrent_groups"], 16) - self.assertEqual(config["storage"]["kafka"]["topic"]["replication_factor"], 3) - self.assertEqual(config["storage"]["kafka"]["consumer"]["group_id"], KAFKA["consumerGroup"]) - self.assertEqual(config["service"]["request"]["timeout"], "30s") - configs.append(config["storage"]) + self.assertNotIn("storage", config) + self.assertNotIn("request", config) + self.assertNotIn("service", config) + if role == "worker": + self.assertEqual(config["consumer"]["group_id"], "mongo-workers") + self.assertNotIn("grpc", config) + sources = pod["spec"]["volumes"][0]["projected"]["sources"] + self.assertEqual(sources[1]["configMap"]["name"], "test-sink-mongo-store") + shared = yaml.safe_load(docs["ConfigMap", "test-sink-mongo-store"]["data"]["store.yaml"]) + self.assertEqual(shared["name"], "mongo") + self.assertEqual(shared["storage"]["mongodb"]["uri_file"], "/etc/sink-secrets/mongodb-uri") + self.assertEqual(shared["storage"]["mongodb"]["max_concurrent_writes"], 64) + self.assertEqual(shared["storage"]["mongodb"]["max_concurrent_groups"], 16) + self.assertNotIn("consumer", shared["kafka"]) + self.assertEqual(shared["kafka"]["replication_factor"], 3) + self.assertEqual(shared["kafka"]["min_insync_replicas"], 1) + self.assertEqual(shared["kafka"]["topic"], {"name": "mongo"}) + configs.append(shared) self.assertEqual(*configs) gateway = docs["Deployment", "test-sink-gateway"] self.assertEqual(len(gateway["spec"]["template"]["spec"]["volumes"]), 1) @@ -314,31 +328,32 @@ def test_shared_secret_projection_and_runtime(self): self.assertNotEqual(rotated["Deployment", name]["spec"]["template"], versioned["Deployment", name]["spec"]["template"]) values["stores"]["mongo"]["storage"]["mongodb"]["maxConcurrentWrites"] = 32 tuned = self.manifests(values) - name = "test-sink-mongo-engine" - config = yaml.safe_load(tuned["ConfigMap", name]["data"]["sink.yaml"]) - self.assertEqual(config["storage"]["mongodb"]["max_concurrent_writes"], 32) - self.assertNotEqual(versioned["Deployment", name]["spec"]["template"]["metadata"]["annotations"], - tuned["Deployment", name]["spec"]["template"]["metadata"]["annotations"]) + shared = yaml.safe_load(tuned["ConfigMap", "test-sink-mongo-store"]["data"]["store.yaml"]) + self.assertEqual(shared["storage"]["mongodb"]["max_concurrent_writes"], 32) + for role in ["engine", "worker"]: + name = f"test-sink-mongo-{role}" + config = yaml.safe_load(tuned["ConfigMap", name]["data"]["sink.yaml"]) + self.assertNotIn("mongodb", config.get("execution", {})) + self.assertNotEqual(versioned["Deployment", name]["spec"]["template"]["metadata"]["annotations"], + tuned["Deployment", name]["spec"]["template"]["metadata"]["annotations"]) def test_advanced_tuning_is_generated_from_values(self): values = copy.deepcopy(BASE) values["metrics"] = {"enabled": True} - values["engineDefaults"] = {"runtime": {"service": {"execution": {"max_requests": 16}}, + values["engineDefaults"] = {"runtime": {"execution": {"merge": {"max_attempts": 5}}, "grpc": {"max_receive_message_bytes": "8MiB"}}} - values["stores"]["mongo"]["engine"] = {"runtime": {"service": {"batching": {"max_wait": "5ms"}}}} - values["stores"]["mongo"]["worker"] = {"enabled": True, "runtime": {"service": {"execution": {"max_requests": 4}}}} - values["stores"]["mongo"]["kafka"] = {**KAFKA, "runtime": {"consumer": {"processing_timeout": "20s"}, - "dead_letter": {"retention": "720h"}}} + values["stores"]["mongo"]["engine"] = {"runtime": {"batching": {"max_wait": "5ms"}}} + values["stores"]["mongo"]["worker"] = {"enabled": True, "runtime": {"execution": {"merge": {"max_attempts": 7}}, "consumer": {"processing_timeout": "20s"}}} + values["stores"]["mongo"]["kafka"] = {**KAFKA, "runtime": {"dead_letter": {"retention": "720h"}}} docs = self.manifests(values) engine = yaml.safe_load(docs["ConfigMap", "test-sink-mongo-engine"]["data"]["sink.yaml"]) worker = yaml.safe_load(docs["ConfigMap", "test-sink-mongo-worker"]["data"]["sink.yaml"]) - self.assertEqual(engine["service"]["execution"]["max_requests"], 16) - self.assertEqual(engine["service"]["batching"]["max_wait"], "5ms") - self.assertEqual(worker["service"]["execution"]["max_requests"], 4) - self.assertNotIn("batching", worker["service"]) + self.assertEqual(engine["execution"]["merge"]["max_attempts"], 5) + self.assertEqual(engine["batching"]["max_wait"], "5ms") + self.assertEqual(worker["execution"]["merge"]["max_attempts"], 7) + self.assertNotIn("batching", worker) self.assertEqual(engine["grpc"]["max_receive_message_bytes"], "8MiB") - self.assertEqual(engine["storage"], worker["storage"]) - self.assertEqual(engine["storage"]["kafka"]["consumer"]["processing_timeout"], "20s") + self.assertEqual(worker["consumer"]["processing_timeout"], "20s") self.assertTrue(engine["prometheus"]["enabled"]) self.assertTrue(worker["prometheus"]["enabled"]) @@ -350,7 +365,7 @@ def test_search_credentials_and_rejected_plaintext(self): search = {"endpoints": ["https://search:9200"], **auth} values = {"stores": {"search": {"state": "active", "storage": {"driver": driver, "search": search}}}} docs = self.manifests(values) - config = yaml.safe_load(docs["ConfigMap", "test-sink-search-engine"]["data"]["sink.yaml"]) + config = yaml.safe_load(docs["ConfigMap", "test-sink-search-store"]["data"]["store.yaml"]) self.assertNotIn("SecretRef", json.dumps(config)) expected = {"endpoints": ["https://search:9200"]} if "username" in auth: @@ -389,8 +404,8 @@ def test_budget_recalculates_with_dns(self): values = {**BASE, "discovery": {"dnsRefreshSeconds": 40, "dnsCacheSeconds": 60}} doc = self.manifests(values)["Deployment", "test-sink-mongo-engine"] container = doc["spec"]["template"]["spec"]["containers"][0] - self.assertEqual(container["lifecycle"]["preStop"]["sleep"]["seconds"], 155) - self.assertEqual(doc["spec"]["template"]["spec"]["terminationGracePeriodSeconds"], 315) + self.assertEqual(container["lifecycle"]["preStop"]["sleep"]["seconds"], 125) + self.assertEqual(doc["spec"]["template"]["spec"]["terminationGracePeriodSeconds"], 285) self.assertEqual(doc["spec"]["minReadySeconds"], 115) def test_controller_owns_replicas_and_hpa_behavior(self): @@ -426,11 +441,11 @@ def test_scale_down_policy_must_cover_termination(self): }}} result = render(values) self.assertNotEqual(result.returncode, 0) - self.assertIn("scaleDown policy periodSeconds must cover termination grace (255)", result.stderr) + self.assertIn("scaleDown policy periodSeconds must cover termination grace (225)", result.stderr) def test_keda_worker_zero_and_partition_trigger(self): values = {"stores": {"mongo": {"storage": STORAGE, "state": "active", "kafka": KAFKA, - "worker": {"enabled": True, "allowScaleToZero": True, "disruptionBudget": {"enabled": False}, + "worker": {"enabled": True, "runtime": {"consumer": {"group_id": "mongo-workers"}}, "allowScaleToZero": True, "disruptionBudget": {"enabled": False}, "autoscaling": {"mode": "keda", "minReplicas": 0, "maxReplicas": 8, "keda": {"initialCooldownPeriod": 60, "restoreToOriginalReplicaCount": True, @@ -476,7 +491,7 @@ def test_keda_production_controls_and_mixed_fallback(self): {"type": "memory", "name": "memory", "metricType": "Utilization", "metadata": {"value": "80"}}, {"type": "prometheus", "name": "execution-budget", "metricType": "AverageValue", "useCachedMetrics": True, - "metadata": {"serverAddress": "http://prometheus", "query": "sum(sink_execution_store_bytes)", + "metadata": {"serverAddress": "http://prometheus", "query": "sum(sink_memory_used_bytes)", "threshold": "134217728", "ignoreNullValues": "false"}}, ] values = {**BASE, "engineDefaults": {"autoscaling": { @@ -524,13 +539,33 @@ def test_kafka_dead_letter_topics_cannot_overlap(self): for topic in ["mongo", "mongo.dlq", " mongo ", " mongo.dlq "]: values = copy.deepcopy(BASE) values["stores"]["mongo"]["kafka"] = copy.deepcopy(KAFKA) - kafka = {**KAFKA, "topic": topic, "consumerGroup": "archive"} + kafka = {**KAFKA, "topic": topic} values["stores"]["archive"] = {"state": "staged", "storage": STORAGE, "kafka": kafka} self.assertNotEqual(render(values).returncode, 0) values = copy.deepcopy(BASE) - values["stores"]["mongo"]["kafka"] = {**KAFKA, "runtime": {"dead_letter": {"topic": "mongo"}}} + values["stores"]["mongo"]["kafka"] = {**KAFKA, "runtime": {"dead_letter": {"name": "mongo"}}} self.assertNotEqual(render(values).returncode, 0) + def test_shared_kafka_policy_and_independent_topics(self): + values = copy.deepcopy(BASE) + values["stores"]["mongo"]["kafka"] = { + **KAFKA, "replicationFactor": 3, "minInSyncReplicas": 2, + "runtime": {"max_record_bytes": "2MiB", "topic": {"retention": "48h"}, + "dead_letter": {"name": "rejected", "retention": "240h"}}, + } + docs = self.manifests(values) + shared = yaml.safe_load(docs["ConfigMap", "test-sink-mongo-store"]["data"]["store.yaml"]) + self.assertEqual(shared["kafka"]["replication_factor"], 3) + self.assertEqual(shared["kafka"]["min_insync_replicas"], 2) + self.assertEqual(shared["kafka"]["max_record_bytes"], "2MiB") + self.assertEqual(shared["kafka"]["topic"], {"name": "mongo", "retention": "48h"}) + self.assertEqual(shared["kafka"]["dead_letter"], {"name": "rejected", "retention": "240h"}) + for runtime in [{"topic": {"partitions": 4}}, {"topic": {"replication_factor": 2}}, + {"topic": {"min_insync_replicas": 1}}, {"topic": {"max_record_bytes": "2MiB"}}, + {"dead_letter": {"topic": "rejected"}}]: + values["stores"]["mongo"]["kafka"] = {**KAFKA, "runtime": runtime} + self.assertNotEqual(render(values).returncode, 0) + def test_examples(self): values = {} self.manifests(values, "-f", str(ROOT / "examples/cluster-values.yaml"), "-f", str(ROOT / "examples/memory-keda-values.yaml"), "--api-versions", "keda.sh/v1alpha1/ScaledObject") @@ -541,7 +576,7 @@ def test_examples(self): def test_unsafe_and_misspelled_settings_fail(self): cases = [ "mode=engine", "engineDefaults.replicaCount=0", "gateway.replicaCount=0", - "engineDefaults.pod.preStopSeconds=94", "gateway.pod.terminationGracePeriodSeconds=100", + "engineDefaults.pod.preStopSeconds=64", "gateway.pod.terminationGracePeriodSeconds=100", "engineDefaults.pod.shutdownBudgetSeconds=60", "requestTimeoutSeconds=301", "stores.Bad.state=active", "stores.mongo.state=deleted", "engineDefaults.replicaCont=3", "engineDefaults.pod.resources.limits.memory=0Gi", "engineDefaults.pod.resources.requests.memory=2Gi", @@ -559,7 +594,7 @@ def test_unsafe_and_misspelled_settings_fail(self): self.assertNotEqual(result.returncode, 0, result.stdout) def test_worker_overpartition_and_implicit_zero_fail(self): - values = {"stores": {"mongo": {"storage": STORAGE, "state": "active", "kafka": KAFKA, "worker": {"enabled": True, "replicaCount": 9}}}} + values = {"stores": {"mongo": {"storage": STORAGE, "state": "active", "kafka": KAFKA, "worker": {"enabled": True, "runtime": {"consumer": {"group_id": "mongo-workers"}}, "replicaCount": 9}}}} self.assertIn("exceeds Kafka partitions", render(values).stderr) values["stores"]["mongo"]["worker"]["replicaCount"] = 0 self.assertIn("requires at least 1", render(values).stderr)