From d5cc17946ee3a31ea99ae6dd35add32e21ed9b72 Mon Sep 17 00:00:00 2001 From: Li Ran Date: Fri, 25 Sep 2026 07:04:13 +0800 Subject: [PATCH 1/2] =?UTF-8?q?=F0=9F=A7=B5=20feat:=20expose=20unified=20S?= =?UTF-8?q?tore=20admission=20settings?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- charts/sink/README.md | 16 ++++++++++++++++ charts/sink/templates/_config.tpl | 2 ++ charts/sink/templates/_store.tpl | 6 +++++- charts/sink/values.schema.json | 22 ++++++++++++++++++++++ charts/sink/values.yaml | 1 + examples/tuning-values.yaml | 2 ++ tests/test_chart.py | 22 +++++++++++++++++++++- 7 files changed, 69 insertions(+), 2 deletions(-) diff --git a/charts/sink/README.md b/charts/sink/README.md index 6cd0ecc..2872704 100644 --- a/charts/sink/README.md +++ b/charts/sink/README.md @@ -19,6 +19,7 @@ defaults: engine: config: batching: {maxOperations: 32, maxWait: 2ms} + executionQueue: {maxTasks: 10000, maxBytes: 128MiB} worker: enabled: false stores: @@ -45,6 +46,21 @@ old paths and settings for the wrong role are rejected. References `{name, key}` point to externally managed Secrets in the release namespace. The chart does not read, create or delete Secret values. +## Unified Engine admission + +`defaults.engine.config.executionQueue` (or the corresponding per-Store Engine +configuration) renders `execution.queue`. `maxTasks` limits ready batches and +individual Native calls in one FIFO; a batch is one task, not one task per +operation. `maxBytes` is shared waiting bytes across batch collection and +admission. It is not a per-method quota or an execution-concurrency limit. +`batching.queue` still bounds each collection queue; `stores..maxConcurrent` +is the independent adaptive execution ceiling. Worker and Gateway reject +`executionQueue`. + +This setting requires the server implementation of unified admission. When +qualifying an unreleased candidate, override the image accordingly; do not enable +it against an older pinned image that does not accept `execution.queue`. + ## Global and Store fields Unless marked required, fields are optional. Defaults below are from this Chart diff --git a/charts/sink/templates/_config.tpl b/charts/sink/templates/_config.tpl index 0f7f02d..3144f8b 100644 --- a/charts/sink/templates/_config.tpl +++ b/charts/sink/templates/_config.tpl @@ -2,6 +2,7 @@ {{- define "sink.configFields" -}} {{- $names := dict "batchSize" "batch_size" + "executionQueue" "execution_queue" "exportTimeout" "export_timeout" "flushInterval" "flush_interval" "groupId" "group_id" @@ -26,6 +27,7 @@ "maxResultBytes" "max_result_bytes" "maxSendMessageBytes" "max_send_message_bytes" "maxSourceBytes" "max_source_bytes" + "maxTasks" "max_tasks" "maxWait" "max_wait" "processingTimeout" "processing_timeout" "queueSize" "queue_size" diff --git a/charts/sink/templates/_store.tpl b/charts/sink/templates/_store.tpl index 82617e5..a318454 100644 --- a/charts/sink/templates/_store.tpl +++ b/charts/sink/templates/_store.tpl @@ -45,8 +45,12 @@ {{- define "sink.roleConfig" -}} {{- $config := include "sink.configFields" .settings.config | fromJson -}} -{{- with $config.merge -}}{{- $_ := set $config "execution" (dict "merge" .) -}}{{- end -}} +{{- $execution := dict -}} +{{- with $config.merge -}}{{- $_ := set $execution "merge" . -}}{{- end -}} +{{- with $config.execution_queue -}}{{- $_ := set $execution "queue" . -}}{{- end -}} +{{- if $execution -}}{{- $_ := set $config "execution" $execution -}}{{- end -}} {{- $_ := unset $config "merge" -}} +{{- $_ := unset $config "execution_queue" -}} {{- $_ := set $config "mode" .role -}} {{- $_ := set $config "health" (dict "address" ":8081") -}} {{- $_ := set $config "prometheus" (dict "enabled" .root.Values.metrics.enabled "address" ":9090") -}} diff --git a/charts/sink/values.schema.json b/charts/sink/values.schema.json index 4bdc6d2..33a88a8 100644 --- a/charts/sink/values.schema.json +++ b/charts/sink/values.schema.json @@ -1222,6 +1222,28 @@ } } } + }, + "executionQueue": { + "type": "object", + "additionalProperties": false, + "properties": { + "maxTasks": { + "type": "integer", + "minimum": 1 + }, + "maxBytes": { + "oneOf": [ + { + "type": "integer", + "minimum": 1024 + }, + { + "type": "string", + "pattern": "^[1-9][0-9]*(KiB|MiB|GiB)?$" + } + ] + } + } } } }, diff --git a/charts/sink/values.yaml b/charts/sink/values.yaml index becb2ec..b27ac7c 100644 --- a/charts/sink/values.yaml +++ b/charts/sink/values.yaml @@ -130,6 +130,7 @@ defaults: logging: {} memory: {} batching: {} + executionQueue: {} # Unified ready-task count and Store-wide waiting bytes. kafkaProducer: {} grpc: {} # Container resources, environment, labels and scheduling. diff --git a/examples/tuning-values.yaml b/examples/tuning-values.yaml index bbc1a3b..a041266 100644 --- a/examples/tuning-values.yaml +++ b/examples/tuning-values.yaml @@ -1,5 +1,6 @@ # Overlay cluster-values.yaml. All application fields are camelCase. # Empty maps in values.yaml inherit Sink defaults; this overlay makes tuning explicit. +# executionQueue requires a server image with unified admission support. gateway: config: grpc: {maxReceiveMessageBytes: 64MiB, maxSendMessageBytes: 64MiB} @@ -11,6 +12,7 @@ defaults: config: grpc: {maxReceiveMessageBytes: 64MiB, maxSendMessageBytes: 64MiB} memory: {maxBytes: 1GiB, highWatermarkPercent: 80, lowWatermarkPercent: 70} + executionQueue: {maxTasks: 10000, maxBytes: 128MiB} batching: maxWait: 2ms maxOperations: 32 diff --git a/tests/test_chart.py b/tests/test_chart.py index 7bd62f2..adefd8b 100644 --- a/tests/test_chart.py +++ b/tests/test_chart.py @@ -92,7 +92,10 @@ def test_tuning_overlay_renders_application_contract(self): } execution = {"merge": merge} for role in ["engine", "worker"]: - self.assertEqual(configs[role]["execution"], execution) + expected = copy.deepcopy(execution) + if role == "engine": + expected["queue"] = {"max_tasks": 10000, "max_bytes": "128MiB"} + self.assertEqual(configs[role]["execution"], expected) self.assertNotIn("merge", configs[role]) store = yaml.safe_load(docs["ConfigMap", "test-sink-mongo-store"]["data"]["store.yaml"]) self.assertEqual(store["max_concurrent"], 96) @@ -116,6 +119,23 @@ def test_tuning_overlay_renders_application_contract(self): } self.assertEqual(configs["worker"]["consumer"], consumer) + def test_execution_queue_is_engine_only_and_independent_of_batching(self): + values = copy.deepcopy(BASE) + values.setdefault("defaults", {}).setdefault("engine", {})["config"] = { + "executionQueue": {"maxTasks": 7, "maxBytes": "256MiB"} + } + docs = self.manifests(values) + config = yaml.safe_load(docs["ConfigMap", "test-sink-mongo-engine"]["data"]["sink.yaml"]) + self.assertEqual(config["execution"], {"queue": {"max_tasks": 7, "max_bytes": "256MiB"}}) + self.assertNotIn("execution_queue", config) + for queue in [{"maxTasks": 0}, {"maxOperations": 10}]: + invalid = copy.deepcopy(values) + invalid["defaults"]["engine"]["config"]["executionQueue"] = queue + self.assertNotEqual(render(invalid).returncode, 0) + invalid = copy.deepcopy(BASE) + invalid["defaults"]["worker"]["config"]["executionQueue"] = {"maxTasks": 7} + self.assertNotEqual(render(invalid).returncode, 0) + def test_kafka_scaler_settings_do_not_change_store_or_pods(self): values = copy.deepcopy(BASE) worker = {"enabled": True} From 0b75dae3f64577d02bda205b24bfdd20009c88b5 Mon Sep 17 00:00:00 2001 From: Li Ran Date: Fri, 25 Sep 2026 10:05:23 +0800 Subject: [PATCH 2/2] =?UTF-8?q?=F0=9F=94=96=20chore:=20prepare=20Sink=20ch?= =?UTF-8?q?art=20v0.11.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- charts/sink/Chart.yaml | 4 ++-- charts/sink/README.md | 4 ++-- charts/sink/values.yaml | 2 +- docs/operations.md | 8 ++++---- tests/integration/README.md | 4 ++-- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 868a534..9fbe2ac 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # Sink Helm charts One release deploys a shared Gateway and independent Engine/Worker Deployments -for each Store. Chart **0.10.0** targets Sink **0.21.0** and Kubernetes **1.30+**. +for each Store. Chart **0.11.0** targets Sink **0.22.0** and Kubernetes **1.30+**. Values are the only configuration source: the chart generates role and Store configuration, while existing Secrets provide credential values. diff --git a/charts/sink/Chart.yaml b/charts/sink/Chart.yaml index 8d6a36b..747befc 100644 --- a/charts/sink/Chart.yaml +++ b/charts/sink/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: sink description: Store-isolated Sink clusters with bounded rollouts and independent autoscaling. type: application -version: 0.10.0 -appVersion: "0.21.0" +version: 0.11.0 +appVersion: "0.22.0" kubeVersion: ">=1.30.0-0" home: https://github.com/batchstream/sink sources: diff --git a/charts/sink/README.md b/charts/sink/README.md index 2872704..e8b1c6d 100644 --- a/charts/sink/README.md +++ b/charts/sink/README.md @@ -1,6 +1,6 @@ # Sink values reference -Chart **0.10.0** targets Sink **0.21.0** and Kubernetes **1.30+**. Values configure +Chart **0.11.0** targets Sink **0.22.0** and Kubernetes **1.30+**. Values configure a shared Gateway and a separate Engine/optional Worker for each Store. The chart generates `sink.yaml` per role and a shared `store.yaml` per Store. There is no separate application configuration file to maintain. @@ -71,7 +71,7 @@ or the targeted Sink version; a different image version may have different appli | `nameOverride` | Replace the Chart name in resource names | Empty | | `fullnameOverride` | Replace the complete release resource prefix | Empty | | `image.repository` | Container repository | `ghcr.io/batchstream/sink` | -| `image.tag` | Version tag | `0.21.0` | +| `image.tag` | Version tag | `0.22.0` | | `image.digest` | Immutable image selector, preferred over tag | Empty or `sha256:<64 hex characters>` | | `image.pullPolicy` | Kubernetes pull policy | `IfNotPresent`; `Always`, `IfNotPresent`, `Never` | | `image.pullSecrets` | Registry Secret references for every component | `[]`; list of `{name}` | diff --git a/charts/sink/values.yaml b/charts/sink/values.yaml index b27ac7c..a7f6764 100644 --- a/charts/sink/values.yaml +++ b/charts/sink/values.yaml @@ -9,7 +9,7 @@ fullnameOverride: '' # One shared image and registry configuration for every Gateway, Engine and Worker. image: repository: ghcr.io/batchstream/sink - tag: 0.21.0 + tag: 0.22.0 digest: '' pullPolicy: IfNotPresent pullSecrets: [] diff --git a/docs/operations.md b/docs/operations.md index 657d6ce..aeeb762 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -2,9 +2,9 @@ ## Configuration and initial admission -Chart 0.10 uses a shared `stores..storage` object for each Engine/Worker +Chart 0.11 uses a shared `stores..storage` object for each Engine/Worker pair. **Maintain only values, not a separate Sink config YAML.** The chart generates -the complete configuration for every role. It requires Sink 0.21.0+. Store map keys remain stable Sink identities; +the complete configuration for every role. It requires Sink 0.22.0+. Store map keys remain stable Sink identities; changing a key creates a different Store. Only active Stores appear in the Gateway routes. Common operational values are an abstraction over Sink configuration: listener @@ -158,8 +158,8 @@ these checks. Store lifecycle checks apply independently. Rolling upgrades require compatible application configuration, forwarding and Kafka formats because old and new Pods coexist during replacement. Qualify the specific image pair using the upgrade scenario. For incompatible versions, use -a separate matching cluster and a coordinated cutover. Chart 0.10 targets Sink -0.21; it does not qualify mixed-version compatibility with 0.20 or older. +a separate matching cluster and a coordinated cutover. Chart 0.11 targets Sink +0.22; it does not qualify mixed-version compatibility with 0.21 or older. ### Drain timing diff --git a/tests/integration/README.md b/tests/integration/README.md index 5cdfdf5..77a356f 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -15,7 +15,7 @@ owned cluster. Its CoreDNS TTL is set to 30 seconds to exercise the chart cache budget; KEDA CRDs are installed only there. No current/default Kubernetes context is used. The cluster and probe image are deleted in `finally`, including on test failures. Logs and JSON reconciliation evidence remain in `.reports/`. -The `all` and `scaling` scenarios require Sink 0.21+ and enable chart-generated +The `all` and `scaling` scenarios require Sink 0.22+ and enable chart-generated logging on all roles with an unreachable local OTLP receiver. Business traffic, readiness, and bounded shutdown must still work. The `upgrades` scenario exercises separate compatible image references; it does not assert compatibility with the retired 0.18 configuration or forwarding protocol. @@ -58,7 +58,7 @@ Use `--scenario scaling` to rerun only the Kafka/KEDA regression in a fresh clus Use `--sink-image repository:tag` to qualify an explicitly built local candidate. Only the disposable test cluster loads that image. Default deployment/scaling -runs require the chart's matching 0.21 release. Before release, pass a locally +runs require the chart's matching 0.22 release. Before release, pass a locally built role-config candidate. `upgrades` requires an explicit baseline candidate. ## Shared image upgrades and rollback