From f4c909ecadeb87503dea0fbb188f7eefac0d1a15 Mon Sep 17 00:00:00 2001 From: Randy <857271370@qq.com> Date: Wed, 23 Sep 2026 23:40:05 +0800 Subject: [PATCH 1/2] =?UTF-8?q?=F0=9F=92=A5=20feat:=20remove=20role-level?= =?UTF-8?q?=20Store=20concurrency=20setting?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/configuration.md | 14 +++++++------- docs/design/store-backpressure.md | 13 ++++++------- internal/config/backpressure_test.go | 5 +++-- internal/config/config_test.go | 18 ++++++++---------- internal/config/file.go | 3 +-- internal/config/load.go | 2 +- internal/config/service.go | 9 ++------- 7 files changed, 28 insertions(+), 36 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index e5b889c..9b466fe 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -11,10 +11,11 @@ sink --config configs/worker.yaml --store-config configs/stores/primary.yaml ``` Both files use standard YAML. There is no include syntax or general field merging -between them. The Store file's `max_concurrent`, when set, overrides the role -file's `execution.store_max_concurrent`; otherwise the role value overrides the -driver-specific default. Paths passed on the command line are relative to the -working directory. +between them. Set per-Store concurrency in the Store file with `max_concurrent`; +when omitted, the driver-specific default is used. The former +`execution.store_max_concurrent` role setting is no longer accepted; move any +explicit value to the Store file. Paths passed on the command line are relative +to the working directory. `--config` is required. `--store-config` is required for Engine/Worker and rejected for Gateway. `sink config check` accepts the same arguments and validates both files offline without opening dependencies or printing configured values. `sink version` @@ -42,7 +43,7 @@ startup. Restart after changes. Configuration files are limited to 4 MiB. | Gateway: `forwarding` | Store routes, discovery and fanout | | Engine: `batching`, `producer` | Collection queues and Kafka publishing buffer | | Worker: `consumer` | Group, polling, processing rounds and retries | -| Engine/Worker: `execution` | Store concurrency fallback, Merge conflict retries and Lua sandbox | +| Engine/Worker: `execution` | Merge conflict retries and Lua sandbox | | Store: `name`, `max_concurrent`, `storage`, `kafka` | Shared identity, backend concurrency ceiling, database connection and Topic/DLQ policy | Request validation belongs to Gateway. Engine has no `request` section and accepts @@ -148,8 +149,7 @@ use the lowercase spelling shown below. Storage names are also case-sensitive. | `storage.search.password` | string | Conditionally | empty | Any password accepted by the search service | Basic-auth password. Must be configured together with `username`. | | `storage.search.api_key` | string | No | empty | Any API key accepted by the search service | API key used instead of basic authentication. | | `request.max_operations` | positive integer | No | `1000` | Integer greater than `0` | Maximum operation count accepted in one Read, Write, or Delete batch request. | -| `max_concurrent` | positive integer | No | Driver default, or `execution.store_max_concurrent` | Integer from `1` through `4096` | Store-specific adaptive-window ceiling per Engine/Worker process; overrides the role setting. Defaults to 64 for MongoDB and 128 for Elasticsearch/OpenSearch. Not a backend capacity estimate or replica-based quota. | -| `execution.store_max_concurrent` | positive integer | No | `64` for MongoDB; `128` for Elasticsearch/OpenSearch | Integer from `1` through `4096` | Role-file fallback adaptive-window ceiling for Engine/Worker. Used when Store `max_concurrent` is omitted; not a backend capacity estimate or replica-based quota. | +| `max_concurrent` | positive integer | No | `64` for MongoDB; `128` for Elasticsearch/OpenSearch | Integer from `1` through `4096` | Optional adaptive-window ceiling per Engine/Worker process for this Store. Not a backend capacity estimate or replica-based quota. | | `execution.merge.max_attempts` | positive integer | No | `3` | Integer greater than `0` | Maximum revision-conflict attempts for Merge and folded conditional Put chains. | | `batching.max_wait` | duration string | No | `2ms` | Positive Go duration within the bounds below | Maximum collection delay measured from the first request in a batch. | | `batching.max_operations` | positive integer | No | `32` | Positive integer | Operation target for one automatically formed batch; a larger valid RPC still executes alone. See [choosing limits](batching.md#choosing-limits). | diff --git a/docs/design/store-backpressure.md b/docs/design/store-backpressure.md index e607aed..318e040 100644 --- a/docs/design/store-backpressure.md +++ b/docs/design/store-backpressure.md @@ -83,13 +83,12 @@ recovery probe. Old responses from before a decrease cannot apply that decrease repeatedly or undo it with stale successes. Growth does not suppress a late overload from an earlier, slower call. -The adaptive window ceiling accepts **1–4096**. Store `max_concurrent` overrides -the Engine/Worker role-file fallback `execution.store_max_concurrent`; when both -are omitted, the default is **64** for MongoDB and **128** for Elasticsearch or -OpenSearch. The ceiling is local to each Engine/Worker process, not a database -capacity estimate or a per-replica allocation of a global quota. The feature is -always wired by application assembly. Internal component tests can omit the -controller to exercise other boundaries independently. +The adaptive window ceiling accepts **1–4096**. Store `max_concurrent` is +optional; when omitted, the default is **64** for MongoDB and **128** for +Elasticsearch or OpenSearch. The ceiling is local to each Engine/Worker process, +not a database capacity estimate or a per-replica allocation of a global quota. +The feature is always wired by application assembly. Internal component tests +can omit the controller to exercise other boundaries independently. ## Feedback and latency diff --git a/internal/config/backpressure_test.go b/internal/config/backpressure_test.go index 19019c9..77e3dc8 100644 --- a/internal/config/backpressure_test.go +++ b/internal/config/backpressure_test.go @@ -19,8 +19,9 @@ func TestStoreConcurrencyCeilingIndependentOfRoleAndMemory(t *testing.T) { t.Fatalf("%s default: %+v %v", mode, loaded.Service, err) } for _, maximum := range []int{-1, 0, 1, 128, 4096, 4097} { - input := base + fmt.Sprintf("execution: {store_max_concurrent: %d}\n", maximum) - loaded, err := Decode(strings.NewReader(input), strings.NewReader(kafkaStore)) + storeConfig := fmt.Sprintf("name: primary\nmax_concurrent: %d\n", maximum) + store := strings.Replace(kafkaStore, "name: primary\n", storeConfig, 1) + loaded, err := Decode(strings.NewReader(base), strings.NewReader(store)) valid := maximum >= 1 && maximum <= 4096 if valid && (err != nil || loaded.Service.StoreMaxConcurrent != maximum) || !valid && err == nil { t.Fatalf("%s maximum=%d: %v", mode, maximum, err) diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 3a5a000..f6f789b 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -101,24 +101,22 @@ func TestStoreConcurrencyDefaultsFollowStorageDriver(t *testing.T) { } } -func TestStoreConcurrencyOverridePrecedence(t *testing.T) { - component := "mode: engine\nexecution: {store_max_concurrent: 32}\n" +func TestStoreConcurrencyOverride(t *testing.T) { store := "name: primary\nmax_concurrent: 256\nstorage:\n driver: elasticsearch\n search:\n endpoints: [http://127.0.0.1:9200]\n" - loaded, err := Decode(strings.NewReader(component), strings.NewReader(store)) + loaded, err := Decode(strings.NewReader("mode: engine\n"), strings.NewReader(store)) if err != nil { t.Fatal(err) } if loaded.Service.StoreMaxConcurrent != 256 { t.Fatalf("Store max_concurrent = %d, want 256", loaded.Service.StoreMaxConcurrent) } +} - store = strings.Replace(store, "max_concurrent: 256\n", "", 1) - loaded, err = Decode(strings.NewReader(component), strings.NewReader(store)) - if err != nil { - t.Fatal(err) - } - if loaded.Service.StoreMaxConcurrent != 32 { - t.Fatalf("role fallback = %d, want 32", loaded.Service.StoreMaxConcurrent) +func TestRoleStoreConcurrencySettingIsRemoved(t *testing.T) { + component := "mode: engine\nexecution: {store_max_concurrent: 32}\n" + _, err := Decode(strings.NewReader(component), strings.NewReader(minimalStorage)) + if err == nil || !strings.Contains(err.Error(), "store_max_concurrent") { + t.Fatalf("removed execution.store_max_concurrent accepted: %v", err) } } diff --git a/internal/config/file.go b/internal/config/file.go index 7d9d33c..0f3ed76 100644 --- a/internal/config/file.go +++ b/internal/config/file.go @@ -41,8 +41,7 @@ type requestFile struct { } type executionFile struct { - StoreMaxConcurrent *int `yaml:"store_max_concurrent"` - Merge mergeFile `yaml:"merge"` + Merge mergeFile `yaml:"merge"` } // Store files contain identity and backend policy shared by Engine and Worker. diff --git a/internal/config/load.go b/internal/config/load.go index fa65720..7ffc451 100644 --- a/internal/config/load.go +++ b/internal/config/load.go @@ -162,7 +162,7 @@ func resolve(file configFile, shared storeFile) (Config, error) { configured.Kafka = resolveKafka("kafka", shared.Kafka, &v) configured.Kafka.Producer = resolveProducer(file.Producer, &v) configured.Kafka.Consumer = resolveConsumer(file.Consumer, &v) - loaded.Service.StoreMaxConcurrent = resolveStoreMaxConcurrent(file.Execution, shared.MaxConcurrent, configured.Driver, &v) + loaded.Service.StoreMaxConcurrent = resolveStoreMaxConcurrent(shared.MaxConcurrent, configured.Driver, &v) if file.Producer != nil && !configured.Kafka.Enabled { v.reject(errors.New("producer requires Kafka enabled in Store config")) } diff --git a/internal/config/service.go b/internal/config/service.go index 4d3d1d1..bdca9e0 100644 --- a/internal/config/service.go +++ b/internal/config/service.go @@ -38,17 +38,12 @@ func resolveService(file configFile, grpc GRPC, v *validator) Service { return loaded } -func resolveStoreMaxConcurrent(execution *executionFile, storeMaxConcurrent *int, driver Driver, v *validator) int { +func resolveStoreMaxConcurrent(storeMaxConcurrent *int, driver Driver, v *validator) int { defaultMaxConcurrent := backpressure.DefaultMaxConcurrent if driver == DriverElasticsearch || driver == DriverOpenSearch { defaultMaxConcurrent = defaultSearchMaxConcurrent } - var executionMaxConcurrent *int - if execution != nil { - executionMaxConcurrent = execution.StoreMaxConcurrent - } - roleMaxConcurrent := v.bounded("execution.store_max_concurrent", executionMaxConcurrent, defaultMaxConcurrent, 4096) - return v.bounded("max_concurrent", storeMaxConcurrent, roleMaxConcurrent, 4096) + return v.bounded("max_concurrent", storeMaxConcurrent, defaultMaxConcurrent, 4096) } func resolveBatching(file batchingFile, grpc GRPC, v *validator) Batching { From c75108856aad39baa3ef27746643d18758654118 Mon Sep 17 00:00:00 2001 From: Randy <857271370@qq.com> Date: Thu, 24 Sep 2026 00:02:26 +0800 Subject: [PATCH 2/2] =?UTF-8?q?=F0=9F=94=A7=20ci:=20pin=20Store=20concurre?= =?UTF-8?q?ncy=20suite=20revision?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 4 ++-- .github/workflows/release-image.yml | 4 ++-- .github/workflows/reliability.yml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bc9d783..8971625 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -147,8 +147,8 @@ jobs: qualification: name: External test suite - uses: batchstream/sink-production-suite/.github/workflows/server-qualification.yml@58b2ece7489c9f31072b325fda51d3956b2e0ba1 + uses: batchstream/sink-production-suite/.github/workflows/server-qualification.yml@ca8862a8531925da8eee74c8b13f77440ef5d827 with: - suite_ref: 58b2ece7489c9f31072b325fda51d3956b2e0ba1 + suite_ref: ca8862a8531925da8eee74c8b13f77440ef5d827 sink_ref: ${{ github.event.pull_request.head.sha || github.sha }} sdk_ref: ${{ github.head_ref || 'main' }} diff --git a/.github/workflows/release-image.yml b/.github/workflows/release-image.yml index e298750..3b9d383 100644 --- a/.github/workflows/release-image.yml +++ b/.github/workflows/release-image.yml @@ -61,9 +61,9 @@ jobs: qualify: name: Public production qualification needs: release - uses: batchstream/sink-production-suite/.github/workflows/release-qualification.yml@58b2ece7489c9f31072b325fda51d3956b2e0ba1 + uses: batchstream/sink-production-suite/.github/workflows/release-qualification.yml@ca8862a8531925da8eee74c8b13f77440ef5d827 with: - suite_ref: 58b2ece7489c9f31072b325fda51d3956b2e0ba1 + suite_ref: ca8862a8531925da8eee74c8b13f77440ef5d827 sink_ref: ${{ needs.release.outputs.revision }} binaries: diff --git a/.github/workflows/reliability.yml b/.github/workflows/reliability.yml index 7068b79..4a2640d 100644 --- a/.github/workflows/reliability.yml +++ b/.github/workflows/reliability.yml @@ -8,7 +8,7 @@ permissions: jobs: soak: - uses: batchstream/sink-production-suite/.github/workflows/reliability.yml@58b2ece7489c9f31072b325fda51d3956b2e0ba1 + uses: batchstream/sink-production-suite/.github/workflows/reliability.yml@ca8862a8531925da8eee74c8b13f77440ef5d827 with: - suite_ref: 58b2ece7489c9f31072b325fda51d3956b2e0ba1 + suite_ref: ca8862a8531925da8eee74c8b13f77440ef5d827 sink_ref: ${{ github.sha }}