From 0df2222e7688f01a993b1ddd05c1c9f7d819bdb2 Mon Sep 17 00:00:00 2001 From: Li Ran Date: Fri, 25 Sep 2026 07:03:51 +0800 Subject: [PATCH 1/4] =?UTF-8?q?=F0=9F=A7=B5=20feat:=20unify=20Store=20exec?= =?UTF-8?q?ution=20admission?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- configs/engine.yaml | 3 + docs/architecture.md | 4 +- docs/batching.md | 13 +- docs/configuration.md | 13 +- docs/design/store-backpressure.md | 140 +++++++---- docs/design/unified-admission-validation.md | 96 +++++++ docs/observability.md | 14 +- internal/app/app.go | 4 +- internal/backpressure/admission.go | 218 ++++++++++++++++ ...ad_admission_test.go => admission_test.go} | 74 +++--- internal/backpressure/controller.go | 134 ++++------ internal/backpressure/controller_test.go | 2 +- internal/backpressure/metrics.go | 21 +- internal/backpressure/pipeline_test.go | 201 +++++++++++++++ internal/backpressure/read_admission.go | 112 --------- internal/backpressure/storage_test.go | 4 +- internal/config/backpressure_test.go | 25 ++ internal/config/config.go | 6 + internal/config/file.go | 8 +- internal/config/service.go | 12 + internal/engine/backpressure_test.go | 4 +- internal/engine/server.go | 2 +- internal/service/backpressure_test.go | 2 +- internal/service/batcher.go | 166 +++++++------ internal/service/batching_regressions_test.go | 2 +- internal/service/native.go | 4 +- internal/service/query.go | 4 +- internal/service/query_admission_test.go | 17 +- internal/service/read.go | 2 +- internal/service/server.go | 4 +- internal/service/unified_admission_test.go | 234 ++++++++++++++++++ 31 files changed, 1146 insertions(+), 399 deletions(-) create mode 100644 docs/design/unified-admission-validation.md create mode 100644 internal/backpressure/admission.go rename internal/backpressure/{read_admission_test.go => admission_test.go} (80%) create mode 100644 internal/backpressure/pipeline_test.go delete mode 100644 internal/backpressure/read_admission.go create mode 100644 internal/service/unified_admission_test.go diff --git a/configs/engine.yaml b/configs/engine.yaml index b69a6b3..184db4b 100644 --- a/configs/engine.yaml +++ b/configs/engine.yaml @@ -21,6 +21,9 @@ batching: producer: max_buffered_bytes: 64MiB execution: + queue: + max_tasks: 10000 # Ready batches and individual native calls; not operation count. + max_bytes: 128MiB # Shared waiting bytes across collection and admission. merge: max_attempts: 3 # Revision-conflict attempts. lua: diff --git a/docs/architecture.md b/docs/architecture.md index 86314ef..eba1d5e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -95,7 +95,9 @@ batches. Automatic mutation batches also share adapter resource and completion mode, limiting refresh waits to their own dataset. Completed write document chains release their scheduling dependencies; each original RPC returns when all of its results are final. MongoDB can use collection-level bulk operations, and Elasticsearch or -OpenSearch can use `_mget` and `_bulk`. Queue limits and adaptive Store admission constrain dispatched work. A shared +OpenSearch can use `_mget` and `_bulk`. Ready batches and Native calls share one bounded FIFO; queue limits and adaptive +Store admission constrain dispatched work. Encoded waiting bytes remain charged +once across batch collection and admission, while a batch counts as one ready task. A shared local window covers record and Native methods; its feedback comes only from real Store operations. See [Store backpressure](design/store-backpressure.md). diff --git a/docs/batching.md b/docs/batching.md index 15d97bc..dbc24f9 100644 --- a/docs/batching.md +++ b/docs/batching.md @@ -6,10 +6,11 @@ is always active; every single-store `Read` uses this path. `Write` and `Delete` use it for `WAIT_UNTIL_APPLIED` and `WAIT_UNTIL_VISIBLE`; `RETURN_AFTER_ACCEPTED` bypasses it because Kafka already batches asynchronous mutations. Read, write, and delete have independent queues within each store. -Record dependencies stay method-local; Store admission is shared across all -methods. A congested Store pauses subsequent dispatch for that Store. +Record dependencies stay method-local. Ready batches and all Native calls enter +one FIFO before obtaining Store execution permits. A congested Store pauses subsequent dispatch for that Store. -`batching` configures batch and queue limits; batching is always active. +`batching` configures formation targets and collection bounds; batching is always +active. `execution.queue` separately bounds ready tasks and shared waiting bytes. The first queued request starts `batching.max_wait`. Collection stops when that timer expires or adding another request would cross @@ -50,7 +51,11 @@ bound Store and never bypasses that check through the batching layer. Independen Store groups follow the bounded-fanout policy in the [runtime guide](store-isolation.md). Read, Write and Delete each have one bounded queue in an Engine process. Queue -operation and byte limits apply per method; the process has three such queues. +operation and byte limits apply per method; the process has three such collection +queues. Each retains at most one ready batch in the shared admission FIFO without +blocking its event loop. If that FIFO is full, already accepted work stays +upstream. `execution.queue.max_bytes` counts waiting bytes across both stages +once, while `execution.queue.max_tasks` counts ready tasks, not operations. Other Stores run in separate Engine processes. A new single-store request that would cross its queue's limit fails with gRPC `RESOURCE_EXHAUSTED` and is not applied. Requests canceled before dispatch are diff --git a/docs/configuration.md b/docs/configuration.md index da967e7..5eed261 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -113,7 +113,8 @@ SDK examples, connection discovery, replica changes and deployment boundaries. ## Synchronous request batching `batching` sets the collection targets and the per-method -`queue` limits. Batching is always active. See [batching behavior](batching.md) +`queue` limits. `execution.queue` sets ready-task capacity and shared waiting +bytes independently. Batching is always active. See [batching behavior](batching.md) for queue admission, ordering, execution budgets, and completion boundaries. ## Prometheus metrics @@ -150,12 +151,14 @@ use the lowercase spelling shown below. Storage names are also case-sensitive. | `storage.search.api_key` | string | No | empty | Any API key accepted by the search service | API key used instead of basic authentication. | | `request.max_operations` | positive integer | No | `1000` | Integer greater than `0` | Maximum operation count accepted in one Read, Write, or Delete batch request. | | `max_concurrent` | positive integer | No | `64` for MongoDB; `128` for Elasticsearch/OpenSearch | Integer from `1` through `4096` | Optional adaptive-window ceiling per Engine/Worker process for this Store. Not a backend capacity estimate or replica-based quota. | +| `execution.queue.max_tasks` | positive integer | No | `10000` | Integer greater than `0`; Engine only | Maximum ready tasks in the single Store admission FIFO. A collected batch is one task, not one task per operation. | +| `execution.queue.max_bytes` | byte size | No | max(`128MiB`, `grpc.max_receive_message_bytes`, `batching.max_bytes`) | Covers one legal gRPC request and one batch; Engine only | Shared encoded waiting bytes across all collection queues and Native/batch admission tasks, charged once. Waiting ends only with execution capacity or caller cancellation/deadline. | | `execution.merge.max_attempts` | positive integer | No | `3` | Integer greater than `0` | Maximum revision-conflict attempts for Merge and folded conditional Put chains. | | `batching.max_wait` | duration string | No | `2ms` | Positive Go duration within the bounds below | Maximum collection delay measured from the first request in a batch. | | `batching.max_operations` | positive integer | No | `32` | Positive integer | Operation target for one automatically formed batch; a larger valid RPC still executes alone. See [choosing limits](batching.md#choosing-limits). | | `batching.max_bytes` | byte size | No | `16MiB` | Size greater than `0B` | Encoded-byte target for one automatically formed batch; one larger valid RPC still runs alone. | -| `batching.queue.max_operations` | positive integer | No | max(`10000`, `batching.max_operations`) | Integer at least `batching.max_operations` | Maximum operations waiting in each batch-method queue; also the request count bound for the shared Query/Count admission FIFO. | -| `batching.queue.max_bytes` | byte size | No | max(`128MiB`, `grpc.max_receive_message_bytes`) | Size at least `grpc.max_receive_message_bytes` and `batching.max_bytes` | Maximum encoded request bytes waiting in each batch-method queue and in the separate Query/Count admission FIFO. Query/Count waiting ends only with capacity or caller-context cancellation/deadline, not a server-added timeout. | +| `batching.queue.max_operations` | positive integer | No | max(`10000`, `batching.max_operations`) | Integer at least `batching.max_operations` | Maximum operations waiting in each batch-method collection queue. Not an admission task limit or concurrency estimate. | +| `batching.queue.max_bytes` | byte size | No | max(`128MiB`, `grpc.max_receive_message_bytes`) | Size at least `grpc.max_receive_message_bytes` and `batching.max_bytes` | Maximum encoded request bytes retained by each batch-method collection queue, including its selected admission candidate. The Store-wide execution.queue.max_bytes budget also applies. | | `execution.merge.lua.timeout` | duration string | No | `100ms` | Positive Go duration within the bounds below | Maximum wall-clock duration of one Lua execution. | | `execution.merge.lua.max_source_bytes` | byte size | No | `64KiB` | Size greater than `0B` | Maximum Lua source size per merge operation. | | `execution.merge.lua.max_result_bytes` | byte size | No | `16MiB` | Size greater than `0B` | Maximum input/current document and encoded result bytes; expanded output is also bounded before conversion. | @@ -224,7 +227,9 @@ available ceiling. This catches undersized configurations; it does not guarantee against OOM under concurrent load. See the [sizing formula and runtime behavior](design/process-memory-admission.md) and [memory metrics and KEDA](observability.md#memory-capacity-and-keda). -Use `memory` for process pressure and `batching.queue` for Engine waiting queues. +Use `memory` for process pressure, `batching.queue` for per-method collection +bounds, and `execution.queue` for ready-task capacity and shared waiting bytes. +Do not derive task limits or execution concurrency from operation counts. Store admission is shared across Read/Write/Delete/Native operations. It counts admitted Storage executions; adapter group fanout remains unchanged. Sink sets diff --git a/docs/design/store-backpressure.md b/docs/design/store-backpressure.md index e2059bb..6008094 100644 --- a/docs/design/store-backpressure.md +++ b/docs/design/store-backpressure.md @@ -18,69 +18,95 @@ guarantee zero overshoot for an arbitrarily large simultaneous deployment. ```mermaid flowchart LR - G[Gateway] --> Q[Engine bounded method queues] - Q -->|TryAcquire before dequeue| C[Local Store controller] - K[Kafka backlog] -->|Wait before poll; Acquire before attempt| C - R[Query / Count] -->|Bounded FIFO; caller context only| C - N[Execute / Scan] -->|Admit or reject before execution| C + G[Gateway] --> B[Read / Write / Delete collection queues] + B -->|Ready batch; nonblocking handoff| Q[One Store admission FIFO] + G --> N[Execute / Query / Count / Scan] + N -->|One ready task| Q + Q -->|FIFO head acquires permit| C[Local Store controller] + K[Kafka backlog in Worker] -->|Wait before poll; Acquire before attempt| C C --> E[Sequential admitted execution] E --> O[Storage observation decorator] O --> S[Storage / NativeStorage] O -->|Real results and execution latency| C ``` -Engine's three existing batch dispatch loops select dependency-ready work, then try admission **before** -releasing queue bytes/operations, recording active records, or launching execution. -When blocked, they continue handling arrivals, cancellation, record completions and -shutdown while waiting for a capacity notification or a cooldown timer. Pending -work remains charged to the original queue, even when represented in the -dispatcher's pending slice. Queue-full behavior remains `RESOURCE_EXHAUSTED`. -Finished executions retain their permits until the dispatcher receives completion, -so goroutines waiting to report completion cannot escape the concurrency bound. - -One permit covers one sequential execution, including parsing, Lua, snapshots, -CAS attempts, and result settlement. Existing batch partitioning produces one -completion mode per dispatch. Mixed internal waves also stay sequential when -admission is present. The Storage decorator **does not acquire or block**. This -avoids moving a bounded queue into unbounded semaphore waiters, and avoids -deadlocking a Merge which already owns its execution slot. - -Read, Write, Delete, Execute, Query, Count and Scan share the same controller. -Query and Count share a bounded admission FIFO. A full execution window or an -overload cooldown leaves those requests waiting rather than rejecting a healthy -burst. `batching.queue.max_operations` bounds waiting requests (one command per -request), and `batching.queue.max_bytes` bounds retained encoded request bytes. -These are the same configured limits as the batch queues, not a second queue on -top of an existing Read/Write/Delete queue. Only the head competes for a Store -permit; no goroutine is spawned by admission. Query/Count are FIFO with respect -to each other, not globally ordered with the independent batch dispatch loops. - -There is **no new server-side admission timeout**. A caller with no deadline can -wait until capacity is available or its context is canceled. A caller deadline -is propagated unchanged, and cancellation removes queue charges without entering -the backend. Only a full waiting queue rejects with `RESOURCE_EXHAUSTED`. -Encoded bytes are a bounded accounting unit, not an exact Go heap estimate; -process memory admission and gRPC message limits remain independent safeguards. -Admission waiting never shortens the lifetime of an already executing permit. - -Execute and Scan keep their fail-fast admission semantics. Engine marks a Store -admission or read-queue rejection `sink-forward-not-started`; a backend error with the same gRPC code -does not receive that evidence. Async publication bypasses Store admission; -Kafka producer bounds still apply, and Worker controls the later application. - -Worker waits **before polling**, outside `consumer.processing_timeout`, with -fetching paused and group heartbeats/rebalances intact. It acquires a permit -before each handler attempt and releases it before Kafka retry backoff and -settlement. A retained poll remains bounded by `consumer.max_poll_records`. -Admission waiting spends no retry attempt and is cancellable by processing -timeout, rebalance, or shutdown. Unresolved records remain uncommitted; it cannot -turn a temporary failure into a DLQ entry. Worker currently processes polls and -mutation waves sequentially; this feature does not add parallel poll processing. +Each Engine has one FIFO of **ready execution tasks** shared by all methods. +A task is a collected batch or an individual non-batch call, not a document, +operation, RPC response frame, or physical database command. Batch producers do +not compete with Native callers through separate `TryAcquire` loops. Even a +new request that could execute immediately cannot overtake an existing ticket. +FIFO is admission order, not original RPC arrival order, completion order, cost +fairness, cross-method record ordering, or a global ordering across replicas. + +The batch layer retains collection targets, completion-mode partitioning and +method-local record dependencies. It selects only dependency-ready work and +keeps at most one selected batch waiting for admission per producer. It cannot +select a later batch until that ticket is dispatched or canceled, preserving +its existing dependency order. Pending work, including a selected batch, stays +charged to the original collection queue until execution or cancellation. +Already running batches continue to release individual record dependencies. + +A `Ticket` is a nonblocking handoff, not a waiting goroutine. The batch event +loop keeps accepting bounded arrivals and handling cancellations, record +completions, execution completions and shutdown while polling its ticket. A full +admission task queue leaves an accepted batch upstream; it is retried on a +capacity notification, not rejected or removed from accounting. Only the FIFO +head may obtain a permit. Other tickets wait for their turn without contending +for execution slots. Completed executions retain permits until completion +notifications are consumed, bounding goroutines awaiting settlement too. + +### Waiting resources and execution resources + +- `execution.queue.max_tasks` bounds ready tasks in the shared admission FIFO. + One batch occupies one task regardless of its operation count. +- `execution.queue.max_bytes` is one Engine-wide budget for encoded request bytes + retained across **all collection queues and admission**. A `Reservation` is + obtained once; transferring its request into a task does not release and + reacquire capacity. Canceling one member releases only that member's bytes. +- `batching.queue.max_operations` and `batching.queue.max_bytes` remain local + collection bounds per batch method. These do not determine Native admission + capacity or the Store execution window. They also bound upstream work when + the shared ready-task FIFO is full. +- Store `max_concurrent` caps simultaneous execution tasks, independently of + waiting capacity. `batching.max_operations` and `batching.max_bytes` are batch + formation targets, not concurrency weights. + +Bytes are an encoded accounting unit, not a Go heap measurement. Process memory +watermarks, transport size limits, bounded adapter fanout, and batch targets +remain independent protection. An immediately executable non-batch request +needs no waiting-byte reservation. There is no new server-side admission +expiry: each caller's original context controls its wait. Queue/resource bounds +can still reject new work with `RESOURCE_EXHAUSTED`; saturation or cooldown +alone does not. Non-batch calls use their existing RPC goroutine for waiting. + +Canceled members are removed before dispatch without canceling surviving members +of their batch. Moving between stages does not reset deadlines or borrow one +caller's deadline for another. Shutdown cancels queued tickets, releases both +stages' charges exactly once, and settles active executions. A canceled caller +never releases an active execution permit before its backend work ends. + +One permit covers the complete sequential execution, including parsing, Lua, +snapshots, CAS attempts, cursor sends and result settlement. Nested service +calls reuse the permit rather than enqueueing themselves again. The Storage +observation decorator never acquires a permit or owns another queue. + +Engine marks a capacity rejection `sink-forward-not-started`; a backend error +with the same gRPC code does not receive that evidence. Health probes bypass the +business FIFO. Async publication remains on Kafka's bounded producer path, not +behind database admission; Worker controls the later application. + +Worker has its own process-local controller. It waits **before polling**, outside +`consumer.processing_timeout`, with fetching paused and group heartbeats/rebalances +intact. It acquires a permit before each handler attempt and releases it before +Kafka retry backoff and settlement. A retained poll remains bounded by +`consumer.max_poll_records`. Admission waiting spends no retry attempt and is +cancellable by processing timeout, rebalance, or shutdown. Unresolved records +remain uncommitted. This change does not add parallel Worker poll processing. ## Window algorithm The window algorithm has fixed-size feedback state and one short mutex-protected -update per Store observation, alongside the bounded Query/Count FIFO. Engine +update per Store observation, alongside the bounded task FIFO and byte reservations. Engine actively completes randomized initialization in `Run` **before serving readiness or RPCs**, without issuing a synthetic database request. Worker retains its first-poll initialization. Subsequent saturation or cooldown does not flap readiness. @@ -88,7 +114,7 @@ first-poll initialization. Subsequent saturation or cooldown does not flap readi | Event | Behavior | | --- | --- | | Initialization | Stagger for a random 0.5–1.5 seconds, then open `min(4, maximum)` executions before Engine readiness | -| Healthy demand | Saturation or queued Query/Count requests request growth; require at least `max(4, window)` healthy samples and a jittered 125–375 ms control interval | +| Healthy demand | Saturation or ready tasks waiting for execution request growth; require at least `max(4, window)` healthy samples and a jittered 125–375 ms control interval | | Initial growth | Add `max(1, window/2)` below a slow-start threshold of 16 | | After congestion | Threshold becomes half the previous window; subsequent growth is additive by one | | Sustained latency inflation | Halve the window, retaining at least one execution so successful work continues and the baseline can adapt | @@ -215,7 +241,11 @@ require progress without rejection, rather than treating a tiny initial window as sufficient protection. Deterministic virtual-time tests hold a caller for an hour without a server-added deadline, cancel the head/middle/tail of the FIFO, exercise byte/count bounds and cooldown, and verify exact permit/queue cleanup. -Race tests concurrently release permits, cancel callers and collect metrics. +Mixed-method tests assert one FIFO dispatch sequence across batch and Native calls, +partial-batch cancellation, a full downstream queue without event-loop deadlock, +and byte ownership transfer without double release. Light-traffic tests prevent +queue traversal or startup readiness from fabricating growth demand. Race tests +concurrently release reservations/permits, cancel tickets and collect metrics. Kafka component tests use an in-process broker to verify that paused intake does not spend processing timeout, report unhealthy, advance offsets, or produce dead letters; another test verifies a real processor's overload/cooldown retry. diff --git a/docs/design/unified-admission-validation.md b/docs/design/unified-admission-validation.md new file mode 100644 index 0000000..525a8c0 --- /dev/null +++ b/docs/design/unified-admission-validation.md @@ -0,0 +1,96 @@ +# Unified Store admission validation + +## Acceptance scope + +An Engine has one FIFO of ready execution tasks. Collection queues retain batch +formation and method-local record ordering; they do not independently compete +for execution permits. Native calls and ready batches share admission. A batch +is one task, irrespective of its operation count. Waiting encoded bytes remain +charged once across collection and admission. + +The initial execution window and feedback growth/decrease parameters are not +increased to hide overload. Startup readiness and uncontended queue traversal +must not create false growth demand. + +## Required gates + +| Area | Required observation | +| --- | --- | +| FIFO | Read, Write, Delete, Execute, Query, Count and Scan reach storage in admission order when only one execution slot is available. This does not promise cross-method record completion ordering with multiple slots. | +| Byte ownership | Transfer from collection into admission neither frees nor duplicates a reservation. Partial cancellation releases only that member; cancellation/dispatch races never make counters negative or leak capacity. | +| Task units | A batch containing 100 reserved operations is one queued task, not 100 execution permits. | +| Full downstream queue | An already accepted batch remains upstream without failing. Its event loop continues handling cancellations and execution completions. | +| Cancellation and deadlines | Every Native method waits while the window is full. Only the caller's context ends waiting. Real gRPC tests cover unary replies and streaming terminal errors, then release capacity to complete the original queued calls. | +| Cold start | Real Elasticsearch and OpenSearch each receive mixed 64-call bursts per Engine with one and four cold Engines, without admission rejection. Read outcomes and Native status are checked, every written record is verified, and task/byte gauges drain. | +| Adaptive protection | Deterministic 1/8/100-controller simulation and real one/four-Engine latency/429/recovery tests retain congestion protection. Light traffic cannot grow the window merely by traversing the FIFO. | +| Worker independence | Kafka retains 32 accepted increments through overload, settles offsets only after recovery, persists exactly 32, and leaves the DLQ empty. | +| Existing contracts | Full candidate/transport coverage gates and public conformance retain record semantics, response limits, partial-result behavior, crash boundaries, native commands, Scan and process lifecycle behavior. | +| Deployment configuration | Helm schema and rendering tests cover Engine-only `executionQueue`, task counts independent of batch operation limits, and rendered configuration accepted by the candidate executable. | + +The cold-start fixture declares the query sort-field mapping directly on the +backend before starting Sink. This does not send warmup traffic through Sink. +Its sort values are unique and immutable, so Scan remains valid while the index +refreshes during the burst. An unmapped sort field is an invalid database +request, not an admission failure. + +## Reproduction + +From the Sink checkout: + +```sh +GOPROXY=off go test -race ./... -count=1 -timeout=180s +GOPROXY=off go vet ./... +GOPROXY=off staticcheck ./... +``` + +From the matching `sink-production-suite` checkout: + +```sh +SINK_SERVER_DIR=../sink make test-candidate +SINK_SERVER_DIR=../sink SINK_GO_DIR=../sink-go make test-conformance +SINK_SERVER_DIR=../sink bash scripts/test-mongodb-integration.sh +``` + +The suite requires named test events and rejects missing/skipped gates. Its +artifacts retain source revisions, diffs, JSON events, process configuration and +logs. Use disposable local infrastructure; these commands do not qualify a +production cluster. The candidate overlay and core queue scenarios must also +pass with the race detector; the key FIFO/ownership/cancellation scenarios are +repeated 50 times during implementation acceptance. + +From `sink-charts`, use its development Python environment: + +```sh +make lint +make test +python tests/check_runtime.py --sink-binary /path/to/candidate/sink +``` + +## Configuration and metrics + +```yaml +execution: + queue: + max_tasks: 10000 + max_bytes: 128MiB +``` + +`max_tasks` is a ready-task bound, not an operation count or physical database +concurrency estimate. `max_bytes` covers waiting data in both stages. Existing +batch collection bounds and the Store `max_concurrent` ceiling remain separate. +Helm exposes the settings as Engine `config.executionQueue.maxTasks/maxBytes`. +Do not configure them on an older image lacking unified admission support. + +`sink_store_admission_queued_tasks` replaces the read-only queued-request gauge. +`sink_store_buffered_bytes` measures shared waiting bytes; +`sink_store_admission_queued_bytes` is its ready-task subset. Do not sum either +with per-method queue bytes as if they were disjoint allocations. Batch request +queue time includes its admission wait; the two histograms are not additive. + +## Limits of acceptance + +These gates do not prove production Kubernetes rolling-upgrade behavior, a +strict cluster-wide database request limit, an OOM guarantee, a workload-specific +latency SLO, or two-hour soak stability. A full waiting budget, process memory +pressure, oversized payload, or real backend overload may still fail a request. +No production deployment or release is performed by this change. diff --git a/docs/observability.md b/docs/observability.md index e732f6b..842476c 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -191,20 +191,24 @@ and memory measurements do not feed these metrics or the congestion algorithm. | `sink_store_feedback_total` | counter | `method`, `signal=healthy,congested,ignored`; one sample per Store call | | `sink_store_backend_duration_seconds_total` | counter | `method`; elapsed Store time excluding admission and downstream Emit waiting | | `sink_store_latency_baseline_seconds` | gauge | `method`, `batch_size=1,2_32,33_128,129_plus`; zero before learning | -| `sink_store_admission_queued_requests` | gauge | Query/Count callers waiting for a Store permit | -| `sink_store_admission_queued_bytes` | gauge | Encoded Query/Count request bytes retained while waiting | -| `sink_store_admission_waits_total` | counter | `outcome=admitted,canceled,rejected`; bounded read-queue outcomes | -| `sink_store_admission_queue_duration_seconds` | histogram | `outcome=admitted,canceled`; caller-controlled read admission wait, not backend time | +| `sink_store_admission_queued_tasks` | gauge | Ready batches and individual Native calls waiting for a Store permit; one batch is one task | +| `sink_store_admission_queued_bytes` | gauge | Encoded request bytes in ready tasks; a subset of `sink_store_buffered_bytes` | +| `sink_store_buffered_bytes` | gauge | Shared encoded waiting bytes across collection and admission, counted once; do not add method-queue bytes to this gauge | +| `sink_store_admission_waits_total` | counter | `outcome=admitted,canceled,rejected`; bounded task-queue outcomes; upstream handoff retries are not rejections | +| `sink_store_admission_queue_duration_seconds` | histogram | `outcome=admitted,canceled`; task admission wait, not collection delay or backend time | | `sink_store_permit_hold_duration_seconds` | histogram | Execution permit lifetime, including retries/cursor sends but excluding admission wait | | `sink_store_emit_wait_duration_seconds_total` | counter | `method`; downstream Query/Scan callback time excluded from backend feedback | Methods are the fixed set `read`, `write`, `write_visible`, `delete`, `delete_visible`, `execute`, `query`, `count`, `scan`. Opaque Execute commands use success/error feedback only, so their latency baseline remains zero. -The collector creates **136 series per process**, independent of request paths, +The collector creates **137 series per process**, independent of request paths, document identities, query text, and error messages. Deployment/target labels and historical Pod churn remain outside that count. +Batch request queue duration includes both collection and admission; do not add +the admission histogram to it as though they were disjoint stages. + Compare backend execution time with `sink_batcher_request_queue_duration_seconds` and `sink_store_admission_queue_duration_seconds` to distinguish backend slowdown from admission waiting. Compare permit-hold time with backend and Emit time to diff --git a/internal/app/app.go b/internal/app/app.go index f0c92d1..87aa2cc 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -82,8 +82,8 @@ func New(ctx context.Context, opts Options) (*Application, error) { }() admissionOptions := backpressure.Options{ Store: loaded.Storage.Name, Role: string(loaded.Mode), MaxConcurrent: loaded.Service.StoreMaxConcurrent, - MaxQueuedRequests: loaded.Service.Batching.Queue.MaxOperations, - MaxQueuedBytes: loaded.Service.Batching.Queue.MaxBytes, + MaxQueuedTasks: loaded.Service.Admission.MaxTasks, + MaxQueuedBytes: loaded.Service.Admission.MaxBytes, } app.admission, err = backpressure.New(admissionOptions) if err != nil { diff --git a/internal/backpressure/admission.go b/internal/backpressure/admission.go new file mode 100644 index 0000000..0db1354 --- /dev/null +++ b/internal/backpressure/admission.go @@ -0,0 +1,218 @@ +package backpressure + +import ( + "container/list" + "context" + "time" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// Reservation charges retained request bytes across collection and admission. +// Moving a request into a batch never releases and reacquires its budget. +type Reservation struct { + controller *Controller + bytes int + ticket *Ticket + released bool +} + +// Ticket is one ready execution, whether it contains one RPC or a whole batch. +// Polling it never blocks a batcher's cancellation/completion event loop. +type Ticket struct { + controller *Controller + entry *list.Element + turn chan struct{} + reservations []*Reservation + enqueued time.Time +} + +func (c *Controller) Reserve(requestBytes int) (*Reservation, error) { + if requestBytes < 0 { + return nil, status.Error(codes.InvalidArgument, "negative admission size") + } + reservation := &Reservation{controller: c, bytes: requestBytes} + if c == nil { + return reservation, nil + } + c.mu.Lock() + defer c.mu.Unlock() + if requestBytes > c.maxQueuedBytes-c.bufferedBytes { + c.rejectQueueLocked() + return nil, ErrQueueFull + } + c.bufferedBytes += requestBytes + return reservation, nil +} + +func (r *Reservation) Release() { + if r == nil || r.controller == nil { + return + } + c := r.controller + c.mu.Lock() + defer c.mu.Unlock() + r.releaseLocked() +} + +func (r *Reservation) releaseLocked() { + if r.released { + return + } + r.released = true + r.controller.bufferedBytes -= r.bytes + if r.ticket != nil { + r.controller.queuedBytes -= r.bytes + r.ticket = nil + } +} + +// Enqueue transfers existing byte reservations into one FIFO task. A full task +// queue leaves ownership with the producer; changed wakes a batcher to retry. +// It does not count such retries as rejected business requests. +func (c *Controller) Enqueue(reservations []*Reservation) (*Ticket, <-chan struct{}) { + ticket := &Ticket{controller: c, reservations: reservations, turn: make(chan struct{}), enqueued: time.Now()} + if c == nil { + return ticket, nil + } + c.mu.Lock() + defer c.mu.Unlock() + if c.waiters.Len() >= c.maxQueuedTasks { + return nil, c.changed + } + for _, reservation := range reservations { + if reservation.controller != c || reservation.ticket != nil { + panic("admission reservation has a different owner") + } + if !reservation.released { + reservation.ticket = ticket + c.queuedBytes += reservation.bytes + } + } + ticket.entry = c.waiters.PushBack(ticket) + if c.waiters.Len()+c.inFlight > c.limit { + c.demand = true + } + if ticket.entry == c.waiters.Front() { + close(ticket.turn) + } + return ticket, nil +} + +// Poll grants a permit only to the FIFO head. Timers only wake that head; they +// neither execute work nor impose an admission deadline. +func (t *Ticket) Poll() (*Permit, <-chan struct{}, time.Duration) { + c := t.controller + if c == nil { + permit := &Permit{} + return permit, nil, 0 + } + c.mu.Lock() + defer c.mu.Unlock() + if t.entry == nil { + panic("admission ticket was already settled") + } + if t.entry != c.waiters.Front() { + return nil, t.turn, 0 + } + permit, changed, delay := c.tryAcquireLocked(time.Now()) + if permit != nil { + t.finishLocked(true) + } + return permit, changed, delay +} + +func (t *Ticket) Cancel() { + if t == nil || t.controller == nil { + return + } + c := t.controller + c.mu.Lock() + defer c.mu.Unlock() + if t.entry != nil { + t.finishLocked(false) + } +} + +func (t *Ticket) finishLocked(admitted bool) { + c := t.controller + head := t.entry == c.waiters.Front() + c.waiters.Remove(t.entry) + t.entry = nil + for _, reservation := range t.reservations { + reservation.releaseLocked() + } + t.reservations = nil + outcome := "canceled" + if admitted { + c.observed.queueAdmitted++ + outcome = "admitted" + } else { + c.observed.queueCanceled++ + } + c.observed.queueDuration.WithLabelValues(outcome).Observe(time.Since(t.enqueued).Seconds()) + if next := c.waiters.Front(); head && next != nil { + close(next.Value.(*Ticket).turn) + } + c.signal() +} + +func (c *Controller) rejectQueueLocked() { + c.observed.rejected++ + c.observed.queueRejected++ +} + +// Admit reuses an execution's permit, or waits in the same FIFO as ready +// batches. Only the caller's original context ends admission waiting. +func (c *Controller) Admit(ctx context.Context, requestBytes int) (context.Context, *Permit, error) { + if err := ctx.Err(); err != nil { + return ctx, nil, status.FromContextError(err).Err() + } + if requestBytes < 0 { + return ctx, nil, status.Error(codes.InvalidArgument, "negative admission size") + } + if c == nil { + return ctx, nil, nil + } + if permit, ok := ctx.Value(permitKey{}).(*Permit); ok && permit.controller == c { + return ctx, nil, nil + } + if permit, _, _ := c.TryAcquire(); permit != nil { + if err := ctx.Err(); err != nil { + permit.Release() + return ctx, nil, status.FromContextError(err).Err() + } + return permit.Context(ctx), permit, nil + } + reservation, err := c.Reserve(requestBytes) + if err != nil { + return ctx, nil, err + } + defer reservation.Release() + reservations := []*Reservation{reservation} + ticket, _ := c.Enqueue(reservations) + if ticket == nil { + c.mu.Lock() + c.rejectQueueLocked() + c.mu.Unlock() + return ctx, nil, ErrQueueFull + } + defer ticket.Cancel() + for { + if err := ctx.Err(); err != nil { + return ctx, nil, status.FromContextError(err).Err() + } + permit, changed, delay := ticket.Poll() + if permit != nil { + if err := ctx.Err(); err != nil { + permit.Release() + return ctx, nil, status.FromContextError(err).Err() + } + return permit.Context(ctx), permit, nil + } + if err := wait(ctx, changed, delay); err != nil { + return ctx, nil, status.FromContextError(err).Err() + } + } +} diff --git a/internal/backpressure/read_admission_test.go b/internal/backpressure/admission_test.go similarity index 80% rename from internal/backpressure/read_admission_test.go rename to internal/backpressure/admission_test.go index e95f87a..fb15d64 100644 --- a/internal/backpressure/read_admission_test.go +++ b/internal/backpressure/admission_test.go @@ -23,7 +23,7 @@ type readAdmissionResult struct { func readController(t *testing.T, maximum, requests, bytes int) *Controller { t.Helper() - opts := Options{Store: "primary", Role: "engine", MaxConcurrent: maximum, MaxQueuedRequests: requests, MaxQueuedBytes: bytes} + opts := Options{Store: "primary", Role: "engine", MaxConcurrent: maximum, MaxQueuedTasks: requests, MaxQueuedBytes: bytes} c, err := New(opts) if err != nil { t.Fatal(err) @@ -31,7 +31,7 @@ func readController(t *testing.T, maximum, requests, bytes int) *Controller { return c } -func TestReadAdmissionFIFOAndCancellationReleaseEveryQueuePosition(t *testing.T) { +func TestAdmissionFIFOAndCancellationReleaseEveryQueuePosition(t *testing.T) { for _, canceled := range []int{0, 1, 2} { t.Run(strconv.Itoa(canceled), func(t *testing.T) { synctest.Test(t, func(t *testing.T) { @@ -48,18 +48,18 @@ func TestReadAdmissionFIFOAndCancellationReleaseEveryQueuePosition(t *testing.T) cancels = append(cancels, cancel) defer cancel() go func() { - admitted, permit, err := c.AdmitRead(ctx, 16) + admitted, permit, err := c.Admit(ctx, 16) result := readAdmissionResult{index: index, ctx: admitted, permit: permit, err: err} results <- result }() synctest.Wait() } - if c.readWaiters.Len() != 3 || c.queuedBytes != 48 { - t.Fatalf("queue is not bounded/accounted: requests=%d bytes=%d", c.readWaiters.Len(), c.queuedBytes) + if c.waiters.Len() != 3 || c.queuedBytes != 48 { + t.Fatalf("queue is not bounded/accounted: requests=%d bytes=%d", c.waiters.Len(), c.queuedBytes) } goroutines := runtime.NumGoroutine() for range 1000 { - _, permit, err := c.AdmitRead(t.Context(), 1) + _, permit, err := c.Admit(t.Context(), 1) if permit != nil || err != ErrQueueFull { t.Fatalf("full queue: permit=%v error=%v", permit, err) } @@ -73,7 +73,7 @@ func TestReadAdmissionFIFOAndCancellationReleaseEveryQueuePosition(t *testing.T) if result.index != canceled || status.Code(result.err) != codes.Canceled || result.permit != nil { t.Fatalf("cancellation changed: %+v", result) } - if c.readWaiters.Len() != 2 || c.queuedBytes != 32 || c.inFlight != 1 { + if c.waiters.Len() != 2 || c.queuedBytes != 32 || c.inFlight != 1 { t.Fatal("cancellation leaked queue capacity or revoked active work") } held.Release() @@ -88,7 +88,7 @@ func TestReadAdmissionFIFOAndCancellationReleaseEveryQueuePosition(t *testing.T) result.permit.Release() } synctest.Wait() - if c.readWaiters.Len() != 0 || c.queuedBytes != 0 || c.inFlight != 0 || c.observed.queueCanceled != 1 || c.observed.queueAdmitted != 2 { + if c.waiters.Len() != 0 || c.queuedBytes != 0 || c.inFlight != 0 || c.observed.queueCanceled != 1 || c.observed.queueAdmitted != 2 { t.Fatal("completed queue leaked state") } }) @@ -96,7 +96,7 @@ func TestReadAdmissionFIFOAndCancellationReleaseEveryQueuePosition(t *testing.T) } } -func TestReadAdmissionByteLimitAndCapacityReuse(t *testing.T) { +func TestAdmissionByteLimitAndCapacityReuse(t *testing.T) { synctest.Test(t, func(t *testing.T) { c := readController(t, 1, 10, 32) held, err := c.Acquire(t.Context()) @@ -107,11 +107,11 @@ func TestReadAdmissionByteLimitAndCapacityReuse(t *testing.T) { ctx, cancel := context.WithCancel(t.Context()) defer cancel() result := make(chan error, 1) - go func() { _, permit, err := c.AdmitRead(ctx, 32); permit.Release(); result <- err }() + go func() { _, permit, err := c.Admit(ctx, 32); permit.Release(); result <- err }() synctest.Wait() for _, bytes := range []int{1, 33, int(^uint(0) >> 1)} { - _, permit, err := c.AdmitRead(t.Context(), bytes) - if err != ErrQueueFull || permit != nil || c.queuedBytes != 32 || c.readWaiters.Len() != 1 { + _, permit, err := c.Admit(t.Context(), bytes) + if err != ErrQueueFull || permit != nil || c.queuedBytes != 32 || c.waiters.Len() != 1 { t.Fatalf("encoded-byte bound failed for %d: %v", bytes, err) } } @@ -120,19 +120,19 @@ func TestReadAdmissionByteLimitAndCapacityReuse(t *testing.T) { t.Fatal(err) } held.Release() - _, permit, err := c.AdmitRead(t.Context(), 32) + _, permit, err := c.Admit(t.Context(), 32) if err != nil { t.Fatal(err) } permit.Release() permit.Release() - if c.queuedBytes != 0 || c.readWaiters.Len() != 0 || c.inFlight != 0 { + if c.queuedBytes != 0 || c.waiters.Len() != 0 || c.inFlight != 0 { t.Fatal("byte reservations or permits leaked") } }) } -func TestReadAdmissionPreservesContextAndOnlyCallerEndsWaiting(t *testing.T) { +func TestAdmissionPreservesContextAndOnlyCallerEndsWaiting(t *testing.T) { synctest.Test(t, func(t *testing.T) { c := readController(t, 1, 4, 64) held, err := c.Acquire(t.Context()) @@ -147,7 +147,7 @@ func TestReadAdmissionPreservesContextAndOnlyCallerEndsWaiting(t *testing.T) { wantDeadline, _ := ctx.Deadline() result := make(chan readAdmissionResult, 1) go func() { - admitted, permit, err := c.AdmitRead(ctx, 16) + admitted, permit, err := c.Admit(ctx, 16) value := readAdmissionResult{ctx: admitted, permit: permit, err: err} result <- value }() @@ -174,8 +174,8 @@ func TestReadAdmissionPreservesContextAndOnlyCallerEndsWaiting(t *testing.T) { defer held.Release() short, shortCancel := context.WithDeadline(t.Context(), time.Now().Add(time.Minute)) defer shortCancel() - _, permit, err := c.AdmitRead(short, 16) - if status.Code(err) != codes.DeadlineExceeded || permit != nil || c.readWaiters.Len() != 0 || c.queuedBytes != 0 { + _, permit, err := c.Admit(short, 16) + if status.Code(err) != codes.DeadlineExceeded || permit != nil || c.waiters.Len() != 0 || c.queuedBytes != 0 { t.Fatalf("caller deadline did not release queue: %v", err) } }) @@ -193,7 +193,7 @@ func TestQueuedHealthyReadsGrowWithoutRejectingRequests(t *testing.T) { active, peak := 0, 0 for range 512 { go func() { - _, permit, err := c.AdmitRead(t.Context(), 16) + _, permit, err := c.Admit(t.Context(), 16) if err != nil { results <- err return @@ -218,13 +218,13 @@ func TestQueuedHealthyReadsGrowWithoutRejectingRequests(t *testing.T) { } } synctest.Wait() - if peak < 3 || peak > 16 || active != 0 || c.observed.increases == 0 || c.observed.rejected != 0 || c.readWaiters.Len() != 0 || c.queuedBytes != 0 || c.inFlight != 0 { + if peak < 3 || peak > 16 || active != 0 || c.observed.increases == 0 || c.observed.rejected != 0 || c.waiters.Len() != 0 || c.queuedBytes != 0 || c.inFlight != 0 { t.Fatalf("healthy demand did not grow safely: peak=%d increases=%d rejected=%d", peak, c.observed.increases, c.observed.rejected) } }) } -func TestReadAdmissionWaitsThroughCooldownWithoutRevokingActiveWork(t *testing.T) { +func TestAdmissionWaitsThroughCooldownWithoutRevokingActiveWork(t *testing.T) { synctest.Test(t, func(t *testing.T) { c := readController(t, 4, 2, 32) held, err := c.Acquire(t.Context()) @@ -237,12 +237,12 @@ func TestReadAdmissionWaitsThroughCooldownWithoutRevokingActiveWork(t *testing.T resume := c.resumeAt result := make(chan readAdmissionResult, 1) go func() { - ctx, permit, err := c.AdmitRead(t.Context(), 16) + ctx, permit, err := c.Admit(t.Context(), 16) value := readAdmissionResult{ctx: ctx, permit: permit, err: err} result <- value }() synctest.Wait() - if c.limit != 0 || c.inFlight != 1 || c.readWaiters.Len() != 1 { + if c.limit != 0 || c.inFlight != 1 || c.waiters.Len() != 1 { t.Fatal("cooldown revoked admitted work or let waiting work bypass") } time.Sleep(time.Until(resume) / 2) @@ -260,26 +260,26 @@ func TestReadAdmissionWaitsThroughCooldownWithoutRevokingActiveWork(t *testing.T }) } -func TestReadAdmissionValidationNilAndReusedPermit(t *testing.T) { +func TestAdmissionValidationNilAndReusedPermit(t *testing.T) { synctest.Test(t, func(t *testing.T) { - for _, opts := range []Options{{MaxQueuedRequests: -1}, {MaxQueuedBytes: -1}} { + for _, opts := range []Options{{MaxQueuedTasks: -1}, {MaxQueuedBytes: -1}} { if _, err := New(opts); err == nil { t.Fatal("negative queue limit accepted") } } c := readController(t, 1, 1, 16) - _, permit, err := c.AdmitRead(t.Context(), -1) + _, permit, err := c.Admit(t.Context(), -1) if status.Code(err) != codes.InvalidArgument || permit != nil { t.Fatal("negative encoded size accepted") } canceled, cancel := context.WithCancel(t.Context()) cancel() - _, permit, err = c.AdmitRead(canceled, 1) + _, permit, err = c.Admit(canceled, 1) if status.Code(err) != codes.Canceled || permit != nil || c.observed.admitted != 0 { t.Fatal("already canceled request was admitted") } var absent *Controller - admitted, permit, err := absent.AdmitRead(t.Context(), 1) + admitted, permit, err := absent.Admit(t.Context(), 1) if err != nil || permit != nil || admitted != t.Context() { t.Fatal("nil admission changed component behavior") } @@ -289,14 +289,14 @@ func TestReadAdmissionValidationNilAndReusedPermit(t *testing.T) { } defer held.Release() ctx := held.Context(t.Context()) - admitted, permit, err = c.AdmitRead(ctx, 1) - if err != nil || permit != nil || admitted != ctx || c.inFlight != 1 || c.readWaiters.Len() != 0 { + admitted, permit, err = c.Admit(ctx, 1) + if err != nil || permit != nil || admitted != ctx || c.inFlight != 1 || c.waiters.Len() != 0 { t.Fatal("nested admission deadlocked or double-counted an existing permit") } }) } -func TestReadAdmissionMetricsSeparateWaitingAndPermitHold(t *testing.T) { +func TestAdmissionMetricsSeparateWaitingAndPermitHold(t *testing.T) { synctest.Test(t, func(t *testing.T) { c := readController(t, 1, 1, 16) held, err := c.Acquire(t.Context()) @@ -310,7 +310,7 @@ func TestReadAdmissionMetricsSeparateWaitingAndPermitHold(t *testing.T) { } result := make(chan *Permit, 1) go func() { - _, permit, err := c.AdmitRead(t.Context(), 16) + _, permit, err := c.Admit(t.Context(), 16) if err != nil { t.Error(err) } @@ -323,7 +323,7 @@ func TestReadAdmissionMetricsSeparateWaitingAndPermitHold(t *testing.T) { } for _, family := range families { switch family.GetName() { - case "sink_store_admission_queued_requests": + case "sink_store_admission_queued_tasks": if family.Metric[0].GetGauge().GetValue() != 1 { t.Fatal("missing waiting request gauge") } @@ -362,7 +362,7 @@ func TestReadAdmissionMetricsSeparateWaitingAndPermitHold(t *testing.T) { }) } -func TestReadAdmissionCancelReleaseContentionDoesNotLeak(t *testing.T) { +func TestAdmissionCancelReleaseContentionDoesNotLeak(t *testing.T) { c := readController(t, 4, 256, 4096) if err := c.Wait(t.Context()); err != nil { t.Fatal(err) @@ -377,7 +377,7 @@ func TestReadAdmissionCancelReleaseContentionDoesNotLeak(t *testing.T) { } else if (index+iteration)%3 == 1 { go cancel() } - _, permit, err := c.AdmitRead(ctx, 16) + _, permit, err := c.Admit(ctx, 16) if err != nil && status.Code(err) != codes.Canceled { t.Errorf("bounded contention lost a request: %v", err) } @@ -387,7 +387,7 @@ func TestReadAdmissionCancelReleaseContentionDoesNotLeak(t *testing.T) { }) } workers.Wait() - if c.inFlight != 0 || c.readWaiters.Len() != 0 || c.queuedBytes != 0 { - t.Fatalf("race leaked admission: active=%d queued=%d bytes=%d", c.inFlight, c.readWaiters.Len(), c.queuedBytes) + if c.inFlight != 0 || c.waiters.Len() != 0 || c.queuedBytes != 0 { + t.Fatalf("race leaked admission: active=%d queued=%d bytes=%d", c.inFlight, c.waiters.Len(), c.queuedBytes) } } diff --git a/internal/backpressure/controller.go b/internal/backpressure/controller.go index 279994d..5ce9d55 100644 --- a/internal/backpressure/controller.go +++ b/internal/backpressure/controller.go @@ -1,6 +1,6 @@ // Package backpressure controls Store dispatch using feedback from real work. -// Read admission has a bounded queue; no background goroutine, dependency probe -// or retry policy is owned by the controller. +// All ready Store executions share one bounded FIFO. The controller owns no +// background goroutine, dependency probe or retry policy. package backpressure import ( @@ -18,53 +18,51 @@ import ( const DefaultMaxConcurrent = 64 const ( - initialConcurrent = 4 - defaultMaxQueuedRequests = 10_000 - defaultMaxQueuedBytes = 128 << 20 - startupWindow = time.Second - controlInterval = 250 * time.Millisecond - minimumCooldown = 200 * time.Millisecond - maximumCooldown = 10 * time.Second + initialConcurrent = 4 + defaultMaxQueuedTasks = 10_000 + defaultMaxQueuedBytes = 128 << 20 + startupWindow = time.Second + controlInterval = 250 * time.Millisecond + minimumCooldown = 200 * time.Millisecond + maximumCooldown = 10 * time.Second ) -// ErrBusy proves that this admission did not start backend work. -var ErrBusy = status.Error(codes.ResourceExhausted, "Store admission is temporarily full or cooling down") - -// ErrQueueFull proves that read admission did not start backend work. -var ErrQueueFull = status.Error(codes.ResourceExhausted, "Store read admission queue is full") +// ErrQueueFull proves that admission did not start backend work. +var ErrQueueFull = status.Error(codes.ResourceExhausted, "Store waiting capacity is full") type Options struct { Store string Role string // MaxConcurrent is a local resource ceiling, not an estimate of DB capacity. - MaxConcurrent int - MaxQueuedRequests int - MaxQueuedBytes int + MaxConcurrent int + MaxQueuedTasks int + MaxQueuedBytes int } type Controller struct { - mu sync.Mutex - limit int - inFlight int - maximum int - threshold int - resumeLimit int - epoch uint64 - clean int - recovered int - failures int - demand bool - started bool - resumeAt time.Time - adjustAt time.Time - changed chan struct{} - random *rand.Rand - latency [methodCount][sizeClasses]latency - observed observations - readWaiters list.List - queuedBytes int - maxQueuedRequests int - maxQueuedBytes int + mu sync.Mutex + limit int + inFlight int + maximum int + threshold int + resumeLimit int + epoch uint64 + clean int + recovered int + failures int + demand bool + started bool + resumeAt time.Time + adjustAt time.Time + changed chan struct{} + random *rand.Rand + latency [methodCount][sizeClasses]latency + observed observations + waiters list.List + queuedBytes int + bufferedBytes int + maxQueuedTasks int + maxQueuedBytes int } type latency struct { @@ -91,11 +89,11 @@ func New(opts Options) (*Controller, error) { if opts.MaxConcurrent < 1 || opts.MaxConcurrent > 4096 { return nil, errors.New("store max_concurrent must be between 1 and 4096") } - if opts.MaxQueuedRequests < 0 || opts.MaxQueuedBytes < 0 { - return nil, errors.New("store read admission queue limits cannot be negative") + if opts.MaxQueuedTasks < 0 || opts.MaxQueuedBytes < 0 { + return nil, errors.New("store admission queue limits cannot be negative") } - if opts.MaxQueuedRequests == 0 { - opts.MaxQueuedRequests = defaultMaxQueuedRequests + if opts.MaxQueuedTasks == 0 { + opts.MaxQueuedTasks = defaultMaxQueuedTasks } if opts.MaxQueuedBytes == 0 { opts.MaxQueuedBytes = defaultMaxQueuedBytes @@ -103,14 +101,14 @@ func New(opts Options) (*Controller, error) { c := &Controller{ maximum: opts.MaxConcurrent, threshold: min(16, opts.MaxConcurrent), resumeLimit: min(initialConcurrent, opts.MaxConcurrent), changed: make(chan struct{}), random: rand.New(rand.NewPCG(rand.Uint64(), rand.Uint64())), - observed: newObservations(opts), - maxQueuedRequests: opts.MaxQueuedRequests, maxQueuedBytes: opts.MaxQueuedBytes, + observed: newObservations(opts), + maxQueuedTasks: opts.MaxQueuedTasks, maxQueuedBytes: opts.MaxQueuedBytes, } return c, nil } -// TryAcquire is used by bounded dispatchers before removing queued work or -// starting a goroutine. On failure, changed and delay describe when to retry; +// TryAcquire is used by the bounded Worker loop; it cannot bypass queued tasks. +// Engine dispatchers enqueue a Ticket instead. On failure, changed and delay describe when to retry; // delay == 0 means only a completion or another state change can free capacity. func (c *Controller) TryAcquire() (*Permit, <-chan struct{}, time.Duration) { return c.tryAcquire(time.Now()) @@ -123,12 +121,17 @@ func (c *Controller) tryAcquire(now time.Time) (*Permit, <-chan struct{}, time.D } c.mu.Lock() defer c.mu.Unlock() + if c.waiters.Len() != 0 { + c.demand = true + return nil, c.changed, 0 + } return c.tryAcquireLocked(now) } func (c *Controller) tryAcquireLocked(now time.Time) (*Permit, <-chan struct{}, time.Duration) { ready, changed, delay := c.availability(now) if !ready { + c.demand = true return nil, changed, delay } c.inFlight++ @@ -137,9 +140,8 @@ func (c *Controller) tryAcquireLocked(now time.Time) (*Permit, <-chan struct{}, return permit, nil, 0 } -// Ready lets a dispatcher avoid rebuilding a batch while capacity is occupied, -// and lets Worker avoid pausing prefetch on every healthy poll. It reserves no -// slot; callers must still use TryAcquire/Acquire before dispatch. +// Ready lets startup and Worker prefetch observe capacity without reserving a +// slot or inventing demand. Engine execution tickets are polled separately. func (c *Controller) Ready() (bool, <-chan struct{}, time.Duration) { if c == nil { return true, nil, 0 @@ -152,15 +154,11 @@ func (c *Controller) Ready() (bool, <-chan struct{}, time.Duration) { func (c *Controller) availability(now time.Time) (bool, <-chan struct{}, time.Duration) { c.resume(now) ready := c.limit > c.inFlight - if !ready { - c.demand = true - } return ready, c.changed, max(0, c.resumeAt.Sub(now)) } -// Acquire may wait only at an already bounded caller (the Kafka processing -// loop). Query/Count use AdmitRead; other Engine RPCs use their existing batch -// queue or fail-fast Admit instead. +// Acquire waits only at the already bounded Kafka processing loop. Engine +// requests use Admit or enqueue a batch Ticket, never this unqueued wait path. func (c *Controller) Acquire(ctx context.Context) (*Permit, error) { for { if err := ctx.Err(); err != nil { @@ -232,28 +230,6 @@ func (p *Permit) Release() { }) } -// Admit reuses the dispatch permit within a sequential execution. A direct -// request without a queue is rejected before execution if capacity is absent. -func (c *Controller) Admit(ctx context.Context) (context.Context, *Permit, error) { - if err := ctx.Err(); err != nil { - return ctx, nil, status.FromContextError(err).Err() - } - if c == nil { - return ctx, nil, nil - } - if permit, ok := ctx.Value(permitKey{}).(*Permit); ok && permit.controller == c { - return ctx, nil, nil - } - permit, _, _ := c.TryAcquire() - if permit == nil { - c.mu.Lock() - c.observed.rejected++ - c.mu.Unlock() - return ctx, nil, ErrBusy - } - return permit.Context(ctx), permit, nil -} - // All state transitions below run with mu held. No timers mutate the state. func (c *Controller) resume(now time.Time) { if !c.started { @@ -347,7 +323,7 @@ func (c *Controller) observeAt(started sample, duration time.Duration, result fe if c.recovered == 4 { c.failures = 0 } - if !started.saturated && !c.demand && c.readWaiters.Len() == 0 { + if !started.saturated && !c.demand && c.waiters.Len() == 0 { return } c.clean++ diff --git a/internal/backpressure/controller_test.go b/internal/backpressure/controller_test.go index 5428440..0fd9897 100644 --- a/internal/backpressure/controller_test.go +++ b/internal/backpressure/controller_test.go @@ -242,7 +242,7 @@ func TestAdmissionCancellationAndNoLostWakeup(t *testing.T) { permit.Release() next := <-done admitted := next.Context(t.Context()) - _, nested, err := c.Admit(admitted) + _, nested, err := c.Admit(admitted, 0) if err != nil || nested != nil { t.Fatal("sequential execution tried to reacquire its permit") } diff --git a/internal/backpressure/metrics.go b/internal/backpressure/metrics.go index bb71294..d2ec5a5 100644 --- a/internal/backpressure/metrics.go +++ b/internal/backpressure/metrics.go @@ -15,8 +15,9 @@ type observations struct { duration *prometheus.Desc baseline *prometheus.Desc cooldown *prometheus.Desc - queueRequests *prometheus.Desc + queueTasks *prometheus.Desc queueBytes *prometheus.Desc + bufferedBytes *prometheus.Desc queueOutcomes *prometheus.Desc queueDuration *prometheus.HistogramVec permitDuration prometheus.Histogram @@ -36,20 +37,21 @@ type observations struct { func newObservations(opts Options) observations { labels := prometheus.Labels{"role": opts.Role, "store": opts.Store} - queueOptions := prometheus.HistogramOpts{Name: "sink_store_admission_queue_duration_seconds", Help: "Query/Count wait time before admission or caller cancellation; excludes backend work.", ConstLabels: labels} + queueOptions := prometheus.HistogramOpts{Name: "sink_store_admission_queue_duration_seconds", Help: "Store task wait time before admission or caller cancellation; excludes backend work.", ConstLabels: labels} permitOptions := prometheus.HistogramOpts{Name: "sink_store_permit_hold_duration_seconds", Help: "Time holding an execution permit, including retries and cursor sends but excluding admission wait.", ConstLabels: labels} observed := observations{ limit: prometheus.NewDesc("sink_store_concurrency_limit", "Current local Store execution window; zero pauses new work.", nil, labels), inFlight: prometheus.NewDesc("sink_store_executions_in_flight", "Admitted sequential Store executions, including existing retries and cursor sends.", nil, labels), changes: prometheus.NewDesc("sink_store_window_changes_total", "Store window adjustments by reason.", []string{"reason"}, labels), - admissions: prometheus.NewDesc("sink_store_admissions_total", "Store executions admitted or direct requests rejected before execution.", []string{"outcome"}, labels), + admissions: prometheus.NewDesc("sink_store_admissions_total", "Store executions admitted or new work rejected before execution.", []string{"outcome"}, labels), feedback: prometheus.NewDesc("sink_store_feedback_total", "Real Store calls classified once, regardless of batch result count.", []string{"method", "signal"}, labels), duration: prometheus.NewDesc("sink_store_backend_duration_seconds_total", "Store call time excluding admission and downstream Emit time.", []string{"method"}, labels), baseline: prometheus.NewDesc("sink_store_latency_baseline_seconds", "Learned Store latency per operation and batch-size class; zero until sampled.", []string{"method", "batch_size"}, labels), cooldown: prometheus.NewDesc("sink_store_cooldown_seconds", "Remaining delay before real work may probe a paused Store.", nil, labels), - queueRequests: prometheus.NewDesc("sink_store_admission_queued_requests", "Query/Count requests waiting for a shared Store execution permit.", nil, labels), - queueBytes: prometheus.NewDesc("sink_store_admission_queued_bytes", "Encoded request bytes retained by the Query/Count admission queue.", nil, labels), - queueOutcomes: prometheus.NewDesc("sink_store_admission_waits_total", "Query/Count admission waits completed or rejected by the bounded queue.", []string{"outcome"}, labels), + queueTasks: prometheus.NewDesc("sink_store_admission_queued_tasks", "Ready Store tasks waiting for a shared Store execution permit.", nil, labels), + queueBytes: prometheus.NewDesc("sink_store_admission_queued_bytes", "Encoded request bytes retained by ready Store tasks, also included in buffered bytes.", nil, labels), + bufferedBytes: prometheus.NewDesc("sink_store_buffered_bytes", "Encoded request bytes retained across batch collection and admission, charged once.", nil, labels), + queueOutcomes: prometheus.NewDesc("sink_store_admission_waits_total", "Store task admission waits completed or rejected by the bounded queue.", []string{"outcome"}, labels), queueDuration: prometheus.NewHistogramVec(queueOptions, []string{"outcome"}), permitDuration: prometheus.NewHistogram(permitOptions), emitDuration: prometheus.NewDesc("sink_store_emit_wait_duration_seconds_total", "Time waiting in Query/Scan downstream Emit callbacks, excluded from backend feedback.", []string{"method"}, labels), @@ -63,7 +65,7 @@ func (c *Controller) Describe(ch chan<- *prometheus.Desc) { for _, desc := range []*prometheus.Desc{c.observed.limit, c.observed.inFlight, c.observed.changes, c.observed.admissions, c.observed.feedback, c.observed.duration, c.observed.baseline, c.observed.cooldown} { ch <- desc } - for _, desc := range []*prometheus.Desc{c.observed.queueRequests, c.observed.queueBytes, c.observed.queueOutcomes, c.observed.emitDuration} { + for _, desc := range []*prometheus.Desc{c.observed.queueTasks, c.observed.queueBytes, c.observed.bufferedBytes, c.observed.queueOutcomes, c.observed.emitDuration} { ch <- desc } c.observed.queueDuration.Describe(ch) @@ -74,9 +76,10 @@ func (c *Controller) Collect(ch chan<- prometheus.Metric) { c.mu.Lock() observed, limit, inFlight := c.observed, c.limit, c.inFlight latency, cooldown := c.latency, max(0, time.Until(c.resumeAt).Seconds()) - queuedRequests, queuedBytes := c.readWaiters.Len(), c.queuedBytes + queuedTasks, queuedBytes, bufferedBytes := c.waiters.Len(), c.queuedBytes, c.bufferedBytes c.mu.Unlock() - ch <- prometheus.MustNewConstMetric(observed.queueRequests, prometheus.GaugeValue, float64(queuedRequests)) + ch <- prometheus.MustNewConstMetric(observed.bufferedBytes, prometheus.GaugeValue, float64(bufferedBytes)) + ch <- prometheus.MustNewConstMetric(observed.queueTasks, prometheus.GaugeValue, float64(queuedTasks)) ch <- prometheus.MustNewConstMetric(observed.queueBytes, prometheus.GaugeValue, float64(queuedBytes)) ch <- prometheus.MustNewConstMetric(observed.queueOutcomes, prometheus.CounterValue, float64(observed.queueAdmitted), "admitted") ch <- prometheus.MustNewConstMetric(observed.queueOutcomes, prometheus.CounterValue, float64(observed.queueCanceled), "canceled") diff --git a/internal/backpressure/pipeline_test.go b/internal/backpressure/pipeline_test.go new file mode 100644 index 0000000..1dea98e --- /dev/null +++ b/internal/backpressure/pipeline_test.go @@ -0,0 +1,201 @@ +package backpressure + +import ( + "context" + "sync" + "testing" + "testing/synctest" + "time" +) + +func TestBatchTicketCountsTasksNotOperationsAndTransfersBytes(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + c := readController(t, 1, 1, 100) + held, err := c.Acquire(t.Context()) + if err != nil { + t.Fatal(err) + } + defer held.Release() + reservations := make([]*Reservation, 0, 100) + for range 100 { + reservation, err := c.Reserve(1) + if err != nil { + t.Fatal(err) + } + reservations = append(reservations, reservation) + } + if c.bufferedBytes != 100 || c.queuedBytes != 0 { + t.Fatal("collection did not retain its shared byte budget") + } + ticket, _ := c.Enqueue(reservations) + if ticket == nil || c.waiters.Len() != 1 || c.bufferedBytes != 100 || c.queuedBytes != 100 { + t.Fatal("batch transfer changed bytes or counted its members as tasks") + } + if _, err := c.Reserve(1); err != ErrQueueFull { + t.Fatal("collection and admission did not share one byte budget") + } + reservations[0].Release() + reservations[0].Release() + if c.bufferedBytes != 99 || c.queuedBytes != 99 { + t.Fatal("canceled batch member did not release exactly its own bytes") + } + held.Release() + if permit, _, _ := c.TryAcquire(); permit != nil { + permit.Release() + t.Fatal("unqueued work overtook a ready batch") + } + permit, _, _ := ticket.Poll() + if permit == nil || c.waiters.Len() != 0 || c.queuedBytes != 0 || c.bufferedBytes != 0 { + t.Fatal("dispatch did not atomically release waiting reservations") + } + ticket.Cancel() + for _, reservation := range reservations { + reservation.Release() + } + permit.Release() + if c.inFlight != 0 || c.bufferedBytes != 0 { + t.Fatal("settlement released capacity twice") + } + }) +} + +func TestFullAdmissionQueueBackpressuresProducerWithoutRejection(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + c := readController(t, 1, 1, 32) + held, err := c.Acquire(t.Context()) + if err != nil { + t.Fatal(err) + } + defer held.Release() + first, err := c.Reserve(16) + if err != nil { + t.Fatal(err) + } + second, err := c.Reserve(16) + if err != nil { + t.Fatal(err) + } + firstReservations := []*Reservation{first} + secondReservations := []*Reservation{second} + ticket, _ := c.Enqueue(firstReservations) + for range 1000 { + next, _ := c.Enqueue(secondReservations) + if next != nil || c.observed.rejected != 0 || c.bufferedBytes != 32 { + t.Fatal("full queue rejected or released already accepted upstream work") + } + } + _, changed := c.Enqueue(secondReservations) + ticket.Cancel() + select { + case <-changed: + default: + t.Fatal("producer lost the queue-capacity wakeup") + } + next, _ := c.Enqueue(secondReservations) + if next == nil || c.queuedBytes != 16 || c.bufferedBytes != 16 { + t.Fatal("upstream reservation could not transfer after queue capacity freed") + } + next.Cancel() + if c.waiters.Len() != 0 || c.bufferedBytes != 0 || c.queuedBytes != 0 { + t.Fatal("canceled producer leaked waiting capacity") + } + }) +} + +func TestBatchAndDirectAdmissionShareFIFO(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + c := readController(t, 1, 4, 64) + held, err := c.Acquire(t.Context()) + if err != nil { + t.Fatal(err) + } + defer held.Release() + reservation, err := c.Reserve(16) + if err != nil { + t.Fatal(err) + } + reservations := []*Reservation{reservation} + batch, _ := c.Enqueue(reservations) + done := make(chan *Permit, 1) + go func() { + _, permit, err := c.Admit(t.Context(), 16) + if err != nil { + t.Error(err) + } + done <- permit + }() + synctest.Wait() + held.Release() + synctest.Wait() + select { + case permit := <-done: + permit.Release() + t.Fatal("direct request overtook queued batch") + default: + } + permit, _, _ := batch.Poll() + if permit == nil { + t.Fatal("FIFO head failed to obtain free capacity") + } + permit.Release() + next := <-done + next.Release() + if c.bufferedBytes != 0 || c.waiters.Len() != 0 || c.inFlight != 0 { + t.Fatal("mixed admission leaked state") + } + }) +} + +func TestReservationReleaseRacesWithTicketCancellation(t *testing.T) { + c := readController(t, 1, 100, 100) + for range 100 { + reservation, err := c.Reserve(1) + if err != nil { + t.Fatal(err) + } + reservations := []*Reservation{reservation} + ticket, _ := c.Enqueue(reservations) + var group sync.WaitGroup + group.Go(reservation.Release) + group.Go(ticket.Cancel) + group.Go(ticket.Cancel) + group.Wait() + } + if c.bufferedBytes != 0 || c.queuedBytes != 0 || c.waiters.Len() != 0 { + t.Fatal("concurrent cancellation leaked queue ownership") + } + ctx, cancel := context.WithCancel(t.Context()) + cancel() + if _, permit, err := c.Admit(ctx, 1); err == nil || permit != nil { + t.Fatal("canceled request received a permit") + } +} + +func TestUncontendedQueuedBatchesDoNotInventGrowthDemand(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + c := readController(t, 64, 10, 64) + if err := c.Wait(t.Context()); err != nil { + t.Fatal(err) + } + for range 32 { + reservation, err := c.Reserve(1) + if err != nil { + t.Fatal(err) + } + reservations := []*Reservation{reservation} + ticket, _ := c.Enqueue(reservations) + permit, _, _ := ticket.Poll() + if permit == nil { + t.Fatal("uncontended batch unexpectedly waited") + } + sample := c.begin(read, 1) + time.Sleep(20 * time.Millisecond) + c.observe(sample, 20*time.Millisecond, healthy) + permit.Release() + time.Sleep(time.Second) + } + if c.limit != initialConcurrent || c.observed.increases != 0 { + t.Fatal("merely passing through the queue inflated Store concurrency") + } + }) +} diff --git a/internal/backpressure/read_admission.go b/internal/backpressure/read_admission.go deleted file mode 100644 index 62b8abf..0000000 --- a/internal/backpressure/read_admission.go +++ /dev/null @@ -1,112 +0,0 @@ -package backpressure - -import ( - "container/list" - "context" - "time" - - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" -) - -type readWaiter struct { - bytes int - turn chan struct{} - enqueued time.Time -} - -// AdmitRead waits in a shared FIFO for Query and Count. Only the caller's context -// ends waiting; cooldown timers wake the head without imposing a new deadline. -// Waiting payloads remain bounded separately from executing Store work. -func (c *Controller) AdmitRead(ctx context.Context, requestBytes int) (context.Context, *Permit, error) { - if err := ctx.Err(); err != nil { - return ctx, nil, status.FromContextError(err).Err() - } - if requestBytes < 0 { - return ctx, nil, status.Error(codes.InvalidArgument, "negative read admission size") - } - if c == nil { - return ctx, nil, nil - } - if permit, ok := ctx.Value(permitKey{}).(*Permit); ok && permit.controller == c { - return ctx, nil, nil - } - c.mu.Lock() - if c.readWaiters.Len() == 0 { - permit, _, _ := c.tryAcquireLocked(time.Now()) - if permit != nil { - c.mu.Unlock() - if err := ctx.Err(); err != nil { - permit.Release() - return ctx, nil, status.FromContextError(err).Err() - } - return permit.Context(ctx), permit, nil - } - } - if c.readWaiters.Len() >= c.maxQueuedRequests || requestBytes > c.maxQueuedBytes-c.queuedBytes { - c.observed.rejected++ - c.observed.queueRejected++ - c.mu.Unlock() - return ctx, nil, ErrQueueFull - } - waiter := &readWaiter{bytes: requestBytes, turn: make(chan struct{}), enqueued: time.Now()} - entry := c.readWaiters.PushBack(waiter) - c.queuedBytes += requestBytes - c.demand = true - if entry == c.readWaiters.Front() { - close(waiter.turn) - } - c.mu.Unlock() - - select { - case <-ctx.Done(): - c.mu.Lock() - c.finishReadWaitLocked(entry, false) - c.mu.Unlock() - return ctx, nil, status.FromContextError(ctx.Err()).Err() - case <-waiter.turn: - } - for { - c.mu.Lock() - if err := ctx.Err(); err != nil { - c.finishReadWaitLocked(entry, false) - c.mu.Unlock() - return ctx, nil, status.FromContextError(err).Err() - } - permit, changed, delay := c.tryAcquireLocked(time.Now()) - if permit != nil { - c.finishReadWaitLocked(entry, true) - c.mu.Unlock() - if err := ctx.Err(); err != nil { - permit.Release() - return ctx, nil, status.FromContextError(err).Err() - } - return permit.Context(ctx), permit, nil - } - c.mu.Unlock() - if err := wait(ctx, changed, delay); err != nil { - c.mu.Lock() - c.finishReadWaitLocked(entry, false) - c.mu.Unlock() - return ctx, nil, status.FromContextError(err).Err() - } - } -} - -func (c *Controller) finishReadWaitLocked(entry *list.Element, admitted bool) { - waiter := entry.Value.(*readWaiter) - head := entry == c.readWaiters.Front() - c.readWaiters.Remove(entry) - c.queuedBytes -= waiter.bytes - outcome := "canceled" - if admitted { - c.observed.queueAdmitted++ - outcome = "admitted" - } else { - c.observed.queueCanceled++ - } - c.observed.queueDuration.WithLabelValues(outcome).Observe(time.Since(waiter.enqueued).Seconds()) - if next := c.readWaiters.Front(); head && next != nil { - close(next.Value.(*readWaiter).turn) - } -} diff --git a/internal/backpressure/storage_test.go b/internal/backpressure/storage_test.go index 91b5a53..b0cf3e3 100644 --- a/internal/backpressure/storage_test.go +++ b/internal/backpressure/storage_test.go @@ -270,7 +270,7 @@ func TestControllerMetricsAndValidation(t *testing.T) { t.Fatal(err) } families, err := registry.Gather() - if err != nil || len(families) != 14 { + if err != nil || len(families) != 15 { t.Fatalf("invalid metrics: %d families, %v", len(families), err) } series := 0 @@ -283,7 +283,7 @@ func TestControllerMetricsAndValidation(t *testing.T) { } } } - if series != 136 { + if series != 137 { t.Fatalf("Store collector cardinality changed: %d series", series) } } diff --git a/internal/config/backpressure_test.go b/internal/config/backpressure_test.go index 77e3dc8..592d31b 100644 --- a/internal/config/backpressure_test.go +++ b/internal/config/backpressure_test.go @@ -29,3 +29,28 @@ func TestStoreConcurrencyCeilingIndependentOfRoleAndMemory(t *testing.T) { } } } + +func TestAdmissionQueueIndependentOfBatchOperationLimits(t *testing.T) { + component := "mode: engine\nbatching: {max_operations: 32, queue: {max_operations: 4000}}\nexecution: {queue: {max_tasks: 7, max_bytes: 256MiB}}\n" + loaded, err := Decode(strings.NewReader(component), strings.NewReader(minimalStorage)) + if err != nil { + t.Fatal(err) + } + if loaded.Service.Admission.MaxTasks != 7 || loaded.Service.Admission.MaxBytes != 256<<20 || loaded.Service.Batching.Queue.MaxOperations != 4000 { + t.Fatalf("queue task and operation limits were conflated: %+v", loaded.Service) + } + defaults, err := Decode(strings.NewReader("mode: engine"), strings.NewReader(minimalStorage)) + if err != nil || defaults.Service.Admission.MaxTasks != 10000 || defaults.Service.Admission.MaxBytes != 128<<20 { + t.Fatalf("admission defaults: %+v %v", defaults.Service.Admission, err) + } + for _, queue := range []string{"max_tasks: 0", "max_tasks: -1", "max_bytes: 0", "max_bytes: 1MiB", "max_operations: 10"} { + input := "mode: engine\nexecution: {queue: {" + queue + "}}\n" + if _, err := Decode(strings.NewReader(input), strings.NewReader(minimalStorage)); err == nil { + t.Fatalf("invalid admission queue accepted: %s", queue) + } + } + worker := "mode: worker\nconsumer: {group_id: workers}\nexecution: {queue: {max_tasks: 10}}\n" + if _, err := Decode(strings.NewReader(worker), strings.NewReader(kafkaStore)); err == nil { + t.Fatal("Engine admission queue accepted by Worker") + } +} diff --git a/internal/config/config.go b/internal/config/config.go index 2a21eba..4405b63 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -60,9 +60,15 @@ type Service struct { StoreMaxConcurrent int Request Request Batching Batching + Admission AdmissionQueue Merge Merge } +type AdmissionQueue struct { + MaxTasks int + MaxBytes int +} + type Request struct { MaxOperations int MaxReadBytes int diff --git a/internal/config/file.go b/internal/config/file.go index 0f3ed76..741e151 100644 --- a/internal/config/file.go +++ b/internal/config/file.go @@ -41,7 +41,13 @@ type requestFile struct { } type executionFile struct { - Merge mergeFile `yaml:"merge"` + Merge mergeFile `yaml:"merge"` + Queue *admissionQueueFile `yaml:"queue"` +} + +type admissionQueueFile struct { + MaxTasks *int `yaml:"max_tasks"` + MaxBytes *byteSize `yaml:"max_bytes"` } // Store files contain identity and backend policy shared by Engine and Worker. diff --git a/internal/config/service.go b/internal/config/service.go index bdca9e0..d68ef8f 100644 --- a/internal/config/service.go +++ b/internal/config/service.go @@ -34,6 +34,18 @@ func resolveService(file configFile, grpc GRPC, v *validator) Service { batching = &batchingFile{} } loaded.Batching = resolveBatching(*batching, grpc, v) + queue := execution.Queue + if queue == nil { + queue = &admissionQueueFile{} + } + loaded.Admission.MaxTasks = v.integer("execution.queue.max_tasks", queue.MaxTasks, 10_000) + minimumBytes := max(grpc.MaxReceiveMessageBytes, loaded.Batching.MaxBytes) + loaded.Admission.MaxBytes = v.bytes("execution.queue.max_bytes", queue.MaxBytes, max(128<<20, minimumBytes), math.MaxInt) + if loaded.Admission.MaxBytes < minimumBytes { + v.reject(errors.New("execution.queue.max_bytes must cover one gRPC request and one batch")) + } + } else if execution.Queue != nil { + v.reject(errors.New("execution.queue is only valid for engine mode")) } return loaded } diff --git a/internal/engine/backpressure_test.go b/internal/engine/backpressure_test.go index e3623e6..bf9b735 100644 --- a/internal/engine/backpressure_test.go +++ b/internal/engine/backpressure_test.go @@ -22,7 +22,7 @@ func (s *admissionFailureService) Execute(context.Context, *sink.ExecuteRequest) } func TestEngineDistinguishesAdmissionFromUnknownNativeOutcome(t *testing.T) { - for _, failure := range []error{backpressure.ErrBusy, backpressure.ErrQueueFull, status.Error(codes.ResourceExhausted, "backend outcome unknown")} { + for _, failure := range []error{backpressure.ErrQueueFull, status.Error(codes.ResourceExhausted, "backend outcome unknown")} { service := &admissionFailureService{err: failure} opts := Options{Service: service, Store: "primary", MaxReadBytes: 1 << 20} server, err := New(opts) @@ -36,7 +36,7 @@ func TestEngineDistinguishesAdmissionFromUnknownNativeOutcome(t *testing.T) { stream := &forwardRecorder{ctx: t.Context()} err = server.Forward(request, stream) marker := stream.trailer.Get(forwarding.NotStartedTrailer) - notStarted := failure == backpressure.ErrBusy || failure == backpressure.ErrQueueFull + notStarted := failure == backpressure.ErrQueueFull if err != failure || (len(marker) == 1) != notStarted || stream.response != nil { t.Fatalf("unknown outcome or admission evidence changed: error=%v marker=%v", err, marker) } diff --git a/internal/engine/server.go b/internal/engine/server.go index 35d5a96..014d56c 100644 --- a/internal/engine/server.go +++ b/internal/engine/server.go @@ -58,7 +58,7 @@ func (s *Server) Forward(req *forward.ForwardRequest, stream grpc.ServerStreamin defer func() { // Direct Native admission can reject after validation, but still before // execution. Do not turn that known rejection into an unknown mutation. - if err == backpressure.ErrBusy || err == backpressure.ErrQueueFull { + if err == backpressure.ErrQueueFull { stream.SetTrailer(metadata.Pairs(forwarding.NotStartedTrailer, "true")) } }() diff --git a/internal/service/backpressure_test.go b/internal/service/backpressure_test.go index 45641b6..961032e 100644 --- a/internal/service/backpressure_test.go +++ b/internal/service/backpressure_test.go @@ -124,7 +124,7 @@ func TestFinishedBatchRetainsPermitUntilDispatcherReceivesCompletion(t *testing. defer cancel() pending := &batchCall[int, int]{ctx: ctx, request: 2, operationCount: 1, encodedBytes: 1, enqueuedAt: time.Now(), result: make(chan batchResult[int], 1)} pending.stopWake = func() bool { close(blocked); <-resume; return true } - if err := batcher.reserve(1, 1); err != nil { + if err := batcher.reserve(pending); err != nil { t.Fatal(err) } cancel() diff --git a/internal/service/batcher.go b/internal/service/batcher.go index 0b232f1..d51e019 100644 --- a/internal/service/batcher.go +++ b/internal/service/batcher.go @@ -27,6 +27,7 @@ type batchCall[Request any, Response any] struct { records []recordIdentity stopWake func() bool partition batchPartition + reservation *backpressure.Reservation resultOnce sync.Once recordMu sync.Mutex pendingRecords map[recordIdentity]bool @@ -118,15 +119,6 @@ func (b *requestBatcher[Request, Response]) Submit( if err := contextError(ctx); err != nil { return empty, err } - if err := b.reserve(operationCount, encodedBytes); err != nil { - reason := "queue_full" - if status.Code(err) == codes.Unavailable { - reason = "shutdown" - } - b.metrics.ObserveBatchRejected(b.store, b.method, reason) - return empty, err - } - call := &batchCall[Request, Response]{ ctx: ctx, request: request, @@ -135,6 +127,15 @@ func (b *requestBatcher[Request, Response]) Submit( enqueuedAt: time.Now(), result: make(chan batchResult[Response], 1), } + if err := b.reserve(call); err != nil { + reason := "queue_full" + if status.Code(err) == codes.Unavailable { + reason = "shutdown" + } + b.metrics.ObserveBatchRejected(b.store, b.method, reason) + return empty, err + } + if b.records != nil { call.records = b.records(request) } @@ -186,7 +187,8 @@ func (b *requestBatcher[Request, Response]) wait() { b.waitGroup.Wait() } -func (b *requestBatcher[Request, Response]) reserve(operationCount int, encodedBytes int) error { +func (b *requestBatcher[Request, Response]) reserve(call *batchCall[Request, Response]) error { + operationCount, encodedBytes := call.operationCount, call.encodedBytes b.queueMu.Lock() defer b.queueMu.Unlock() if b.ctx.Err() != nil { @@ -197,6 +199,11 @@ func (b *requestBatcher[Request, Response]) reserve(operationCount int, encodedB encodedBytes > b.maxQueuedBytes || b.queuedBytes > b.maxQueuedBytes-encodedBytes { return status.Error(codes.ResourceExhausted, "synchronous batch queue is full") } + reservation, err := b.admission.Reserve(encodedBytes) + if err != nil { + return err + } + call.reservation = reservation b.queuedCalls++ b.queuedOperations += operationCount b.queuedBytes += encodedBytes @@ -205,6 +212,7 @@ func (b *requestBatcher[Request, Response]) reserve(operationCount int, encodedB } func (b *requestBatcher[Request, Response]) release(call *batchCall[Request, Response]) { + call.reservation.Release() if call.stopWake != nil { call.stopWake() } @@ -227,13 +235,19 @@ func (b *requestBatcher[Request, Response]) release(call *batchCall[Request, Res func (b *requestBatcher[Request, Response]) run() { defer b.waitGroup.Done() pending := make([]*batchCall[Request, Response], 0) - // Each active record stays occupied until every selected caller releases it. + var selected []*batchCall[Request, Response] + var ticket *backpressure.Ticket + var reason string + // A producer retains at most one ready batch in admission. Its event loop + // keeps settling cancellations and completions while that batch waits. active := make(map[recordIdentity]int) recordsDone := make(chan []recordIdentity, max(1, min(1024, b.maxConcurrent))) completed := make(chan *backpressure.Permit, max(1, min(1024, b.maxConcurrent))) running := 0 for { if b.ctx.Err() != nil { + ticket.Cancel() + b.failCalls(selected) b.failCalls(pending) b.drain() for running > 0 { @@ -243,8 +257,6 @@ func (b *requestBatcher[Request, Response]) run() { } return } - // Drain the bounded input queue before selecting a batch, including aged - // backlog accumulated while all execution slots were occupied. drainInput: for { select { @@ -254,74 +266,72 @@ func (b *requestBatcher[Request, Response]) run() { break drainInput } } - live := pending[:0] - for _, call := range pending { - if err := contextError(call.ctx); err != nil { - b.release(call) - completeCall(call, emptyResponse[Response](), err) - } else { - live = append(live, call) - } + pending = b.liveCalls(pending) + selected = b.liveCalls(selected) + if len(selected) == 0 { + ticket.Cancel() + ticket = nil } - clear(pending[len(live):]) - pending = live var timer *time.Timer var deadline <-chan time.Time var admissionChanged <-chan struct{} - ready := true - if len(pending) > 0 { - var retryAfter time.Duration - ready, admissionChanged, retryAfter = b.admission.Ready() - if ready { - admissionChanged = nil - } else if retryAfter > 0 { - timer = time.NewTimer(retryAfter) - deadline = timer.C + if len(selected) == 0 && (b.maxConcurrent == 0 || running < b.maxConcurrent) { + candidate, remaining, selectedReason := b.selectReady(pending, active) + if len(candidate) > 0 { + wait := b.maxWait - time.Since(candidate[0].enqueuedAt) + if selectedReason != "max_wait" || wait <= 0 { + selected, pending, reason = candidate, remaining, selectedReason + } else { + timer = time.NewTimer(wait) + deadline = timer.C + } } } - if ready && (b.maxConcurrent == 0 || running < b.maxConcurrent) { - selected, remaining, reason := b.selectReady(pending, active) - if len(selected) > 0 { - wait := b.maxWait - time.Since(selected[0].enqueuedAt) - if reason != "max_wait" || wait <= 0 { - permit, changed, retryAfter := b.admission.TryAcquire() - if permit == nil { - admissionChanged = changed - if retryAfter > 0 { - timer = time.NewTimer(retryAfter) - deadline = timer.C - } - } else { - pending = remaining - for _, call := range selected { - b.release(call) - call.pendingRecords = make(map[recordIdentity]bool, len(call.records)) - for _, key := range call.records { - if call.pendingRecords[key] { - continue - } - call.pendingRecords[key] = true - active[key]++ + if len(selected) > 0 { + if ticket == nil { + reservations := make([]*backpressure.Reservation, 0, len(selected)) + for _, call := range selected { + reservations = append(reservations, call.reservation) + } + ticket, admissionChanged = b.admission.Enqueue(reservations) + } + if ticket != nil { + permit, changed, retryAfter := ticket.Poll() + if permit == nil { + admissionChanged = changed + if retryAfter > 0 { + timer = time.NewTimer(retryAfter) + deadline = timer.C + } + } else { + ticket = nil + calls := selected + selected = nil + for _, call := range calls { + b.release(call) + call.pendingRecords = make(map[recordIdentity]bool, len(call.records)) + for _, key := range call.records { + if call.pendingRecords[key] { + continue } - call.onRecordsDone = func(keys []recordIdentity) { - select { - case recordsDone <- keys: - case <-b.ctx.Done(): - } + call.pendingRecords[key] = true + active[key]++ + } + call.onRecordsDone = func(keys []recordIdentity) { + select { + case recordsDone <- keys: + case <-b.ctx.Done(): } } - running++ - go func() { - b.executeBatch(selected, reason, permit) - // Keep the slot until the dispatcher receives completion, - // bounding goroutines waiting to report finished work too. - completed <- permit - }() - continue } - } else { - timer = time.NewTimer(wait) - deadline = timer.C + running++ + flushReason := reason + go func() { + b.executeBatch(calls, flushReason, permit) + // Completion delivery stays inside the execution bound. + completed <- permit + }() + continue } } } @@ -349,6 +359,20 @@ func (b *requestBatcher[Request, Response]) run() { } } +func (b *requestBatcher[Request, Response]) liveCalls(calls []*batchCall[Request, Response]) []*batchCall[Request, Response] { + live := calls[:0] + for _, call := range calls { + if err := contextError(call.ctx); err != nil { + b.release(call) + completeCall(call, emptyResponse[Response](), err) + } else { + live = append(live, call) + } + } + clear(calls[len(live):]) + return live +} + // A blocked earlier RPC reserves all its keys in the dependency graph. Later // independent RPCs can pass it, while multi-record chains retain queue order. func (b *requestBatcher[Request, Response]) selectReady(pending []*batchCall[Request, Response], active map[recordIdentity]int) ([]*batchCall[Request, Response], []*batchCall[Request, Response], string) { diff --git a/internal/service/batching_regressions_test.go b/internal/service/batching_regressions_test.go index 7d3d6d8..6363030 100644 --- a/internal/service/batching_regressions_test.go +++ b/internal/service/batching_regressions_test.go @@ -356,7 +356,7 @@ func queueCompletionCall[Request interface{ SizeVT() int }, Response any](t *tes call.records = batcher.records(call.request) call.partition = batcher.partition(call.request) call.enqueuedAt = time.Now() - if err := batcher.reserve(call.operationCount, call.encodedBytes); err != nil { + if err := batcher.reserve(call); err != nil { t.Fatal(err) } // Submit reserves queue capacity before sending to input. Observing that diff --git a/internal/service/native.go b/internal/service/native.go index 5df55fc..7d4064b 100644 --- a/internal/service/native.go +++ b/internal/service/native.go @@ -103,7 +103,7 @@ func (s *Server) Execute(ctx context.Context, req *sink.ExecuteRequest) (*sink.E if !ok { return nil, nativeStatus(storage.ErrNativeUnsupported) } - ctx, permit, err := s.admission.Admit(ctx) + ctx, permit, err := s.admission.Admit(ctx, req.SizeVT()) if err != nil { return nil, err } @@ -189,7 +189,7 @@ func (s *Server) scan(ctx context.Context, req *sink.ScanRequest, send func(*sin } return send(frame) } - ctx, permit, err := s.admission.Admit(ctx) + ctx, permit, err := s.admission.Admit(ctx, req.SizeVT()) if err != nil { return err } diff --git a/internal/service/query.go b/internal/service/query.go index e19a797..1e5c276 100644 --- a/internal/service/query.go +++ b/internal/service/query.go @@ -57,7 +57,7 @@ func (s *Server) query(ctx context.Context, req *sink.QueryRequest, send func(*s } return send(frame) } - ctx, permit, err := s.admission.AdmitRead(ctx, req.SizeVT()) + ctx, permit, err := s.admission.Admit(ctx, req.SizeVT()) if err != nil { return err } @@ -88,7 +88,7 @@ func (s *Server) Count(ctx context.Context, req *sink.CountRequest) (*sink.Count if !ok { return nil, nativeStatus(storage.ErrNativeUnsupported) } - ctx, permit, err := s.admission.AdmitRead(ctx, req.SizeVT()) + ctx, permit, err := s.admission.Admit(ctx, req.SizeVT()) if err != nil { return nil, err } diff --git a/internal/service/query_admission_test.go b/internal/service/query_admission_test.go index 2cce587..70d1666 100644 --- a/internal/service/query_admission_test.go +++ b/internal/service/query_admission_test.go @@ -10,8 +10,8 @@ import ( "github.com/batchstream/sink/internal/storage/memory" ) -func TestQueryAndCountWaitForStoreCapacityWithoutNewDeadline(t *testing.T) { - for _, method := range []string{"Query", "Count"} { +func TestAllNativeMethodsWaitForStoreCapacityWithoutNewDeadline(t *testing.T) { + for _, method := range []string{"Query", "Count", "Execute", "Scan"} { t.Run(method, func(t *testing.T) { synctest.Test(t, func(t *testing.T) { backend := &nativeBoundaryStore{Store: memory.New()} @@ -27,14 +27,23 @@ func TestQueryAndCountWaitForStoreCapacityWithoutNewDeadline(t *testing.T) { command := &sink.Command{Uri: "sink://primary", Method: "GET", Path: "/_search"} result := make(chan error, 1) go func() { - if method == "Query" { + switch method { + case "Query": request := &sink.QueryRequest{Command: command} _, err := batching.server.Query(ctx, request) result <- err - } else { + case "Count": request := &sink.CountRequest{Command: command} _, err := batching.Count(ctx, request) result <- err + case "Execute": + request := &sink.ExecuteRequest{Command: command} + _, err := batching.Execute(ctx, request) + result <- err + case "Scan": + request := &sink.ScanRequest{Command: command} + _, err := batching.server.Scan(ctx, request) + result <- err } }() synctest.Wait() diff --git a/internal/service/read.go b/internal/service/read.go index 73fe59b..851e985 100644 --- a/internal/service/read.go +++ b/internal/service/read.go @@ -61,7 +61,7 @@ func (s *Server) read(ctx context.Context, req *sink.ReadRequest, budgets *respo if len(storageOperations) == 0 { return response, nil } - ctx, permit, err := s.admission.Admit(ctx) + ctx, permit, err := s.admission.Admit(ctx, req.SizeVT()) if err != nil { return nil, err } diff --git a/internal/service/server.go b/internal/service/server.go index 1999391..eb56290 100644 --- a/internal/service/server.go +++ b/internal/service/server.go @@ -100,7 +100,7 @@ func (s *Server) write(ctx context.Context, req *sink.WriteRequest, budgets *res return response, nil } - ctx, permit, err := s.admission.Admit(ctx) + ctx, permit, err := s.admission.Admit(ctx, req.SizeVT()) if err != nil { return nil, err } @@ -208,7 +208,7 @@ func (s *Server) delete(ctx context.Context, req *sink.DeleteRequest, budgets *r return response, nil } - ctx, permit, err := s.admission.Admit(ctx) + ctx, permit, err := s.admission.Admit(ctx, req.SizeVT()) if err != nil { return nil, err } diff --git a/internal/service/unified_admission_test.go b/internal/service/unified_admission_test.go new file mode 100644 index 0000000..8820de0 --- /dev/null +++ b/internal/service/unified_admission_test.go @@ -0,0 +1,234 @@ +package service + +import ( + "context" + "slices" + "testing" + "testing/synctest" + "time" + + sink "github.com/batchstream/sink-protocol/sink/v1" + "github.com/batchstream/sink/internal/backpressure" + "github.com/batchstream/sink/internal/storage" + "github.com/batchstream/sink/internal/storage/memory" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type orderedAdmissionStore struct { + *nativeBoundaryStore + events chan string +} + +func (s *orderedAdmissionStore) Read(ctx context.Context, req storage.ReadRequest) (storage.ReadResponse, error) { + s.events <- "Read" + return s.Store.Read(ctx, req) +} + +func (s *orderedAdmissionStore) Write(ctx context.Context, req storage.WriteRequest) (storage.WriteResponse, error) { + s.events <- "Write" + return s.Store.Write(ctx, req) +} + +func (s *orderedAdmissionStore) Delete(ctx context.Context, req storage.DeleteRequest) (storage.DeleteResponse, error) { + s.events <- "Delete" + return s.Store.Delete(ctx, req) +} + +func (s *orderedAdmissionStore) Execute(ctx context.Context, req storage.NativeRequest) (storage.NativeResponse, error) { + s.events <- "Execute" + return s.nativeBoundaryStore.Execute(ctx, req) +} + +func (s *orderedAdmissionStore) Query(ctx context.Context, req storage.QueryRequest) (storage.QueryResponse, error) { + s.events <- "Query" + return s.nativeBoundaryStore.Query(ctx, req) +} + +func (s *orderedAdmissionStore) Count(ctx context.Context, req storage.CountRequest) (storage.CountResponse, error) { + s.events <- "Count" + return s.nativeBoundaryStore.Count(ctx, req) +} + +func (s *orderedAdmissionStore) Scan(ctx context.Context, req storage.ScanRequest) (storage.ScanResponse, error) { + s.events <- "Scan" + return s.nativeBoundaryStore.Scan(ctx, req) +} + +func TestBatchAndNativeMethodsShareOneDispatchFIFO(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + native := &nativeBoundaryStore{Store: memory.New()} + backend := &orderedAdmissionStore{nativeBoundaryStore: native, events: make(chan string, 16)} + batching, controller := admissionServer(t, backend, 1) + defer batching.Close() + held, err := controller.Acquire(t.Context()) + if err != nil { + t.Fatal(err) + } + defer held.Release() + command := &sink.Command{Uri: "sink://primary", Method: "GET", Path: "/_search"} + methods := []string{"Write", "Count", "Read", "Execute", "Delete", "Query", "Scan"} + results := make(chan error, len(methods)) + for _, method := range methods { + go func() { + var err error + switch method { + case "Write": + request := &sink.WriteRequest{Operations: []*sink.WriteOperation{completionPut("record", 1)}, CompletionMode: sink.CompletionMode_COMPLETION_MODE_WAIT_UNTIL_APPLIED} + _, err = batching.Write(t.Context(), request) + case "Read": + operation := &sink.ReadOperation{Address: completionAddress("record")} + request := &sink.ReadRequest{Operations: []*sink.ReadOperation{operation}} + _, err = batching.Read(t.Context(), request) + case "Delete": + operation := &sink.DeleteOperation{Address: completionAddress("record")} + request := &sink.DeleteRequest{Operations: []*sink.DeleteOperation{operation}, CompletionMode: sink.CompletionMode_COMPLETION_MODE_WAIT_UNTIL_APPLIED} + _, err = batching.Delete(t.Context(), request) + case "Count": + request := &sink.CountRequest{Command: command} + _, err = batching.Count(t.Context(), request) + case "Execute": + request := &sink.ExecuteRequest{Command: command} + _, err = batching.Execute(t.Context(), request) + case "Query": + request := &sink.QueryRequest{Command: command} + _, err = batching.server.Query(t.Context(), request) + case "Scan": + request := &sink.ScanRequest{Command: command} + _, err = batching.server.Scan(t.Context(), request) + } + results <- err + }() + synctest.Wait() + time.Sleep(2 * time.Millisecond) + synctest.Wait() + } + if len(backend.events) != 0 { + t.Fatal("a method bypassed the shared admission window") + } + held.Release() + for range methods { + if err := <-results; err != nil { + t.Fatal(err) + } + } + synctest.Wait() + var got []string + for range methods { + got = append(got, <-backend.events) + } + if !slices.Equal(got, methods) { + t.Fatalf("Store dispatch was not FIFO: got %v, want %v", got, methods) + } + permit, _, _ := controller.TryAcquire() + if permit == nil { + t.Fatal("mixed dispatch leaked a permit") + } + permit.Release() + }) +} + +func TestBatchProducerWaitsForAdmissionSpaceWithoutBlockingCancellation(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + opts := backpressure.Options{MaxConcurrent: 1, MaxQueuedTasks: 1, MaxQueuedBytes: 64} + controller, err := backpressure.New(opts) + if err != nil { + t.Fatal(err) + } + held, err := controller.Acquire(t.Context()) + if err != nil { + t.Fatal(err) + } + defer held.Release() + nativeContext, cancelNative := context.WithCancel(t.Context()) + defer cancelNative() + nativeDone := make(chan error, 1) + go func() { + _, permit, err := controller.Admit(nativeContext, 16) + permit.Release() + nativeDone <- err + }() + synctest.Wait() + executed := make(chan int, 4) + execute := func(_ context.Context, calls []*batchCall[int, int]) { + for _, call := range calls { + executed <- call.request + completeCall(call, call.request, nil) + } + } + batchOptions := requestBatcherOptions[int, int]{Admission: controller, Unlimited: true, MaxWait: time.Millisecond, MaxOperations: 1, MaxBytes: 16, MaxQueuedOperations: 4, MaxQueuedBytes: 64, Execute: execute} + batcher := newRequestBatcher(batchOptions) + defer batcher.Close() + canceledContext, cancel := context.WithCancel(t.Context()) + defer cancel() + canceled := make(chan error, 1) + go func() { _, err := batcher.Submit(canceledContext, 1, 1, 16); canceled <- err }() + synctest.Wait() + live := make(chan error, 1) + go func() { _, err := batcher.Submit(t.Context(), 2, 1, 16); live <- err }() + synctest.Wait() + cancel() + if err := <-canceled; status.Code(err) != codes.Canceled { + t.Fatalf("full downstream queue blocked cancellation: %v", err) + } + synctest.Wait() + if batcher.queuedCallCount() != 1 || len(executed) != 0 { + t.Fatal("cancellation lost live upstream work or executed canceled work") + } + cancelNative() + if err := <-nativeDone; status.Code(err) != codes.Canceled { + t.Fatal(err) + } + held.Release() + if err := <-live; err != nil { + t.Fatalf("accepted batch was rejected by downstream queue: %v", err) + } + if got := <-executed; got != 2 || len(executed) != 0 { + t.Fatalf("canceled batch reached execution: %d", got) + } + }) +} + +func TestCanceledMemberOfQueuedBatchDoesNotCancelOtherCallers(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + controller := admissionController(t, 1) + held, err := controller.Acquire(t.Context()) + if err != nil { + t.Fatal(err) + } + defer held.Release() + executed := make(chan int, 2) + execute := func(_ context.Context, calls []*batchCall[int, int]) { + for _, call := range calls { + executed <- call.request + completeCall(call, call.request, nil) + } + } + opts := requestBatcherOptions[int, int]{Admission: controller, Unlimited: true, MaxWait: time.Second, MaxOperations: 2, MaxBytes: 32, MaxQueuedOperations: 4, MaxQueuedBytes: 64, Execute: execute} + batcher := newRequestBatcher(opts) + defer batcher.Close() + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + first := make(chan error, 1) + second := make(chan error, 1) + go func() { _, err := batcher.Submit(ctx, 1, 1, 16); first <- err }() + synctest.Wait() + go func() { _, err := batcher.Submit(t.Context(), 2, 1, 16); second <- err }() + synctest.Wait() + cancel() + if err := <-first; status.Code(err) != codes.Canceled { + t.Fatal(err) + } + synctest.Wait() + if batcher.queuedBytes != 16 || batcher.queuedCallCount() != 1 { + t.Fatal("queued batch member cancellation lost or duplicated queue charges") + } + held.Release() + if err := <-second; err != nil { + t.Fatal(err) + } + if got := <-executed; got != 2 || len(executed) != 0 { + t.Fatalf("batch cancellation reached the wrong calls: %d", got) + } + }) +} From 1c7aed1c1afa4589be8b6fa3c4345417c31d041c Mon Sep 17 00:00:00 2001 From: Li Ran Date: Fri, 25 Sep 2026 09:11:33 +0800 Subject: [PATCH 2/4] =?UTF-8?q?=E2=9C=85=20test:=20retrigger=20Sink=20reli?= =?UTF-8?q?ability=20gate?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From 1ee1c53878724113a46651703f91626d6bd8ef44 Mon Sep 17 00:00:00 2001 From: Li Ran Date: Fri, 25 Sep 2026 09:52:28 +0800 Subject: [PATCH 3/4] =?UTF-8?q?=F0=9F=94=A7=20ci:=20pin=20qualification=20?= =?UTF-8?q?workflows=20to=20current=20suite?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 4 ++-- .github/workflows/release-image.yml | 4 ++-- .github/workflows/reliability.yml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 24f9930..67583e1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -147,8 +147,8 @@ jobs: qualification: name: External test suite - uses: batchstream/sink-production-suite/.github/workflows/server-qualification.yml@c95932c11d922345deec8e03895129c33ee885b6 + uses: batchstream/sink-production-suite/.github/workflows/server-qualification.yml@5ec11a158a96829521b8640f460d864e6000050f with: - suite_ref: c95932c11d922345deec8e03895129c33ee885b6 + suite_ref: 5ec11a158a96829521b8640f460d864e6000050f sink_ref: ${{ github.event.pull_request.head.sha || github.sha }} sdk_ref: ${{ github.head_ref || 'main' }} diff --git a/.github/workflows/release-image.yml b/.github/workflows/release-image.yml index 7466bb1..6100261 100644 --- a/.github/workflows/release-image.yml +++ b/.github/workflows/release-image.yml @@ -61,9 +61,9 @@ jobs: qualify: name: Public production qualification needs: release - uses: batchstream/sink-production-suite/.github/workflows/release-qualification.yml@c95932c11d922345deec8e03895129c33ee885b6 + uses: batchstream/sink-production-suite/.github/workflows/release-qualification.yml@5ec11a158a96829521b8640f460d864e6000050f with: - suite_ref: c95932c11d922345deec8e03895129c33ee885b6 + suite_ref: 5ec11a158a96829521b8640f460d864e6000050f sink_ref: ${{ needs.release.outputs.revision }} binaries: diff --git a/.github/workflows/reliability.yml b/.github/workflows/reliability.yml index d3f93b1..31647ea 100644 --- a/.github/workflows/reliability.yml +++ b/.github/workflows/reliability.yml @@ -8,7 +8,7 @@ permissions: jobs: soak: - uses: batchstream/sink-production-suite/.github/workflows/reliability.yml@c95932c11d922345deec8e03895129c33ee885b6 + uses: batchstream/sink-production-suite/.github/workflows/reliability.yml@5ec11a158a96829521b8640f460d864e6000050f with: - suite_ref: c95932c11d922345deec8e03895129c33ee885b6 + suite_ref: 5ec11a158a96829521b8640f460d864e6000050f sink_ref: ${{ github.sha }} From 1c2f1f0153b00398284c0e31646badf2c3381255 Mon Sep 17 00:00:00 2001 From: Li Ran Date: Fri, 25 Sep 2026 10:03:28 +0800 Subject: [PATCH 4/4] =?UTF-8?q?=F0=9F=94=A7=20ci:=20pin=20qualification=20?= =?UTF-8?q?to=20unified=20admission=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 4 ++-- .github/workflows/release-image.yml | 4 ++-- .github/workflows/reliability.yml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 67583e1..fba11e8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -147,8 +147,8 @@ jobs: qualification: name: External test suite - uses: batchstream/sink-production-suite/.github/workflows/server-qualification.yml@5ec11a158a96829521b8640f460d864e6000050f + uses: batchstream/sink-production-suite/.github/workflows/server-qualification.yml@bb4db31e534339a4d0da7134fd703ccdb84f4707 with: - suite_ref: 5ec11a158a96829521b8640f460d864e6000050f + suite_ref: bb4db31e534339a4d0da7134fd703ccdb84f4707 sink_ref: ${{ github.event.pull_request.head.sha || github.sha }} sdk_ref: ${{ github.head_ref || 'main' }} diff --git a/.github/workflows/release-image.yml b/.github/workflows/release-image.yml index 6100261..f173639 100644 --- a/.github/workflows/release-image.yml +++ b/.github/workflows/release-image.yml @@ -61,9 +61,9 @@ jobs: qualify: name: Public production qualification needs: release - uses: batchstream/sink-production-suite/.github/workflows/release-qualification.yml@5ec11a158a96829521b8640f460d864e6000050f + uses: batchstream/sink-production-suite/.github/workflows/release-qualification.yml@bb4db31e534339a4d0da7134fd703ccdb84f4707 with: - suite_ref: 5ec11a158a96829521b8640f460d864e6000050f + suite_ref: bb4db31e534339a4d0da7134fd703ccdb84f4707 sink_ref: ${{ needs.release.outputs.revision }} binaries: diff --git a/.github/workflows/reliability.yml b/.github/workflows/reliability.yml index 31647ea..88d1c07 100644 --- a/.github/workflows/reliability.yml +++ b/.github/workflows/reliability.yml @@ -8,7 +8,7 @@ permissions: jobs: soak: - uses: batchstream/sink-production-suite/.github/workflows/reliability.yml@5ec11a158a96829521b8640f460d864e6000050f + uses: batchstream/sink-production-suite/.github/workflows/reliability.yml@bb4db31e534339a4d0da7134fd703ccdb84f4707 with: - suite_ref: 5ec11a158a96829521b8640f460d864e6000050f + suite_ref: bb4db31e534339a4d0da7134fd703ccdb84f4707 sink_ref: ${{ github.sha }}