diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8971625..24f9930 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -147,8 +147,8 @@ jobs: qualification: name: External test suite - uses: batchstream/sink-production-suite/.github/workflows/server-qualification.yml@ca8862a8531925da8eee74c8b13f77440ef5d827 + uses: batchstream/sink-production-suite/.github/workflows/server-qualification.yml@c95932c11d922345deec8e03895129c33ee885b6 with: - suite_ref: ca8862a8531925da8eee74c8b13f77440ef5d827 + suite_ref: c95932c11d922345deec8e03895129c33ee885b6 sink_ref: ${{ github.event.pull_request.head.sha || github.sha }} sdk_ref: ${{ github.head_ref || 'main' }} diff --git a/.github/workflows/release-image.yml b/.github/workflows/release-image.yml index 3b9d383..7466bb1 100644 --- a/.github/workflows/release-image.yml +++ b/.github/workflows/release-image.yml @@ -61,9 +61,9 @@ jobs: qualify: name: Public production qualification needs: release - uses: batchstream/sink-production-suite/.github/workflows/release-qualification.yml@ca8862a8531925da8eee74c8b13f77440ef5d827 + uses: batchstream/sink-production-suite/.github/workflows/release-qualification.yml@c95932c11d922345deec8e03895129c33ee885b6 with: - suite_ref: ca8862a8531925da8eee74c8b13f77440ef5d827 + suite_ref: c95932c11d922345deec8e03895129c33ee885b6 sink_ref: ${{ needs.release.outputs.revision }} binaries: diff --git a/.github/workflows/reliability.yml b/.github/workflows/reliability.yml index 4a2640d..d3f93b1 100644 --- a/.github/workflows/reliability.yml +++ b/.github/workflows/reliability.yml @@ -8,7 +8,7 @@ permissions: jobs: soak: - uses: batchstream/sink-production-suite/.github/workflows/reliability.yml@ca8862a8531925da8eee74c8b13f77440ef5d827 + uses: batchstream/sink-production-suite/.github/workflows/reliability.yml@c95932c11d922345deec8e03895129c33ee885b6 with: - suite_ref: ca8862a8531925da8eee74c8b13f77440ef5d827 + suite_ref: c95932c11d922345deec8e03895129c33ee885b6 sink_ref: ${{ github.sha }} diff --git a/docs/configuration.md b/docs/configuration.md index 9b466fe..da967e7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -154,8 +154,8 @@ use the lowercase spelling shown below. Storage names are also case-sensitive. | `batching.max_wait` | duration string | No | `2ms` | Positive Go duration within the bounds below | Maximum collection delay measured from the first request in a batch. | | `batching.max_operations` | positive integer | No | `32` | Positive integer | Operation target for one automatically formed batch; a larger valid RPC still executes alone. See [choosing limits](batching.md#choosing-limits). | | `batching.max_bytes` | byte size | No | `16MiB` | Size greater than `0B` | Encoded-byte target for one automatically formed batch; one larger valid RPC still runs alone. | -| `batching.queue.max_operations` | positive integer | No | max(`10000`, `batching.max_operations`) | Integer at least `batching.max_operations` | Maximum operations waiting in each method queue. | -| `batching.queue.max_bytes` | byte size | No | max(`128MiB`, `grpc.max_receive_message_bytes`) | Size at least `grpc.max_receive_message_bytes` and `batching.max_bytes` | Maximum encoded request bytes waiting in each method queue. | +| `batching.queue.max_operations` | positive integer | No | max(`10000`, `batching.max_operations`) | Integer at least `batching.max_operations` | Maximum operations waiting in each batch-method queue; also the request count bound for the shared Query/Count admission FIFO. | +| `batching.queue.max_bytes` | byte size | No | max(`128MiB`, `grpc.max_receive_message_bytes`) | Size at least `grpc.max_receive_message_bytes` and `batching.max_bytes` | Maximum encoded request bytes waiting in each batch-method queue and in the separate Query/Count admission FIFO. Query/Count waiting ends only with capacity or caller-context cancellation/deadline, not a server-added timeout. | | `execution.merge.lua.timeout` | duration string | No | `100ms` | Positive Go duration within the bounds below | Maximum wall-clock duration of one Lua execution. | | `execution.merge.lua.max_source_bytes` | byte size | No | `64KiB` | Size greater than `0B` | Maximum Lua source size per merge operation. | | `execution.merge.lua.max_result_bytes` | byte size | No | `16MiB` | Size greater than `0B` | Maximum input/current document and encoded result bytes; expanded output is also bounded before conversion. | diff --git a/docs/design/store-backpressure.md b/docs/design/store-backpressure.md index 318e040..e2059bb 100644 --- a/docs/design/store-backpressure.md +++ b/docs/design/store-backpressure.md @@ -2,7 +2,8 @@ Gateway/Engine/Worker scaling adds application processing capacity. Store backpressure independently regulates the work those processes send to the -database. A growing request queue or Kafka lag never raises the Store window. +database. Queue demand can request growth, but only healthy backend completions +raise the Store window; queue growth or Kafka lag alone cannot do so. The controller does not consume replica counts, HPA signals, CPU, memory, connection counts, Ping results, or discovery information. @@ -20,15 +21,15 @@ flowchart LR G[Gateway] --> Q[Engine bounded method queues] Q -->|TryAcquire before dequeue| C[Local Store controller] K[Kafka backlog] -->|Wait before poll; Acquire before attempt| C - N[Native request] -->|Admit or reject before execution| C + R[Query / Count] -->|Bounded FIFO; caller context only| C + N[Execute / Scan] -->|Admit or reject before execution| C C --> E[Sequential admitted execution] E --> O[Storage observation decorator] O --> S[Storage / NativeStorage] O -->|Real results and execution latency| C ``` -There is no new work queue and no goroutine in the controller. Engine's three -existing dispatch loops select dependency-ready work, then try admission **before** +Engine's three existing batch dispatch loops select dependency-ready work, then try admission **before** releasing queue bytes/operations, recording active records, or launching execution. When blocked, they continue handling arrivals, cancellation, record completions and shutdown while waiting for a capacity notification or a cooldown timer. Pending @@ -45,9 +46,25 @@ avoids moving a bounded queue into unbounded semaphore waiters, and avoids deadlocking a Merge which already owns its execution slot. Read, Write, Delete, Execute, Query, Count and Scan share the same controller. -Native methods have no existing bounded waiting queue: they fail admission -immediately when the window is occupied or paused. Engine marks this specific -rejection `sink-forward-not-started`; a backend error with the same gRPC code +Query and Count share a bounded admission FIFO. A full execution window or an +overload cooldown leaves those requests waiting rather than rejecting a healthy +burst. `batching.queue.max_operations` bounds waiting requests (one command per +request), and `batching.queue.max_bytes` bounds retained encoded request bytes. +These are the same configured limits as the batch queues, not a second queue on +top of an existing Read/Write/Delete queue. Only the head competes for a Store +permit; no goroutine is spawned by admission. Query/Count are FIFO with respect +to each other, not globally ordered with the independent batch dispatch loops. + +There is **no new server-side admission timeout**. A caller with no deadline can +wait until capacity is available or its context is canceled. A caller deadline +is propagated unchanged, and cancellation removes queue charges without entering +the backend. Only a full waiting queue rejects with `RESOURCE_EXHAUSTED`. +Encoded bytes are a bounded accounting unit, not an exact Go heap estimate; +process memory admission and gRPC message limits remain independent safeguards. +Admission waiting never shortens the lifetime of an already executing permit. + +Execute and Scan keep their fail-fast admission semantics. Engine marks a Store +admission or read-queue rejection `sink-forward-not-started`; a backend error with the same gRPC code does not receive that evidence. Async publication bypasses Store admission; Kafka producer bounds still apply, and Worker controls the later application. @@ -62,14 +79,16 @@ mutation waves sequentially; this feature does not add parallel poll processing. ## Window algorithm -The algorithm has a fixed-size state and one short mutex-protected update per -Store observation. It starts on the first admission demand, so constructing an -idle process early cannot defeat startup staggering. +The window algorithm has fixed-size feedback state and one short mutex-protected +update per Store observation, alongside the bounded Query/Count FIFO. Engine +actively completes randomized initialization in `Run` **before serving readiness +or RPCs**, without issuing a synthetic database request. Worker retains its +first-poll initialization. Subsequent saturation or cooldown does not flap readiness. | Event | Behavior | | --- | --- | -| First demand | Window zero for a random 0.5–1.5 seconds, then one execution | -| Healthy, saturated work | Grow only after at least `max(4, window)` healthy samples and a jittered 125–375 ms control interval | +| Initialization | Stagger for a random 0.5–1.5 seconds, then open `min(4, maximum)` executions before Engine readiness | +| Healthy demand | Saturation or queued Query/Count requests request growth; require at least `max(4, window)` healthy samples and a jittered 125–375 ms control interval | | Initial growth | Add `max(1, window/2)` below a slow-start threshold of 16 | | After congestion | Threshold becomes half the previous window; subsequent growth is additive by one | | Sustained latency inflation | Halve the window, retaining at least one execution so successful work continues and the baseline can adapt | @@ -190,6 +209,13 @@ A seeded discrete-event simulation runs 1, 8 and 100 independent controllers against a shared backend with capacity 8 → 2 → 8, checking recovery, bounded overload traffic, startup staggering and progress across instances. It is a repeatable algorithm regression, not a production throughput benchmark. +The initial window of four retains this startup-overshoot gate: an eight-slot +initial window failed the 100-instance regression. Healthy burst tests also +require progress without rejection, rather than treating a tiny initial window +as sufficient protection. Deterministic virtual-time tests hold a caller for an +hour without a server-added deadline, cancel the head/middle/tail of the FIFO, +exercise byte/count bounds and cooldown, and verify exact permit/queue cleanup. +Race tests concurrently release permits, cancel callers and collect metrics. Kafka component tests use an in-process broker to verify that paused intake does not spend processing timeout, report unhealthy, advance offsets, or produce dead letters; another test verifies a real processor's overload/cooldown retry. diff --git a/docs/observability.md b/docs/observability.md index 888419c..e732f6b 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -191,16 +191,26 @@ and memory measurements do not feed these metrics or the congestion algorithm. | `sink_store_feedback_total` | counter | `method`, `signal=healthy,congested,ignored`; one sample per Store call | | `sink_store_backend_duration_seconds_total` | counter | `method`; elapsed Store time excluding admission and downstream Emit waiting | | `sink_store_latency_baseline_seconds` | gauge | `method`, `batch_size=1,2_32,33_128,129_plus`; zero before learning | +| `sink_store_admission_queued_requests` | gauge | Query/Count callers waiting for a Store permit | +| `sink_store_admission_queued_bytes` | gauge | Encoded Query/Count request bytes retained while waiting | +| `sink_store_admission_waits_total` | counter | `outcome=admitted,canceled,rejected`; bounded read-queue outcomes | +| `sink_store_admission_queue_duration_seconds` | histogram | `outcome=admitted,canceled`; caller-controlled read admission wait, not backend time | +| `sink_store_permit_hold_duration_seconds` | histogram | Execution permit lifetime, including retries/cursor sends but excluding admission wait | +| `sink_store_emit_wait_duration_seconds_total` | counter | `method`; downstream Query/Scan callback time excluded from backend feedback | Methods are the fixed set `read`, `write`, `write_visible`, `delete`, `delete_visible`, `execute`, `query`, `count`, `scan`. Opaque Execute commands use success/error feedback only, so their latency baseline remains zero. -The collector creates **80 series per process**, independent of request paths, +The collector creates **136 series per process**, independent of request paths, document identities, query text, and error messages. Deployment/target labels and historical Pod churn remain outside that count. -Compare backend execution time with `sink_batcher_request_queue_duration_seconds` to -distinguish backend slowdown from admission waiting. A mean Store duration is +Compare backend execution time with `sink_batcher_request_queue_duration_seconds` +and `sink_store_admission_queue_duration_seconds` to distinguish backend slowdown +from admission waiting. Compare permit-hold time with backend and Emit time to +identify slow consumers that retain open cursors without overloading the backend. +These counters describe different scopes; do not add Gateway/Engine observations +as if they were independent requests. A mean Store duration is `rate(sink_store_backend_duration_seconds_total[5m])` divided by `sum without (signal) (rate(sink_store_feedback_total[5m]))`. These count interface calls, not database commands or records. In-flight executions may temporarily diff --git a/internal/app/admission_startup_test.go b/internal/app/admission_startup_test.go new file mode 100644 index 0000000..5937a0d --- /dev/null +++ b/internal/app/admission_startup_test.go @@ -0,0 +1,111 @@ +package app + +import ( + "context" + "net" + "net/http" + "sync" + "testing" + "testing/synctest" + "time" + + "github.com/batchstream/sink/internal/backpressure" + "github.com/batchstream/sink/internal/config" +) + +type startupListener struct { + entered chan time.Time + closed chan struct{} + once sync.Once +} + +func (l *startupListener) Accept() (net.Conn, error) { + l.entered <- time.Now() + <-l.closed + return nil, net.ErrClosed +} + +func (l *startupListener) Close() error { + l.once.Do(func() { close(l.closed) }) + return nil +} + +func (l *startupListener) Addr() net.Addr { + address := &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1)} + return address +} + +func TestEngineCompletesStoreStartupBeforeServingReadiness(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + opts := backpressure.Options{Store: "primary", Role: "engine", MaxConcurrent: 64} + controller, err := backpressure.New(opts) + if err != nil { + t.Fatal(err) + } + listener := &startupListener{entered: make(chan time.Time, 1), closed: make(chan struct{})} + healthServer := &http.Server{Handler: http.NotFoundHandler()} + settings := config.Config{Mode: config.ModeEngine, ShutdownTimeout: time.Second} + app := &Application{config: settings, admission: controller, healthListener: listener, healthServer: healthServer} + defer app.Close() + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + done := make(chan error, 1) + start := time.Now() + go func() { done <- app.Run(ctx) }() + synctest.Wait() + time.Sleep(400 * time.Millisecond) + select { + case <-listener.entered: + t.Fatal("readiness was served before Store startup completed") + default: + } + served := <-listener.entered + if served.Sub(start) < 500*time.Millisecond || served.Sub(start) > 1500*time.Millisecond { + t.Fatalf("startup staggering changed: %s", served.Sub(start)) + } + var permits []*backpressure.Permit + for range 4 { + permit, _, _ := controller.TryAcquire() + if permit == nil { + t.Fatal("ready Engine still has a zero/one startup window") + } + permits = append(permits, permit) + } + for _, permit := range permits { + permit.Release() + } + cancel() + if err := <-done; err != nil { + t.Fatal(err) + } + }) +} + +func TestEngineStartupHonorsProcessCancellation(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + opts := backpressure.Options{Store: "primary", Role: "engine"} + controller, err := backpressure.New(opts) + if err != nil { + t.Fatal(err) + } + listener := &startupListener{entered: make(chan time.Time, 1), closed: make(chan struct{})} + healthServer := &http.Server{Handler: http.NotFoundHandler()} + settings := config.Config{Mode: config.ModeEngine, ShutdownTimeout: time.Second} + app := &Application{config: settings, admission: controller, healthListener: listener, healthServer: healthServer} + defer app.Close() + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + done := make(chan error, 1) + go func() { done <- app.Run(ctx) }() + synctest.Wait() + cancel() + if err := <-done; err != nil { + t.Fatal(err) + } + select { + case <-listener.entered: + t.Fatal("canceled Engine started serving during initialization") + default: + } + }) +} diff --git a/internal/app/app.go b/internal/app/app.go index a5b5a20..f0c92d1 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -80,7 +80,11 @@ func New(ctx context.Context, opts Options) (*Application, error) { app.Close() } }() - admissionOptions := backpressure.Options{Store: loaded.Storage.Name, Role: string(loaded.Mode), MaxConcurrent: loaded.Service.StoreMaxConcurrent} + admissionOptions := backpressure.Options{ + Store: loaded.Storage.Name, Role: string(loaded.Mode), MaxConcurrent: loaded.Service.StoreMaxConcurrent, + MaxQueuedRequests: loaded.Service.Batching.Queue.MaxOperations, + MaxQueuedBytes: loaded.Service.Batching.Queue.MaxBytes, + } app.admission, err = backpressure.New(admissionOptions) if err != nil { return nil, err diff --git a/internal/app/lifecycle.go b/internal/app/lifecycle.go index 56c3bd0..f9aadce 100644 --- a/internal/app/lifecycle.go +++ b/internal/app/lifecycle.go @@ -8,6 +8,7 @@ import ( "net/http" "time" + "github.com/batchstream/sink/internal/config" "github.com/batchstream/sink/internal/logging" searchstorage "github.com/batchstream/sink/internal/storage/search" "google.golang.org/grpc" @@ -23,6 +24,13 @@ func (app *Application) Run(ctx context.Context) error { cancel() app.background.Wait() }() + // Finish randomized Store startup before either readiness or RPCs are served. + // Routine saturation and subsequent cooldowns must not flap readiness. + if app.admission != nil && app.config.Mode == config.ModeEngine { + if err := app.admission.Wait(runContext); err != nil { + return nil + } + } if app.gateway != nil { app.background.Go(func() { app.gateway.Run(runContext) }) } diff --git a/internal/backpressure/cold_start_regression_test.go b/internal/backpressure/cold_start_regression_test.go new file mode 100644 index 0000000..1c7f22c --- /dev/null +++ b/internal/backpressure/cold_start_regression_test.go @@ -0,0 +1,27 @@ +package backpressure + +import ( + "strconv" + "testing" + "testing/synctest" +) + +func TestHealthyStartupOpensBoundedWindow(t *testing.T) { + for _, maximum := range []int{1, 4, 64} { + t.Run(strconv.Itoa(maximum), func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + opts := Options{Store: "primary", Role: "engine", MaxConcurrent: maximum} + controller, err := New(opts) + if err != nil { + t.Fatal(err) + } + if err := controller.Wait(t.Context()); err != nil { + t.Fatal(err) + } + if want := min(4, maximum); controller.limit != want { + t.Fatalf("healthy initial window = %d, want %d", controller.limit, want) + } + }) + }) + } +} diff --git a/internal/backpressure/controller.go b/internal/backpressure/controller.go index 78ff568..279994d 100644 --- a/internal/backpressure/controller.go +++ b/internal/backpressure/controller.go @@ -1,8 +1,10 @@ // Package backpressure controls Store dispatch using feedback from real work. -// It owns no queue, background goroutine, dependency probe or retry policy. +// Read admission has a bounded queue; no background goroutine, dependency probe +// or retry policy is owned by the controller. package backpressure import ( + "container/list" "context" "errors" "math/rand/v2" @@ -16,41 +18,53 @@ import ( const DefaultMaxConcurrent = 64 const ( - startupWindow = time.Second - controlInterval = 250 * time.Millisecond - minimumCooldown = 200 * time.Millisecond - maximumCooldown = 10 * time.Second + initialConcurrent = 4 + defaultMaxQueuedRequests = 10_000 + defaultMaxQueuedBytes = 128 << 20 + startupWindow = time.Second + controlInterval = 250 * time.Millisecond + minimumCooldown = 200 * time.Millisecond + maximumCooldown = 10 * time.Second ) // ErrBusy proves that this admission did not start backend work. var ErrBusy = status.Error(codes.ResourceExhausted, "Store admission is temporarily full or cooling down") +// ErrQueueFull proves that read admission did not start backend work. +var ErrQueueFull = status.Error(codes.ResourceExhausted, "Store read admission queue is full") + type Options struct { Store string Role string // MaxConcurrent is a local resource ceiling, not an estimate of DB capacity. - MaxConcurrent int + MaxConcurrent int + MaxQueuedRequests int + MaxQueuedBytes int } type Controller struct { - mu sync.Mutex - limit int - inFlight int - maximum int - threshold int - resumeLimit int - epoch uint64 - clean int - recovered int - failures int - demand bool - started bool - resumeAt time.Time - adjustAt time.Time - changed chan struct{} - random *rand.Rand - latency [methodCount][sizeClasses]latency - observed observations + mu sync.Mutex + limit int + inFlight int + maximum int + threshold int + resumeLimit int + epoch uint64 + clean int + recovered int + failures int + demand bool + started bool + resumeAt time.Time + adjustAt time.Time + changed chan struct{} + random *rand.Rand + latency [methodCount][sizeClasses]latency + observed observations + readWaiters list.List + queuedBytes int + maxQueuedRequests int + maxQueuedBytes int } type latency struct { @@ -65,6 +79,7 @@ type latency struct { type Permit struct { controller *Controller once sync.Once + started time.Time } type permitKey struct{} @@ -76,10 +91,20 @@ func New(opts Options) (*Controller, error) { if opts.MaxConcurrent < 1 || opts.MaxConcurrent > 4096 { return nil, errors.New("store max_concurrent must be between 1 and 4096") } + if opts.MaxQueuedRequests < 0 || opts.MaxQueuedBytes < 0 { + return nil, errors.New("store read admission queue limits cannot be negative") + } + if opts.MaxQueuedRequests == 0 { + opts.MaxQueuedRequests = defaultMaxQueuedRequests + } + if opts.MaxQueuedBytes == 0 { + opts.MaxQueuedBytes = defaultMaxQueuedBytes + } c := &Controller{ - maximum: opts.MaxConcurrent, threshold: min(16, opts.MaxConcurrent), resumeLimit: 1, + maximum: opts.MaxConcurrent, threshold: min(16, opts.MaxConcurrent), resumeLimit: min(initialConcurrent, opts.MaxConcurrent), changed: make(chan struct{}), random: rand.New(rand.NewPCG(rand.Uint64(), rand.Uint64())), - observed: newObservations(opts), + observed: newObservations(opts), + maxQueuedRequests: opts.MaxQueuedRequests, maxQueuedBytes: opts.MaxQueuedBytes, } return c, nil } @@ -98,13 +123,17 @@ func (c *Controller) tryAcquire(now time.Time) (*Permit, <-chan struct{}, time.D } c.mu.Lock() defer c.mu.Unlock() + return c.tryAcquireLocked(now) +} + +func (c *Controller) tryAcquireLocked(now time.Time) (*Permit, <-chan struct{}, time.Duration) { ready, changed, delay := c.availability(now) if !ready { return nil, changed, delay } c.inFlight++ c.observed.admitted++ - permit := &Permit{controller: c} + permit := &Permit{controller: c, started: now} return permit, nil, 0 } @@ -130,7 +159,8 @@ func (c *Controller) availability(now time.Time) (bool, <-chan struct{}, time.Du } // Acquire may wait only at an already bounded caller (the Kafka processing -// loop). Engine RPCs use the existing batch queue or fail-fast Admit instead. +// loop). Query/Count use AdmitRead; other Engine RPCs use their existing batch +// queue or fail-fast Admit instead. func (c *Controller) Acquire(ctx context.Context) (*Permit, error) { for { if err := ctx.Err(); err != nil { @@ -146,8 +176,9 @@ func (c *Controller) Acquire(ctx context.Context) (*Permit, error) { } } -// Wait leaves the next poll in Kafka. It reserves nothing while polling and -// cannot prevent consumer group heartbeats or rebalance callbacks. +// Wait observes available capacity without reserving a permit. Engine uses it +// before serving; Worker uses it before polling so backlog stays in Kafka while +// consumer group heartbeats and rebalance callbacks continue. func (c *Controller) Wait(ctx context.Context) error { if c == nil { return ctx.Err() @@ -196,6 +227,7 @@ func (p *Permit) Release() { c.mu.Lock() defer c.mu.Unlock() c.inFlight-- + c.observed.permitDuration.Observe(max(0, time.Since(p.started).Seconds())) c.signal() }) } @@ -250,6 +282,7 @@ type sample struct { size int epoch uint64 saturated bool + emitWait time.Duration } func (c *Controller) begin(method method, operations int) sample { @@ -268,6 +301,7 @@ func (c *Controller) observeAt(started sample, duration time.Duration, result fe defer c.mu.Unlock() c.observed.samples[started.method][result]++ c.observed.seconds[started.method] += max(0, duration.Seconds()) + c.observed.emitSeconds[started.method] += max(0, started.emitWait.Seconds()) // Replies from the same pre-decrease flight cannot repeatedly collapse the // window, or undo the decrease with old successes. if started.epoch != c.epoch || c.limit == 0 { @@ -313,7 +347,7 @@ func (c *Controller) observeAt(started sample, duration time.Duration, result fe if c.recovered == 4 { c.failures = 0 } - if !started.saturated && !c.demand { + if !started.saturated && !c.demand && c.readWaiters.Len() == 0 { return } c.clean++ diff --git a/internal/backpressure/metrics.go b/internal/backpressure/metrics.go index d592cac..bb71294 100644 --- a/internal/backpressure/metrics.go +++ b/internal/backpressure/metrics.go @@ -7,35 +7,55 @@ import ( ) type observations struct { - limit *prometheus.Desc - inFlight *prometheus.Desc - changes *prometheus.Desc - admissions *prometheus.Desc - feedback *prometheus.Desc - duration *prometheus.Desc - baseline *prometheus.Desc - cooldown *prometheus.Desc - admitted uint64 - rejected uint64 - increases uint64 - overloads uint64 - slowdowns uint64 - samples [methodCount][feedbackCount]uint64 - seconds [methodCount]float64 + limit *prometheus.Desc + inFlight *prometheus.Desc + changes *prometheus.Desc + admissions *prometheus.Desc + feedback *prometheus.Desc + duration *prometheus.Desc + baseline *prometheus.Desc + cooldown *prometheus.Desc + queueRequests *prometheus.Desc + queueBytes *prometheus.Desc + queueOutcomes *prometheus.Desc + queueDuration *prometheus.HistogramVec + permitDuration prometheus.Histogram + emitDuration *prometheus.Desc + admitted uint64 + rejected uint64 + increases uint64 + overloads uint64 + slowdowns uint64 + queueAdmitted uint64 + queueCanceled uint64 + queueRejected uint64 + samples [methodCount][feedbackCount]uint64 + seconds [methodCount]float64 + emitSeconds [methodCount]float64 } func newObservations(opts Options) observations { labels := prometheus.Labels{"role": opts.Role, "store": opts.Store} + queueOptions := prometheus.HistogramOpts{Name: "sink_store_admission_queue_duration_seconds", Help: "Query/Count wait time before admission or caller cancellation; excludes backend work.", ConstLabels: labels} + permitOptions := prometheus.HistogramOpts{Name: "sink_store_permit_hold_duration_seconds", Help: "Time holding an execution permit, including retries and cursor sends but excluding admission wait.", ConstLabels: labels} observed := observations{ - limit: prometheus.NewDesc("sink_store_concurrency_limit", "Current local Store execution window; zero pauses new work.", nil, labels), - inFlight: prometheus.NewDesc("sink_store_executions_in_flight", "Admitted sequential Store executions, including existing retries and cursor sends.", nil, labels), - changes: prometheus.NewDesc("sink_store_window_changes_total", "Store window adjustments by reason.", []string{"reason"}, labels), - admissions: prometheus.NewDesc("sink_store_admissions_total", "Store executions admitted or direct requests rejected before execution.", []string{"outcome"}, labels), - feedback: prometheus.NewDesc("sink_store_feedback_total", "Real Store calls classified once, regardless of batch result count.", []string{"method", "signal"}, labels), - duration: prometheus.NewDesc("sink_store_backend_duration_seconds_total", "Store call time excluding admission and downstream Emit time.", []string{"method"}, labels), - baseline: prometheus.NewDesc("sink_store_latency_baseline_seconds", "Learned Store latency per operation and batch-size class; zero until sampled.", []string{"method", "batch_size"}, labels), - cooldown: prometheus.NewDesc("sink_store_cooldown_seconds", "Remaining delay before real work may probe a paused Store.", nil, labels), + limit: prometheus.NewDesc("sink_store_concurrency_limit", "Current local Store execution window; zero pauses new work.", nil, labels), + inFlight: prometheus.NewDesc("sink_store_executions_in_flight", "Admitted sequential Store executions, including existing retries and cursor sends.", nil, labels), + changes: prometheus.NewDesc("sink_store_window_changes_total", "Store window adjustments by reason.", []string{"reason"}, labels), + admissions: prometheus.NewDesc("sink_store_admissions_total", "Store executions admitted or direct requests rejected before execution.", []string{"outcome"}, labels), + feedback: prometheus.NewDesc("sink_store_feedback_total", "Real Store calls classified once, regardless of batch result count.", []string{"method", "signal"}, labels), + duration: prometheus.NewDesc("sink_store_backend_duration_seconds_total", "Store call time excluding admission and downstream Emit time.", []string{"method"}, labels), + baseline: prometheus.NewDesc("sink_store_latency_baseline_seconds", "Learned Store latency per operation and batch-size class; zero until sampled.", []string{"method", "batch_size"}, labels), + cooldown: prometheus.NewDesc("sink_store_cooldown_seconds", "Remaining delay before real work may probe a paused Store.", nil, labels), + queueRequests: prometheus.NewDesc("sink_store_admission_queued_requests", "Query/Count requests waiting for a shared Store execution permit.", nil, labels), + queueBytes: prometheus.NewDesc("sink_store_admission_queued_bytes", "Encoded request bytes retained by the Query/Count admission queue.", nil, labels), + queueOutcomes: prometheus.NewDesc("sink_store_admission_waits_total", "Query/Count admission waits completed or rejected by the bounded queue.", []string{"outcome"}, labels), + queueDuration: prometheus.NewHistogramVec(queueOptions, []string{"outcome"}), + permitDuration: prometheus.NewHistogram(permitOptions), + emitDuration: prometheus.NewDesc("sink_store_emit_wait_duration_seconds_total", "Time waiting in Query/Scan downstream Emit callbacks, excluded from backend feedback.", []string{"method"}, labels), } + observed.queueDuration.WithLabelValues("admitted") + observed.queueDuration.WithLabelValues("canceled") return observed } @@ -43,13 +63,26 @@ func (c *Controller) Describe(ch chan<- *prometheus.Desc) { for _, desc := range []*prometheus.Desc{c.observed.limit, c.observed.inFlight, c.observed.changes, c.observed.admissions, c.observed.feedback, c.observed.duration, c.observed.baseline, c.observed.cooldown} { ch <- desc } + for _, desc := range []*prometheus.Desc{c.observed.queueRequests, c.observed.queueBytes, c.observed.queueOutcomes, c.observed.emitDuration} { + ch <- desc + } + c.observed.queueDuration.Describe(ch) + c.observed.permitDuration.Describe(ch) } func (c *Controller) Collect(ch chan<- prometheus.Metric) { c.mu.Lock() observed, limit, inFlight := c.observed, c.limit, c.inFlight latency, cooldown := c.latency, max(0, time.Until(c.resumeAt).Seconds()) + queuedRequests, queuedBytes := c.readWaiters.Len(), c.queuedBytes c.mu.Unlock() + ch <- prometheus.MustNewConstMetric(observed.queueRequests, prometheus.GaugeValue, float64(queuedRequests)) + ch <- prometheus.MustNewConstMetric(observed.queueBytes, prometheus.GaugeValue, float64(queuedBytes)) + ch <- prometheus.MustNewConstMetric(observed.queueOutcomes, prometheus.CounterValue, float64(observed.queueAdmitted), "admitted") + ch <- prometheus.MustNewConstMetric(observed.queueOutcomes, prometheus.CounterValue, float64(observed.queueCanceled), "canceled") + ch <- prometheus.MustNewConstMetric(observed.queueOutcomes, prometheus.CounterValue, float64(observed.queueRejected), "rejected") + observed.queueDuration.Collect(ch) + observed.permitDuration.Collect(ch) ch <- prometheus.MustNewConstMetric(observed.limit, prometheus.GaugeValue, float64(limit)) ch <- prometheus.MustNewConstMetric(observed.inFlight, prometheus.GaugeValue, float64(inFlight)) ch <- prometheus.MustNewConstMetric(observed.cooldown, prometheus.GaugeValue, cooldown) @@ -63,6 +96,7 @@ func (c *Controller) Collect(ch chan<- prometheus.Metric) { ch <- prometheus.MustNewConstMetric(observed.feedback, prometheus.CounterValue, float64(observed.samples[kind][signal]), name, label) } ch <- prometheus.MustNewConstMetric(observed.duration, prometheus.CounterValue, observed.seconds[kind], name) + ch <- prometheus.MustNewConstMetric(observed.emitDuration, prometheus.CounterValue, observed.emitSeconds[kind], name) for size, label := range []string{"1", "2_32", "33_128", "129_plus"} { ch <- prometheus.MustNewConstMetric(observed.baseline, prometheus.GaugeValue, latency[kind][size].baseline/float64(time.Second), name, label) } diff --git a/internal/backpressure/read_admission.go b/internal/backpressure/read_admission.go new file mode 100644 index 0000000..62b8abf --- /dev/null +++ b/internal/backpressure/read_admission.go @@ -0,0 +1,112 @@ +package backpressure + +import ( + "container/list" + "context" + "time" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type readWaiter struct { + bytes int + turn chan struct{} + enqueued time.Time +} + +// AdmitRead waits in a shared FIFO for Query and Count. Only the caller's context +// ends waiting; cooldown timers wake the head without imposing a new deadline. +// Waiting payloads remain bounded separately from executing Store work. +func (c *Controller) AdmitRead(ctx context.Context, requestBytes int) (context.Context, *Permit, error) { + if err := ctx.Err(); err != nil { + return ctx, nil, status.FromContextError(err).Err() + } + if requestBytes < 0 { + return ctx, nil, status.Error(codes.InvalidArgument, "negative read admission size") + } + if c == nil { + return ctx, nil, nil + } + if permit, ok := ctx.Value(permitKey{}).(*Permit); ok && permit.controller == c { + return ctx, nil, nil + } + c.mu.Lock() + if c.readWaiters.Len() == 0 { + permit, _, _ := c.tryAcquireLocked(time.Now()) + if permit != nil { + c.mu.Unlock() + if err := ctx.Err(); err != nil { + permit.Release() + return ctx, nil, status.FromContextError(err).Err() + } + return permit.Context(ctx), permit, nil + } + } + if c.readWaiters.Len() >= c.maxQueuedRequests || requestBytes > c.maxQueuedBytes-c.queuedBytes { + c.observed.rejected++ + c.observed.queueRejected++ + c.mu.Unlock() + return ctx, nil, ErrQueueFull + } + waiter := &readWaiter{bytes: requestBytes, turn: make(chan struct{}), enqueued: time.Now()} + entry := c.readWaiters.PushBack(waiter) + c.queuedBytes += requestBytes + c.demand = true + if entry == c.readWaiters.Front() { + close(waiter.turn) + } + c.mu.Unlock() + + select { + case <-ctx.Done(): + c.mu.Lock() + c.finishReadWaitLocked(entry, false) + c.mu.Unlock() + return ctx, nil, status.FromContextError(ctx.Err()).Err() + case <-waiter.turn: + } + for { + c.mu.Lock() + if err := ctx.Err(); err != nil { + c.finishReadWaitLocked(entry, false) + c.mu.Unlock() + return ctx, nil, status.FromContextError(err).Err() + } + permit, changed, delay := c.tryAcquireLocked(time.Now()) + if permit != nil { + c.finishReadWaitLocked(entry, true) + c.mu.Unlock() + if err := ctx.Err(); err != nil { + permit.Release() + return ctx, nil, status.FromContextError(err).Err() + } + return permit.Context(ctx), permit, nil + } + c.mu.Unlock() + if err := wait(ctx, changed, delay); err != nil { + c.mu.Lock() + c.finishReadWaitLocked(entry, false) + c.mu.Unlock() + return ctx, nil, status.FromContextError(err).Err() + } + } +} + +func (c *Controller) finishReadWaitLocked(entry *list.Element, admitted bool) { + waiter := entry.Value.(*readWaiter) + head := entry == c.readWaiters.Front() + c.readWaiters.Remove(entry) + c.queuedBytes -= waiter.bytes + outcome := "canceled" + if admitted { + c.observed.queueAdmitted++ + outcome = "admitted" + } else { + c.observed.queueCanceled++ + } + c.observed.queueDuration.WithLabelValues(outcome).Observe(time.Since(waiter.enqueued).Seconds()) + if next := c.readWaiters.Front(); head && next != nil { + close(next.Value.(*readWaiter).turn) + } +} diff --git a/internal/backpressure/read_admission_test.go b/internal/backpressure/read_admission_test.go new file mode 100644 index 0000000..e95f87a --- /dev/null +++ b/internal/backpressure/read_admission_test.go @@ -0,0 +1,393 @@ +package backpressure + +import ( + "context" + "runtime" + "strconv" + "sync" + "testing" + "testing/synctest" + "time" + + "github.com/prometheus/client_golang/prometheus" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type readAdmissionResult struct { + index int + ctx context.Context + permit *Permit + err error +} + +func readController(t *testing.T, maximum, requests, bytes int) *Controller { + t.Helper() + opts := Options{Store: "primary", Role: "engine", MaxConcurrent: maximum, MaxQueuedRequests: requests, MaxQueuedBytes: bytes} + c, err := New(opts) + if err != nil { + t.Fatal(err) + } + return c +} + +func TestReadAdmissionFIFOAndCancellationReleaseEveryQueuePosition(t *testing.T) { + for _, canceled := range []int{0, 1, 2} { + t.Run(strconv.Itoa(canceled), func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + c := readController(t, 1, 3, 48) + held, err := c.Acquire(t.Context()) + if err != nil { + t.Fatal(err) + } + defer held.Release() + results := make(chan readAdmissionResult, 3) + var cancels []context.CancelFunc + for index := range 3 { + ctx, cancel := context.WithCancel(t.Context()) + cancels = append(cancels, cancel) + defer cancel() + go func() { + admitted, permit, err := c.AdmitRead(ctx, 16) + result := readAdmissionResult{index: index, ctx: admitted, permit: permit, err: err} + results <- result + }() + synctest.Wait() + } + if c.readWaiters.Len() != 3 || c.queuedBytes != 48 { + t.Fatalf("queue is not bounded/accounted: requests=%d bytes=%d", c.readWaiters.Len(), c.queuedBytes) + } + goroutines := runtime.NumGoroutine() + for range 1000 { + _, permit, err := c.AdmitRead(t.Context(), 1) + if permit != nil || err != ErrQueueFull { + t.Fatalf("full queue: permit=%v error=%v", permit, err) + } + } + synctest.Wait() + if runtime.NumGoroutine() > goroutines { + t.Fatal("rejected requests created waiting goroutines") + } + cancels[canceled]() + result := <-results + if result.index != canceled || status.Code(result.err) != codes.Canceled || result.permit != nil { + t.Fatalf("cancellation changed: %+v", result) + } + if c.readWaiters.Len() != 2 || c.queuedBytes != 32 || c.inFlight != 1 { + t.Fatal("cancellation leaked queue capacity or revoked active work") + } + held.Release() + for index := range 3 { + if index == canceled { + continue + } + result := <-results + if result.err != nil || result.index != index || result.permit == nil { + t.Fatalf("FIFO admission changed: %+v, want %d", result, index) + } + result.permit.Release() + } + synctest.Wait() + if c.readWaiters.Len() != 0 || c.queuedBytes != 0 || c.inFlight != 0 || c.observed.queueCanceled != 1 || c.observed.queueAdmitted != 2 { + t.Fatal("completed queue leaked state") + } + }) + }) + } +} + +func TestReadAdmissionByteLimitAndCapacityReuse(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + c := readController(t, 1, 10, 32) + held, err := c.Acquire(t.Context()) + if err != nil { + t.Fatal(err) + } + defer held.Release() + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + result := make(chan error, 1) + go func() { _, permit, err := c.AdmitRead(ctx, 32); permit.Release(); result <- err }() + synctest.Wait() + for _, bytes := range []int{1, 33, int(^uint(0) >> 1)} { + _, permit, err := c.AdmitRead(t.Context(), bytes) + if err != ErrQueueFull || permit != nil || c.queuedBytes != 32 || c.readWaiters.Len() != 1 { + t.Fatalf("encoded-byte bound failed for %d: %v", bytes, err) + } + } + cancel() + if err := <-result; status.Code(err) != codes.Canceled { + t.Fatal(err) + } + held.Release() + _, permit, err := c.AdmitRead(t.Context(), 32) + if err != nil { + t.Fatal(err) + } + permit.Release() + permit.Release() + if c.queuedBytes != 0 || c.readWaiters.Len() != 0 || c.inFlight != 0 { + t.Fatal("byte reservations or permits leaked") + } + }) +} + +func TestReadAdmissionPreservesContextAndOnlyCallerEndsWaiting(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + c := readController(t, 1, 4, 64) + held, err := c.Acquire(t.Context()) + if err != nil { + t.Fatal(err) + } + defer held.Release() + type contextKey struct{} + ctx := context.WithValue(t.Context(), contextKey{}, "retained") + ctx, cancel := context.WithDeadline(ctx, time.Now().Add(2*time.Hour)) + defer cancel() + wantDeadline, _ := ctx.Deadline() + result := make(chan readAdmissionResult, 1) + go func() { + admitted, permit, err := c.AdmitRead(ctx, 16) + value := readAdmissionResult{ctx: admitted, permit: permit, err: err} + result <- value + }() + synctest.Wait() + time.Sleep(time.Hour) + synctest.Wait() + select { + case value := <-result: + t.Fatalf("an internal timeout ended waiting: %v", value.err) + default: + } + held.Release() + value := <-result + deadline, _ := value.ctx.Deadline() + if value.err != nil || deadline != wantDeadline || value.ctx.Value(contextKey{}) != "retained" || c.queuedBytes != 0 { + t.Fatalf("caller context or queue lifetime changed: %+v", value) + } + value.permit.Release() + + held, err = c.Acquire(t.Context()) + if err != nil { + t.Fatal(err) + } + defer held.Release() + short, shortCancel := context.WithDeadline(t.Context(), time.Now().Add(time.Minute)) + defer shortCancel() + _, permit, err := c.AdmitRead(short, 16) + if status.Code(err) != codes.DeadlineExceeded || permit != nil || c.readWaiters.Len() != 0 || c.queuedBytes != 0 { + t.Fatalf("caller deadline did not release queue: %v", err) + } + }) +} + +func TestQueuedHealthyReadsGrowWithoutRejectingRequests(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + c := readController(t, 16, 512, 8192) + if err := c.Wait(t.Context()); err != nil { + t.Fatal(err) + } + c.limit = 1 + results := make(chan error, 512) + var mu sync.Mutex + active, peak := 0, 0 + for range 512 { + go func() { + _, permit, err := c.AdmitRead(t.Context(), 16) + if err != nil { + results <- err + return + } + mu.Lock() + active++ + peak = max(peak, active) + mu.Unlock() + started := c.begin(query, 1) + time.Sleep(20 * time.Millisecond) + c.observe(started, 20*time.Millisecond, healthy) + mu.Lock() + active-- + mu.Unlock() + permit.Release() + results <- nil + }() + } + for range 512 { + if err := <-results; err != nil { + t.Fatal(err) + } + } + synctest.Wait() + if peak < 3 || peak > 16 || active != 0 || c.observed.increases == 0 || c.observed.rejected != 0 || c.readWaiters.Len() != 0 || c.queuedBytes != 0 || c.inFlight != 0 { + t.Fatalf("healthy demand did not grow safely: peak=%d increases=%d rejected=%d", peak, c.observed.increases, c.observed.rejected) + } + }) +} + +func TestReadAdmissionWaitsThroughCooldownWithoutRevokingActiveWork(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + c := readController(t, 4, 2, 32) + held, err := c.Acquire(t.Context()) + if err != nil { + t.Fatal(err) + } + defer held.Release() + started := c.begin(query, 1) + c.observe(started, time.Millisecond, congested) + resume := c.resumeAt + result := make(chan readAdmissionResult, 1) + go func() { + ctx, permit, err := c.AdmitRead(t.Context(), 16) + value := readAdmissionResult{ctx: ctx, permit: permit, err: err} + result <- value + }() + synctest.Wait() + if c.limit != 0 || c.inFlight != 1 || c.readWaiters.Len() != 1 { + t.Fatal("cooldown revoked admitted work or let waiting work bypass") + } + time.Sleep(time.Until(resume) / 2) + select { + case value := <-result: + t.Fatalf("read did not wait for overload recovery: %v", value.err) + default: + } + held.Release() + value := <-result + if value.err != nil || time.Now().Before(resume) || c.limit != 2 || c.observed.rejected != 0 { + t.Fatal("cooldown recovery was bypassed or rejected healthy waiting work") + } + value.permit.Release() + }) +} + +func TestReadAdmissionValidationNilAndReusedPermit(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + for _, opts := range []Options{{MaxQueuedRequests: -1}, {MaxQueuedBytes: -1}} { + if _, err := New(opts); err == nil { + t.Fatal("negative queue limit accepted") + } + } + c := readController(t, 1, 1, 16) + _, permit, err := c.AdmitRead(t.Context(), -1) + if status.Code(err) != codes.InvalidArgument || permit != nil { + t.Fatal("negative encoded size accepted") + } + canceled, cancel := context.WithCancel(t.Context()) + cancel() + _, permit, err = c.AdmitRead(canceled, 1) + if status.Code(err) != codes.Canceled || permit != nil || c.observed.admitted != 0 { + t.Fatal("already canceled request was admitted") + } + var absent *Controller + admitted, permit, err := absent.AdmitRead(t.Context(), 1) + if err != nil || permit != nil || admitted != t.Context() { + t.Fatal("nil admission changed component behavior") + } + held, err := c.Acquire(t.Context()) + if err != nil { + t.Fatal(err) + } + defer held.Release() + ctx := held.Context(t.Context()) + admitted, permit, err = c.AdmitRead(ctx, 1) + if err != nil || permit != nil || admitted != ctx || c.inFlight != 1 || c.readWaiters.Len() != 0 { + t.Fatal("nested admission deadlocked or double-counted an existing permit") + } + }) +} + +func TestReadAdmissionMetricsSeparateWaitingAndPermitHold(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + c := readController(t, 1, 1, 16) + held, err := c.Acquire(t.Context()) + if err != nil { + t.Fatal(err) + } + defer held.Release() + registry := prometheus.NewPedanticRegistry() + if err := registry.Register(c); err != nil { + t.Fatal(err) + } + result := make(chan *Permit, 1) + go func() { + _, permit, err := c.AdmitRead(t.Context(), 16) + if err != nil { + t.Error(err) + } + result <- permit + }() + synctest.Wait() + families, err := registry.Gather() + if err != nil { + t.Fatal(err) + } + for _, family := range families { + switch family.GetName() { + case "sink_store_admission_queued_requests": + if family.Metric[0].GetGauge().GetValue() != 1 { + t.Fatal("missing waiting request gauge") + } + case "sink_store_admission_queued_bytes": + if family.Metric[0].GetGauge().GetValue() != 16 { + t.Fatal("missing waiting bytes gauge") + } + } + } + time.Sleep(time.Minute) + held.Release() + permit := <-result + time.Sleep(time.Second) + permit.Release() + permit.Release() + families, err = registry.Gather() + if err != nil { + t.Fatal(err) + } + for _, family := range families { + switch family.GetName() { + case "sink_store_admission_queue_duration_seconds": + for _, metric := range family.Metric { + for _, label := range metric.Label { + if label.GetName() == "outcome" && label.GetValue() == "admitted" && (metric.GetHistogram().GetSampleCount() != 1 || metric.GetHistogram().GetSampleSum() != 60) { + t.Fatal("queue time includes execution or was lost") + } + } + } + case "sink_store_permit_hold_duration_seconds": + if family.Metric[0].GetHistogram().GetSampleCount() != 2 || family.Metric[0].GetHistogram().GetSampleSum() != 61 { + t.Fatal("permit hold time includes queue time or double release") + } + } + } + }) +} + +func TestReadAdmissionCancelReleaseContentionDoesNotLeak(t *testing.T) { + c := readController(t, 4, 256, 4096) + if err := c.Wait(t.Context()); err != nil { + t.Fatal(err) + } + var workers sync.WaitGroup + for index := range 256 { + workers.Go(func() { + for iteration := range 16 { + ctx, cancel := context.WithCancel(t.Context()) + if (index+iteration)%3 == 0 { + cancel() + } else if (index+iteration)%3 == 1 { + go cancel() + } + _, permit, err := c.AdmitRead(ctx, 16) + if err != nil && status.Code(err) != codes.Canceled { + t.Errorf("bounded contention lost a request: %v", err) + } + permit.Release() + cancel() + } + }) + } + workers.Wait() + if c.inFlight != 0 || c.readWaiters.Len() != 0 || c.queuedBytes != 0 { + t.Fatalf("race leaked admission: active=%d queued=%d bytes=%d", c.inFlight, c.readWaiters.Len(), c.queuedBytes) + } +} diff --git a/internal/backpressure/storage.go b/internal/backpressure/storage.go index 5bd3318..5a68761 100644 --- a/internal/backpressure/storage.go +++ b/internal/backpressure/storage.go @@ -231,6 +231,7 @@ func (s *observedNative) finishStream(started sample, at time.Time, stream *stre if stream.failed || errors.Is(stream.ctx.Err(), context.Canceled) || (timeout && stream.wait > duration) { result = ignored } + started.emitWait = stream.wait s.controller.observe(started, duration, result) } diff --git a/internal/backpressure/storage_test.go b/internal/backpressure/storage_test.go index 92c2efa..91b5a53 100644 --- a/internal/backpressure/storage_test.go +++ b/internal/backpressure/storage_test.go @@ -228,6 +228,9 @@ func TestStreamFeedbackExcludesSlowClientAndCallbackFailures(t *testing.T) { if c.limit != 8 || c.observed.seconds[kind] > 0.081 || c.latency[kind][0].baseline > float64(11*time.Millisecond) { t.Fatal("slow client time leaked into backend latency") } + if c.observed.emitSeconds[kind] != 8 { + t.Fatalf("slow-client time was not recorded separately: %g", c.observed.emitSeconds[kind]) + } ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond) emit := func(storage.Document) error { <-ctx.Done() @@ -267,7 +270,20 @@ func TestControllerMetricsAndValidation(t *testing.T) { t.Fatal(err) } families, err := registry.Gather() - if err != nil || len(families) != 8 { + if err != nil || len(families) != 14 { t.Fatalf("invalid metrics: %d families, %v", len(families), err) } + series := 0 + for _, family := range families { + for _, metric := range family.Metric { + if histogram := metric.GetHistogram(); histogram != nil { + series += len(histogram.Bucket) + 3 // Finite buckets, +Inf, sum, count. + } else { + series++ + } + } + } + if series != 136 { + t.Fatalf("Store collector cardinality changed: %d series", series) + } } diff --git a/internal/engine/backpressure_test.go b/internal/engine/backpressure_test.go index 45efd88..e3623e6 100644 --- a/internal/engine/backpressure_test.go +++ b/internal/engine/backpressure_test.go @@ -22,7 +22,7 @@ func (s *admissionFailureService) Execute(context.Context, *sink.ExecuteRequest) } func TestEngineDistinguishesAdmissionFromUnknownNativeOutcome(t *testing.T) { - for _, failure := range []error{backpressure.ErrBusy, status.Error(codes.ResourceExhausted, "backend outcome unknown")} { + for _, failure := range []error{backpressure.ErrBusy, backpressure.ErrQueueFull, status.Error(codes.ResourceExhausted, "backend outcome unknown")} { service := &admissionFailureService{err: failure} opts := Options{Service: service, Store: "primary", MaxReadBytes: 1 << 20} server, err := New(opts) @@ -36,7 +36,8 @@ func TestEngineDistinguishesAdmissionFromUnknownNativeOutcome(t *testing.T) { stream := &forwardRecorder{ctx: t.Context()} err = server.Forward(request, stream) marker := stream.trailer.Get(forwarding.NotStartedTrailer) - if err != failure || (len(marker) == 1) != (failure == backpressure.ErrBusy) || stream.response != nil { + notStarted := failure == backpressure.ErrBusy || failure == backpressure.ErrQueueFull + if err != failure || (len(marker) == 1) != notStarted || stream.response != nil { t.Fatalf("unknown outcome or admission evidence changed: error=%v marker=%v", err, marker) } origin := stream.trailer.Get(forwarding.ErrorStatusTrailer) diff --git a/internal/engine/server.go b/internal/engine/server.go index 5ea5b9a..35d5a96 100644 --- a/internal/engine/server.go +++ b/internal/engine/server.go @@ -58,7 +58,7 @@ func (s *Server) Forward(req *forward.ForwardRequest, stream grpc.ServerStreamin defer func() { // Direct Native admission can reject after validation, but still before // execution. Do not turn that known rejection into an unknown mutation. - if err == backpressure.ErrBusy { + if err == backpressure.ErrBusy || err == backpressure.ErrQueueFull { stream.SetTrailer(metadata.Pairs(forwarding.NotStartedTrailer, "true")) } }() diff --git a/internal/service/backpressure_test.go b/internal/service/backpressure_test.go index 27bdd6d..45641b6 100644 --- a/internal/service/backpressure_test.go +++ b/internal/service/backpressure_test.go @@ -217,8 +217,18 @@ func TestSharedStoreAdmissionPreservesOrderingAndNativeCapability(t *testing.T) } command := &sink.Command{Uri: "sink://primary", Method: "GET", Path: "/_search"} countRequest := &sink.CountRequest{Command: command} - if _, err := batching.Count(t.Context(), countRequest); err != backpressure.ErrBusy || native.calls != 0 { - t.Fatal("Native bypassed Store budget") + countContext, cancelCount := context.WithCancel(t.Context()) + countResult := make(chan error, 1) + go func() { _, err := batching.Count(countContext, countRequest); countResult <- err }() + synctest.Wait() + select { + case err := <-countResult: + t.Fatalf("Count did not wait for the shared Store budget: %v", err) + default: + } + cancelCount() + if err := <-countResult; status.Code(err) != codes.Canceled || native.calls != 0 { + t.Fatal("canceled queued Count reached Store") } if permit, _, _ := controller.TryAcquire(); permit != nil { permit.Release() diff --git a/internal/service/query.go b/internal/service/query.go index 820b276..e19a797 100644 --- a/internal/service/query.go +++ b/internal/service/query.go @@ -57,7 +57,7 @@ func (s *Server) query(ctx context.Context, req *sink.QueryRequest, send func(*s } return send(frame) } - ctx, permit, err := s.admission.Admit(ctx) + ctx, permit, err := s.admission.AdmitRead(ctx, req.SizeVT()) if err != nil { return err } @@ -88,7 +88,7 @@ func (s *Server) Count(ctx context.Context, req *sink.CountRequest) (*sink.Count if !ok { return nil, nativeStatus(storage.ErrNativeUnsupported) } - ctx, permit, err := s.admission.Admit(ctx) + ctx, permit, err := s.admission.AdmitRead(ctx, req.SizeVT()) if err != nil { return nil, err } diff --git a/internal/service/query_admission_test.go b/internal/service/query_admission_test.go new file mode 100644 index 0000000..2cce587 --- /dev/null +++ b/internal/service/query_admission_test.go @@ -0,0 +1,61 @@ +package service + +import ( + "context" + "testing" + "testing/synctest" + "time" + + sink "github.com/batchstream/sink-protocol/sink/v1" + "github.com/batchstream/sink/internal/storage/memory" +) + +func TestQueryAndCountWaitForStoreCapacityWithoutNewDeadline(t *testing.T) { + for _, method := range []string{"Query", "Count"} { + t.Run(method, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + backend := &nativeBoundaryStore{Store: memory.New()} + batching, controller := admissionServer(t, backend, 1) + defer batching.Close() + permit, err := controller.Acquire(t.Context()) + if err != nil { + t.Fatal(err) + } + defer permit.Release() + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + command := &sink.Command{Uri: "sink://primary", Method: "GET", Path: "/_search"} + result := make(chan error, 1) + go func() { + if method == "Query" { + request := &sink.QueryRequest{Command: command} + _, err := batching.server.Query(ctx, request) + result <- err + } else { + request := &sink.CountRequest{Command: command} + _, err := batching.Count(ctx, request) + result <- err + } + }() + synctest.Wait() + time.Sleep(time.Hour) + synctest.Wait() + select { + case err := <-result: + t.Fatalf("healthy queued request failed before caller cancellation or capacity: %v", err) + default: + } + if backend.calls != 0 { + t.Fatal("queued request bypassed Store capacity") + } + permit.Release() + if err := <-result; err != nil { + t.Fatal(err) + } + if backend.calls != 1 { + t.Fatalf("backend calls = %d, want exactly one", backend.calls) + } + }) + }) + } +}