From 892d31c7ec3938e93d092ef24a12d861757ccefe Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Sun, 15 Nov 2026 10:42:43 -0800 Subject: [PATCH] fix(engine): back off and cap the service rollup window after an OOM On a memory-limit failure the service rollup narrowed its ingest window toward the one-second floor, but wasted roughly a dozen passes doing it. Two problems compounded: - drainServiceRollup retried an out-of-memory pass back-to-back for the whole tick budget, running the window to the memory limit a dozen times in a row and starving the queries that memory is for. - After rollupChunkWins committed passes the window doubled straight back into the size that had just failed. With no memory of the failing size a dense region failed roughly every fourth pass forever. drainServiceRollup now ends the tick on any error instead of retrying a failing window; the rollup loop's interval (>=1s, 1m by default) spaces the next attempt out, the same cross-tick spacing runBackgroundLoop relies on. The non-OOM path is unchanged: a committed pass with pending work still drains within the budget. rollupChunk gains a ceiling: the smallest window a committed streak grew back into and then found too large. Growth caps strictly below it and holds at the largest known-safe window, so a committed streak never doubles back into a size known to fail. Only a failure the window grew into sets the ceiling -- an initial or shrinking OOM is one episode of pressure, not proof that a grown window is too large -- so a transient spike or a deep backfill still recovers to the full unbounded window once committed passes grow it back. Closes #481 --- internal/engine/duck.go | 52 +++++++++++++++++++++---- internal/engine/rollup_chunk_test.go | 57 ++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+), 8 deletions(-) diff --git a/internal/engine/duck.go b/internal/engine/duck.go index 9876a64d8..ccf5bd8a7 100644 --- a/internal/engine/duck.go +++ b/internal/engine/duck.go @@ -954,9 +954,21 @@ const rollupChunkWins = 3 // every affected bucket is recomputed whole, so a full window failed on // every retry and the rollup never advanced. A pass that did no work leaves // it unchanged: growing on those made it halve, grow and fail again. +// +// ceiling remembers the smallest window a committed streak grew back into and +// then found too large (issue #481). Without it, rollupChunkWins successes +// doubled the window straight back into the size that had just run out of +// memory, so a dense region failed roughly every fourth pass forever. Growth +// now caps strictly below the ceiling and holds at the largest known-safe +// window. Only a failure the window grew into sets the ceiling: an initial or +// a shrinking out-of-memory pass is one episode of pressure, not evidence that +// a grown window is too large, so a transient spike or a deep backfill still +// recovers to the full unbounded window once committed passes grow it back. type rollupChunk struct { - size int64 - wins int + size int64 + wins int + ceiling int64 // smallest grown-into window that ran out of memory; 0 = none + grew bool // the current window was reached by growing, not by shrinking } func (c *rollupChunk) next(committed bool, err error) { @@ -971,14 +983,36 @@ func (c *rollupChunk) next(committed bool, err error) { return } c.wins = 0 - if c.size *= 2; c.size >= rollupChunkNanos { + // Hold at the largest known-safe window instead of doubling back into + // a size that already ran out of memory. + next := c.size * 2 + if c.ceiling != 0 && next >= c.ceiling { + return + } + c.grew = true + if c.size = next; c.size >= rollupChunkNanos { c.size = 0 } case errors.As(err, &engineError) && engineError.Type == duckdb.ErrorTypeOutOfMemory: + // A window a committed streak grew into is now known too large: cap + // future growth strictly below it. A failure at the initial or a + // shrinking window is just ongoing pressure, not a grown window proven + // too large, so it records no ceiling and the window can still recover + // fully once the pressure clears. + if c.grew { + failing := c.size + if failing == 0 { + failing = rollupChunkNanos + } + if c.ceiling == 0 || failing < c.ceiling { + c.ceiling = failing + } + } if c.size == 0 { c.size = rollupChunkNanos } c.size = max(c.size/2, int64(time.Second)) + c.grew = false c.wins = 0 default: c.wins = 0 @@ -991,17 +1025,19 @@ func (d *Duck) RefreshDashboardRollups(ctx context.Context) (int64, error) { return d.drainServiceRollup(ctx, d.cfg.RollupInterval/2) } -// drainServiceRollup repeats passes within a tick budget. An admitted pass -// finishes its transaction; memory pressure narrows the next window. +// drainServiceRollup repeats passes within a tick budget while committed work +// remains. An admitted pass finishes its transaction; an out-of-memory pass +// narrows the next window and ends the tick instead of retrying. Retrying a +// failing window back-to-back ran it to the memory limit a dozen times across +// the budget and starved the queries that memory is for (issue #481); ending +// the tick lets the rollup loop's interval space the next attempt out. func (d *Duck) drainServiceRollup(ctx context.Context, budget time.Duration) (int64, error) { deadline := time.Now().Add(budget) var total int64 for { - before := d.serviceChunk.size n, err := d.refreshServiceRollup(ctx) total += n - retry := err != nil && d.serviceChunk.size != before - if err != nil && !retry || !retry && !d.servicePending || !time.Now().Before(deadline) { + if err != nil || !d.servicePending || !time.Now().Before(deadline) { return total, err } } diff --git a/internal/engine/rollup_chunk_test.go b/internal/engine/rollup_chunk_test.go index d588df421..55aa433f8 100644 --- a/internal/engine/rollup_chunk_test.go +++ b/internal/engine/rollup_chunk_test.go @@ -94,3 +94,60 @@ func TestServiceRollupChunkGrowsOnlyAfterCommittedPasses(t *testing.T) { t.Fatalf("%d out-of-memory passes against %d committed ones", ooms, worked) } } + +// Issue #481: once a committed streak had grown the window back into a size +// that ran out of memory, doubling straight into that same size made a dense +// region fail roughly every fourth pass forever. The window must remember the +// failing size as a ceiling and never climb back to or past it. +func TestServiceRollupChunkDoesNotGrowBackIntoFailingSize(t *testing.T) { + var c rollupChunk + c.next(false, errRollupOOM) // unbounded -> 30m + c.next(false, errRollupOOM) // 30m -> 15m + failing := 2 * c.size // 30m: the window a committed streak grows into + + // Grow back up into the failing window. + for c.size < failing { + for range rollupChunkWins { + c.next(true, nil) + } + } + if c.size != failing { + t.Fatalf("grew to %v, want %v", time.Duration(c.size), time.Duration(failing)) + } + + // The regrown window runs out of memory: that size is now known too large. + c.next(false, errRollupOOM) + + // No committed streak, however long, may climb back to or past it, nor + // snap to the unbounded window above it. + for range 100 { + c.next(true, nil) + if c.size == 0 || c.size >= failing { + t.Fatalf("grew back to %v (failing size %v); the ceiling was ignored", + time.Duration(c.size), time.Duration(failing)) + } + } +} + +// The remembered ceiling survives the shrink-and-grow cycle: a committed +// streak settles at the largest known-safe window rather than oscillating in +// and out of the size that keeps failing. +func TestServiceRollupChunkSettlesBelowRememberedCeiling(t *testing.T) { + var c rollupChunk + c.next(false, errRollupOOM) // unbounded -> 30m + c.next(false, errRollupOOM) // 30m -> 15m + for range rollupChunkWins { + c.next(true, nil) // 15m -> 30m + } + if time.Duration(c.size) != 30*time.Minute { + t.Fatalf("grew to %v, want 30m before it fails", time.Duration(c.size)) + } + c.next(false, errRollupOOM) // 30m fails -> ceiling 30m, window 15m + + for range 50 { + c.next(true, nil) + } + if got := time.Duration(c.size); got != 15*time.Minute { + t.Fatalf("settled at %v, want 15m held just below the 30m ceiling", got) + } +}