Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 44 additions & 8 deletions internal/engine/duck.go
Original file line number Diff line number Diff line change
Expand Up @@ -954,9 +954,21 @@ const rollupChunkWins = 3
// every affected bucket is recomputed whole, so a full window failed on
// every retry and the rollup never advanced. A pass that did no work leaves
// it unchanged: growing on those made it halve, grow and fail again.
//
// ceiling remembers the smallest window a committed streak grew back into and
// then found too large (issue #481). Without it, rollupChunkWins successes
// doubled the window straight back into the size that had just run out of
// memory, so a dense region failed roughly every fourth pass forever. Growth
// now caps strictly below the ceiling and holds at the largest known-safe
// window. Only a failure the window grew into sets the ceiling: an initial or
// a shrinking out-of-memory pass is one episode of pressure, not evidence that
// a grown window is too large, so a transient spike or a deep backfill still
// recovers to the full unbounded window once committed passes grow it back.
type rollupChunk struct {
size int64
wins int
size int64
wins int
ceiling int64 // smallest grown-into window that ran out of memory; 0 = none
grew bool // the current window was reached by growing, not by shrinking
}

func (c *rollupChunk) next(committed bool, err error) {
Expand All @@ -971,14 +983,36 @@ func (c *rollupChunk) next(committed bool, err error) {
return
}
c.wins = 0
if c.size *= 2; c.size >= rollupChunkNanos {
// Hold at the largest known-safe window instead of doubling back into
// a size that already ran out of memory.
next := c.size * 2
if c.ceiling != 0 && next >= c.ceiling {
return
}
c.grew = true
if c.size = next; c.size >= rollupChunkNanos {
c.size = 0
}
case errors.As(err, &engineError) && engineError.Type == duckdb.ErrorTypeOutOfMemory:
// A window a committed streak grew into is now known too large: cap
// future growth strictly below it. A failure at the initial or a
// shrinking window is just ongoing pressure, not a grown window proven
// too large, so it records no ceiling and the window can still recover
// fully once the pressure clears.
if c.grew {
failing := c.size
if failing == 0 {
failing = rollupChunkNanos
}
if c.ceiling == 0 || failing < c.ceiling {
c.ceiling = failing
}
}
if c.size == 0 {
c.size = rollupChunkNanos
}
c.size = max(c.size/2, int64(time.Second))
c.grew = false
c.wins = 0
default:
c.wins = 0
Expand All @@ -991,17 +1025,19 @@ func (d *Duck) RefreshDashboardRollups(ctx context.Context) (int64, error) {
return d.drainServiceRollup(ctx, d.cfg.RollupInterval/2)
}

// drainServiceRollup repeats passes within a tick budget. An admitted pass
// finishes its transaction; memory pressure narrows the next window.
// drainServiceRollup repeats passes within a tick budget while committed work
// remains. An admitted pass finishes its transaction; an out-of-memory pass
// narrows the next window and ends the tick instead of retrying. Retrying a
// failing window back-to-back ran it to the memory limit a dozen times across
// the budget and starved the queries that memory is for (issue #481); ending
// the tick lets the rollup loop's interval space the next attempt out.
func (d *Duck) drainServiceRollup(ctx context.Context, budget time.Duration) (int64, error) {
deadline := time.Now().Add(budget)
var total int64
for {
before := d.serviceChunk.size
n, err := d.refreshServiceRollup(ctx)
total += n
retry := err != nil && d.serviceChunk.size != before
if err != nil && !retry || !retry && !d.servicePending || !time.Now().Before(deadline) {
if err != nil || !d.servicePending || !time.Now().Before(deadline) {
return total, err
}
}
Expand Down
57 changes: 57 additions & 0 deletions internal/engine/rollup_chunk_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -94,3 +94,60 @@ func TestServiceRollupChunkGrowsOnlyAfterCommittedPasses(t *testing.T) {
t.Fatalf("%d out-of-memory passes against %d committed ones", ooms, worked)
}
}

// Issue #481: once a committed streak had grown the window back into a size
// that ran out of memory, doubling straight into that same size made a dense
// region fail roughly every fourth pass forever. The window must remember the
// failing size as a ceiling and never climb back to or past it.
func TestServiceRollupChunkDoesNotGrowBackIntoFailingSize(t *testing.T) {
var c rollupChunk
c.next(false, errRollupOOM) // unbounded -> 30m
c.next(false, errRollupOOM) // 30m -> 15m
failing := 2 * c.size // 30m: the window a committed streak grows into

// Grow back up into the failing window.
for c.size < failing {
for range rollupChunkWins {
c.next(true, nil)
}
}
if c.size != failing {
t.Fatalf("grew to %v, want %v", time.Duration(c.size), time.Duration(failing))
}

// The regrown window runs out of memory: that size is now known too large.
c.next(false, errRollupOOM)

// No committed streak, however long, may climb back to or past it, nor
// snap to the unbounded window above it.
for range 100 {
c.next(true, nil)
if c.size == 0 || c.size >= failing {
t.Fatalf("grew back to %v (failing size %v); the ceiling was ignored",
time.Duration(c.size), time.Duration(failing))
}
}
}

// The remembered ceiling survives the shrink-and-grow cycle: a committed
// streak settles at the largest known-safe window rather than oscillating in
// and out of the size that keeps failing.
func TestServiceRollupChunkSettlesBelowRememberedCeiling(t *testing.T) {
var c rollupChunk
c.next(false, errRollupOOM) // unbounded -> 30m
c.next(false, errRollupOOM) // 30m -> 15m
for range rollupChunkWins {
c.next(true, nil) // 15m -> 30m
}
if time.Duration(c.size) != 30*time.Minute {
t.Fatalf("grew to %v, want 30m before it fails", time.Duration(c.size))
}
c.next(false, errRollupOOM) // 30m fails -> ceiling 30m, window 15m

for range 50 {
c.next(true, nil)
}
if got := time.Duration(c.size); got != 15*time.Minute {
t.Fatalf("settled at %v, want 15m held just below the 30m ceiling", got)
}
}
Loading