From 4914787e38e88d0c8714683356cd98e9dd58337e Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Sun, 15 Nov 2026 16:12:13 -0800 Subject: [PATCH] fix(engine): self-heal a missed rollup-cache checkpoint instead of degrading readiness for an hour The rollup-cache checkpoint folds the write-ahead log back into its file. Its non-FORCE form refuses while a read that began before the last write commit is still open, and the old 250ms retry budget was shorter than almost any such read, so a busy instance reliably missed the fold. The miss folded into the whole maintenance error, which flipped /readyz to degraded until the next maintenance pass -- an hour away by default. A missed fold harms nothing a restart or a page would fix: the WAL stays correct and queryable, just larger until the next successful fold. Separate checkpoint health from maintenance health. A checkpoint-only failure is recorded in new CheckpointHealth state (lastCheckpointOK/At/Err, checkpointFailures, a pending flag) and never touches maintenance health. Retention DELETEs, prune, compaction, and recovery still fail maintenance as before -- those let storage grow unchecked, so they must degrade readiness immediately. The retention DELETEs move into their own gated helper (pruneRollupCache) so the fold can be driven separately. Drive the checkpoint off a pending flag that both the hourly pass and the 10s compaction cycle honor, rather than checkpointing every 10s. When nothing is pending the cycle takes no write gate, so an idle WAL adds no contention with the 60s rollup tick; when a fold is pending it holds both write gates and the background slot exactly as the hourly pass did. The hourly pass marks the fold pending (an interval of rollup commits plus the retention DELETEs always leave work) and attempts it; a miss stays pending and the next ~10s cycle retries, so a transient conflict self-heals in seconds instead of an hour. Only a context cancellation propagates, so the maintenance loop still stops promptly. Raise the retry budget from 250ms to 3s. Both write gates are idle-cheap relative to the 60s rollup tick, so giving an older reader a few seconds to drain lets the common transient conflict clear in place. The exact-message retry gate (checkpointRetryable/checkpointBusyMessage) is unchanged. Report checkpoint separately in readiness via a new checkpoint check. Because the fold retries every 10s, a single recent miss is not degraded; only a fold that keeps failing past several cycles (checkpointDegradeAfterFailures) is. It never reports unhealthy -- the instance serves while the WAL grows -- and a genuinely wedged engine is still caught by the independent telemetry SELECT-1 probe. Closes #482 --- internal/api/health.go | 57 +++++++ internal/api/health_test.go | 33 ++++ internal/engine/checkpoint_retry_test.go | 93 +++++++++- internal/engine/duck.go | 209 +++++++++++++++++++---- internal/engine/duck_test.go | 54 +++++- 5 files changed, 397 insertions(+), 49 deletions(-) diff --git a/internal/api/health.go b/internal/api/health.go index f3babe10..fea8bbb4 100644 --- a/internal/api/health.go +++ b/internal/api/health.go @@ -122,6 +122,7 @@ collect: resp.Checks["data"] = h.checkDataDir() resp.Checks["maintenance"] = h.checkMaintenance() + resp.Checks["checkpoint"] = h.checkCheckpoint() resp.Checks["queries"] = h.checkQueries() if h.oidc != nil { resp.Checks["oidc"] = h.oidc.readiness() @@ -366,6 +367,62 @@ func maintenanceResult(lastOK, lastAt time.Time, lastErr error, consecutiveFailu return res } +// checkpointDegradeAfterFailures is how many rollup-cache checkpoint attempts +// may fail in a row before readiness degrades. A missed fold is retried on the +// 10s compaction cycle, so a transient conflict — an older reader that drains +// within a second or two — clears after a cycle or two and never approaches +// this count; the failure counter resets to zero on the next success. Reaching +// this many failures is roughly a minute of continuous refusal, i.e. a fold +// that is genuinely stuck (a reader that never drains) and whose WAL is growing +// unbounded — worth surfacing. A single recent miss stays ok, so one transient +// conflict does not flap /readyz out of rotation. A genuinely wedged engine is +// caught independently by the telemetry SELECT-1 probe, not here. +const checkpointDegradeAfterFailures = 6 + +// checkCheckpoint surfaces the rollup-cache checkpoint's own health, separately +// from maintenance. A checkpoint folds the write-ahead log back into the cache +// file; the non-FORCE form refuses while a read that began before the last +// write commit is still open. That refusal is transient and self-heals on the +// 10s compaction cycle, so it reports "degraded" (never "unhealthy", and only +// after the fold has kept failing past checkpointDegradeAfterFailures) rather +// than pulling the instance from rotation: ingest and queries work while the +// WAL merely grows, and a restart would not help. +func (h *HealthHandler) checkCheckpoint() CheckResult { + if h.duck == nil { + return CheckResult{ + Status: "unhealthy", + Error: "duckdb not initialized", + } + } + lastOK, lastAt, failures, lastErr := h.duck.CheckpointHealth() + return checkpointResult(lastOK, lastAt, lastErr, failures) +} + +// checkpointResult classifies checkpoint health. It is pure so every branch — +// never attempted, a transient miss, and a persistently failing fold — is +// unit-testable without a running Duck. +func checkpointResult(lastOK, lastAt time.Time, lastErr error, consecutiveFailures int) CheckResult { + res := CheckResult{Status: "ok"} + if !lastAt.IsZero() { + res.UpdatedAt = lastAt.UTC().Format(time.RFC3339) + } + if lastErr == nil { + // The last fold succeeded, or none has been attempted yet. + return res + } + res.Error = lastErr.Error() + res.Detail = fmt.Sprintf("%d consecutive failed checkpoints", consecutiveFailures) + if !lastOK.IsZero() { + res.Detail += "; last clean fold: " + lastOK.UTC().Format(time.RFC3339) + } + // Tolerate a transient miss: the 10s cycle retries, so degrade only once the + // fold has failed enough times in a row that a retry would have cleared it. + if consecutiveFailures >= checkpointDegradeAfterFailures { + res.Status = "degraded" + } + return res +} + func (h *HealthHandler) checkRollups(ctx context.Context) CheckResult { if h.duck == nil { return CheckResult{ diff --git a/internal/api/health_test.go b/internal/api/health_test.go index 41022ac2..976be7d5 100644 --- a/internal/api/health_test.go +++ b/internal/api/health_test.go @@ -387,6 +387,39 @@ func TestMaintenanceStaleThreshold(t *testing.T) { } } +// A single transient checkpoint miss — retried every 10s — must not degrade +// readiness, while a fold that keeps failing past the cadence must. Checkpoint +// health never reports unhealthy: the instance still serves while the WAL grows. +func TestReadinessToleratesTransientCheckpointMiss(t *testing.T) { + now := time.Now() + boom := errors.New("Cannot CHECKPOINT: there are other write transactions active") + cases := []struct { + name string + lastOK, lastAt time.Time + lastErr error + failures int + wantStatus string + }{ + {"never attempted", time.Time{}, time.Time{}, nil, 0, "ok"}, + {"clean fold", now.Add(-30 * time.Second), now.Add(-30 * time.Second), nil, 0, "ok"}, + {"single transient miss", now.Add(-time.Minute), now.Add(-5 * time.Second), boom, 1, "ok"}, + {"a few misses still tolerated", now.Add(-time.Minute), now.Add(-5 * time.Second), boom, 3, "ok"}, + {"persistently failing fold", now.Add(-2 * time.Minute), now.Add(-5 * time.Second), boom, checkpointDegradeAfterFailures, "degraded"}, + {"never succeeded, still failing", time.Time{}, now.Add(-time.Second), boom, checkpointDegradeAfterFailures + 2, "degraded"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + res := checkpointResult(c.lastOK, c.lastAt, c.lastErr, c.failures) + if res.Status != c.wantStatus { + t.Fatalf("status = %q, want %q (detail=%q err=%q)", res.Status, c.wantStatus, res.Detail, res.Error) + } + if res.Status == "unhealthy" { + t.Fatalf("checkpoint must never report unhealthy: %+v", res) + } + }) + } +} + func TestMaintenanceResult(t *testing.T) { now := time.Now() rollupEvery := 60 * time.Second diff --git a/internal/engine/checkpoint_retry_test.go b/internal/engine/checkpoint_retry_test.go index 9c004e32..6ddbab4e 100644 --- a/internal/engine/checkpoint_retry_test.go +++ b/internal/engine/checkpoint_retry_test.go @@ -92,10 +92,16 @@ func TestMaintenanceCheckpointWaitsForOldReaderToDrain(t *testing.T) { if err := <-result; err != nil { t.Fatalf("checkpoint failed after the old reader drained: %v", err) } + // The maintenance pass itself stays clean: the checkpoint retry lives in + // checkpoint health, never maintenance health. lastSuccess, _, failures, err := d.MaintenanceHealth() if lastSuccess.IsZero() || failures != 0 || err != nil { t.Fatalf("successful retry reported maintenance failure: success=%s failures=%d err=%v", lastSuccess, failures, err) } + cpOK, _, cpFailures, cpErr := d.CheckpointHealth() + if cpOK.IsZero() || cpFailures != 0 || cpErr != nil { + t.Fatalf("drained checkpoint not recorded clean in checkpoint health: ok=%s failures=%d err=%v", cpOK, cpFailures, cpErr) + } } func TestMaintenanceCheckpointRetryHonorsCallerCancellation(t *testing.T) { @@ -123,23 +129,32 @@ func TestMaintenanceCheckpointRetryExhaustionIsExplicitAndBounded(t *testing.T) d, repo := newBatchCacheTest(t) rows := checkpointOldReader(t, d, repo) defer rows.Close() - ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second) + // A budget well short of the caller's deadline so the fold exhausts its own + // retry window rather than the whole pass timing out. + ctx, cancel := context.WithTimeout(t.Context(), checkpointRetryBudget+10*time.Second) defer cancel() before := testutil.ToFloat64(metrics.DuckDBCheckpointFailures) started := time.Now() - err := d.runRepositoryMaintenance(ctx) - if !errors.Is(err, context.DeadlineExceeded) || !strings.Contains(err.Error(), "checkpoint retry budget exhausted") || !strings.Contains(err.Error(), "other write transactions active") { - t.Fatalf("exhausted checkpoint was hidden or mislabeled: %v", err) + // The reader never drains, so the fold exhausts its retry budget. Under the + // new contract that is a checkpoint failure, not a maintenance failure: the + // pass returns nil and the miss is retried on the compaction cycle. + if err := d.runRepositoryMaintenance(ctx); err != nil { + t.Fatalf("a missed checkpoint failed the whole maintenance pass: %v", err) } - if elapsed := time.Since(started); elapsed > time.Second { - t.Fatalf("250ms retry budget was not bounded: %s", elapsed) + if elapsed := time.Since(started); elapsed > checkpointRetryBudget+5*time.Second { + t.Fatalf("checkpoint retry budget was not bounded: %s", elapsed) } if after := testutil.ToFloat64(metrics.DuckDBCheckpointFailures); after != before+1 { t.Fatalf("attempts counted as logical checkpoint failures: before=%v after=%v", before, after) } - _, _, failures, healthErr := d.MaintenanceHealth() - if failures != 1 || !errors.Is(healthErr, context.DeadlineExceeded) { - t.Fatalf("checkpoint exhaustion not reflected in maintenance health: failures=%d error=%v", failures, healthErr) + // Maintenance health stays clean; the exhausted fold is recorded only in + // checkpoint health, explicitly and with the pinned native diagnostic. + if _, _, maintFailures, maintErr := d.MaintenanceHealth(); maintFailures != 0 || maintErr != nil { + t.Fatalf("checkpoint miss leaked into maintenance health: failures=%d error=%v", maintFailures, maintErr) + } + _, _, cpFailures, cpErr := d.CheckpointHealth() + if cpFailures != 1 || cpErr == nil || !strings.Contains(cpErr.Error(), "checkpoint retry budget exhausted") || !strings.Contains(cpErr.Error(), "other write transactions active") { + t.Fatalf("exhausted checkpoint not recorded explicitly in checkpoint health: failures=%d error=%v", cpFailures, cpErr) } } @@ -188,6 +203,66 @@ func TestCheckpointRetriesConflictWithoutCountingTransientFailure(t *testing.T) } } +// A checkpoint that fails inside a maintenance pass must leave maintenance +// health clean and be recorded only in checkpoint health, so one missed fold +// cannot degrade /readyz for an hour. +func TestCheckpointMissDoesNotDegradeMaintenanceHealth(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + defer db.Close() + mock.ExpectExec("CHECKPOINT").WillReturnError(errors.New("injected checkpoint failure")) + d := &Duck{DB: db, writeDB: db, repository: emptyRepository(t)} + if err := d.runRepositoryMaintenance(t.Context()); err != nil { + t.Fatalf("a missed checkpoint failed the whole maintenance pass: %v", err) + } + lastOK, _, maintFailures, maintErr := d.MaintenanceHealth() + if lastOK.IsZero() || maintFailures != 0 || maintErr != nil { + t.Fatalf("checkpoint miss leaked into maintenance health: lastOK=%s failures=%d err=%v", lastOK, maintFailures, maintErr) + } + _, cpAt, cpFailures, cpErr := d.CheckpointHealth() + if cpAt.IsZero() || cpFailures != 1 || cpErr == nil { + t.Fatalf("checkpoint miss not recorded in checkpoint health: at=%s failures=%d err=%v", cpAt, cpFailures, cpErr) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} + +// A missed fold stays pending and the next 10s compaction cycle retries it, so +// a transient miss self-heals in seconds instead of waiting an hour. Driving +// runParquetCompactionCycle directly proves the self-heal without a real clock. +func TestMissedCheckpointRetriesOnCompactionCycle(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + defer db.Close() + mock.ExpectExec("CHECKPOINT").WillReturnError(errors.New("injected checkpoint failure")) + mock.ExpectExec("CHECKPOINT").WillReturnResult(sqlmock.NewResult(0, 0)) + d := &Duck{DB: db, writeDB: db, repository: emptyRepository(t)} + // A pass left a fold pending but could not complete it. + d.markCheckpointPending() + if _, err := d.runParquetCompactionCycle(t.Context()); err != nil { + t.Fatalf("compaction cycle surfaced the checkpoint miss as its own error: %v", err) + } + if _, _, cpFailures, cpErr := d.CheckpointHealth(); cpFailures != 1 || cpErr == nil { + t.Fatalf("first cycle did not record the miss: failures=%d err=%v", cpFailures, cpErr) + } + // The miss is still pending, so the next cycle retries and clears it. + if _, err := d.runParquetCompactionCycle(t.Context()); err != nil { + t.Fatal(err) + } + lastOK, _, cpFailures, cpErr := d.CheckpointHealth() + if lastOK.IsZero() || cpFailures != 0 || cpErr != nil { + t.Fatalf("checkpoint did not self-heal on the compaction cycle: lastOK=%s failures=%d err=%v", lastOK, cpFailures, cpErr) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} + func TestCheckpointDoesNotRetryUnrelatedError(t *testing.T) { db, mock, err := sqlmock.New() if err != nil { diff --git a/internal/engine/duck.go b/internal/engine/duck.go index 9876a64d..1c819d37 100644 --- a/internal/engine/duck.go +++ b/internal/engine/duck.go @@ -104,13 +104,36 @@ type Duck struct { // Protected by writeGate; controls periodic version-history retirement. versionRollupPasses uint64 versionRollupFailures map[string]versionRollupFailure - // maintHealthMu guards the maintenance health fields below, which the - // readiness probe reads while the maintenance pass writes them. + // maintHealthMu guards the maintenance and checkpoint health fields below, + // which the readiness probe reads while the maintenance pass and the + // compaction cycle write them. maintHealthMu sync.Mutex lastMaintenanceOK time.Time lastMaintenanceAt time.Time lastMaintenanceErr error maintenanceFailures int + // Checkpoint health is tracked separately from maintenance health on + // purpose. Folding the rollup-cache WAL back into its file (CHECKPOINT) is + // the one maintenance step the engine can refuse on a busy instance — the + // non-FORCE form will not run while a read that began before the last write + // commit is still open — yet a missed fold harms nothing a restart or an + // operator page would fix: the WAL stays correct and queryable, it is just + // larger until the next successful fold. Rolling that miss into the whole + // maintenance error made a single transient refusal flip /readyz to degraded + // until the next hourly pass. Keeping it separate lets the 10s compaction + // cycle retry the fold so a miss self-heals in seconds, while retention, + // prune, and compaction failures — which do let storage grow unchecked — + // still degrade maintenance readiness immediately. + // + // checkpointPending means the WAL has unfolded work and the last attempt did + // not complete; both the hourly pass and the 10s cycle honor it, and a + // success clears it. When it is false the cycle takes no write gate at all, + // so an idle WAL adds no contention with the 60s rollup tick. + checkpointPending bool + lastCheckpointOK time.Time + lastCheckpointAt time.Time + lastCheckpointErr error + checkpointFailures int } // MaintenanceHealth reports the maintenance loop's own health: when it last @@ -127,6 +150,94 @@ func (d *Duck) MaintenanceHealth() (lastOK, lastAt time.Time, consecutiveFailure return d.lastMaintenanceOK, d.lastMaintenanceAt, d.maintenanceFailures, d.lastMaintenanceErr } +// CheckpointHealth reports the rollup-cache checkpoint's own health, mirroring +// MaintenanceHealth: when a fold last succeeded, when one was last attempted, +// how many have failed in a row, and the most recent error (nil after a clean +// fold). It is surfaced via /readyz separately from maintenance so a transient +// refusal — retried on the 10s compaction cycle — does not degrade readiness, +// while a fold that keeps failing well past that cadence still does. A zero +// lastAt means no checkpoint has been attempted since the process started. +func (d *Duck) CheckpointHealth() (lastOK, lastAt time.Time, consecutiveFailures int, lastErr error) { + d.maintHealthMu.Lock() + defer d.maintHealthMu.Unlock() + return d.lastCheckpointOK, d.lastCheckpointAt, d.checkpointFailures, d.lastCheckpointErr +} + +// markCheckpointPending records that the rollup-cache WAL has work to fold, so +// the next hourly pass or 10s compaction cycle attempts a checkpoint. +func (d *Duck) markCheckpointPending() { + d.maintHealthMu.Lock() + d.checkpointPending = true + d.maintHealthMu.Unlock() +} + +// recordCheckpointHealth folds one checkpoint outcome into checkpoint health. +// A success clears the pending flag; a failure leaves it set so a later cycle +// retries, and never touches maintenance health. +func (d *Duck) recordCheckpointHealth(err error) { + finished := time.Now() + d.maintHealthMu.Lock() + defer d.maintHealthMu.Unlock() + d.lastCheckpointAt = finished + d.lastCheckpointErr = err + if err == nil { + d.lastCheckpointOK = finished + d.checkpointFailures = 0 + d.checkpointPending = false + return + } + d.checkpointFailures++ +} + +// runPendingCheckpoint folds the rollup-cache WAL back into its file when a +// checkpoint is pending, recording the outcome in checkpoint health. Both the +// hourly maintenance pass and the 10s compaction cycle call it, so a fold the +// hourly pass could not complete self-heals within one cycle (~10s) instead of +// waiting a full maintenance interval. +// +// When nothing is pending it reads a flag and returns, taking no write gate, so +// the 10s cycle adds no contention on an idle WAL. When a fold is pending it +// holds both write gates and the background slot — the same critical section +// the hourly pass used — which is idle-cheap relative to the 60s rollup tick. +// +// It returns a non-nil error only when the caller's context was cancelled or +// expired, so the maintenance pass and loop stop promptly; a checkpoint refusal +// or other checkpoint error is recorded in checkpoint health and retried later, +// not surfaced to the caller as a maintenance failure. +func (d *Duck) runPendingCheckpoint(ctx context.Context) error { + d.maintHealthMu.Lock() + pending := d.checkpointPending + d.maintHealthMu.Unlock() + if !pending { + return nil + } + err := func() error { + unlock, err := d.writeGate.LockContext(ctx, writegate.Maintenance) + if err != nil { + return err + } + defer unlock() + unlockCache, err := d.cacheGate.LockContext(ctx, writegate.Maintenance) + if err != nil { + return err + } + defer unlockCache() + background, err := d.lockBackground(ctx) + if err != nil { + return err + } + defer background() + return d.checkpoint(ctx) + }() + if ctx.Err() != nil { + // The caller gave up (deadline or shutdown). This is not a checkpoint + // failure: leave it pending for a later cycle and surface cancellation. + return ctx.Err() + } + d.recordCheckpointHealth(err) + return nil +} + const ( // Rollup progress keys in rollup_state. readCacheVersion drops that table // with the rollups, so a version change also restarts their backfill; the @@ -565,9 +676,9 @@ func (d *Duck) writer() *sql.DB { return d.writeDB } // that took. Checkpoint work can contend with reads, so it is a candidate // explanation for a stalled query and has to be measurable. // -// Maintenance holds both write gates throughout this operation. The pinned -// engine can still return its transaction-conflict diagnostic when an older -// reader remains open after a newer write commits. Give that reader a bounded +// runPendingCheckpoint holds both write gates throughout this operation. The +// pinned engine can still return its transaction-conflict diagnostic when an +// older reader remains open after a newer write commits. Give that reader a bounded // opportunity to drain; keep one writer connection and admit no new writers. // Other errors are not retried. Metrics describe the logical operation: one // failure on exhaustion/error, or one successful duration including retry wait. @@ -631,7 +742,12 @@ func (d *Duck) checkpoint(ctx context.Context) (retErr error) { } const ( - checkpointRetryBudget = 250 * time.Millisecond + // A few seconds, not milliseconds: the old 250ms budget was shorter than any + // read that began before the last write commit and was still running, so a + // busy instance reliably missed the fold. Both write gates are idle-cheap + // relative to the 60s rollup tick, so giving an older reader a few seconds to + // drain costs little and lets the common transient conflict clear in place. + checkpointRetryBudget = 3 * time.Second checkpointRetryDelay = 10 * time.Millisecond checkpointBusyMessage = "TransactionContext Error: Cannot CHECKPOINT: there are other write transactions active. Try using FORCE CHECKPOINT to wait until all active transactions are finished" ) @@ -870,6 +986,13 @@ func (d *Duck) runParquetCompactionCycle(ctx context.Context) (int, error) { result = metrics.TelemetrySuccess } metrics.RecordTelemetryOperation(metrics.TelemetryCompaction, result, time.Since(started).Seconds()) + // Retry a checkpoint the hourly pass (or an earlier cycle) could not + // complete, so a missed WAL fold self-heals within one ~10s cycle instead of + // waiting for the next hourly maintenance pass. This takes no write gate when + // nothing is pending, and its outcome lives in checkpoint health — never + // folded into the compaction result this returns. The cycle has its own + // ctx.Done() path, so the cancellation signal is discarded here. + _ = d.runPendingCheckpoint(ctx) return compacted, err } @@ -894,33 +1017,12 @@ func (d *Duck) runRepositoryMaintenance(ctx context.Context) error { } metrics.RecordTelemetryOperation(metrics.TelemetryCompaction, compactResult, time.Since(compactStart).Seconds()) pruneErr := errors.Join(recoveryErr, cleanupErr, parquetErr, compactErr) - cacheErr := func() error { - unlock, err := d.writeGate.LockContext(ctx, writegate.Maintenance) - if err != nil { - return err - } - defer unlock() - unlockCache, err := d.cacheGate.LockContext(ctx, writegate.Maintenance) - if err != nil { - return err - } - defer unlockCache() - background, err := d.lockBackground(ctx) - if err != nil { - return err - } - defer background() - var errs []error - if d.cfg.RetentionDays > 0 { - for _, table := range []string{"service_rollup", "span_rollup", "log_rollup", "endpoint_rollup", "receiver_rollup", "resource_rollup", "receiver_entity_rollup", "edge_rollup"} { - if _, err := d.writer().ExecContext(ctx, fmt.Sprintf("DELETE FROM %s WHERE bucket < (now() - INTERVAL %d DAY)::TIMESTAMP_NS", table, d.cfg.RetentionDays)); err != nil { - errs = append(errs, fmt.Errorf("prune %s: %w", table, err)) - } - } - } - return errors.Join(errors.Join(errs...), d.checkpoint(ctx)) - }() - err := errors.Join(pruneErr, cacheErr) + // Retention DELETEs on the rollup-cache tables are maintenance work: they + // reclaim storage, so a failure here still degrades maintenance readiness. + // They hold both write gates and the background slot, as the checkpoint + // does, but are kept separate from it so the fold can be retried on its own. + retentionErr := d.pruneRollupCache(ctx) + err := errors.Join(pruneErr, retentionErr) maintenanceResult := metrics.TelemetrySuccess if err != nil { maintenanceResult = metrics.TelemetryError @@ -940,9 +1042,50 @@ func (d *Duck) runRepositoryMaintenance(ctx context.Context) error { if err == nil { slog.Info("telemetry maintenance complete", "duration", time.Since(start)) } + // Fold the rollup-cache WAL back into its file. An interval of rollup + // commits plus the retention DELETEs above have grown it, so there is always + // work to fold; mark it pending and attempt it now. A refusal from a + // still-open older reader is recorded in checkpoint health (not maintenance) + // and retried on the 10s compaction cycle, so it does not fail this pass. + // Only a context cancellation propagates, so the loop stops promptly. + d.markCheckpointPending() + if cancelErr := d.runPendingCheckpoint(ctx); cancelErr != nil { + return errors.Join(err, cancelErr) + } return err } +// pruneRollupCache deletes retention-expired rows from the rollup-cache tables +// under both write gates and the background slot. It is a no-op taking no gate +// when retention is disabled. +func (d *Duck) pruneRollupCache(ctx context.Context) error { + if d.cfg.RetentionDays <= 0 { + return nil + } + unlock, err := d.writeGate.LockContext(ctx, writegate.Maintenance) + if err != nil { + return err + } + defer unlock() + unlockCache, err := d.cacheGate.LockContext(ctx, writegate.Maintenance) + if err != nil { + return err + } + defer unlockCache() + background, err := d.lockBackground(ctx) + if err != nil { + return err + } + defer background() + var errs []error + for _, table := range []string{"service_rollup", "span_rollup", "log_rollup", "endpoint_rollup", "receiver_rollup", "resource_rollup", "receiver_entity_rollup", "edge_rollup"} { + if _, err := d.writer().ExecContext(ctx, fmt.Sprintf("DELETE FROM %s WHERE bucket < (now() - INTERVAL %d DAY)::TIMESTAMP_NS", table, d.cfg.RetentionDays)); err != nil { + errs = append(errs, fmt.Errorf("prune %s: %w", table, err)) + } + } + return errors.Join(errs...) +} + // rollupChunkWins is how many committed passes in a row grow a narrowed // service rollup window. const rollupChunkWins = 3 diff --git a/internal/engine/duck_test.go b/internal/engine/duck_test.go index c71537ed..a20592bc 100644 --- a/internal/engine/duck_test.go +++ b/internal/engine/duck_test.go @@ -7,6 +7,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "testing" "time" @@ -401,7 +402,10 @@ func TestMaintenanceRetriesRetiredDirectoryCleanup(t *testing.T) { } } -func TestMaintenanceTracksConsecutiveFailures(t *testing.T) { +// Consecutive checkpoint failures accrue in checkpoint health, not maintenance +// health, and a later success clears them. A missed fold must never fail the +// maintenance pass, since it self-heals on the compaction cycle. +func TestCheckpointTracksConsecutiveFailures(t *testing.T) { db, mock, err := sqlmock.New() if err != nil { t.Fatal(err) @@ -413,20 +417,56 @@ func TestMaintenanceTracksConsecutiveFailures(t *testing.T) { mock.ExpectExec("CHECKPOINT").WillReturnResult(sqlmock.NewResult(0, 0)) d := &Duck{DB: db, writeDB: db, repository: emptyRepository(t)} for range 3 { - if err := d.runRepositoryMaintenance(context.Background()); err == nil { - t.Fatal("maintenance succeeded despite checkpoint failure") + if err := d.runRepositoryMaintenance(context.Background()); err != nil { + t.Fatalf("checkpoint failure degraded the maintenance pass: %v", err) } } - _, _, failures, _ := d.MaintenanceHealth() + if _, _, maintFailures, maintErr := d.MaintenanceHealth(); maintFailures != 0 || maintErr != nil { + t.Fatalf("checkpoint failures leaked into maintenance health: failures=%d err=%v", maintFailures, maintErr) + } + _, _, failures, _ := d.CheckpointHealth() if failures != 3 { - t.Fatalf("consecutive failures = %d, want 3", failures) + t.Fatalf("consecutive checkpoint failures = %d, want 3", failures) } if err := d.runRepositoryMaintenance(context.Background()); err != nil { t.Fatal(err) } - _, _, failures, _ = d.MaintenanceHealth() + _, _, failures, _ = d.CheckpointHealth() if failures != 0 { - t.Fatalf("consecutive failures after recovery = %d, want 0", failures) + t.Fatalf("consecutive checkpoint failures after recovery = %d, want 0", failures) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} + +// Retention DELETEs on the rollup cache are real maintenance work: a failure +// there still degrades maintenance health and consecutive-failure tracking, +// while leaving checkpoint health clean when the fold itself succeeds. +func TestRetentionFailureDegradesMaintenance(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + defer db.Close() + for _, table := range []string{"service_rollup", "span_rollup", "log_rollup", "endpoint_rollup", "receiver_rollup", "resource_rollup", "receiver_entity_rollup", "edge_rollup"} { + e := mock.ExpectExec("DELETE FROM " + table) + if table == "service_rollup" { + e.WillReturnError(errors.New("injected retention failure")) + } else { + e.WillReturnResult(sqlmock.NewResult(0, 0)) + } + } + mock.ExpectExec("CHECKPOINT").WillReturnResult(sqlmock.NewResult(0, 0)) + d := &Duck{DB: db, writeDB: db, repository: emptyRepository(t), cfg: config.Config{RetentionDays: 1}} + if err := d.runRepositoryMaintenance(context.Background()); err == nil || !strings.Contains(err.Error(), "injected retention failure") { + t.Fatalf("retention failure did not fail maintenance: %v", err) + } + if _, _, maintFailures, maintErr := d.MaintenanceHealth(); maintFailures != 1 || maintErr == nil { + t.Fatalf("retention failure not tracked in maintenance health: failures=%d err=%v", maintFailures, maintErr) + } + if _, _, cpFailures, cpErr := d.CheckpointHealth(); cpFailures != 0 || cpErr != nil { + t.Fatalf("retention failure leaked into checkpoint health: failures=%d err=%v", cpFailures, cpErr) } if err := mock.ExpectationsWereMet(); err != nil { t.Fatal(err)