diff --git a/internal/api/health.go b/internal/api/health.go index f3babe10..fea8bbb4 100644 --- a/internal/api/health.go +++ b/internal/api/health.go @@ -122,6 +122,7 @@ collect: resp.Checks["data"] = h.checkDataDir() resp.Checks["maintenance"] = h.checkMaintenance() + resp.Checks["checkpoint"] = h.checkCheckpoint() resp.Checks["queries"] = h.checkQueries() if h.oidc != nil { resp.Checks["oidc"] = h.oidc.readiness() @@ -366,6 +367,62 @@ func maintenanceResult(lastOK, lastAt time.Time, lastErr error, consecutiveFailu return res } +// checkpointDegradeAfterFailures is how many rollup-cache checkpoint attempts +// may fail in a row before readiness degrades. A missed fold is retried on the +// 10s compaction cycle, so a transient conflict — an older reader that drains +// within a second or two — clears after a cycle or two and never approaches +// this count; the failure counter resets to zero on the next success. Reaching +// this many failures is roughly a minute of continuous refusal, i.e. a fold +// that is genuinely stuck (a reader that never drains) and whose WAL is growing +// unbounded — worth surfacing. A single recent miss stays ok, so one transient +// conflict does not flap /readyz out of rotation. A genuinely wedged engine is +// caught independently by the telemetry SELECT-1 probe, not here. +const checkpointDegradeAfterFailures = 6 + +// checkCheckpoint surfaces the rollup-cache checkpoint's own health, separately +// from maintenance. A checkpoint folds the write-ahead log back into the cache +// file; the non-FORCE form refuses while a read that began before the last +// write commit is still open. That refusal is transient and self-heals on the +// 10s compaction cycle, so it reports "degraded" (never "unhealthy", and only +// after the fold has kept failing past checkpointDegradeAfterFailures) rather +// than pulling the instance from rotation: ingest and queries work while the +// WAL merely grows, and a restart would not help. +func (h *HealthHandler) checkCheckpoint() CheckResult { + if h.duck == nil { + return CheckResult{ + Status: "unhealthy", + Error: "duckdb not initialized", + } + } + lastOK, lastAt, failures, lastErr := h.duck.CheckpointHealth() + return checkpointResult(lastOK, lastAt, lastErr, failures) +} + +// checkpointResult classifies checkpoint health. It is pure so every branch — +// never attempted, a transient miss, and a persistently failing fold — is +// unit-testable without a running Duck. +func checkpointResult(lastOK, lastAt time.Time, lastErr error, consecutiveFailures int) CheckResult { + res := CheckResult{Status: "ok"} + if !lastAt.IsZero() { + res.UpdatedAt = lastAt.UTC().Format(time.RFC3339) + } + if lastErr == nil { + // The last fold succeeded, or none has been attempted yet. + return res + } + res.Error = lastErr.Error() + res.Detail = fmt.Sprintf("%d consecutive failed checkpoints", consecutiveFailures) + if !lastOK.IsZero() { + res.Detail += "; last clean fold: " + lastOK.UTC().Format(time.RFC3339) + } + // Tolerate a transient miss: the 10s cycle retries, so degrade only once the + // fold has failed enough times in a row that a retry would have cleared it. + if consecutiveFailures >= checkpointDegradeAfterFailures { + res.Status = "degraded" + } + return res +} + func (h *HealthHandler) checkRollups(ctx context.Context) CheckResult { if h.duck == nil { return CheckResult{ diff --git a/internal/api/health_test.go b/internal/api/health_test.go index 41022ac2..976be7d5 100644 --- a/internal/api/health_test.go +++ b/internal/api/health_test.go @@ -387,6 +387,39 @@ func TestMaintenanceStaleThreshold(t *testing.T) { } } +// A single transient checkpoint miss — retried every 10s — must not degrade +// readiness, while a fold that keeps failing past the cadence must. Checkpoint +// health never reports unhealthy: the instance still serves while the WAL grows. +func TestReadinessToleratesTransientCheckpointMiss(t *testing.T) { + now := time.Now() + boom := errors.New("Cannot CHECKPOINT: there are other write transactions active") + cases := []struct { + name string + lastOK, lastAt time.Time + lastErr error + failures int + wantStatus string + }{ + {"never attempted", time.Time{}, time.Time{}, nil, 0, "ok"}, + {"clean fold", now.Add(-30 * time.Second), now.Add(-30 * time.Second), nil, 0, "ok"}, + {"single transient miss", now.Add(-time.Minute), now.Add(-5 * time.Second), boom, 1, "ok"}, + {"a few misses still tolerated", now.Add(-time.Minute), now.Add(-5 * time.Second), boom, 3, "ok"}, + {"persistently failing fold", now.Add(-2 * time.Minute), now.Add(-5 * time.Second), boom, checkpointDegradeAfterFailures, "degraded"}, + {"never succeeded, still failing", time.Time{}, now.Add(-time.Second), boom, checkpointDegradeAfterFailures + 2, "degraded"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + res := checkpointResult(c.lastOK, c.lastAt, c.lastErr, c.failures) + if res.Status != c.wantStatus { + t.Fatalf("status = %q, want %q (detail=%q err=%q)", res.Status, c.wantStatus, res.Detail, res.Error) + } + if res.Status == "unhealthy" { + t.Fatalf("checkpoint must never report unhealthy: %+v", res) + } + }) + } +} + func TestMaintenanceResult(t *testing.T) { now := time.Now() rollupEvery := 60 * time.Second diff --git a/internal/engine/checkpoint_retry_test.go b/internal/engine/checkpoint_retry_test.go index 9c004e32..6ddbab4e 100644 --- a/internal/engine/checkpoint_retry_test.go +++ b/internal/engine/checkpoint_retry_test.go @@ -92,10 +92,16 @@ func TestMaintenanceCheckpointWaitsForOldReaderToDrain(t *testing.T) { if err := <-result; err != nil { t.Fatalf("checkpoint failed after the old reader drained: %v", err) } + // The maintenance pass itself stays clean: the checkpoint retry lives in + // checkpoint health, never maintenance health. lastSuccess, _, failures, err := d.MaintenanceHealth() if lastSuccess.IsZero() || failures != 0 || err != nil { t.Fatalf("successful retry reported maintenance failure: success=%s failures=%d err=%v", lastSuccess, failures, err) } + cpOK, _, cpFailures, cpErr := d.CheckpointHealth() + if cpOK.IsZero() || cpFailures != 0 || cpErr != nil { + t.Fatalf("drained checkpoint not recorded clean in checkpoint health: ok=%s failures=%d err=%v", cpOK, cpFailures, cpErr) + } } func TestMaintenanceCheckpointRetryHonorsCallerCancellation(t *testing.T) { @@ -123,23 +129,32 @@ func TestMaintenanceCheckpointRetryExhaustionIsExplicitAndBounded(t *testing.T) d, repo := newBatchCacheTest(t) rows := checkpointOldReader(t, d, repo) defer rows.Close() - ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second) + // A budget well short of the caller's deadline so the fold exhausts its own + // retry window rather than the whole pass timing out. + ctx, cancel := context.WithTimeout(t.Context(), checkpointRetryBudget+10*time.Second) defer cancel() before := testutil.ToFloat64(metrics.DuckDBCheckpointFailures) started := time.Now() - err := d.runRepositoryMaintenance(ctx) - if !errors.Is(err, context.DeadlineExceeded) || !strings.Contains(err.Error(), "checkpoint retry budget exhausted") || !strings.Contains(err.Error(), "other write transactions active") { - t.Fatalf("exhausted checkpoint was hidden or mislabeled: %v", err) + // The reader never drains, so the fold exhausts its retry budget. Under the + // new contract that is a checkpoint failure, not a maintenance failure: the + // pass returns nil and the miss is retried on the compaction cycle. + if err := d.runRepositoryMaintenance(ctx); err != nil { + t.Fatalf("a missed checkpoint failed the whole maintenance pass: %v", err) } - if elapsed := time.Since(started); elapsed > time.Second { - t.Fatalf("250ms retry budget was not bounded: %s", elapsed) + if elapsed := time.Since(started); elapsed > checkpointRetryBudget+5*time.Second { + t.Fatalf("checkpoint retry budget was not bounded: %s", elapsed) } if after := testutil.ToFloat64(metrics.DuckDBCheckpointFailures); after != before+1 { t.Fatalf("attempts counted as logical checkpoint failures: before=%v after=%v", before, after) } - _, _, failures, healthErr := d.MaintenanceHealth() - if failures != 1 || !errors.Is(healthErr, context.DeadlineExceeded) { - t.Fatalf("checkpoint exhaustion not reflected in maintenance health: failures=%d error=%v", failures, healthErr) + // Maintenance health stays clean; the exhausted fold is recorded only in + // checkpoint health, explicitly and with the pinned native diagnostic. + if _, _, maintFailures, maintErr := d.MaintenanceHealth(); maintFailures != 0 || maintErr != nil { + t.Fatalf("checkpoint miss leaked into maintenance health: failures=%d error=%v", maintFailures, maintErr) + } + _, _, cpFailures, cpErr := d.CheckpointHealth() + if cpFailures != 1 || cpErr == nil || !strings.Contains(cpErr.Error(), "checkpoint retry budget exhausted") || !strings.Contains(cpErr.Error(), "other write transactions active") { + t.Fatalf("exhausted checkpoint not recorded explicitly in checkpoint health: failures=%d error=%v", cpFailures, cpErr) } } @@ -188,6 +203,66 @@ func TestCheckpointRetriesConflictWithoutCountingTransientFailure(t *testing.T) } } +// A checkpoint that fails inside a maintenance pass must leave maintenance +// health clean and be recorded only in checkpoint health, so one missed fold +// cannot degrade /readyz for an hour. +func TestCheckpointMissDoesNotDegradeMaintenanceHealth(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + defer db.Close() + mock.ExpectExec("CHECKPOINT").WillReturnError(errors.New("injected checkpoint failure")) + d := &Duck{DB: db, writeDB: db, repository: emptyRepository(t)} + if err := d.runRepositoryMaintenance(t.Context()); err != nil { + t.Fatalf("a missed checkpoint failed the whole maintenance pass: %v", err) + } + lastOK, _, maintFailures, maintErr := d.MaintenanceHealth() + if lastOK.IsZero() || maintFailures != 0 || maintErr != nil { + t.Fatalf("checkpoint miss leaked into maintenance health: lastOK=%s failures=%d err=%v", lastOK, maintFailures, maintErr) + } + _, cpAt, cpFailures, cpErr := d.CheckpointHealth() + if cpAt.IsZero() || cpFailures != 1 || cpErr == nil { + t.Fatalf("checkpoint miss not recorded in checkpoint health: at=%s failures=%d err=%v", cpAt, cpFailures, cpErr) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} + +// A missed fold stays pending and the next 10s compaction cycle retries it, so +// a transient miss self-heals in seconds instead of waiting an hour. Driving +// runParquetCompactionCycle directly proves the self-heal without a real clock. +func TestMissedCheckpointRetriesOnCompactionCycle(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + defer db.Close() + mock.ExpectExec("CHECKPOINT").WillReturnError(errors.New("injected checkpoint failure")) + mock.ExpectExec("CHECKPOINT").WillReturnResult(sqlmock.NewResult(0, 0)) + d := &Duck{DB: db, writeDB: db, repository: emptyRepository(t)} + // A pass left a fold pending but could not complete it. + d.markCheckpointPending() + if _, err := d.runParquetCompactionCycle(t.Context()); err != nil { + t.Fatalf("compaction cycle surfaced the checkpoint miss as its own error: %v", err) + } + if _, _, cpFailures, cpErr := d.CheckpointHealth(); cpFailures != 1 || cpErr == nil { + t.Fatalf("first cycle did not record the miss: failures=%d err=%v", cpFailures, cpErr) + } + // The miss is still pending, so the next cycle retries and clears it. + if _, err := d.runParquetCompactionCycle(t.Context()); err != nil { + t.Fatal(err) + } + lastOK, _, cpFailures, cpErr := d.CheckpointHealth() + if lastOK.IsZero() || cpFailures != 0 || cpErr != nil { + t.Fatalf("checkpoint did not self-heal on the compaction cycle: lastOK=%s failures=%d err=%v", lastOK, cpFailures, cpErr) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} + func TestCheckpointDoesNotRetryUnrelatedError(t *testing.T) { db, mock, err := sqlmock.New() if err != nil { diff --git a/internal/engine/duck.go b/internal/engine/duck.go index 9876a64d..1c819d37 100644 --- a/internal/engine/duck.go +++ b/internal/engine/duck.go @@ -104,13 +104,36 @@ type Duck struct { // Protected by writeGate; controls periodic version-history retirement. versionRollupPasses uint64 versionRollupFailures map[string]versionRollupFailure - // maintHealthMu guards the maintenance health fields below, which the - // readiness probe reads while the maintenance pass writes them. + // maintHealthMu guards the maintenance and checkpoint health fields below, + // which the readiness probe reads while the maintenance pass and the + // compaction cycle write them. maintHealthMu sync.Mutex lastMaintenanceOK time.Time lastMaintenanceAt time.Time lastMaintenanceErr error maintenanceFailures int + // Checkpoint health is tracked separately from maintenance health on + // purpose. Folding the rollup-cache WAL back into its file (CHECKPOINT) is + // the one maintenance step the engine can refuse on a busy instance — the + // non-FORCE form will not run while a read that began before the last write + // commit is still open — yet a missed fold harms nothing a restart or an + // operator page would fix: the WAL stays correct and queryable, it is just + // larger until the next successful fold. Rolling that miss into the whole + // maintenance error made a single transient refusal flip /readyz to degraded + // until the next hourly pass. Keeping it separate lets the 10s compaction + // cycle retry the fold so a miss self-heals in seconds, while retention, + // prune, and compaction failures — which do let storage grow unchecked — + // still degrade maintenance readiness immediately. + // + // checkpointPending means the WAL has unfolded work and the last attempt did + // not complete; both the hourly pass and the 10s cycle honor it, and a + // success clears it. When it is false the cycle takes no write gate at all, + // so an idle WAL adds no contention with the 60s rollup tick. + checkpointPending bool + lastCheckpointOK time.Time + lastCheckpointAt time.Time + lastCheckpointErr error + checkpointFailures int } // MaintenanceHealth reports the maintenance loop's own health: when it last @@ -127,6 +150,94 @@ func (d *Duck) MaintenanceHealth() (lastOK, lastAt time.Time, consecutiveFailure return d.lastMaintenanceOK, d.lastMaintenanceAt, d.maintenanceFailures, d.lastMaintenanceErr } +// CheckpointHealth reports the rollup-cache checkpoint's own health, mirroring +// MaintenanceHealth: when a fold last succeeded, when one was last attempted, +// how many have failed in a row, and the most recent error (nil after a clean +// fold). It is surfaced via /readyz separately from maintenance so a transient +// refusal — retried on the 10s compaction cycle — does not degrade readiness, +// while a fold that keeps failing well past that cadence still does. A zero +// lastAt means no checkpoint has been attempted since the process started. +func (d *Duck) CheckpointHealth() (lastOK, lastAt time.Time, consecutiveFailures int, lastErr error) { + d.maintHealthMu.Lock() + defer d.maintHealthMu.Unlock() + return d.lastCheckpointOK, d.lastCheckpointAt, d.checkpointFailures, d.lastCheckpointErr +} + +// markCheckpointPending records that the rollup-cache WAL has work to fold, so +// the next hourly pass or 10s compaction cycle attempts a checkpoint. +func (d *Duck) markCheckpointPending() { + d.maintHealthMu.Lock() + d.checkpointPending = true + d.maintHealthMu.Unlock() +} + +// recordCheckpointHealth folds one checkpoint outcome into checkpoint health. +// A success clears the pending flag; a failure leaves it set so a later cycle +// retries, and never touches maintenance health. +func (d *Duck) recordCheckpointHealth(err error) { + finished := time.Now() + d.maintHealthMu.Lock() + defer d.maintHealthMu.Unlock() + d.lastCheckpointAt = finished + d.lastCheckpointErr = err + if err == nil { + d.lastCheckpointOK = finished + d.checkpointFailures = 0 + d.checkpointPending = false + return + } + d.checkpointFailures++ +} + +// runPendingCheckpoint folds the rollup-cache WAL back into its file when a +// checkpoint is pending, recording the outcome in checkpoint health. Both the +// hourly maintenance pass and the 10s compaction cycle call it, so a fold the +// hourly pass could not complete self-heals within one cycle (~10s) instead of +// waiting a full maintenance interval. +// +// When nothing is pending it reads a flag and returns, taking no write gate, so +// the 10s cycle adds no contention on an idle WAL. When a fold is pending it +// holds both write gates and the background slot — the same critical section +// the hourly pass used — which is idle-cheap relative to the 60s rollup tick. +// +// It returns a non-nil error only when the caller's context was cancelled or +// expired, so the maintenance pass and loop stop promptly; a checkpoint refusal +// or other checkpoint error is recorded in checkpoint health and retried later, +// not surfaced to the caller as a maintenance failure. +func (d *Duck) runPendingCheckpoint(ctx context.Context) error { + d.maintHealthMu.Lock() + pending := d.checkpointPending + d.maintHealthMu.Unlock() + if !pending { + return nil + } + err := func() error { + unlock, err := d.writeGate.LockContext(ctx, writegate.Maintenance) + if err != nil { + return err + } + defer unlock() + unlockCache, err := d.cacheGate.LockContext(ctx, writegate.Maintenance) + if err != nil { + return err + } + defer unlockCache() + background, err := d.lockBackground(ctx) + if err != nil { + return err + } + defer background() + return d.checkpoint(ctx) + }() + if ctx.Err() != nil { + // The caller gave up (deadline or shutdown). This is not a checkpoint + // failure: leave it pending for a later cycle and surface cancellation. + return ctx.Err() + } + d.recordCheckpointHealth(err) + return nil +} + const ( // Rollup progress keys in rollup_state. readCacheVersion drops that table // with the rollups, so a version change also restarts their backfill; the @@ -565,9 +676,9 @@ func (d *Duck) writer() *sql.DB { return d.writeDB } // that took. Checkpoint work can contend with reads, so it is a candidate // explanation for a stalled query and has to be measurable. // -// Maintenance holds both write gates throughout this operation. The pinned -// engine can still return its transaction-conflict diagnostic when an older -// reader remains open after a newer write commits. Give that reader a bounded +// runPendingCheckpoint holds both write gates throughout this operation. The +// pinned engine can still return its transaction-conflict diagnostic when an +// older reader remains open after a newer write commits. Give that reader a bounded // opportunity to drain; keep one writer connection and admit no new writers. // Other errors are not retried. Metrics describe the logical operation: one // failure on exhaustion/error, or one successful duration including retry wait. @@ -631,7 +742,12 @@ func (d *Duck) checkpoint(ctx context.Context) (retErr error) { } const ( - checkpointRetryBudget = 250 * time.Millisecond + // A few seconds, not milliseconds: the old 250ms budget was shorter than any + // read that began before the last write commit and was still running, so a + // busy instance reliably missed the fold. Both write gates are idle-cheap + // relative to the 60s rollup tick, so giving an older reader a few seconds to + // drain costs little and lets the common transient conflict clear in place. + checkpointRetryBudget = 3 * time.Second checkpointRetryDelay = 10 * time.Millisecond checkpointBusyMessage = "TransactionContext Error: Cannot CHECKPOINT: there are other write transactions active. Try using FORCE CHECKPOINT to wait until all active transactions are finished" ) @@ -870,6 +986,13 @@ func (d *Duck) runParquetCompactionCycle(ctx context.Context) (int, error) { result = metrics.TelemetrySuccess } metrics.RecordTelemetryOperation(metrics.TelemetryCompaction, result, time.Since(started).Seconds()) + // Retry a checkpoint the hourly pass (or an earlier cycle) could not + // complete, so a missed WAL fold self-heals within one ~10s cycle instead of + // waiting for the next hourly maintenance pass. This takes no write gate when + // nothing is pending, and its outcome lives in checkpoint health — never + // folded into the compaction result this returns. The cycle has its own + // ctx.Done() path, so the cancellation signal is discarded here. + _ = d.runPendingCheckpoint(ctx) return compacted, err } @@ -894,33 +1017,12 @@ func (d *Duck) runRepositoryMaintenance(ctx context.Context) error { } metrics.RecordTelemetryOperation(metrics.TelemetryCompaction, compactResult, time.Since(compactStart).Seconds()) pruneErr := errors.Join(recoveryErr, cleanupErr, parquetErr, compactErr) - cacheErr := func() error { - unlock, err := d.writeGate.LockContext(ctx, writegate.Maintenance) - if err != nil { - return err - } - defer unlock() - unlockCache, err := d.cacheGate.LockContext(ctx, writegate.Maintenance) - if err != nil { - return err - } - defer unlockCache() - background, err := d.lockBackground(ctx) - if err != nil { - return err - } - defer background() - var errs []error - if d.cfg.RetentionDays > 0 { - for _, table := range []string{"service_rollup", "span_rollup", "log_rollup", "endpoint_rollup", "receiver_rollup", "resource_rollup", "receiver_entity_rollup", "edge_rollup"} { - if _, err := d.writer().ExecContext(ctx, fmt.Sprintf("DELETE FROM %s WHERE bucket < (now() - INTERVAL %d DAY)::TIMESTAMP_NS", table, d.cfg.RetentionDays)); err != nil { - errs = append(errs, fmt.Errorf("prune %s: %w", table, err)) - } - } - } - return errors.Join(errors.Join(errs...), d.checkpoint(ctx)) - }() - err := errors.Join(pruneErr, cacheErr) + // Retention DELETEs on the rollup-cache tables are maintenance work: they + // reclaim storage, so a failure here still degrades maintenance readiness. + // They hold both write gates and the background slot, as the checkpoint + // does, but are kept separate from it so the fold can be retried on its own. + retentionErr := d.pruneRollupCache(ctx) + err := errors.Join(pruneErr, retentionErr) maintenanceResult := metrics.TelemetrySuccess if err != nil { maintenanceResult = metrics.TelemetryError @@ -940,9 +1042,50 @@ func (d *Duck) runRepositoryMaintenance(ctx context.Context) error { if err == nil { slog.Info("telemetry maintenance complete", "duration", time.Since(start)) } + // Fold the rollup-cache WAL back into its file. An interval of rollup + // commits plus the retention DELETEs above have grown it, so there is always + // work to fold; mark it pending and attempt it now. A refusal from a + // still-open older reader is recorded in checkpoint health (not maintenance) + // and retried on the 10s compaction cycle, so it does not fail this pass. + // Only a context cancellation propagates, so the loop stops promptly. + d.markCheckpointPending() + if cancelErr := d.runPendingCheckpoint(ctx); cancelErr != nil { + return errors.Join(err, cancelErr) + } return err } +// pruneRollupCache deletes retention-expired rows from the rollup-cache tables +// under both write gates and the background slot. It is a no-op taking no gate +// when retention is disabled. +func (d *Duck) pruneRollupCache(ctx context.Context) error { + if d.cfg.RetentionDays <= 0 { + return nil + } + unlock, err := d.writeGate.LockContext(ctx, writegate.Maintenance) + if err != nil { + return err + } + defer unlock() + unlockCache, err := d.cacheGate.LockContext(ctx, writegate.Maintenance) + if err != nil { + return err + } + defer unlockCache() + background, err := d.lockBackground(ctx) + if err != nil { + return err + } + defer background() + var errs []error + for _, table := range []string{"service_rollup", "span_rollup", "log_rollup", "endpoint_rollup", "receiver_rollup", "resource_rollup", "receiver_entity_rollup", "edge_rollup"} { + if _, err := d.writer().ExecContext(ctx, fmt.Sprintf("DELETE FROM %s WHERE bucket < (now() - INTERVAL %d DAY)::TIMESTAMP_NS", table, d.cfg.RetentionDays)); err != nil { + errs = append(errs, fmt.Errorf("prune %s: %w", table, err)) + } + } + return errors.Join(errs...) +} + // rollupChunkWins is how many committed passes in a row grow a narrowed // service rollup window. const rollupChunkWins = 3 diff --git a/internal/engine/duck_test.go b/internal/engine/duck_test.go index c71537ed..a20592bc 100644 --- a/internal/engine/duck_test.go +++ b/internal/engine/duck_test.go @@ -7,6 +7,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "testing" "time" @@ -401,7 +402,10 @@ func TestMaintenanceRetriesRetiredDirectoryCleanup(t *testing.T) { } } -func TestMaintenanceTracksConsecutiveFailures(t *testing.T) { +// Consecutive checkpoint failures accrue in checkpoint health, not maintenance +// health, and a later success clears them. A missed fold must never fail the +// maintenance pass, since it self-heals on the compaction cycle. +func TestCheckpointTracksConsecutiveFailures(t *testing.T) { db, mock, err := sqlmock.New() if err != nil { t.Fatal(err) @@ -413,20 +417,56 @@ func TestMaintenanceTracksConsecutiveFailures(t *testing.T) { mock.ExpectExec("CHECKPOINT").WillReturnResult(sqlmock.NewResult(0, 0)) d := &Duck{DB: db, writeDB: db, repository: emptyRepository(t)} for range 3 { - if err := d.runRepositoryMaintenance(context.Background()); err == nil { - t.Fatal("maintenance succeeded despite checkpoint failure") + if err := d.runRepositoryMaintenance(context.Background()); err != nil { + t.Fatalf("checkpoint failure degraded the maintenance pass: %v", err) } } - _, _, failures, _ := d.MaintenanceHealth() + if _, _, maintFailures, maintErr := d.MaintenanceHealth(); maintFailures != 0 || maintErr != nil { + t.Fatalf("checkpoint failures leaked into maintenance health: failures=%d err=%v", maintFailures, maintErr) + } + _, _, failures, _ := d.CheckpointHealth() if failures != 3 { - t.Fatalf("consecutive failures = %d, want 3", failures) + t.Fatalf("consecutive checkpoint failures = %d, want 3", failures) } if err := d.runRepositoryMaintenance(context.Background()); err != nil { t.Fatal(err) } - _, _, failures, _ = d.MaintenanceHealth() + _, _, failures, _ = d.CheckpointHealth() if failures != 0 { - t.Fatalf("consecutive failures after recovery = %d, want 0", failures) + t.Fatalf("consecutive checkpoint failures after recovery = %d, want 0", failures) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} + +// Retention DELETEs on the rollup cache are real maintenance work: a failure +// there still degrades maintenance health and consecutive-failure tracking, +// while leaving checkpoint health clean when the fold itself succeeds. +func TestRetentionFailureDegradesMaintenance(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + defer db.Close() + for _, table := range []string{"service_rollup", "span_rollup", "log_rollup", "endpoint_rollup", "receiver_rollup", "resource_rollup", "receiver_entity_rollup", "edge_rollup"} { + e := mock.ExpectExec("DELETE FROM " + table) + if table == "service_rollup" { + e.WillReturnError(errors.New("injected retention failure")) + } else { + e.WillReturnResult(sqlmock.NewResult(0, 0)) + } + } + mock.ExpectExec("CHECKPOINT").WillReturnResult(sqlmock.NewResult(0, 0)) + d := &Duck{DB: db, writeDB: db, repository: emptyRepository(t), cfg: config.Config{RetentionDays: 1}} + if err := d.runRepositoryMaintenance(context.Background()); err == nil || !strings.Contains(err.Error(), "injected retention failure") { + t.Fatalf("retention failure did not fail maintenance: %v", err) + } + if _, _, maintFailures, maintErr := d.MaintenanceHealth(); maintFailures != 1 || maintErr == nil { + t.Fatalf("retention failure not tracked in maintenance health: failures=%d err=%v", maintFailures, maintErr) + } + if _, _, cpFailures, cpErr := d.CheckpointHealth(); cpFailures != 0 || cpErr != nil { + t.Fatalf("retention failure leaked into checkpoint health: failures=%d err=%v", cpFailures, cpErr) } if err := mock.ExpectationsWereMet(); err != nil { t.Fatal(err)