Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 57 additions & 0 deletions internal/api/health.go
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,7 @@ collect:

resp.Checks["data"] = h.checkDataDir()
resp.Checks["maintenance"] = h.checkMaintenance()
resp.Checks["checkpoint"] = h.checkCheckpoint()
resp.Checks["queries"] = h.checkQueries()
if h.oidc != nil {
resp.Checks["oidc"] = h.oidc.readiness()
Expand Down Expand Up @@ -366,6 +367,62 @@ func maintenanceResult(lastOK, lastAt time.Time, lastErr error, consecutiveFailu
return res
}

// checkpointDegradeAfterFailures is how many rollup-cache checkpoint attempts
// may fail in a row before readiness degrades. A missed fold is retried on the
// 10s compaction cycle, so a transient conflict — an older reader that drains
// within a second or two — clears after a cycle or two and never approaches
// this count; the failure counter resets to zero on the next success. Reaching
// this many failures is roughly a minute of continuous refusal, i.e. a fold
// that is genuinely stuck (a reader that never drains) and whose WAL is growing
// unbounded — worth surfacing. A single recent miss stays ok, so one transient
// conflict does not flap /readyz out of rotation. A genuinely wedged engine is
// caught independently by the telemetry SELECT-1 probe, not here.
const checkpointDegradeAfterFailures = 6

// checkCheckpoint surfaces the rollup-cache checkpoint's own health, separately
// from maintenance. A checkpoint folds the write-ahead log back into the cache
// file; the non-FORCE form refuses while a read that began before the last
// write commit is still open. That refusal is transient and self-heals on the
// 10s compaction cycle, so it reports "degraded" (never "unhealthy", and only
// after the fold has kept failing past checkpointDegradeAfterFailures) rather
// than pulling the instance from rotation: ingest and queries work while the
// WAL merely grows, and a restart would not help.
func (h *HealthHandler) checkCheckpoint() CheckResult {
if h.duck == nil {
return CheckResult{
Status: "unhealthy",
Error: "duckdb not initialized",
}
}
lastOK, lastAt, failures, lastErr := h.duck.CheckpointHealth()
return checkpointResult(lastOK, lastAt, lastErr, failures)
}

// checkpointResult classifies checkpoint health. It is pure so every branch —
// never attempted, a transient miss, and a persistently failing fold — is
// unit-testable without a running Duck.
func checkpointResult(lastOK, lastAt time.Time, lastErr error, consecutiveFailures int) CheckResult {
res := CheckResult{Status: "ok"}
if !lastAt.IsZero() {
res.UpdatedAt = lastAt.UTC().Format(time.RFC3339)
}
if lastErr == nil {
// The last fold succeeded, or none has been attempted yet.
return res
}
res.Error = lastErr.Error()
res.Detail = fmt.Sprintf("%d consecutive failed checkpoints", consecutiveFailures)
if !lastOK.IsZero() {
res.Detail += "; last clean fold: " + lastOK.UTC().Format(time.RFC3339)
}
// Tolerate a transient miss: the 10s cycle retries, so degrade only once the
// fold has failed enough times in a row that a retry would have cleared it.
if consecutiveFailures >= checkpointDegradeAfterFailures {
res.Status = "degraded"
}
return res
}

func (h *HealthHandler) checkRollups(ctx context.Context) CheckResult {
if h.duck == nil {
return CheckResult{
Expand Down
33 changes: 33 additions & 0 deletions internal/api/health_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -387,6 +387,39 @@ func TestMaintenanceStaleThreshold(t *testing.T) {
}
}

// A single transient checkpoint miss — retried every 10s — must not degrade
// readiness, while a fold that keeps failing past the cadence must. Checkpoint
// health never reports unhealthy: the instance still serves while the WAL grows.
func TestReadinessToleratesTransientCheckpointMiss(t *testing.T) {
now := time.Now()
boom := errors.New("Cannot CHECKPOINT: there are other write transactions active")
cases := []struct {
name string
lastOK, lastAt time.Time
lastErr error
failures int
wantStatus string
}{
{"never attempted", time.Time{}, time.Time{}, nil, 0, "ok"},
{"clean fold", now.Add(-30 * time.Second), now.Add(-30 * time.Second), nil, 0, "ok"},
{"single transient miss", now.Add(-time.Minute), now.Add(-5 * time.Second), boom, 1, "ok"},
{"a few misses still tolerated", now.Add(-time.Minute), now.Add(-5 * time.Second), boom, 3, "ok"},
{"persistently failing fold", now.Add(-2 * time.Minute), now.Add(-5 * time.Second), boom, checkpointDegradeAfterFailures, "degraded"},
{"never succeeded, still failing", time.Time{}, now.Add(-time.Second), boom, checkpointDegradeAfterFailures + 2, "degraded"},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
res := checkpointResult(c.lastOK, c.lastAt, c.lastErr, c.failures)
if res.Status != c.wantStatus {
t.Fatalf("status = %q, want %q (detail=%q err=%q)", res.Status, c.wantStatus, res.Detail, res.Error)
}
if res.Status == "unhealthy" {
t.Fatalf("checkpoint must never report unhealthy: %+v", res)
}
})
}
}

func TestMaintenanceResult(t *testing.T) {
now := time.Now()
rollupEvery := 60 * time.Second
Expand Down
93 changes: 84 additions & 9 deletions internal/engine/checkpoint_retry_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -92,10 +92,16 @@ func TestMaintenanceCheckpointWaitsForOldReaderToDrain(t *testing.T) {
if err := <-result; err != nil {
t.Fatalf("checkpoint failed after the old reader drained: %v", err)
}
// The maintenance pass itself stays clean: the checkpoint retry lives in
// checkpoint health, never maintenance health.
lastSuccess, _, failures, err := d.MaintenanceHealth()
if lastSuccess.IsZero() || failures != 0 || err != nil {
t.Fatalf("successful retry reported maintenance failure: success=%s failures=%d err=%v", lastSuccess, failures, err)
}
cpOK, _, cpFailures, cpErr := d.CheckpointHealth()
if cpOK.IsZero() || cpFailures != 0 || cpErr != nil {
t.Fatalf("drained checkpoint not recorded clean in checkpoint health: ok=%s failures=%d err=%v", cpOK, cpFailures, cpErr)
}
}

func TestMaintenanceCheckpointRetryHonorsCallerCancellation(t *testing.T) {
Expand Down Expand Up @@ -123,23 +129,32 @@ func TestMaintenanceCheckpointRetryExhaustionIsExplicitAndBounded(t *testing.T)
d, repo := newBatchCacheTest(t)
rows := checkpointOldReader(t, d, repo)
defer rows.Close()
ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second)
// A budget well short of the caller's deadline so the fold exhausts its own
// retry window rather than the whole pass timing out.
ctx, cancel := context.WithTimeout(t.Context(), checkpointRetryBudget+10*time.Second)
defer cancel()
before := testutil.ToFloat64(metrics.DuckDBCheckpointFailures)
started := time.Now()
err := d.runRepositoryMaintenance(ctx)
if !errors.Is(err, context.DeadlineExceeded) || !strings.Contains(err.Error(), "checkpoint retry budget exhausted") || !strings.Contains(err.Error(), "other write transactions active") {
t.Fatalf("exhausted checkpoint was hidden or mislabeled: %v", err)
// The reader never drains, so the fold exhausts its retry budget. Under the
// new contract that is a checkpoint failure, not a maintenance failure: the
// pass returns nil and the miss is retried on the compaction cycle.
if err := d.runRepositoryMaintenance(ctx); err != nil {
t.Fatalf("a missed checkpoint failed the whole maintenance pass: %v", err)
}
if elapsed := time.Since(started); elapsed > time.Second {
t.Fatalf("250ms retry budget was not bounded: %s", elapsed)
if elapsed := time.Since(started); elapsed > checkpointRetryBudget+5*time.Second {
t.Fatalf("checkpoint retry budget was not bounded: %s", elapsed)
}
if after := testutil.ToFloat64(metrics.DuckDBCheckpointFailures); after != before+1 {
t.Fatalf("attempts counted as logical checkpoint failures: before=%v after=%v", before, after)
}
_, _, failures, healthErr := d.MaintenanceHealth()
if failures != 1 || !errors.Is(healthErr, context.DeadlineExceeded) {
t.Fatalf("checkpoint exhaustion not reflected in maintenance health: failures=%d error=%v", failures, healthErr)
// Maintenance health stays clean; the exhausted fold is recorded only in
// checkpoint health, explicitly and with the pinned native diagnostic.
if _, _, maintFailures, maintErr := d.MaintenanceHealth(); maintFailures != 0 || maintErr != nil {
t.Fatalf("checkpoint miss leaked into maintenance health: failures=%d error=%v", maintFailures, maintErr)
}
_, _, cpFailures, cpErr := d.CheckpointHealth()
if cpFailures != 1 || cpErr == nil || !strings.Contains(cpErr.Error(), "checkpoint retry budget exhausted") || !strings.Contains(cpErr.Error(), "other write transactions active") {
t.Fatalf("exhausted checkpoint not recorded explicitly in checkpoint health: failures=%d error=%v", cpFailures, cpErr)
}
}

Expand Down Expand Up @@ -188,6 +203,66 @@ func TestCheckpointRetriesConflictWithoutCountingTransientFailure(t *testing.T)
}
}

// A checkpoint that fails inside a maintenance pass must leave maintenance
// health clean and be recorded only in checkpoint health, so one missed fold
// cannot degrade /readyz for an hour.
func TestCheckpointMissDoesNotDegradeMaintenanceHealth(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatal(err)
}
defer db.Close()
mock.ExpectExec("CHECKPOINT").WillReturnError(errors.New("injected checkpoint failure"))
d := &Duck{DB: db, writeDB: db, repository: emptyRepository(t)}
if err := d.runRepositoryMaintenance(t.Context()); err != nil {
t.Fatalf("a missed checkpoint failed the whole maintenance pass: %v", err)
}
lastOK, _, maintFailures, maintErr := d.MaintenanceHealth()
if lastOK.IsZero() || maintFailures != 0 || maintErr != nil {
t.Fatalf("checkpoint miss leaked into maintenance health: lastOK=%s failures=%d err=%v", lastOK, maintFailures, maintErr)
}
_, cpAt, cpFailures, cpErr := d.CheckpointHealth()
if cpAt.IsZero() || cpFailures != 1 || cpErr == nil {
t.Fatalf("checkpoint miss not recorded in checkpoint health: at=%s failures=%d err=%v", cpAt, cpFailures, cpErr)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatal(err)
}
}

// A missed fold stays pending and the next 10s compaction cycle retries it, so
// a transient miss self-heals in seconds instead of waiting an hour. Driving
// runParquetCompactionCycle directly proves the self-heal without a real clock.
func TestMissedCheckpointRetriesOnCompactionCycle(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatal(err)
}
defer db.Close()
mock.ExpectExec("CHECKPOINT").WillReturnError(errors.New("injected checkpoint failure"))
mock.ExpectExec("CHECKPOINT").WillReturnResult(sqlmock.NewResult(0, 0))
d := &Duck{DB: db, writeDB: db, repository: emptyRepository(t)}
// A pass left a fold pending but could not complete it.
d.markCheckpointPending()
if _, err := d.runParquetCompactionCycle(t.Context()); err != nil {
t.Fatalf("compaction cycle surfaced the checkpoint miss as its own error: %v", err)
}
if _, _, cpFailures, cpErr := d.CheckpointHealth(); cpFailures != 1 || cpErr == nil {
t.Fatalf("first cycle did not record the miss: failures=%d err=%v", cpFailures, cpErr)
}
// The miss is still pending, so the next cycle retries and clears it.
if _, err := d.runParquetCompactionCycle(t.Context()); err != nil {
t.Fatal(err)
}
lastOK, _, cpFailures, cpErr := d.CheckpointHealth()
if lastOK.IsZero() || cpFailures != 0 || cpErr != nil {
t.Fatalf("checkpoint did not self-heal on the compaction cycle: lastOK=%s failures=%d err=%v", lastOK, cpFailures, cpErr)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatal(err)
}
}

func TestCheckpointDoesNotRetryUnrelatedError(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
Expand Down
Loading
Loading