From 52c4824c9a85162584427a7e1b1306024a94d629 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Sun, 15 Nov 2026 16:40:03 -0800 Subject: [PATCH] refactor(telemetry): remove dead compaction crash-recovery marker Compaction never produced the state its recovery machinery was built to undo. Nothing creates `.retired-` input directories: inputs stay `.batch` and a published output's `replaces.json` retires them. So the `COMPACTION.json` marker, `RestoreRetiredInputs` and its move branch, the `.retired-` handling, the marker-protected cleanup set, and the `Open` path that read the marker and refused to start were all dead or redundant with recovery that already runs on every boot. A compaction is now purely in-process: it merges into `compaction/` staging and commits by renaming that into the batch namespace, where the output's `replaces.json` retires its inputs. The whole recovery protocol is to discard the `compaction/` staging on boot and let `replaces.json` decide. That is equivalent and crash-safe on both sides of the single rename: a crash before it leaves the inputs live as `.batch` and the staging is dropped; a crash after it leaves the output present with its `replaces.json`, which `loadBatches` applies to retire the inputs. Neither window loses rows or orphans inputs. Deleted: `compactionMarker`, `recoverCompaction`, `completeCompaction`, `RecoverParquet`, `validateCompactionMarker`, `RestoreRetiredInputs`, `protectedRetiredSuffix`, `logUnresolvedCompaction`, `compactionRunbookURL`, the `COMPACTION.json` write, and the unused `pathExists`/`parquetPublishFunc` helpers. `compactParquet` now calls `PublishReplacement` directly after `PrepareReplacement`. The interrupted-merge attempt-file backoff (`COMPACTION-ATTEMPT.json`) and `replaces.json` itself are untouched. The repair quarantine guard that consulted the marker to refuse a batch mid-compaction is removed; it is superseded by the data-directory flock (#485), so repair cannot run against a live data dir and has no in-process compaction to collide with. A crash-matrix test (`TestInterruptedCompaction*`, `TestFailedCompactionPublishKeepsInputsLiveAndRetryPreservesRows`) drives each window on a real filesystem and asserts exact row counts and catalog composition, so it fails if `replaces.json` masking regresses, if boot completes a merge it should discard, or if any row is lost. The removed marker tests' crash-safety coverage is preserved by these. Closes #486 --- internal/engine/duck.go | 22 +- internal/engine/duck_test.go | 2 +- internal/telemetry/format.go | 13 +- internal/telemetry/metric_format_test.go | 4 +- internal/telemetry/parquet.go | 88 ---- internal/telemetry/store/compaction.go | 179 ++----- .../store/compaction_preemption_test.go | 3 - .../store/compaction_recovery_test.go | 291 +++++++++++ internal/telemetry/store/repair.go | 39 +- internal/telemetry/store/repair_test.go | 29 -- internal/telemetry/store/repository.go | 105 +--- internal/telemetry/store/repository_test.go | 450 +----------------- internal/telemetry/variant_test.go | 2 +- .../docs/docs/guides/troubleshooting.mdx | 43 -- .../docs/docs/reference/data-layout.mdx | 6 +- 15 files changed, 382 insertions(+), 894 deletions(-) create mode 100644 internal/telemetry/store/compaction_recovery_test.go diff --git a/internal/engine/duck.go b/internal/engine/duck.go index 20a08f8fc..827572bf9 100644 --- a/internal/engine/duck.go +++ b/internal/engine/duck.go @@ -997,10 +997,6 @@ func runMaintenanceSchedule(ctx context.Context, every time.Duration, run func() func (d *Duck) runParquetCompactionCycle(ctx context.Context) (int, error) { started := time.Now() - if err := d.repository.RecoverParquet(ctx, d); err != nil { - metrics.RecordTelemetryOperation(metrics.TelemetryCompaction, metrics.TelemetryError, time.Since(started).Seconds()) - return 0, err - } compacted, err := d.repository.CompactParquetPass(ctx, d, parquetMaintenanceBatchLimit, parquetCompactionCycleBudget) // Retired inputs wait only for readers that captured them, which query // deadlines bound, so each cycle deletes what earlier cycles retired. @@ -1027,25 +1023,21 @@ func (d *Duck) runParquetCompactionCycle(ctx context.Context) (int, error) { func (d *Duck) runRepositoryMaintenance(ctx context.Context) error { start := time.Now() - recoveryErr := d.repository.RecoverParquet(ctx, d) compactStart := time.Now() - var cleanupErr, parquetErr, compactErr error - compacted := 0 - if recoveryErr == nil { - cleanupErr = d.repository.CleanupParquet() - if d.cfg.RetentionDays > 0 { - _, parquetErr = d.repository.PruneParquetPass(ctx, d, time.Now().Add(-time.Duration(d.cfg.RetentionDays)*24*time.Hour).UnixNano(), parquetMaintenanceBatchLimit, parquetMaintenancePhaseBudget) - } - compacted, compactErr = d.repository.CompactParquetPass(ctx, d, parquetMaintenanceBatchLimit, parquetMaintenancePhaseBudget) + var parquetErr error + cleanupErr := d.repository.CleanupParquet() + if d.cfg.RetentionDays > 0 { + _, parquetErr = d.repository.PruneParquetPass(ctx, d, time.Now().Add(-time.Duration(d.cfg.RetentionDays)*24*time.Hour).UnixNano(), parquetMaintenanceBatchLimit, parquetMaintenancePhaseBudget) } + compacted, compactErr := d.repository.CompactParquetPass(ctx, d, parquetMaintenanceBatchLimit, parquetMaintenancePhaseBudget) compactResult := metrics.TelemetryNoop - if recoveryErr != nil || compactErr != nil { + if compactErr != nil { compactResult = metrics.TelemetryError } else if compacted > 0 { compactResult = metrics.TelemetrySuccess } metrics.RecordTelemetryOperation(metrics.TelemetryCompaction, compactResult, time.Since(compactStart).Seconds()) - pruneErr := errors.Join(recoveryErr, cleanupErr, parquetErr, compactErr) + pruneErr := errors.Join(cleanupErr, parquetErr, compactErr) // Retention DELETEs on the rollup-cache tables are maintenance work: they // reclaim storage, so a failure here still degrades maintenance readiness. // They hold both write gates and the background slot, as the checkpoint diff --git a/internal/engine/duck_test.go b/internal/engine/duck_test.go index f3ff9f211..342b68d68 100644 --- a/internal/engine/duck_test.go +++ b/internal/engine/duck_test.go @@ -421,7 +421,7 @@ func TestMaintenanceRetriesRetiredDirectoryCleanup(t *testing.T) { t.Fatal(err) } defer repository.Close() - retired := filepath.Join(repository.Parquet.BatchesDir(), "stale.retired-old") + retired := filepath.Join(repository.Parquet.BatchesDir(), "stale.retired") if err := os.Mkdir(retired, 0o755); err != nil { t.Fatal(err) } diff --git a/internal/telemetry/format.go b/internal/telemetry/format.go index 7547c28b1..80f9ac0b5 100644 --- a/internal/telemetry/format.go +++ b/internal/telemetry/format.go @@ -21,18 +21,17 @@ func validateBatchFormats(dir string) error { } for _, entry := range entries { name := entry.Name() - // Quarantine is outside the authoritative set. Retired inputs may still - // be needed for compaction recovery and must be checked before cleanup. + // Quarantine is outside the authoritative set. A retired directory is + // retention's interrupted delete and keeps its format until cleanup + // reclaims it, so it is checked before cleanup runs. active := strings.HasSuffix(name, BatchSuffix) - retired := strings.HasSuffix(name, ".retired") || strings.Contains(name, ".retired-") - if !entry.IsDir() || name == SchemaBatch || (!active && (!retired || strings.Contains(name, ".quarantined-"))) { + retired := strings.HasSuffix(name, ".retired") + if !entry.IsDir() || name == SchemaBatch || (!active && !retired) { continue } if _, err := readBatchMetadata(filepath.Join(dir, name, "metadata.json")); err != nil { // A retired directory without metadata was partly removed by - // retention or compaction cleanup and has no format left to - // preserve. Cleanup reclaims it, or compaction recovery reports the - // missing input through its runbook. + // retention and has no format left to preserve; cleanup reclaims it. if retired && errors.Is(err, os.ErrNotExist) { continue } diff --git a/internal/telemetry/metric_format_test.go b/internal/telemetry/metric_format_test.go index 95d097c24..02c9ee1c3 100644 --- a/internal/telemetry/metric_format_test.go +++ b/internal/telemetry/metric_format_test.go @@ -35,7 +35,7 @@ func TestMetricPhysicalRowKeepsFidelityColumns(t *testing.T) { } func TestFormatThreeIsRefusedBeforeCleanup(t *testing.T) { - for _, suffix := range []string{BatchSuffix, ".retired", ".retired-replacement"} { + for _, suffix := range []string{BatchSuffix, ".retired"} { t.Run(suffix, func(t *testing.T) { root := t.TempDir() store, err := OpenParquetStore(root) @@ -161,7 +161,7 @@ func TestFormatFiveRequiresMetricPhysicalSchema(t *testing.T) { } func TestPartiallyRemovedRetiredBatchDoesNotStopOpen(t *testing.T) { - for _, suffix := range []string{".retired", ".retired-replacement"} { + for _, suffix := range []string{".retired"} { for _, missing := range []string{"spans.parquet", "logs.parquet", "metrics.parquet", "trace.fidx", "metadata.json"} { t.Run(suffix+"/"+missing, func(t *testing.T) { root := t.TempDir() diff --git a/internal/telemetry/parquet.go b/internal/telemetry/parquet.go index 3fc444900..a6ae1ca5f 100644 --- a/internal/telemetry/parquet.go +++ b/internal/telemetry/parquet.go @@ -432,94 +432,6 @@ func (p *ParquetStore) lockCommitPublish(ctx context.Context) error { func (p *ParquetStore) unlockPublish() { p.publishGate <- struct{}{} } -// RestoreRetiredInputs rolls back a compaction whose durable output vanished. -// Readers see the inputs only once the catalog installs them. -func (p *ParquetStore) RestoreRetiredInputs(inputs []string, replacementID string, publish func(func(context.Context) error) error) error { - if err := ValidateBatchID(replacementID); err != nil { - return err - } - for _, id := range inputs { - if err := ValidateBatchID(id); err != nil { - return err - } - } - type restoredInput struct { - id string - active string - retired string - batch *storedBatch - move bool - } - prepared := make([]restoredInput, 0, len(inputs)) - for _, id := range inputs { - active := p.BatchPath(id) - path := active - move := false - if _, err := os.Stat(active); errors.Is(err, os.ErrNotExist) { - path = filepath.Join(p.batchesDir, id+".retired-"+replacementID) - move = true - } else if err != nil { - return err - } - batch, err := loadStoredBatch(path) - if err != nil { - if errors.Is(err, os.ErrNotExist) { - return fmt.Errorf("input %s is missing", id) - } - return fmt.Errorf("load input %s: %w", id, err) - } - batch.dir = active - if batch.metadata.Spans > 0 { - batch.traces.path = filepath.Join(active, "trace.fidx") - } - prepared = append(prepared, restoredInput{id: id, active: active, retired: path, batch: batch, move: move}) - } - return publish(func(ctx context.Context) error { - if err := p.lockPublish(ctx); err != nil { - return err - } - defer p.unlockPublish() - installActive := func() { - p.mu.Lock() - defer p.mu.Unlock() - for _, input := range prepared { - if _, err := os.Stat(input.active); err == nil { - // The replacement never reached the namespace, so its - // replaces list no longer covers the restored input. - if lister, ok := p.replacedBy[input.id]; ok && lister == replacementID { - delete(p.replacedBy, input.id) - } - p.setBatchLocked(input.id, input.batch) - } else { - p.setBatchLocked(input.id, nil) - } - } - } - - moved := make([]restoredInput, 0, len(prepared)) - for _, input := range prepared { - if !input.move { - continue - } - if err := os.Rename(input.retired, input.active); err != nil { - var rollbackErr error - for i := len(moved) - 1; i >= 0; i-- { - rollbackErr = errors.Join(rollbackErr, os.Rename(moved[i].active, moved[i].retired)) - } - installActive() - return errors.Join(err, rollbackErr, syncDirectory(p.batchesDir)) - } - moved = append(moved, input) - } - var syncErr error - if len(moved) > 0 { - syncErr = syncDirectory(p.batchesDir) - } - installActive() - return syncErr - }) -} - // TraceTotals describes the whole trace, not the page a limit admitted. // // These come free: readIndexedTrace already decodes every row of the trace and diff --git a/internal/telemetry/store/compaction.go b/internal/telemetry/store/compaction.go index 7abf41c6e..2bbf46b24 100644 --- a/internal/telemetry/store/compaction.go +++ b/internal/telemetry/store/compaction.go @@ -68,11 +68,6 @@ const ( maxDayCompactionRows int64 = 250_000 ) -type compactionMarker struct { - Output telemetry.BatchMetadata `json:"output"` - Inputs []string `json:"inputs"` -} - // compactionAttemptFile records that a merge was started. The completion // marker is written only after PrepareReplacement, so a process killed during // the merge -- which is the likely way a large merge ends -- leaves no trace @@ -172,14 +167,6 @@ func (r *Repository) compactParquet(ctx context.Context, publisher ParquetPublis if err := ctx.Err(); err != nil { return 0, err } - markerPath := filepath.Join(r.root, "COMPACTION.json") - if exists, err := pathExists(markerPath); err != nil { - return 0, err - } else if exists { - if err := r.recoverCompaction(ctx, publisher.PublishParquetReplacement); err != nil { - return 0, fmt.Errorf("recover pending Parquet compaction: %w", err) - } - } // A pass that was killed mid-merge left a record of how much it tried to // take. Take less. batchCap := compactionBatchCap(maxBatches, readCompactionAttempt(r.root)) @@ -197,21 +184,21 @@ func (r *Repository) compactParquet(ctx context.Context, publisher ParquetPublis ID: fmt.Sprintf("compact-%d", time.Now().UnixNano()), MinIngestedNanos: math.MaxInt64, Generation: generation + 1, } - marker := compactionMarker{Output: output, Inputs: make([]string, 0, len(selected))} + inputs := make([]string, 0, len(selected)) for _, batch := range selected { - marker.Inputs = append(marker.Inputs, batch.ID) + inputs = append(inputs, batch.ID) if batch.MinIngestedNanos > 0 { - marker.Output.MinIngestedNanos = min(marker.Output.MinIngestedNanos, batch.MinIngestedNanos) + output.MinIngestedNanos = min(output.MinIngestedNanos, batch.MinIngestedNanos) } - marker.Output.MaxIngestedNanos = max(marker.Output.MaxIngestedNanos, batch.MaxIngestedNanos) - marker.Output.Spans += batch.Spans - marker.Output.Logs += batch.Logs - marker.Output.Metrics += batch.Metrics + output.MaxIngestedNanos = max(output.MaxIngestedNanos, batch.MaxIngestedNanos) + output.Spans += batch.Spans + output.Logs += batch.Logs + output.Metrics += batch.Metrics } - if marker.Output.MinIngestedNanos == math.MaxInt64 { - marker.Output.MinIngestedNanos = 0 + if output.MinIngestedNanos == math.MaxInt64 { + output.MinIngestedNanos = 0 } - stage := r.compactionStage(marker.Output.ID) + stage := r.compactionStage(output.ID) if err := os.RemoveAll(stage); err != nil { return 0, err } @@ -221,12 +208,12 @@ func (r *Repository) compactParquet(ctx context.Context, publisher ParquetPublis if err := errors.Join(syncDirectory(filepath.Dir(stage)), syncDirectory(r.root)); err != nil { return 0, err } - prepared := false - defer func() { - if !prepared { - _ = os.RemoveAll(stage) - } - }() + // The staging directory is the whole crash-recovery story. If anything below + // fails before PublishReplacement renames it into the batch namespace, it is + // removed here and, equivalently, discarded on the next boot; the selected + // inputs stay live and a later pass redoes the merge. Once the rename lands, + // this path no longer exists and the removal is a no-op. + defer func() { _ = os.RemoveAll(stage) }() type mergePlan struct { signal string inputs []string @@ -249,9 +236,9 @@ func (r *Repository) compactParquet(ctx context.Context, publisher ParquetPublis } plans = append(plans, mergePlan{signal: signal, inputs: inputs, output: filepath.Join(stage, signal+".parquet")}) } - // Written before the merge, on purpose: the completion marker below is - // only reached if the merge returns, and the failure this guards against - // is the merge not returning. + // Written before the merge, on purpose: the attempt record is cleared only + // after the merge returns, and the failure this guards against is the merge + // not returning. if err := writeCompactionAttempt(r.root, len(selected)); err != nil { return 0, err } @@ -264,11 +251,11 @@ func (r *Repository) compactParquet(ctx context.Context, publisher ParquetPublis return fmt.Errorf("compact %s Parquet: %w", plan.signal, err) } if plan.signal == "spans" { - if stats.Rows != int64(marker.Output.Spans) { - return fmt.Errorf("compacted span count: got %d want %d", stats.Rows, marker.Output.Spans) + if stats.Rows != int64(output.Spans) { + return fmt.Errorf("compacted span count: got %d want %d", stats.Rows, output.Spans) } - marker.Output.MinSpanStartNanos = stats.MinSpanStartNanos - marker.Output.MaxSpanStartNanos = stats.MaxSpanStartNanos + output.MinSpanStartNanos = stats.MinSpanStartNanos + output.MaxSpanStartNanos = stats.MaxSpanStartNanos } if err := mergeCtx.Err(); err != nil { return err @@ -284,22 +271,27 @@ func (r *Repository) compactParquet(ctx context.Context, publisher ParquetPublis if err != nil { return 0, err } - if err := r.Parquet.PrepareReplacement(stage, marker.Output); err != nil { + if err := r.Parquet.PrepareReplacement(stage, output); err != nil { return 0, err } - data, err := json.Marshal(marker) - if err != nil { - return 0, err - } - if err := writeDurableFile(markerPath, data); err != nil { - return 0, err - } - if err := syncDirectory(r.root); err != nil { + // Completion is a single atomic step: PublishReplacement writes the output's + // replaces.json into the stage, renames the stage into the batch namespace, + // and retires the inputs. There is no separate durable marker to recover + // from -- the output's presence plus its replaces.json is the whole record, + // which boot's loadBatches reapplies, so a crash on either side of the + // rename is consistent: before it the inputs are live and the stage is + // discarded, after it the output is live and its replaces.json retires the + // inputs. + if err := r.Parquet.PublishReplacement(stage, output, inputs, func(swap func(context.Context) error) error { + return publisher.PublishParquetReplacement(ctx, output, inputs, swap) + }); err != nil { return 0, err } - prepared = true - if err := r.completeCompaction(ctx, marker, publisher.PublishParquetReplacement); err != nil { - return 0, err + // A completed higher-generation output clears the debt-preemption size; a + // generation-0 or minimum-pair completion must leave it, so an earlier + // preempted higher generation still shrinks its next attempt. + if output.Generation > 1 { + r.preemptedCompactionInputs = 0 } if err := clearCompactionAttempt(r.root); err != nil { return 0, err @@ -589,90 +581,10 @@ func (r *Repository) CompactParquetPass(ctx context.Context, publisher ParquetPu } } -type parquetPublishFunc func(context.Context, telemetry.BatchMetadata, []string, func(context.Context) error) error - -// RecoverParquet resolves a pending compaction marker before any cleanup, -// retention, or new compaction can mutate its rollback set. -func (r *Repository) RecoverParquet(ctx context.Context, publisher ParquetPublisher) error { - r.compactionMu.Lock() - defer r.compactionMu.Unlock() - return r.recoverCompaction(ctx, publisher.PublishParquetReplacement) -} - -func (r *Repository) recoverCompaction(ctx context.Context, publish parquetPublishFunc) error { - data, err := os.ReadFile(filepath.Join(r.root, "COMPACTION.json")) - if errors.Is(err, os.ErrNotExist) { - return nil - } - if err != nil { - return err - } - var marker compactionMarker - if err := json.Unmarshal(data, &marker); err != nil { - return err - } - if err := validateCompactionMarker(marker); err != nil { - return err - } - stageExists, err := pathExists(r.compactionStage(marker.Output.ID)) - if err != nil { - return err - } - finalExists, err := pathExists(r.Parquet.BatchPath(marker.Output.ID)) - if err != nil { - return err - } - if !stageExists && !finalExists { - if err := r.Parquet.RestoreRetiredInputs(marker.Inputs, marker.Output.ID, func(swap func(context.Context) error) error { - return publish(ctx, telemetry.BatchMetadata{}, nil, swap) - }); err != nil { - return fmt.Errorf("restore compaction %s inputs: %w", marker.Output.ID, err) - } - if err := os.Remove(filepath.Join(r.root, "COMPACTION.json")); err != nil && !errors.Is(err, os.ErrNotExist) { - return err - } - return syncDirectory(r.root) - } - return r.completeCompaction(ctx, marker, publish) -} - -func (r *Repository) completeCompaction(ctx context.Context, marker compactionMarker, publish parquetPublishFunc) error { - if err := r.Parquet.PublishReplacement(r.compactionStage(marker.Output.ID), marker.Output, marker.Inputs, func(swap func(context.Context) error) error { - return publish(ctx, marker.Output, marker.Inputs, swap) - }); err != nil { - return err - } - // Recovery can finish publication too; only higher-generation outputs - // reset the debt-preemption size. - if marker.Output.Generation > 1 { - r.preemptedCompactionInputs = 0 - } - markerPath := filepath.Join(r.root, "COMPACTION.json") - if err := os.Remove(markerPath); err != nil && !errors.Is(err, os.ErrNotExist) { - return err - } - return syncDirectory(r.root) -} - func (r *Repository) compactionStage(id string) string { return filepath.Join(r.root, "compaction", id) } -func validateCompactionMarker(marker compactionMarker) error { - if err := telemetry.ValidateBatchID(marker.Output.ID); err != nil { - return fmt.Errorf("invalid compaction output: %w", err) - } - if len(marker.Inputs) == 0 { - return errors.New("compaction marker has no inputs") - } - for _, id := range marker.Inputs { - if err := telemetry.ValidateBatchID(id); err != nil { - return fmt.Errorf("invalid compaction input: %w", err) - } - } - return nil -} - func syncFile(path string) error { f, err := os.OpenFile(path, os.O_RDWR, 0) if err != nil { @@ -694,17 +606,6 @@ func syncDirectory(path string) error { return dir.Sync() } -func pathExists(path string) (bool, error) { - _, err := os.Stat(path) - if err == nil { - return true, nil - } - if errors.Is(err, os.ErrNotExist) { - return false, nil - } - return false, err -} - func writeDurableFile(path string, data []byte) error { tmp := path + ".tmp" f, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600) diff --git a/internal/telemetry/store/compaction_preemption_test.go b/internal/telemetry/store/compaction_preemption_test.go index 22104cbb3..f3dcff264 100644 --- a/internal/telemetry/store/compaction_preemption_test.go +++ b/internal/telemetry/store/compaction_preemption_test.go @@ -73,9 +73,6 @@ func TestHigherGenerationMergeYieldsWithoutBlockingDebtRetirement(t *testing.T) if r.CompactionDebt() >= telemetry.CompactionDebtLimit { t.Fatal("debt did not retire") } - if err := r.RecoverParquet(ctx, &testParquetCompactor{}); err != nil { - t.Fatal(err) - } if err := r.CleanupParquet(); err != nil { t.Fatal(err) } diff --git a/internal/telemetry/store/compaction_recovery_test.go b/internal/telemetry/store/compaction_recovery_test.go new file mode 100644 index 000000000..ebeee34f1 --- /dev/null +++ b/internal/telemetry/store/compaction_recovery_test.go @@ -0,0 +1,291 @@ +package store + +import ( + "context" + "errors" + "fmt" + "math" + "os" + "path/filepath" + "testing" + + "github.com/labstack/pitot/internal/telemetry" +) + +// Compaction keeps no durable marker and performs no boot-time rollback. A +// compaction merges its inputs into compaction/ staging and commits by +// renaming that staging into the batch namespace, where the output's +// replaces.json retires the inputs. These tests pin that both sides of that +// single rename are crash-safe, by abandoning a repository in each window and +// reopening the same directory. Each one asserts exact row counts and catalog +// composition, so it fails if recovery stops masking inputs by replaces.json, +// completes a merge it should have discarded, or loses rows. + +// seedCompactionInputs commits n single-row-per-signal batches that all belong +// to trace-1, so the trace query returns one span per surviving input. +func seedCompactionInputs(t *testing.T, repository *Repository, n int) []string { + t.Helper() + ids := make([]string, n) + for i := range n { + batch := testBatch() + batch.ID = fmt.Sprintf("in-%d", i) + batch.Spans[0].SpanID = fmt.Sprintf("span-%d", i) + batch.Spans[0].StartUnixNanos = int64(100 + i) + if err := repository.Commit(context.Background(), batch); err != nil { + t.Fatal(err) + } + ids[i] = batch.ID + } + return ids +} + +// stageCompactionMerge reproduces a compaction that merged a complete output +// into compaction/ staging but has not yet renamed it into the batch +// namespace -- the exact on-disk state when a process dies after the merge and +// before the commit. It writes no marker, no replaces.json, and never renames. +func stageCompactionMerge(t *testing.T, repository *Repository, outID string, inputIDs []string) telemetry.BatchMetadata { + t.Helper() + byID := map[string]telemetry.BatchMetadata{} + for _, m := range repository.Parquet.BatchMetadata() { + byID[m.ID] = m + } + output := telemetry.BatchMetadata{ID: outID, Generation: 1, MinIngestedNanos: math.MaxInt64} + for _, id := range inputIDs { + m := byID[id] + if m.MinIngestedNanos > 0 { + output.MinIngestedNanos = min(output.MinIngestedNanos, m.MinIngestedNanos) + } + output.MaxIngestedNanos = max(output.MaxIngestedNanos, m.MaxIngestedNanos) + output.Spans += m.Spans + output.Logs += m.Logs + output.Metrics += m.Metrics + } + if output.MinIngestedNanos == math.MaxInt64 { + output.MinIngestedNanos = 0 + } + stage := repository.compactionStage(outID) + if err := os.MkdirAll(stage, 0o755); err != nil { + t.Fatal(err) + } + for _, s := range telemetry.Signals { + var paths []string + for _, id := range inputIDs { + path := filepath.Join(repository.Parquet.BatchPath(id), s.Name+".parquet") + if _, err := os.Stat(path); err == nil { + paths = append(paths, path) + } else if err != nil { + t.Fatal(err) + } + } + if len(paths) == 0 { + continue + } + stats, err := repository.Parquet.MergeParquet(context.Background(), s.Name, paths, filepath.Join(stage, s.Name+".parquet")) + if err != nil { + t.Fatal(err) + } + if s.Name == "spans" { + output.MinSpanStartNanos = stats.MinSpanStartNanos + output.MaxSpanStartNanos = stats.MaxSpanStartNanos + } + } + if err := repository.Parquet.PrepareReplacement(stage, output); err != nil { + t.Fatal(err) + } + return output +} + +// W1: a merge produced a complete compaction/ stage but the process +// died before the stage->final rename. No output reached the batch namespace, +// so the inputs are still the authoritative copy. Reopening must keep every +// input live and queryable, must not surface the output, and must discard the +// staging directory. +func TestInterruptedCompactionBeforeRenameKeepsInputsLive(t *testing.T) { + dir := t.TempDir() + repository, err := Open(dir) + if err != nil { + t.Fatal(err) + } + const inputs = minCompactionInputs + ids := seedCompactionInputs(t, repository, inputs) + output := stageCompactionMerge(t, repository, "compact-before-rename", ids) + // Abandon the process with the stage prepared but never committed. + if err := repository.Close(); err != nil { + t.Fatal(err) + } + + reopened, err := Open(dir) + if err != nil { + t.Fatal(err) + } + defer reopened.Close() + metadata := reopened.Parquet.BatchMetadata() + if len(metadata) != inputs { + t.Fatalf("catalog holds %d batches, want the %d live inputs", len(metadata), inputs) + } + for _, m := range metadata { + if m.ID == output.ID { + t.Fatalf("an unpublished compaction output became visible: %s", output.ID) + } + } + for _, id := range ids { + if _, err := os.Stat(reopened.Parquet.BatchPath(id)); err != nil { + t.Fatalf("input %s was lost: %v", id, err) + } + } + if got := reopened.RowCount(); got != uint64(3*inputs) { + t.Fatalf("rows after discarding the stage = %d, want %d", got, 3*inputs) + } + spans, err := traceAll(reopened, "trace-1") + if err != nil || len(spans) != inputs { + t.Fatalf("trace spans after discarding the stage = %d, %v", len(spans), err) + } + if _, err := os.Stat(filepath.Join(dir, "compaction")); !os.IsNotExist(err) { + t.Fatalf("compaction staging was not discarded: %v", err) + } +} + +// W2: the stage->final rename landed, so the output and its replaces.json are +// durable, but the process died before the retired input directories were +// deleted. Reopening must surface the output, retire the inputs named by +// replaces.json without double-counting them, and reclaim their directories. +func TestInterruptedCompactionAfterRenameRetiresInputsFromReplaces(t *testing.T) { + dir := t.TempDir() + repository, err := Open(dir) + if err != nil { + t.Fatal(err) + } + const inputs = minCompactionInputs + ids := seedCompactionInputs(t, repository, inputs) + // The real compaction path renames the output into place and writes its + // replaces.json; deferred deletion (CollectRetiredParquet) is deliberately + // not run, so the retired input directories are still on disk -- the window + // this test reopens in. + compacted, err := repository.CompactParquet(context.Background(), &testParquetCompactor{}, minCompactionInputs) + if err != nil || compacted != inputs { + t.Fatalf("compaction = %d, %v", compacted, err) + } + for _, id := range ids { + if _, err := os.Stat(repository.Parquet.BatchPath(id)); err != nil { + t.Fatalf("input %s should still be on disk before deferred deletion: %v", id, err) + } + } + if err := repository.Close(); err != nil { + t.Fatal(err) + } + + reopened, err := Open(dir) + if err != nil { + t.Fatal(err) + } + defer reopened.Close() + metadata := reopened.Parquet.BatchMetadata() + if len(metadata) != 1 || metadata[0].Generation != 1 { + t.Fatalf("catalog = %#v, want the single compacted output", metadata) + } + if got := reopened.RowCount(); got != uint64(3*inputs) { + t.Fatalf("rows after recovery = %d, want %d with inputs masked once", got, 3*inputs) + } + spans, err := traceAll(reopened, "trace-1") + if err != nil || len(spans) != inputs { + t.Fatalf("trace spans after recovery = %d, %v", len(spans), err) + } + for _, id := range ids { + if _, err := os.Stat(reopened.Parquet.BatchPath(id)); !os.IsNotExist(err) { + t.Fatalf("retired input %s was not reclaimed: %v", id, err) + } + } +} + +// W3: the scenario the deleted TestRepositoryRecoversInterruptedCompactionSwap +// covered, re-expressed against replaces.json recovery. The output is durable +// and deferred deletion got halfway: some retired input directories are gone, +// some remain. Reopening must not expose a half-old/half-new file set -- only +// the output is authoritative, every row survives exactly once, and the +// leftover input directories are reclaimed. +func TestInterruptedCompactionMidInputDeletionRetiresRemainder(t *testing.T) { + dir := t.TempDir() + repository, err := Open(dir) + if err != nil { + t.Fatal(err) + } + const inputs = minCompactionInputs + ids := seedCompactionInputs(t, repository, inputs) + compacted, err := repository.CompactParquet(context.Background(), &testParquetCompactor{}, minCompactionInputs) + if err != nil || compacted != inputs { + t.Fatalf("compaction = %d, %v", compacted, err) + } + // Reproduce a crash partway through deleting the retired inputs: half the + // directories are already gone, half still linger beside the output. + for _, id := range ids[:inputs/2] { + if err := os.RemoveAll(repository.Parquet.BatchPath(id)); err != nil { + t.Fatal(err) + } + } + if err := repository.Close(); err != nil { + t.Fatal(err) + } + + reopened, err := Open(dir) + if err != nil { + t.Fatal(err) + } + defer reopened.Close() + metadata := reopened.Parquet.BatchMetadata() + if len(metadata) != 1 || metadata[0].ID == "" || metadata[0].Generation != 1 { + t.Fatalf("catalog = %#v, want only the compacted output", metadata) + } + if got := reopened.RowCount(); got != uint64(3*inputs) { + t.Fatalf("rows after partial deletion recovery = %d, want %d", got, 3*inputs) + } + spans, err := traceAll(reopened, "trace-1") + if err != nil || len(spans) != inputs { + t.Fatalf("trace spans after partial deletion recovery = %d, %v", len(spans), err) + } + leftover, err := filepath.Glob(filepath.Join(reopened.Parquet.BatchesDir(), "in-*")) + if err != nil || len(leftover) != 0 { + t.Fatalf("retired inputs after recovery = %v, %v", leftover, err) + } +} + +// A compaction whose publish fails leaves no marker and no partial output: the +// staging is discarded and the inputs stay live, so no row is lost and a later +// pass simply redoes the merge. +func TestFailedCompactionPublishKeepsInputsLiveAndRetryPreservesRows(t *testing.T) { + dir := t.TempDir() + repository, err := Open(dir) + if err != nil { + t.Fatal(err) + } + defer repository.Close() + const inputs = minCompactionInputs + seedCompactionInputs(t, repository, inputs) + + compactor := &testParquetCompactor{publishErr: errors.New("publication unavailable")} + if _, err := repository.CompactParquet(context.Background(), compactor, minCompactionInputs); err == nil { + t.Fatal("compaction reported success despite a publication failure") + } + if metadata := repository.Parquet.BatchMetadata(); len(metadata) != inputs { + t.Fatalf("catalog = %d batches after a failed publish, want the %d inputs untouched", len(metadata), inputs) + } + if got := repository.RowCount(); got != uint64(3*inputs) { + t.Fatalf("rows after a failed publish = %d, want %d", got, 3*inputs) + } + spans, err := traceAll(repository, "trace-1") + if err != nil || len(spans) != inputs { + t.Fatalf("trace spans after a failed publish = %d, %v", len(spans), err) + } + + compactor.publishErr = nil + compacted, err := repository.CompactParquet(context.Background(), compactor, minCompactionInputs) + if err != nil || compacted < minCompactionMergeInputs { + t.Fatalf("retry compaction = %d, %v", compacted, err) + } + if got := repository.RowCount(); got != uint64(3*inputs) { + t.Fatalf("rows after a successful retry = %d, want %d", got, 3*inputs) + } + spans, err = traceAll(repository, "trace-1") + if err != nil || len(spans) != inputs { + t.Fatalf("trace spans after a successful retry = %d, %v", len(spans), err) + } +} diff --git a/internal/telemetry/store/repair.go b/internal/telemetry/store/repair.go index 89fd44a2b..d9cf44b99 100644 --- a/internal/telemetry/store/repair.go +++ b/internal/telemetry/store/repair.go @@ -2,7 +2,6 @@ package store import ( "context" - "encoding/json" "errors" "fmt" "os" @@ -90,9 +89,11 @@ func VerifyBatches(root string) ([]BatchIssue, error) { } // QuarantineBatch atomically sets aside one specifically named unreadable -// batch. Valid data and any batch protected by a live compaction transaction -// are refused. The returned directory remains beside the authoritative set so -// an operator can recover it from a backup or rename it back after repair. +// batch. Valid data is refused. The returned directory remains beside the +// authoritative set so an operator can recover it from a backup or rename it +// back after repair. Repair runs only against a data directory no live +// instance holds -- the data-directory lock enforces that -- so there is no +// in-process compaction for it to collide with. func QuarantineBatch(root, id string) (string, error) { // Refuse to rename batches out from under a live store, which has no // cross-process coordination with this command. @@ -111,9 +112,6 @@ func quarantineBatch(root, id string, validate func(string) error) (string, erro if id+telemetry.BatchSuffix == telemetry.SchemaBatch { return "", errors.New("the Parquet schema batch cannot be quarantined") } - if err := ensureBatchOutsideLiveCompaction(root, id); err != nil { - return "", err - } batchesDir := filepath.Join(root, "parquet", "batches") source := filepath.Join(batchesDir, id+telemetry.BatchSuffix) info, err := os.Lstat(source) @@ -137,30 +135,3 @@ func quarantineBatch(root, id string, validate func(string) error) (string, erro } return destination, nil } - -func ensureBatchOutsideLiveCompaction(root, id string) error { - path := filepath.Join(root, "COMPACTION.json") - data, err := os.ReadFile(path) - if errors.Is(err, os.ErrNotExist) { - return nil - } - if err != nil { - return fmt.Errorf("inspect live compaction marker: %w", err) - } - var marker compactionMarker - if err := json.Unmarshal(data, &marker); err != nil { - return fmt.Errorf("read live compaction marker: %w", err) - } - if err := validateCompactionMarker(marker); err != nil { - return fmt.Errorf("read live compaction marker: %w", err) - } - if marker.Output.ID == id { - return fmt.Errorf("telemetry batch %s belongs to a live compaction; resolve that transaction before repair", id) - } - for _, input := range marker.Inputs { - if input == id { - return fmt.Errorf("telemetry batch %s belongs to a live compaction; resolve that transaction before repair", id) - } - } - return nil -} diff --git a/internal/telemetry/store/repair_test.go b/internal/telemetry/store/repair_test.go index 2144696c8..1949f9009 100644 --- a/internal/telemetry/store/repair_test.go +++ b/internal/telemetry/store/repair_test.go @@ -2,7 +2,6 @@ package store import ( "context" - "encoding/json" "errors" "fmt" "os" @@ -172,34 +171,6 @@ func TestConcurrentVerifyReportsUnsupportedFormat(t *testing.T) { } } -func TestQuarantineBatchRefusesLiveCompactionMembers(t *testing.T) { - root := t.TempDir() - repository, err := Open(root) - if err != nil { - t.Fatal(err) - } - if err := repository.Commit(context.Background(), Batch{ - ID: "input", Spans: []telemetry.Span{{TraceID: "trace", SpanID: "span"}}, - }); err != nil { - t.Fatal(err) - } - if err := repository.Close(); err != nil { - t.Fatal(err) - } - corruptBatchFile(t, root, "input", "spans.parquet") - marker := compactionMarker{Output: telemetry.BatchMetadata{ID: "replacement"}, Inputs: []string{"input"}} - data, err := json.Marshal(marker) - if err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(root, "COMPACTION.json"), data, 0o644); err != nil { - t.Fatal(err) - } - if _, err := QuarantineBatch(root, "input"); err == nil || !strings.Contains(err.Error(), "live compaction") { - t.Fatalf("QuarantineBatch live input error = %v", err) - } -} - func TestQuarantineBatchRejectsUnsafeID(t *testing.T) { if _, err := QuarantineBatch(t.TempDir(), "../outside"); err == nil { t.Fatal("unsafe batch ID was accepted") diff --git a/internal/telemetry/store/repository.go b/internal/telemetry/store/repository.go index 79e4b8048..ec62ec3f2 100644 --- a/internal/telemetry/store/repository.go +++ b/internal/telemetry/store/repository.go @@ -2,10 +2,8 @@ package store import ( "context" - "encoding/json" "errors" "fmt" - "log/slog" "os" "path/filepath" "strings" @@ -57,14 +55,16 @@ func Open(root string) (*Repository, error) { return nil, err } r := &Repository{root: root, Parquet: parquetStore, lock: lock} - if err := r.recoverCompaction(context.Background(), func(ctx context.Context, _ telemetry.BatchMetadata, _ []string, publish func(context.Context) error) error { - return publish(ctx) - }); err != nil { - r.logUnresolvedCompaction(err) - _ = r.Close() - return nil, fmt.Errorf("recover Parquet compaction (unresolved marker at %s): %w", - filepath.Join(root, "COMPACTION.json"), err) - } + // Compaction has no durable marker and no boot-time rollback. A compaction + // is purely in-process: it merges into compaction/ staging and + // commits by renaming that into the batch namespace, where the output's + // replaces.json retires its inputs. So the entire recovery protocol is to + // discard the staging directory and let replaces.json decide. A crash + // before the rename leaves the inputs live as .batch and the staging is + // dropped here; a crash after it leaves the output present with its + // replaces.json, which loadBatches has already applied to retire the + // inputs. Either way the on-disk state is consistent and nothing refuses + // to start. if err := r.cleanupCompactionArtifacts(); err != nil { _ = r.Close() return nil, fmt.Errorf("clean Parquet compaction staging: %w", err) @@ -88,51 +88,15 @@ func Open(root string) (*Repository, error) { return r, nil } -// compactionRunbookURL is the manual rollback procedure. The startup log links -// to it rather than restating it, because the procedure has preconditions that -// are refined as the failure modes are understood, and a copy here drifts from -// the guide silently — each revision so far has corrected one location and left -// the other saying something the code no longer guarantees. Guidance an -// operator can act on wrongly is worse than guidance in one place they have to -// open. TestUnresolvedCompactionRunbookExists keeps the link honest. -const compactionRunbookURL = "https://pitot.dev/docs/guides/troubleshooting#an-unresolved-compaction-blocks-startup" - -// logUnresolvedCompaction spells out the operator's options for a marker that -// blocks startup. -// -// Refusing to boot is only the safe half of the decision. A marker names the -// retired inputs a rollback still needs, and those directories hold the only -// copy of the rows the compaction was merging, so a startup that guessed at -// the rollback set could delete them — which is why recovery fails closed. The -// other half is saying where to look, because the reachable next step for an -// instance that will not start is rm -rf of the data directory, and that -// destroys exactly what failing closed preserved. -// -// The guidance is logged rather than wrapped into the error: the paths and the -// ordering constraint do not fit an error string that composes, and the -// rollback warning is the kind of thing an operator has to be able to read -// once, in full, at the moment the process refuses to come up. -func (r *Repository) logUnresolvedCompaction(err error) { - slog.Error("Parquet compaction is unresolved; Pitot will not start", - "err", err, - "marker", filepath.Join(r.root, "COMPACTION.json"), - "compaction_staging", filepath.Join(r.root, "compaction"), - "retired_inputs", r.Parquet.BatchesDir(), - "preserved_state", "the live marker is retained and startup cleanup will not run after this failure", - "retry", "clear the underlying cause and start again; recovery re-runs on its own", - "runbook", compactionRunbookURL, - "warning", "deleting the marker on its own is not a rollback; cleanup then treats the retired inputs as reclaimable and removes them") -} - -// cleanupCompactionArtifacts drops staging left after compaction recovery has -// completed and consumed its live marker. +// cleanupCompactionArtifacts discards the compaction staging directory on +// boot. A compaction that did not commit never renamed its staging into the +// batch namespace, so the staging holds no authoritative rows; one that did +// commit already left the output in the namespace, so the staging is spent. +// Dropping it is the whole boot-time recovery for compaction. func (r *Repository) cleanupCompactionArtifacts() error { if err := os.RemoveAll(filepath.Join(r.root, "compaction")); err != nil { return err } - if err := os.Remove(filepath.Join(r.root, "COMPACTION.json.tmp")); err != nil && !errors.Is(err, os.ErrNotExist) { - return err - } return syncDirectory(r.root) } @@ -170,11 +134,11 @@ func (r *Repository) CollectRetiredParquet() (int, error) { return r.Parquet.CollectRetired() } +// cleanupRetired reclaims retention's .retired directories whose removal +// was interrupted after the rename. Compaction no longer produces any retired +// directory of its own: it retires inputs through each output's replaces.json, +// and CollectRetired deletes those input directories in place. func (r *Repository) cleanupRetired() error { - protected, err := r.protectedRetiredSuffix() - if err != nil { - return err - } entries, err := os.ReadDir(r.Parquet.BatchesDir()) if err != nil { return err @@ -183,16 +147,7 @@ func (r *Repository) cleanupRetired() error { var cleanupErr error for _, entry := range entries { name := entry.Name() - if !entry.IsDir() || strings.HasSuffix(name, telemetry.BatchSuffix) || !strings.Contains(name, ".retired") { - continue - } - // The live marker is the only thing that makes a retired directory - // unreclaimable, and it protects exactly one output's inputs. So - // removing COMPACTION.json without first renaming its - // .retired- directories back to .batch does not - // roll the compaction back — it makes the rows here deletable, and - // this loop deletes them on the next pass. - if protected != "" && strings.HasSuffix(name, protected) { + if !entry.IsDir() || !strings.HasSuffix(name, ".retired") { continue } if err := os.RemoveAll(filepath.Join(r.Parquet.BatchesDir(), name)); err != nil { @@ -207,26 +162,6 @@ func (r *Repository) cleanupRetired() error { return cleanupErr } -// protectedRetiredSuffix names the retired-input set a live compaction marker -// may still need. An unreadable marker fails cleanup closed. -func (r *Repository) protectedRetiredSuffix() (string, error) { - data, err := os.ReadFile(filepath.Join(r.root, "COMPACTION.json")) - if errors.Is(err, os.ErrNotExist) { - return "", nil - } - if err != nil { - return "", err - } - var marker compactionMarker - if err := json.Unmarshal(data, &marker); err != nil { - return "", fmt.Errorf("read live compaction marker: %w", err) - } - if err := validateCompactionMarker(marker); err != nil { - return "", err - } - return ".retired-" + marker.Output.ID, nil -} - // beginPublication reserves a strictly increasing attempt stamp before any row // can become visible. The same mutex protects reservations and the horizon. func (r *Repository) beginPublication(now int64) int64 { diff --git a/internal/telemetry/store/repository_test.go b/internal/telemetry/store/repository_test.go index 2b351231f..8496b1d0f 100644 --- a/internal/telemetry/store/repository_test.go +++ b/internal/telemetry/store/repository_test.go @@ -3,7 +3,6 @@ package store import ( "context" "database/sql" - "encoding/json" "errors" "fmt" "os" @@ -144,25 +143,22 @@ func TestRepositoryHasOneAuthoritativeStorageLayout(t *testing.T) { } } -func TestRepositoryCleansUnpublishedCompactionArtifacts(t *testing.T) { +func TestRepositoryBootDiscardsCompactionStaging(t *testing.T) { dir := t.TempDir() staging := filepath.Join(dir, "compaction", "orphan") if err := os.MkdirAll(staging, 0o755); err != nil { t.Fatal(err) } - temporaryMarker := filepath.Join(dir, "COMPACTION.json.tmp") - if err := os.WriteFile(temporaryMarker, []byte("partial"), 0o600); err != nil { - t.Fatal(err) - } repository, err := Open(dir) if err != nil { t.Fatal(err) } defer repository.Close() - for _, path := range []string{filepath.Join(dir, "compaction"), temporaryMarker} { - if _, err := os.Stat(path); !os.IsNotExist(err) { - t.Fatalf("unpublished compaction artifact remains at %s: %v", path, err) - } + // Discarding the staging directory is the whole boot-time recovery for an + // interrupted compaction: a merge that never renamed its output into the + // batch namespace left no authoritative rows behind. + if _, err := os.Stat(filepath.Join(dir, "compaction")); !os.IsNotExist(err) { + t.Fatalf("compaction staging remains after boot: %v", err) } } @@ -221,103 +217,6 @@ func TestRepositoryPrunesOnlyExpiredBatches(t *testing.T) { } } -func TestRepositoryDiscardsRecoverableMarkerWithoutOutput(t *testing.T) { - dir := t.TempDir() - repository, err := Open(dir) - if err != nil { - t.Fatal(err) - } - batch := testBatch() - batch.ID = "intact-input" - if err := repository.Commit(context.Background(), batch); err != nil { - t.Fatal(err) - } - marker := compactionMarker{ - Output: telemetry.BatchMetadata{ID: "missing-output", Generation: 1}, - Inputs: []string{batch.ID}, - } - data, err := json.Marshal(marker) - if err != nil { - t.Fatal(err) - } - if err := writeDurableFile(filepath.Join(dir, "COMPACTION.json"), data); err != nil { - t.Fatal(err) - } - retired := filepath.Join(repository.Parquet.BatchesDir(), batch.ID+".retired-"+marker.Output.ID) - if err := os.Rename(repository.Parquet.BatchPath(batch.ID), retired); err != nil { - t.Fatal(err) - } - if err := repository.Close(); err != nil { - t.Fatal(err) - } - - recovered, err := Open(dir) - if err != nil { - t.Fatal(err) - } - defer recovered.Close() - if got := recovered.RowCount(); got != 3 { - t.Fatalf("rows after marker recovery = %d, want 3", got) - } - if _, err := os.Stat(filepath.Join(dir, "COMPACTION.json")); !os.IsNotExist(err) { - t.Fatalf("recoverable marker remains: %v", err) - } -} - -func TestRepositoryRestoresInputsThroughPublicationGate(t *testing.T) { - dir := t.TempDir() - repository, err := Open(dir) - if err != nil { - t.Fatal(err) - } - defer repository.Close() - batch := testBatch() - batch.ID = "retired-input" - if err := repository.Commit(context.Background(), batch); err != nil { - t.Fatal(err) - } - marker := compactionMarker{ - Output: telemetry.BatchMetadata{ID: "missing-output", Generation: 1}, - Inputs: []string{batch.ID}, - } - data, err := json.Marshal(marker) - if err != nil { - t.Fatal(err) - } - if err := writeDurableFile(filepath.Join(dir, "COMPACTION.json"), data); err != nil { - t.Fatal(err) - } - retired := filepath.Join(repository.Parquet.BatchesDir(), batch.ID+".retired-"+marker.Output.ID) - if err := os.Rename(repository.Parquet.BatchPath(batch.ID), retired); err != nil { - t.Fatal(err) - } - - entered := make(chan struct{}) - release := make(chan struct{}) - publisher := testParquetPublisherFunc(func(ctx context.Context, publish func(context.Context) error) error { - close(entered) - <-release - return publish(ctx) - }) - done := make(chan error, 1) - go func() { done <- repository.RecoverParquet(context.Background(), publisher) }() - select { - case <-entered: - case <-time.After(time.Second): - t.Fatal("recovery did not enter the publication gate") - } - if _, err := os.Stat(repository.Parquet.BatchPath(batch.ID)); !os.IsNotExist(err) { - t.Fatalf("input became visible before publication: %v", err) - } - close(release) - if err := <-done; err != nil { - t.Fatal(err) - } - if _, err := os.Stat(repository.Parquet.BatchPath(batch.ID)); err != nil { - t.Fatalf("input was not restored after publication: %v", err) - } -} - func TestRepositoryPrunePassDrainsWithinBudgetAndOldestFirst(t *testing.T) { repository, err := Open(t.TempDir()) if err != nil { @@ -389,7 +288,7 @@ func TestRepositoryCleanupOwnsRetiredDirectories(t *testing.T) { t.Fatal(err) } defer repository.Close() - retired := filepath.Join(repository.Parquet.BatchesDir(), "old.retired-compacted") + retired := filepath.Join(repository.Parquet.BatchesDir(), "old.retired") if err := os.Mkdir(retired, 0o755); err != nil { t.Fatal(err) } @@ -493,9 +392,6 @@ func TestRepositoryCompactsParquetWithoutChangingRows(t *testing.T) { t.Fatalf("compacted %s rows = %d", signal, count) } } - if _, err := os.Stat(filepath.Join(dir, "COMPACTION.json")); !os.IsNotExist(err) { - t.Fatalf("completed compaction marker remains: %v", err) - } } func TestSelectCompactionBatchesRequiresFullSmallFileGroup(t *testing.T) { @@ -575,126 +471,6 @@ func TestSelectCompactionBatchesSkipsUncompactableOlderGroup(t *testing.T) { } } -func TestRepositoryRecoversPendingCompactionWithoutRestart(t *testing.T) { - dir := t.TempDir() - repository, err := Open(dir) - if err != nil { - t.Fatal(err) - } - defer repository.Close() - for i := range minCompactionInputs { - batch := testBatch() - batch.ID = fmt.Sprintf("pending-%d", i) - batch.Spans[0].SpanID = fmt.Sprintf("span-%d", i) - if err := repository.Commit(context.Background(), batch); err != nil { - t.Fatal(err) - } - } - compactor := &testParquetCompactor{publishErr: errors.New("publication unavailable")} - if _, err := repository.CompactParquet(context.Background(), compactor, minCompactionInputs); err == nil { - t.Fatal("compaction succeeded despite publication failure") - } - if _, err := os.Stat(filepath.Join(dir, "COMPACTION.json")); err != nil { - t.Fatalf("pending marker: %v", err) - } - compactor.publishErr = nil - if compacted, err := repository.CompactParquet(context.Background(), compactor, minCompactionInputs); err != nil || compacted != 0 { - t.Fatalf("resume compaction = %d, %v", compacted, err) - } - if got := repository.Parquet.BatchMetadata(); len(got) != 1 || got[0].Generation != 1 { - t.Fatalf("recovered metadata = %#v", got) - } - if _, err := os.Stat(filepath.Join(dir, "COMPACTION.json")); !os.IsNotExist(err) { - t.Fatalf("recovered compaction marker remains: %v", err) - } -} - -func TestRepositoryRecoversInterruptedCompactionSwap(t *testing.T) { - dir := t.TempDir() - repository, err := Open(dir) - if err != nil { - t.Fatal(err) - } - for i := range minCompactionInputs { - batch := testBatch() - batch.ID = fmt.Sprintf("recover-%d", i) - batch.Spans[0].SpanID = fmt.Sprintf("span-%d", i) - if err := repository.Commit(context.Background(), batch); err != nil { - t.Fatal(err) - } - } - output := telemetry.BatchMetadata{ - ID: "compact-recovery", MinIngestedNanos: 100, MaxIngestedNanos: 300, Generation: 1, - Spans: minCompactionInputs, Logs: minCompactionInputs, Metrics: minCompactionInputs, - } - stage := repository.compactionStage(output.ID) - if err := os.MkdirAll(stage, 0o755); err != nil { - t.Fatal(err) - } - for _, s := range telemetry.Signals { - signal := s.Name - paths := make([]string, minCompactionInputs) - for i := range paths { - paths[i] = filepath.Join(repository.Parquet.BatchPath(fmt.Sprintf("recover-%d", i)), signal+".parquet") - } - stats, err := repository.Parquet.MergeParquet(context.Background(), signal, paths, filepath.Join(stage, signal+".parquet")) - if err != nil { - t.Fatal(err) - } - if signal == "spans" { - output.MinSpanStartNanos = stats.MinSpanStartNanos - output.MaxSpanStartNanos = stats.MaxSpanStartNanos - } - } - if err := repository.Parquet.PrepareReplacement(stage, output); err != nil { - t.Fatal(err) - } - inputs := make([]string, minCompactionInputs) - for i := range inputs { - inputs[i] = fmt.Sprintf("recover-%d", i) - } - marker := compactionMarker{Output: output, Inputs: inputs} - data, err := json.Marshal(marker) - if err != nil { - t.Fatal(err) - } - if err := writeDurableFile(filepath.Join(dir, "COMPACTION.json"), data); err != nil { - t.Fatal(err) - } - // Reproduce a crash halfway through retiring inputs. Startup must finish - // the intended publication, not expose a half-old/half-new file set. - for _, id := range inputs[:4] { - if err := os.Rename(repository.Parquet.BatchPath(id), filepath.Join(repository.Parquet.BatchesDir(), id+".retired-"+output.ID)); err != nil { - t.Fatal(err) - } - } - // The crashed process exits, releasing its data-directory lock while the - // marker, staged output, and retired inputs stay on disk for recovery. - if err := repository.Close(); err != nil { - t.Fatal(err) - } - - recovered, err := Open(dir) - if err != nil { - t.Fatal(err) - } - defer recovered.Close() - metadata := recovered.Parquet.BatchMetadata() - if len(metadata) != 1 || metadata[0].ID != output.ID || recovered.RowCount() != 3*minCompactionInputs { - t.Fatalf("recovered metadata = %#v rows=%d", metadata, recovered.RowCount()) - } - spans, err := traceAll(recovered, "trace-1") - if err != nil || len(spans) != minCompactionInputs { - t.Fatalf("recovered trace spans = %d, %v", len(spans), err) - } - if retired, err := filepath.Glob(filepath.Join(recovered.Parquet.BatchesDir(), "*.retired*")); err != nil || len(retired) != 0 { - t.Fatalf("retired inputs after recovery = %v, %v", retired, err) - } - if _, err := os.Stat(filepath.Join(dir, "COMPACTION.json")); !os.IsNotExist(err) { - t.Fatalf("compaction marker remains after recovery: %v", err) - } -} - func openTestDuckDB(t *testing.T) *sql.DB { t.Helper() db, err := sql.Open("duckdb", "") @@ -711,217 +487,6 @@ func traceAll(repository *Repository, traceID string) ([]telemetry.IndexedSpan, return spans, err } -// seedFailedCompaction leaves a live COMPACTION.json whose recovery always -// fails, which is the state that used to latch maintenance off permanently. -func seedFailedCompaction(t *testing.T, dir string, repository *Repository) { - t.Helper() - for i := range minCompactionInputs { - batch := testBatch() - batch.ID = fmt.Sprintf("latched-%d", i) - batch.Spans[0].SpanID = fmt.Sprintf("span-%d", i) - if err := repository.Commit(context.Background(), batch); err != nil { - t.Fatal(err) - } - } - compactor := &testParquetCompactor{publishErr: errors.New("publication unavailable")} - if _, err := repository.CompactParquet(context.Background(), compactor, minCompactionInputs); err == nil { - t.Fatal("compaction succeeded despite publication failure") - } - if _, err := os.Stat(filepath.Join(dir, "COMPACTION.json")); err != nil { - t.Fatalf("expected a live marker: %v", err) - } -} - -// TestRepositoryRecoveryFailureStaysLive pins fail-closed recovery. The marker -// and rollback set remain authoritative until recovery succeeds; silently -// bypassing them can make old inputs eligible for deletion. -func TestRepositoryRecoveryFailureStaysLive(t *testing.T) { - dir := t.TempDir() - repository, err := Open(dir) - if err != nil { - t.Fatal(err) - } - defer repository.Close() - seedFailedCompaction(t, dir, repository) - - failing := testParquetPublisherFunc(func(context.Context, func(context.Context) error) error { - return errors.New("publication unavailable") - }) - for attempt := 1; attempt <= 5; attempt++ { - if err := repository.RecoverParquet(context.Background(), failing); err == nil { - t.Fatalf("attempt %d: recovery reported success", attempt) - } - if _, err := os.Stat(filepath.Join(dir, "COMPACTION.json")); err != nil { - t.Fatalf("attempt %d: live marker was removed: %v", attempt, err) - } - } - if _, err := os.Stat(filepath.Join(dir, "COMPACTION.json.failed")); !os.IsNotExist(err) { - t.Fatalf("recovery created a fallback marker: %v", err) - } -} - -// TestRepositoryCleanupPreservesMarkerRollbackSet pins that cleanup never -// deletes the retired inputs a pending marker still needs. Deleting one is -// unrecoverable: the input is gone and its rows were never published under the -// replacement, so the rows exist in no queryable batch at all. -func TestRepositoryCleanupPreservesMarkerRollbackSet(t *testing.T) { - dir := t.TempDir() - repository, err := Open(dir) - if err != nil { - t.Fatal(err) - } - defer repository.Close() - - marker := compactionMarker{ - Output: telemetry.BatchMetadata{ID: "compact-live", Generation: 1}, - Inputs: []string{"input-a"}, - } - data, err := json.Marshal(marker) - if err != nil { - t.Fatal(err) - } - if err := writeDurableFile(filepath.Join(dir, "COMPACTION.json"), data); err != nil { - t.Fatal(err) - } - batches := repository.Parquet.BatchesDir() - needed := filepath.Join(batches, "input-a.retired-compact-live") - stale := filepath.Join(batches, "input-b.retired") - for _, path := range []string{needed, stale} { - if err := os.MkdirAll(path, 0o755); err != nil { - t.Fatal(err) - } - } - - if err := repository.CleanupParquet(); err != nil { - t.Fatal(err) - } - if _, err := os.Stat(needed); err != nil { - t.Fatalf("cleanup deleted the rollback set of a live marker: %v", err) - } - if _, err := os.Stat(stale); !os.IsNotExist(err) { - t.Fatalf("cleanup left an unreferenced retired directory: %v", err) - } -} - -func TestRepositoryOpenFailsClosedOnUnrecoverableMarker(t *testing.T) { - dir := t.TempDir() - repository, err := Open(dir) - if err != nil { - t.Fatal(err) - } - seedFailedCompaction(t, dir, repository) - if err := repository.Close(); err != nil { - t.Fatal(err) - } - // Corrupt the staged output so recovery cannot complete or roll back. - stage := filepath.Join(dir, "compaction") - entries, err := os.ReadDir(stage) - if err != nil { - t.Fatal(err) - } - for _, entry := range entries { - if err := os.WriteFile(filepath.Join(stage, entry.Name(), "metadata.json"), []byte("{"), 0o644); err != nil { - t.Fatal(err) - } - } - - if reopened, err := Open(dir); err == nil { - _ = reopened.Close() - t.Fatal("Open succeeded with an unrecoverable marker") - } - if _, err := os.Stat(filepath.Join(dir, "COMPACTION.json")); err != nil { - t.Fatalf("Open removed the live marker: %v", err) - } - if _, err := os.Stat(stage); err != nil { - t.Fatalf("Open destroyed the staged output: %v", err) - } -} - -func TestRepositoryOpenFailsOnUnreadableMarker(t *testing.T) { - dir := t.TempDir() - repository, err := Open(dir) - if err != nil { - t.Fatal(err) - } - if err := repository.Close(); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(dir, "COMPACTION.json"), []byte("not json"), 0o644); err != nil { - t.Fatal(err) - } - for boot := 1; boot <= 3; boot++ { - if reopened, err := Open(dir); err == nil { - _ = reopened.Close() - t.Fatalf("boot %d accepted an unreadable marker", boot) - } - if _, err := os.Stat(filepath.Join(dir, "COMPACTION.json")); err != nil { - t.Fatalf("boot %d removed the unreadable marker: %v", boot, err) - } - } -} - -func TestRepositoryOpenRejectsUnsafeCompactionMarker(t *testing.T) { - dir := t.TempDir() - repository, err := Open(dir) - if err != nil { - t.Fatal(err) - } - if err := repository.Close(); err != nil { - t.Fatal(err) - } - marker := compactionMarker{ - Output: telemetry.BatchMetadata{ID: "../outside"}, - Inputs: []string{"input"}, - } - data, err := json.Marshal(marker) - if err != nil { - t.Fatal(err) - } - markerPath := filepath.Join(dir, "COMPACTION.json") - if err := os.WriteFile(markerPath, data, 0o600); err != nil { - t.Fatal(err) - } - if reopened, err := Open(dir); err == nil { - _ = reopened.Close() - t.Fatal("Open accepted a marker-controlled path outside storage") - } - if _, err := os.Stat(markerPath); err != nil { - t.Fatalf("Open removed the invalid live marker: %v", err) - } -} - -// TestUnresolvedCompactionRunbookExists keeps the startup log's runbook link -// pointing at a section that exists. The log stopped restating the rollback -// procedure because maintaining it in two places produced a string of -// revisions that corrected one copy and left the other unsafe; a link only -// removes that risk while it resolves, so the anchor is checked here rather -// than trusted. -func TestUnresolvedCompactionRunbookExists(t *testing.T) { - const guide = "../../../site/src/content/docs/docs/guides/troubleshooting.mdx" - data, err := os.ReadFile(guide) - if err != nil { - t.Fatalf("read the runbook the startup log links to: %v", err) - } - _, anchor, found := strings.Cut(compactionRunbookURL, "#") - if !found { - t.Fatalf("runbook URL %q has no anchor", compactionRunbookURL) - } - var headings []string - for _, line := range strings.Split(string(data), "\n") { - title, isHeading := strings.CutPrefix(line, "## ") - if !isHeading { - continue - } - slug := strings.ToLower(strings.TrimSpace(title)) - slug = strings.Join(strings.Fields(slug), "-") - headings = append(headings, slug) - if slug == anchor { - return - } - } - t.Fatalf("runbook anchor %q is not a section in %s; sections are %v", anchor, guide, headings) -} - // A live repository owns its data directory exclusively: it deletes, renames, // and rewrites files there with no cross-process coordination. A second process // opened on the same directory must fail fast rather than race the first into @@ -952,7 +517,6 @@ func TestOpenRejectsSecondProcessOnSameDataDir(t *testing.T) { t.Fatal(err) } } - func (f testParquetPublisherFunc) PublishParquetReplacement(ctx context.Context, _ telemetry.BatchMetadata, _ []string, publish func(context.Context) error) error { return f.PublishParquet(ctx, publish) } diff --git a/internal/telemetry/variant_test.go b/internal/telemetry/variant_test.go index c4b0a6394..af7f1508f 100644 --- a/internal/telemetry/variant_test.go +++ b/internal/telemetry/variant_test.go @@ -169,7 +169,7 @@ func TestShreddedResourcesAreSharedAcrossFilesAndBounded(t *testing.T) { } func TestUnsupportedFormatLeavesStoreUntouched(t *testing.T) { - for _, suffix := range []string{BatchSuffix, ".retired", ".retired-replacement"} { + for _, suffix := range []string{BatchSuffix, ".retired"} { t.Run(suffix, func(t *testing.T) { testUnsupportedFormatLeavesStoreUntouched(t, suffix) }) } } diff --git a/site/src/content/docs/docs/guides/troubleshooting.mdx b/site/src/content/docs/docs/guides/troubleshooting.mdx index de1647598..40dc01c08 100644 --- a/site/src/content/docs/docs/guides/troubleshooting.mdx +++ b/site/src/content/docs/docs/guides/troubleshooting.mdx @@ -18,49 +18,6 @@ settings. Unknown `PITOT_` setting names fail startup, except platform-injected service discovery names. Check spelling against the [configuration reference](/docs/reference/configuration). -## An unresolved compaction blocks startup - -One startup refusal is not about configuration. If a compaction was interrupted -and Pitot can neither finish nor undo it, recovery fails closed and the error -names `telemetry/COMPACTION.json`. - -The marker identifies the authoritative batch set during a swap. The replacement -is staged under `telemetry/compaction/` or published as -`telemetry/parquet/batches/.batch`. A published replacement lists -its inputs in `replaces.json`, and its presence alone retires them: inputs keep -their `.batch` directories until no reader can still open them, and startup -deletes any that remain. Releases before deferred deletion named retired inputs -`.retired-`. A completed publish can already have deleted some -inputs; verify every input before attempting a rollback. - -**Clear the underlying cause, such as a full disk, and start Pitot again.** -Recovery runs automatically at startup. - -If recovery keeps failing, roll the compaction back by hand. With the process -stopped, use `telemetry/parquet/batches/` as the batch directory: - -1. Read `telemetry/COMPACTION.json`. It is JSON, with an `output` object and an - `inputs` list of batch ids. -2. Back up `telemetry/`, then check every id in `inputs`. Each must exist in - exactly one form: `.batch` or `.retired-`. **If an id - exists in both forms or neither form, stop here.** Both forms make the - authoritative copy ambiguous; neither means its rows may survive only inside - the replacement. Reconcile the batch from the backup, or complete the - compaction instead of rolling it back. -3. Rename each `.retired-` back to `.batch`. -4. Delete `telemetry/compaction/` if it exists. -5. Delete `telemetry/parquet/batches/.batch` if it exists. While - it exists its `replaces.json` retires the inputs, and startup deletes them; - a corrupt replacement continues to block startup. -6. Delete `telemetry/COMPACTION.json`. -7. Start Pitot. - -:::caution[Verify inputs before deleting the replacement] -Restore the retired inputs before removing the compaction marker. The marker -protects those directories from startup cleanup. Keep the replacement until -every input is restored; it may hold the only surviving copy of some rows. -::: - ## An unreadable telemetry batch blocks startup Pitot also refuses to start when an authoritative `.batch` directory has diff --git a/site/src/content/docs/docs/reference/data-layout.mdx b/site/src/content/docs/docs/reference/data-layout.mdx index 53cd3e0bc..ca9d9c658 100644 --- a/site/src/content/docs/docs/reference/data-layout.mdx +++ b/site/src/content/docs/docs/reference/data-layout.mdx @@ -15,12 +15,10 @@ Everything Pitot persists lives under `PITOT_DATA_DIR` (`./data` by default, |---|---| | `telemetry/parquet/batches/*.batch/` | Authoritative atomic batches containing `metadata.json`, present signal Parquet files, and `trace.fidx` when spans are present. A compacted batch also lists the inputs it replaced in `replaces.json`; those inputs are no longer authoritative and keep their directories only until readers that captured them finish | | `telemetry/parquet/batches/_schema.batch/` | Empty schema anchors that keep every DuckDB view queryable | -| `telemetry/parquet/batches/*.retired/` | Replaced or expired batches being deleted; startup removes any left by a crash | -| `telemetry/parquet/batches/*.retired-*/` | Compaction inputs from releases before deferred deletion; a live marker protects its own set | +| `telemetry/parquet/batches/*.retired/` | Expired batches being deleted; startup removes any left by a crash | | `telemetry/parquet/batches/*.quarantined-*/` | Unreadable batches explicitly set aside by `pitot repair quarantine`; preserved but not queryable | | `telemetry/parquet/staging/` | Unacknowledged writes being prepared for atomic publication | -| `telemetry/compaction/` | Prepared replacement batches during compaction | -| `telemetry/COMPACTION.json` | Durable marker for an interrupted compaction swap. Present only while one is unresolved, and blocks startup until it is — see [troubleshoot](/docs/guides/troubleshooting) | +| `telemetry/compaction/` | Prepared replacement batches during compaction; startup discards any left by a crash | | `query/catalog.duckdb` | Rebuildable DuckDB rollups and query state | | `query/tmp/` | Spill space for queries that exceed the memory cap | | `control/pitot.sqlite` | Users, sessions, dashboards, alert rules, agent history |