From 0d2c8e93a5931da0c3a9c8324be04f9166670c93 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Mon, 16 Nov 2026 05:50:21 -0800 Subject: [PATCH] fix(telemetry): publish the compaction swap catalog atomically PublishReplacement swaps a compacted output for its inputs under p.mu, but it did so by calling setBatchLocked once per input and once for the output. Every setBatchLocked call republishes the lock-free metadataSnapshot that readers use to build their read_parquet file lists, so a swap of N inputs republished the catalog N+1 times and stepped through intermediate states in which the inputs were already gone but the replacement was not yet installed. A reader that captured BatchMetadata in that gap built a file list missing a whole compaction group and under-counted rows that the swap conserves. TestCompactionAndRetentionUnderConcurrentReads reproduced this under -race: windowed reads saw fewer than the committed rows, and unwindowed reads dropped well below the retention-tolerant range. Split setBatchLocked into a mutate-only setBatchLockedNoPublish plus a single publishBatchesLocked step. setBatchLocked keeps its exact behavior for every other caller (commit, restoration, rollback, startup, retention). PublishReplacement now mutates all inputs and the output with the no-publish variant and publishes once after recording the replaces list, so readers only ever observe the pre-swap or post-swap catalog. The pin, retire, deletion, fsync, marker and on-disk rename sequences are unchanged; only the in-memory snapshot publication becomes atomic. Fixes #499 --- internal/telemetry/compaction_debt.go | 24 +++++++++++++++++++++--- internal/telemetry/parquet.go | 13 +++++++++++-- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/internal/telemetry/compaction_debt.go b/internal/telemetry/compaction_debt.go index 3342a959..9c293317 100644 --- a/internal/telemetry/compaction_debt.go +++ b/internal/telemetry/compaction_debt.go @@ -65,9 +65,13 @@ func (p *ParquetStore) changeGeneration0Locked(b BatchMetadata, delta int) { } } -// setBatchLocked is the sole live-catalog mutation. Replacement, restoration, -// rollback, startup and retention all use the same inventory accounting. -func (p *ParquetStore) setBatchLocked(id string, batch *storedBatch) { +// setBatchLockedNoPublish mutates the live catalog and its inventory accounting +// without republishing the reader snapshot or firing the debt gate. A caller +// that changes several batches as one logical step takes mu, calls this for +// each, then publishes once with publishBatchesLocked, so lock-free readers +// never observe an intermediate catalog. The generation-0 snapshot it stores is +// a backlog gauge, not query state, so leaving it per-mutation is harmless. +func (p *ParquetStore) setBatchLockedNoPublish(id string, batch *storedBatch) { if old := p.batches[id]; old != nil { p.changeGeneration0Locked(old.metadata, -1) } @@ -78,6 +82,12 @@ func (p *ParquetStore) setBatchLocked(id string, batch *storedBatch) { p.changeGeneration0Locked(batch.metadata, 1) } p.generation0Snapshot.Store(int64(p.generation0Batches)) +} + +// publishBatchesLocked republishes the canonical reader snapshot and wakes a +// waiter once debt has crossed its limit. It is the single publish step that +// follows one or more setBatchLockedNoPublish mutations under mu. +func (p *ParquetStore) publishBatchesLocked() { p.publishMetadataLocked() if p.compactionDebt >= CompactionDebtLimit && p.debtLimitReached != nil { close(p.debtLimitReached) @@ -85,6 +95,14 @@ func (p *ParquetStore) setBatchLocked(id string, batch *storedBatch) { } } +// setBatchLocked is the sole single-batch live-catalog mutation. Commit, +// restoration, rollback, startup and retention all use the same inventory +// accounting and publish the new catalog atomically on every call. +func (p *ParquetStore) setBatchLocked(id string, batch *storedBatch) { + p.setBatchLockedNoPublish(id, batch) + p.publishBatchesLocked() +} + // WaitForCompactionLimit wakes higher-generation work when ingest needs the // compactor. Registration and the threshold check share the catalog lock, so // debt cannot cross the limit between the check and subscribing. No lock is diff --git a/internal/telemetry/parquet.go b/internal/telemetry/parquet.go index 3fa74b99..3fc44490 100644 --- a/internal/telemetry/parquet.go +++ b/internal/telemetry/parquet.go @@ -987,17 +987,26 @@ func (p *ParquetStore) PublishReplacement(stage string, metadata BatchMetadata, // output is durable; until then marker recovery retries the fsync. retired := make([]BatchMetadata, 0, len(inputs)) p.mu.Lock() + // Retire the inputs and install the output as one catalog edit. Each + // setBatchLocked would otherwise republish the snapshot per batch, and a + // lock-free reader that captured BatchMetadata between the input removals + // and the output install would build its file list from a catalog that + // has already lost a whole compaction group but not yet gained its + // replacement, under-counting rows that are conserved by the swap (#499). for _, id := range inputs { if batch, exists := p.batches[id]; exists { retired = append(retired, batch.metadata) } - p.setBatchLocked(id, nil) + p.setBatchLockedNoPublish(id, nil) } - p.setBatchLocked(metadata.ID, replacement) + p.setBatchLockedNoPublish(metadata.ID, replacement) // Whether or not the fsync succeeded, the output's replaces list now // covers its inputs: a retried commit of one must neither adopt its // directory nor write it again once it is deleted. p.recordReplacesLocked(metadata.ID, inputs) + // A single publish exposes the pre-swap or post-swap catalog, never one + // of the intermediate states the per-batch mutations stepped through. + p.publishBatchesLocked() p.mu.Unlock() // Count catalog retirements once, even if marker cleanup or fsync is // retried. Already absent inputs must not inflate retirement throughput.