From e68fd0d2ae6710ad4d7a847e0a811e5049022d0a Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Sun, 15 Nov 2026 10:23:41 -0800 Subject: [PATCH] fix(store): lock the telemetry data directory against a second process A telemetry data directory has no flock or lockfile, so nothing stops a second process from opening the same directory as a live one. When it does, the newcomer deletes the live process's staging/ and compaction/ directories mid-write, removes _schema.batch, and runs CollectRetired against an empty pin table; pitot repair quarantine likewise renames a .batch directory out from under a running store with no liveness check. DuckDB's own file lock only stops a second server incidentally, depending on open order, so it is not a guarantee. Take an exclusive, non-blocking flock (LOCK_EX|LOCK_NB) on a dedicated root/LOCK file in store.Open, held for the lifetime of the Repository and released in Close. A second Open on the same directory now fails fast with an error naming the directory and saying another process holds it, instead of racing into corruption. The same lock guards the repair CLI entrypoints (VerifyBatches and QuarantineBatch), so repair refuses to run against a live data directory and releases the lock when it returns. The lock is a per-process guard taken once at Open, off every hot path. A crashed process releases its flock on exit, so the normal Open -> Close -> reopen path and crash-recovery startup are unaffected; the interrupted-compaction-swap test now closes the crashed repository before the recovering reopen, modelling that process exit. Closes #485 --- internal/telemetry/store/lock.go | 56 +++++++++++++++++++++ internal/telemetry/store/repair.go | 14 ++++++ internal/telemetry/store/repository.go | 19 ++++++- internal/telemetry/store/repository_test.go | 36 +++++++++++++ 4 files changed, 123 insertions(+), 2 deletions(-) create mode 100644 internal/telemetry/store/lock.go diff --git a/internal/telemetry/store/lock.go b/internal/telemetry/store/lock.go new file mode 100644 index 00000000..a29c4935 --- /dev/null +++ b/internal/telemetry/store/lock.go @@ -0,0 +1,56 @@ +package store + +import ( + "errors" + "fmt" + "os" + "path/filepath" + + "golang.org/x/sys/unix" +) + +// lockFileName is a dedicated file at the data-directory root whose open +// descriptor carries the exclusive flock. It is never read, renamed, or +// removed; its presence on disk means nothing without a live process holding +// the lock, so it survives restarts harmlessly. +const lockFileName = "LOCK" + +// dataDirLock is an exclusive, advisory, whole-process lock on a telemetry data +// directory. A live store deletes, renames, and rewrites files under this root +// with no cross-process coordination, so a second process on the same directory +// would delete the first's staging and compaction work mid-write. The lock +// serializes those processes: the second acquirer fails fast instead of racing. +type dataDirLock struct { + file *os.File +} + +// acquireDataDirLock takes an exclusive, non-blocking flock on root/LOCK. When +// another process already holds it the call fails immediately with an error +// naming the directory, rather than blocking until the other process exits. +func acquireDataDirLock(root string) (*dataDirLock, error) { + path := filepath.Join(root, lockFileName) + file, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0o644) + if err != nil { + return nil, fmt.Errorf("open telemetry data directory lock %s: %w", path, err) + } + if err := unix.Flock(int(file.Fd()), unix.LOCK_EX|unix.LOCK_NB); err != nil { + _ = file.Close() + if errors.Is(err, unix.EWOULDBLOCK) { + return nil, fmt.Errorf("telemetry data directory %s is held by another process; stop the running Pitot (or repair) using this directory before starting here", root) + } + return nil, fmt.Errorf("lock telemetry data directory %s: %w", root, err) + } + return &dataDirLock{file: file}, nil +} + +// Close drops the flock and closes the descriptor. Closing alone releases the +// lock; the explicit unlock makes the release independent of close ordering. +func (l *dataDirLock) Close() error { + if l == nil || l.file == nil { + return nil + } + unlockErr := unix.Flock(int(l.file.Fd()), unix.LOCK_UN) + closeErr := l.file.Close() + l.file = nil + return errors.Join(unlockErr, closeErr) +} diff --git a/internal/telemetry/store/repair.go b/internal/telemetry/store/repair.go index 195c383d..89fd44a2 100644 --- a/internal/telemetry/store/repair.go +++ b/internal/telemetry/store/repair.go @@ -29,6 +29,13 @@ type BatchIssue struct { // large set uses the machine while no query grows with it; GOMAXPROCS bounds // both. Issues keep directory order. func VerifyBatches(root string) ([]BatchIssue, error) { + // Refuse to run against a live data directory: a running store can retire + // and rewrite batches underneath the verifier and produce false reports. + lock, err := acquireDataDirLock(root) + if err != nil { + return nil, err + } + defer lock.Close() batchesDir := filepath.Join(root, "parquet", "batches") entries, err := os.ReadDir(batchesDir) if err != nil { @@ -87,6 +94,13 @@ func VerifyBatches(root string) ([]BatchIssue, error) { // are refused. The returned directory remains beside the authoritative set so // an operator can recover it from a backup or rename it back after repair. func QuarantineBatch(root, id string) (string, error) { + // Refuse to rename batches out from under a live store, which has no + // cross-process coordination with this command. + lock, err := acquireDataDirLock(root) + if err != nil { + return "", err + } + defer lock.Close() return quarantineBatch(root, id, telemetry.ValidatePublishedBatch) } diff --git a/internal/telemetry/store/repository.go b/internal/telemetry/store/repository.go index b4a7e6c3..79e4b804 100644 --- a/internal/telemetry/store/repository.go +++ b/internal/telemetry/store/repository.go @@ -27,6 +27,7 @@ type Batch struct { // directory rename is the transaction and the filesystem is the catalog. type Repository struct { releaseBacklogGauge func() + lock *dataDirLock root string Parquet *telemetry.ParquetStore compactionMu sync.Mutex @@ -44,11 +45,18 @@ func Open(root string) (*Repository, error) { if err := os.MkdirAll(root, 0o755); err != nil { return nil, err } + // Take the directory lock before touching any batch file, so a second + // process cannot delete this one's staging and compaction work mid-write. + lock, err := acquireDataDirLock(root) + if err != nil { + return nil, err + } parquetStore, err := telemetry.OpenParquetStore(filepath.Join(root, "parquet")) if err != nil { + _ = lock.Close() return nil, err } - r := &Repository{root: root, Parquet: parquetStore} + r := &Repository{root: root, Parquet: parquetStore, lock: lock} if err := r.recoverCompaction(context.Background(), func(ctx context.Context, _ telemetry.BatchMetadata, _ []string, publish func(context.Context) error) error { return publish(ctx) }); err != nil { @@ -132,7 +140,14 @@ func (r *Repository) Close() error { if r.releaseBacklogGauge != nil { r.releaseBacklogGauge() } - return r.Parquet.Close() + // Close the Parquet handles before releasing the lock, so the directory is + // quiescent by the time another process may acquire it. + err := r.Parquet.Close() + if r.lock != nil { + err = errors.Join(err, r.lock.Close()) + r.lock = nil + } + return err } func (r *Repository) CompactionDebt() int { return r.Parquet.CompactionDebt() } diff --git a/internal/telemetry/store/repository_test.go b/internal/telemetry/store/repository_test.go index ac5c6b4e..2b351231 100644 --- a/internal/telemetry/store/repository_test.go +++ b/internal/telemetry/store/repository_test.go @@ -668,6 +668,11 @@ func TestRepositoryRecoversInterruptedCompactionSwap(t *testing.T) { t.Fatal(err) } } + // The crashed process exits, releasing its data-directory lock while the + // marker, staged output, and retired inputs stay on disk for recovery. + if err := repository.Close(); err != nil { + t.Fatal(err) + } recovered, err := Open(dir) if err != nil { @@ -917,6 +922,37 @@ func TestUnresolvedCompactionRunbookExists(t *testing.T) { t.Fatalf("runbook anchor %q is not a section in %s; sections are %v", anchor, guide, headings) } +// A live repository owns its data directory exclusively: it deletes, renames, +// and rewrites files there with no cross-process coordination. A second process +// opened on the same directory must fail fast rather than race the first into +// corruption, and the hold must end at Close so the normal reopen path works. +func TestOpenRejectsSecondProcessOnSameDataDir(t *testing.T) { + dir := t.TempDir() + first, err := Open(dir) + if err != nil { + t.Fatal(err) + } + second, err := Open(dir) + if err == nil { + _ = second.Close() + _ = first.Close() + t.Fatal("second Open succeeded while the first still held the data directory") + } + if !strings.Contains(err.Error(), dir) || !strings.Contains(err.Error(), "another process") { + t.Fatalf("lock error must name the data directory and another process: %v", err) + } + if err := first.Close(); err != nil { + t.Fatal(err) + } + reopened, err := Open(dir) + if err != nil { + t.Fatalf("reopen after Close: %v", err) + } + if err := reopened.Close(); err != nil { + t.Fatal(err) + } +} + func (f testParquetPublisherFunc) PublishParquetReplacement(ctx context.Context, _ telemetry.BatchMetadata, _ []string, publish func(context.Context) error) error { return f.PublishParquet(ctx, publish) }