diff --git a/internal/telemetry/store/lock.go b/internal/telemetry/store/lock.go new file mode 100644 index 00000000..a29c4935 --- /dev/null +++ b/internal/telemetry/store/lock.go @@ -0,0 +1,56 @@ +package store + +import ( + "errors" + "fmt" + "os" + "path/filepath" + + "golang.org/x/sys/unix" +) + +// lockFileName is a dedicated file at the data-directory root whose open +// descriptor carries the exclusive flock. It is never read, renamed, or +// removed; its presence on disk means nothing without a live process holding +// the lock, so it survives restarts harmlessly. +const lockFileName = "LOCK" + +// dataDirLock is an exclusive, advisory, whole-process lock on a telemetry data +// directory. A live store deletes, renames, and rewrites files under this root +// with no cross-process coordination, so a second process on the same directory +// would delete the first's staging and compaction work mid-write. The lock +// serializes those processes: the second acquirer fails fast instead of racing. +type dataDirLock struct { + file *os.File +} + +// acquireDataDirLock takes an exclusive, non-blocking flock on root/LOCK. When +// another process already holds it the call fails immediately with an error +// naming the directory, rather than blocking until the other process exits. +func acquireDataDirLock(root string) (*dataDirLock, error) { + path := filepath.Join(root, lockFileName) + file, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0o644) + if err != nil { + return nil, fmt.Errorf("open telemetry data directory lock %s: %w", path, err) + } + if err := unix.Flock(int(file.Fd()), unix.LOCK_EX|unix.LOCK_NB); err != nil { + _ = file.Close() + if errors.Is(err, unix.EWOULDBLOCK) { + return nil, fmt.Errorf("telemetry data directory %s is held by another process; stop the running Pitot (or repair) using this directory before starting here", root) + } + return nil, fmt.Errorf("lock telemetry data directory %s: %w", root, err) + } + return &dataDirLock{file: file}, nil +} + +// Close drops the flock and closes the descriptor. Closing alone releases the +// lock; the explicit unlock makes the release independent of close ordering. +func (l *dataDirLock) Close() error { + if l == nil || l.file == nil { + return nil + } + unlockErr := unix.Flock(int(l.file.Fd()), unix.LOCK_UN) + closeErr := l.file.Close() + l.file = nil + return errors.Join(unlockErr, closeErr) +} diff --git a/internal/telemetry/store/repair.go b/internal/telemetry/store/repair.go index 195c383d..89fd44a2 100644 --- a/internal/telemetry/store/repair.go +++ b/internal/telemetry/store/repair.go @@ -29,6 +29,13 @@ type BatchIssue struct { // large set uses the machine while no query grows with it; GOMAXPROCS bounds // both. Issues keep directory order. func VerifyBatches(root string) ([]BatchIssue, error) { + // Refuse to run against a live data directory: a running store can retire + // and rewrite batches underneath the verifier and produce false reports. + lock, err := acquireDataDirLock(root) + if err != nil { + return nil, err + } + defer lock.Close() batchesDir := filepath.Join(root, "parquet", "batches") entries, err := os.ReadDir(batchesDir) if err != nil { @@ -87,6 +94,13 @@ func VerifyBatches(root string) ([]BatchIssue, error) { // are refused. The returned directory remains beside the authoritative set so // an operator can recover it from a backup or rename it back after repair. func QuarantineBatch(root, id string) (string, error) { + // Refuse to rename batches out from under a live store, which has no + // cross-process coordination with this command. + lock, err := acquireDataDirLock(root) + if err != nil { + return "", err + } + defer lock.Close() return quarantineBatch(root, id, telemetry.ValidatePublishedBatch) } diff --git a/internal/telemetry/store/repository.go b/internal/telemetry/store/repository.go index b4a7e6c3..79e4b804 100644 --- a/internal/telemetry/store/repository.go +++ b/internal/telemetry/store/repository.go @@ -27,6 +27,7 @@ type Batch struct { // directory rename is the transaction and the filesystem is the catalog. type Repository struct { releaseBacklogGauge func() + lock *dataDirLock root string Parquet *telemetry.ParquetStore compactionMu sync.Mutex @@ -44,11 +45,18 @@ func Open(root string) (*Repository, error) { if err := os.MkdirAll(root, 0o755); err != nil { return nil, err } + // Take the directory lock before touching any batch file, so a second + // process cannot delete this one's staging and compaction work mid-write. + lock, err := acquireDataDirLock(root) + if err != nil { + return nil, err + } parquetStore, err := telemetry.OpenParquetStore(filepath.Join(root, "parquet")) if err != nil { + _ = lock.Close() return nil, err } - r := &Repository{root: root, Parquet: parquetStore} + r := &Repository{root: root, Parquet: parquetStore, lock: lock} if err := r.recoverCompaction(context.Background(), func(ctx context.Context, _ telemetry.BatchMetadata, _ []string, publish func(context.Context) error) error { return publish(ctx) }); err != nil { @@ -132,7 +140,14 @@ func (r *Repository) Close() error { if r.releaseBacklogGauge != nil { r.releaseBacklogGauge() } - return r.Parquet.Close() + // Close the Parquet handles before releasing the lock, so the directory is + // quiescent by the time another process may acquire it. + err := r.Parquet.Close() + if r.lock != nil { + err = errors.Join(err, r.lock.Close()) + r.lock = nil + } + return err } func (r *Repository) CompactionDebt() int { return r.Parquet.CompactionDebt() } diff --git a/internal/telemetry/store/repository_test.go b/internal/telemetry/store/repository_test.go index ac5c6b4e..2b351231 100644 --- a/internal/telemetry/store/repository_test.go +++ b/internal/telemetry/store/repository_test.go @@ -668,6 +668,11 @@ func TestRepositoryRecoversInterruptedCompactionSwap(t *testing.T) { t.Fatal(err) } } + // The crashed process exits, releasing its data-directory lock while the + // marker, staged output, and retired inputs stay on disk for recovery. + if err := repository.Close(); err != nil { + t.Fatal(err) + } recovered, err := Open(dir) if err != nil { @@ -917,6 +922,37 @@ func TestUnresolvedCompactionRunbookExists(t *testing.T) { t.Fatalf("runbook anchor %q is not a section in %s; sections are %v", anchor, guide, headings) } +// A live repository owns its data directory exclusively: it deletes, renames, +// and rewrites files there with no cross-process coordination. A second process +// opened on the same directory must fail fast rather than race the first into +// corruption, and the hold must end at Close so the normal reopen path works. +func TestOpenRejectsSecondProcessOnSameDataDir(t *testing.T) { + dir := t.TempDir() + first, err := Open(dir) + if err != nil { + t.Fatal(err) + } + second, err := Open(dir) + if err == nil { + _ = second.Close() + _ = first.Close() + t.Fatal("second Open succeeded while the first still held the data directory") + } + if !strings.Contains(err.Error(), dir) || !strings.Contains(err.Error(), "another process") { + t.Fatalf("lock error must name the data directory and another process: %v", err) + } + if err := first.Close(); err != nil { + t.Fatal(err) + } + reopened, err := Open(dir) + if err != nil { + t.Fatalf("reopen after Close: %v", err) + } + if err := reopened.Close(); err != nil { + t.Fatal(err) + } +} + func (f testParquetPublisherFunc) PublishParquetReplacement(ctx context.Context, _ telemetry.BatchMetadata, _ []string, publish func(context.Context) error) error { return f.PublishParquet(ctx, publish) }