Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 56 additions & 0 deletions internal/telemetry/store/lock.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
package store

import (
"errors"
"fmt"
"os"
"path/filepath"

"golang.org/x/sys/unix"
)

// lockFileName is a dedicated file at the data-directory root whose open
// descriptor carries the exclusive flock. It is never read, renamed, or
// removed; its presence on disk means nothing without a live process holding
// the lock, so it survives restarts harmlessly.
const lockFileName = "LOCK"

// dataDirLock is an exclusive, advisory, whole-process lock on a telemetry data
// directory. A live store deletes, renames, and rewrites files under this root
// with no cross-process coordination, so a second process on the same directory
// would delete the first's staging and compaction work mid-write. The lock
// serializes those processes: the second acquirer fails fast instead of racing.
type dataDirLock struct {
file *os.File
}

// acquireDataDirLock takes an exclusive, non-blocking flock on root/LOCK. When
// another process already holds it the call fails immediately with an error
// naming the directory, rather than blocking until the other process exits.
func acquireDataDirLock(root string) (*dataDirLock, error) {
path := filepath.Join(root, lockFileName)
file, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0o644)
if err != nil {
return nil, fmt.Errorf("open telemetry data directory lock %s: %w", path, err)
}
if err := unix.Flock(int(file.Fd()), unix.LOCK_EX|unix.LOCK_NB); err != nil {
_ = file.Close()
if errors.Is(err, unix.EWOULDBLOCK) {
return nil, fmt.Errorf("telemetry data directory %s is held by another process; stop the running Pitot (or repair) using this directory before starting here", root)
}
return nil, fmt.Errorf("lock telemetry data directory %s: %w", root, err)
}
return &dataDirLock{file: file}, nil
}

// Close drops the flock and closes the descriptor. Closing alone releases the
// lock; the explicit unlock makes the release independent of close ordering.
func (l *dataDirLock) Close() error {
if l == nil || l.file == nil {
return nil
}
unlockErr := unix.Flock(int(l.file.Fd()), unix.LOCK_UN)
closeErr := l.file.Close()
l.file = nil
return errors.Join(unlockErr, closeErr)
}
14 changes: 14 additions & 0 deletions internal/telemetry/store/repair.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,13 @@ type BatchIssue struct {
// large set uses the machine while no query grows with it; GOMAXPROCS bounds
// both. Issues keep directory order.
func VerifyBatches(root string) ([]BatchIssue, error) {
// Refuse to run against a live data directory: a running store can retire
// and rewrite batches underneath the verifier and produce false reports.
lock, err := acquireDataDirLock(root)
if err != nil {
return nil, err
}
defer lock.Close()
batchesDir := filepath.Join(root, "parquet", "batches")
entries, err := os.ReadDir(batchesDir)
if err != nil {
Expand Down Expand Up @@ -87,6 +94,13 @@ func VerifyBatches(root string) ([]BatchIssue, error) {
// are refused. The returned directory remains beside the authoritative set so
// an operator can recover it from a backup or rename it back after repair.
func QuarantineBatch(root, id string) (string, error) {
// Refuse to rename batches out from under a live store, which has no
// cross-process coordination with this command.
lock, err := acquireDataDirLock(root)
if err != nil {
return "", err
}
defer lock.Close()
return quarantineBatch(root, id, telemetry.ValidatePublishedBatch)
}

Expand Down
19 changes: 17 additions & 2 deletions internal/telemetry/store/repository.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ type Batch struct {
// directory rename is the transaction and the filesystem is the catalog.
type Repository struct {
releaseBacklogGauge func()
lock *dataDirLock
root string
Parquet *telemetry.ParquetStore
compactionMu sync.Mutex
Expand All @@ -44,11 +45,18 @@ func Open(root string) (*Repository, error) {
if err := os.MkdirAll(root, 0o755); err != nil {
return nil, err
}
// Take the directory lock before touching any batch file, so a second
// process cannot delete this one's staging and compaction work mid-write.
lock, err := acquireDataDirLock(root)
if err != nil {
return nil, err
}
parquetStore, err := telemetry.OpenParquetStore(filepath.Join(root, "parquet"))
if err != nil {
_ = lock.Close()
return nil, err
}
r := &Repository{root: root, Parquet: parquetStore}
r := &Repository{root: root, Parquet: parquetStore, lock: lock}
if err := r.recoverCompaction(context.Background(), func(ctx context.Context, _ telemetry.BatchMetadata, _ []string, publish func(context.Context) error) error {
return publish(ctx)
}); err != nil {
Expand Down Expand Up @@ -132,7 +140,14 @@ func (r *Repository) Close() error {
if r.releaseBacklogGauge != nil {
r.releaseBacklogGauge()
}
return r.Parquet.Close()
// Close the Parquet handles before releasing the lock, so the directory is
// quiescent by the time another process may acquire it.
err := r.Parquet.Close()
if r.lock != nil {
err = errors.Join(err, r.lock.Close())
r.lock = nil
}
return err
}

func (r *Repository) CompactionDebt() int { return r.Parquet.CompactionDebt() }
Expand Down
36 changes: 36 additions & 0 deletions internal/telemetry/store/repository_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -668,6 +668,11 @@ func TestRepositoryRecoversInterruptedCompactionSwap(t *testing.T) {
t.Fatal(err)
}
}
// The crashed process exits, releasing its data-directory lock while the
// marker, staged output, and retired inputs stay on disk for recovery.
if err := repository.Close(); err != nil {
t.Fatal(err)
}

recovered, err := Open(dir)
if err != nil {
Expand Down Expand Up @@ -917,6 +922,37 @@ func TestUnresolvedCompactionRunbookExists(t *testing.T) {
t.Fatalf("runbook anchor %q is not a section in %s; sections are %v", anchor, guide, headings)
}

// A live repository owns its data directory exclusively: it deletes, renames,
// and rewrites files there with no cross-process coordination. A second process
// opened on the same directory must fail fast rather than race the first into
// corruption, and the hold must end at Close so the normal reopen path works.
func TestOpenRejectsSecondProcessOnSameDataDir(t *testing.T) {
dir := t.TempDir()
first, err := Open(dir)
if err != nil {
t.Fatal(err)
}
second, err := Open(dir)
if err == nil {
_ = second.Close()
_ = first.Close()
t.Fatal("second Open succeeded while the first still held the data directory")
}
if !strings.Contains(err.Error(), dir) || !strings.Contains(err.Error(), "another process") {
t.Fatalf("lock error must name the data directory and another process: %v", err)
}
if err := first.Close(); err != nil {
t.Fatal(err)
}
reopened, err := Open(dir)
if err != nil {
t.Fatalf("reopen after Close: %v", err)
}
if err := reopened.Close(); err != nil {
t.Fatal(err)
}
}

func (f testParquetPublisherFunc) PublishParquetReplacement(ctx context.Context, _ telemetry.BatchMetadata, _ []string, publish func(context.Context) error) error {
return f.PublishParquet(ctx, publish)
}
Expand Down
Loading