diff --git a/CHANGELOG.md b/CHANGELOG.md index 251102e96..0266da57e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,39 +11,22 @@ conflict with a third party's registered US trademark in cloud services. - Container images are `ghcr.io/labstack/pitot` and `docker.io/labstack/pitot`. Update deployment manifests, service names, install URLs and MCP client entries. The MCP Registry identity is `io.github.labstack/pitot`. -- The server and installer refuse to start if any variable with the exact, - case-sensitive pre-rename prefix is set, including empty assignments, - installer overrides and secret-file settings. The error lists names only. - Rename those variables to `PITOT_*` before upgrading; they no longer silently - use defaults. +- Configure the server and installer with `PITOT_*` environment variables. - The query language is now **PitotQL**. Existing structured dashboard queries, MCP `query_telemetry` arguments (`text`, `panel.query`, `panel.sql`), query keywords, Explore URL parameters and browser storage keys are unchanged. Previous query HTTP paths, MCP reference URI and error-code reader aliases - are removed. Clients must use canonical PitotQL names. Permanent website - redirects preserve existing documentation links. See - [rename compatibility](site/src/content/docs/docs/reference/rename-compatibility.mdx) for the cutover requirements. -- Before upgrading, stop the service and take a cold backup of the complete - data directory. On startup, Pitot automatically renames the previous control - database and its SQLite WAL, shared-memory and rollback-journal sidecars in - `control/` to `pitot.sqlite` and matching sidecar names, then syncs the directory. - No manual filename change is needed, including for unattended agent upgrades. - If both database names or an incomplete sidecar move exist, startup refuses - before opening SQLite; stop the instance and recover a consistent file set - from the cold backup before retrying. Never delete a database just to bypass - this refusal or create a fresh volume by accident. For rollback to the previous - binary, restore its complete cold backup before starting it; older binaries - do not move the database name back. -- Telemetry spill footer keys retain their persisted format-5 spelling so - existing Parquet files remain valid and byte compatibility stays unchanged. + are removed. Clients must use canonical PitotQL names. +- The control database is `control/pitot.sqlite`; startup opens only this name. +- Telemetry spill footer keys use `pitot.spill.`. Batches with previous spill + keys fail verification and must be replaced before use. - Sessions use the `pitot_session` cookie (with the `__Host-` prefix over HTTPS), so users must sign in again. Keyboard shortcut preferences reset. Saved chat activities using the previous MCP App URI no longer render; begin a new chat. Reserved diagnostic context metadata now uses `_pitot_diagnostic_context`; discard pre-rename chat history before resuming chat. -- Published SQL migrations remain unchanged. Existing ingest and OAuth secrets - and their token prefixes remain valid. The image defaults to - `/var/lib/pitot/data`: mount the existing volume there, or set `PITOT_DATA_DIR` +- Existing ingest and OAuth secrets and their token prefixes remain valid. + The image defaults to `/var/lib/pitot/data`: mount the existing volume there, or set `PITOT_DATA_DIR` to its existing mount location. - HTTP request headers and metrics use the Pitot name. Update custom clients using the product request header to `Pitot-Request` (and dashboard deletion confirmation to diff --git a/cmd/pitot/main.go b/cmd/pitot/main.go index 20f025b3f..63786a916 100644 --- a/cmd/pitot/main.go +++ b/cmd/pitot/main.go @@ -64,11 +64,6 @@ type loginLinkCommand struct { } func main() { - if err := store.RejectPreviousEnvironment(os.Environ()); err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } - showVersion, loginLink, healthURL, derivedProbe, repair, err := parseCommandLine(os.Args[1:], os.Stderr) if errors.Is(err, flag.ErrHelp) { return @@ -716,9 +711,6 @@ func createLoginLink(cfg config.Config, rawEmail string, breakGlass bool, output if err != nil { return err } - if err := store.MigrateControlDatabase(cfg.ControlSQLitePath()); err != nil { - return err - } if _, err := os.Stat(cfg.ControlSQLitePath()); err != nil { if errors.Is(err, os.ErrNotExist) { return fmt.Errorf("control database does not exist at %s", cfg.ControlSQLitePath()) diff --git a/cmd/pitot/main_test.go b/cmd/pitot/main_test.go index 9b24c0aa1..274d61c0a 100644 --- a/cmd/pitot/main_test.go +++ b/cmd/pitot/main_test.go @@ -1,7 +1,6 @@ package main import ( - "bufio" "bytes" "context" "crypto/ecdsa" @@ -825,70 +824,3 @@ func TestApplyConfiguredIngestTokenLogsWithoutToken(t *testing.T) { t.Fatalf("logs must report both outcomes by setting name: %s", output) } } - -func TestStartupRefusesRetiredEnvironmentBeforeStores(t *testing.T) { - if os.Getenv("TEST_STARTUP_CHILD") == "1" { - os.Args = []string{"pitot"} - main() - return - } - for _, retired := range []bool{true, false} { - t.Run(map[bool]string{true: "retired", false: "current"}[retired], func(t *testing.T) { - root := filepath.Join(t.TempDir(), "data") - ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) - defer cancel() - command := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestStartupRefusesRetiredEnvironmentBeforeStores$") - command.Env = []string{"TEST_STARTUP_CHILD=1", "PATH=" + os.Getenv("PATH"), "PITOT_DATA_DIR=" + root, "PITOT_ADDR=127.0.0.1:0", "PITOT_MCP_ENABLED=false", "PITOT_AUTH_CODE_SECRET=" + strings.Repeat("x", 32)} - // Test fixture for the exact retired prefix; production spelling is isolated. - const previous = "FANOUT_" - if retired { - command.Env = append(command.Env, previous+"DATA_DIR=private-value", previous+"TOKEN=") - output, err := command.CombinedOutput() - if err == nil || !strings.Contains(string(output), previous+"DATA_DIR") || !strings.Contains(string(output), previous+"TOKEN") || !strings.Contains(string(output), "rename them to PITOT_") || strings.Contains(string(output), "private-value") { - t.Fatalf("refusal: %s, %v", output, err) - } - if _, err := os.Stat(root); !os.IsNotExist(err) { - t.Fatalf("data directory touched before refusal: %v", err) - } - return - } - pipe, err := command.StderrPipe() - if err != nil { - t.Fatal(err) - } - if err := command.Start(); err != nil { - t.Fatal(err) - } - ready := make(chan bool, 1) - go func() { - scanner := bufio.NewScanner(pipe) - for scanner.Scan() { - if strings.Contains(scanner.Text(), "HTTP and OTLP listening") { - ready <- true - return - } - } - ready <- false - }() - select { - case started := <-ready: - if !started { - _ = command.Wait() - t.Fatal("current environment did not start") - } - if err := command.Process.Signal(os.Interrupt); err != nil { - t.Fatal(err) - } - if err := command.Wait(); err != nil { - t.Fatal(err) - } - if _, err := os.Stat(filepath.Join(root, "control", "pitot.sqlite")); err != nil { - t.Fatal(err) - } - case <-ctx.Done(): - _ = command.Wait() - t.Fatal("startup timed out") - } - }) - } -} diff --git a/docs/release.md b/docs/release.md index b1afce1ab..33cf3c549 100644 --- a/docs/release.md +++ b/docs/release.md @@ -144,7 +144,7 @@ After merging the rename pull request: separately according to registry policy. Do not publish from this branch. 6. Verify the upgrade steps in `CHANGELOG.md`, including the control database filename, session cookies and saved chat metadata. Telemetry footer keys - retain their persisted format-5 spelling. + use the `pitot.spill.` namespace. The pre-rename environment variable prefix is no longer read. ## Refresh landing stills diff --git a/internal/api/pitotql_test.go b/internal/api/pitotql_test.go index 441ea93cd..c59421fc7 100644 --- a/internal/api/pitotql_test.go +++ b/internal/api/pitotql_test.go @@ -458,20 +458,3 @@ func TestPitotQLParseOffersSeenDottedKey(t *testing.T) { } } } - -func TestRetiredQueryRoutesAreNotRegistered(t *testing.T) { - for _, action := range []string{"parse", "format", "complete", "reference"} { - method := http.MethodPost - if action == "reference" { - method = http.MethodGet - } - path := "/api/fanql/" + action - if _, ok := classifyRoute(method, path); ok { - t.Fatalf("retired route classified: %s", path) - } - response := servePitotQL(t, &fakePitotQLParser{}, &fakePitotQLPanels{}, method, path, `{}`) - if response.Code != http.StatusNotFound { - t.Fatalf("retired route: %d %s", response.Code, response.Body) - } - } -} diff --git a/internal/config/config.go b/internal/config/config.go index 8d9352a2e..6394c7a37 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -287,7 +287,6 @@ func (c Config) ControlDir() string { } func (c Config) ControlSQLitePath() string { - // Startup migrates the previous filename before opening this control database. return filepath.Join(c.ControlDir(), "pitot.sqlite") } diff --git a/internal/db/migrations/20260930000000_initial.sql b/internal/db/migrations/20260930000000_initial.sql index bb7ef9ddf..9ac64d48d 100644 --- a/internal/db/migrations/20260930000000_initial.sql +++ b/internal/db/migrations/20260930000000_initial.sql @@ -1,5 +1,5 @@ -- +goose Up --- Fanout control schema. sqlc reads these migrations to generate query bindings. +-- Pitot control schema. sqlc reads these migrations to generate query bindings. -- DuckDB maintains its own telemetry query schema. CREATE TABLE alert_rules ( diff --git a/internal/db/migrations/20261004000000_dashboard_specs.sql b/internal/db/migrations/20261004000000_dashboard_specs.sql index 2133109b1..7a1ca4714 100644 --- a/internal/db/migrations/20261004000000_dashboard_specs.sql +++ b/internal/db/migrations/20261004000000_dashboard_specs.sql @@ -1,6 +1,6 @@ -- +goose Up -- Dashboards become typed v1 specs with a version history. The widget model --- is replaced outright: Fanout is pre-release, so existing dashboards are +-- is replaced outright: Pitot is pre-release, so existing dashboards are -- dropped and each owner receives the new default dashboard on next visit. DROP TABLE dashboard_widgets; DROP TABLE dashboards; diff --git a/internal/db/migrations_test.go b/internal/db/migrations_test.go index 67afb6fe1..c60fd5e58 100644 --- a/internal/db/migrations_test.go +++ b/internal/db/migrations_test.go @@ -11,8 +11,8 @@ import ( // never change or remove a checksum for a migration already published. func TestMigrationChecksums(t *testing.T) { checksums := map[string]string{ - "migrations/20260930000000_initial.sql": "2f256d45df0a604c198e165af00b66f63d9f36b13e27f2898c485da524883203", - "migrations/20261004000000_dashboard_specs.sql": "11894224e7c3ef0efcacae9a6f5f8f6d4af27ddd81ed8868b96b44fb9bbf7d36", + "migrations/20260930000000_initial.sql": "4e944361f785f869c855a9677987dd73c15edc694c2c0e41e4fcc05569711961", + "migrations/20261004000000_dashboard_specs.sql": "8b51ab3a02cf1acc9709bae2b29cfaba9320555293d17cd30b7092caf4107704", "migrations/20261008000001_dashboard_origins.sql": "6e7e89adf6e51633da0d73b052b12eec08ae3e7dd21ca315a4ac2fecfa703501", "migrations/20261009000000_user_profile_status.sql": "aed4c0a316a5fb67b1e60c82d45c6b3ec34a2fbb44721ef772f6b5d3df546659", "migrations/20261030000000_alert_last_evaluated_at.sql": "577ef95a1964aebf72d8ecb9610c310910121f7bb735dbb287966525b2fb537a", diff --git a/internal/mcp/pitotql_test.go b/internal/mcp/pitotql_test.go index 1d108ec53..21fc3b942 100644 --- a/internal/mcp/pitotql_test.go +++ b/internal/mcp/pitotql_test.go @@ -285,24 +285,3 @@ func TestQueryTelemetryKeepsInvalidPanelProblems(t *testing.T) { } } - -func TestRetiredQueryReferenceIsUnavailable(t *testing.T) { - const previousURI = "pitot://fanql/reference" - client := connectTestClient(t, newPanelServer(t), nil) - listed, err := client.ListResources(t.Context(), nil) - if err != nil { - t.Fatal(err) - } - for _, resource := range listed.Resources { - if resource.URI == previousURI { - t.Fatal("read alias advertised") - } - } - if _, err := client.ReadResource(t.Context(), &mcp.ReadResourceParams{URI: previousURI}); err == nil { - t.Fatal("retired resource resolved") - } - result, err := client.ReadResource(t.Context(), &mcp.ReadResourceParams{URI: pitotqlReferenceURI}) - if err != nil || len(result.Contents) != 1 || result.Contents[0].Text != pitotql.Reference() { - t.Fatalf("current resource = %+v, %v", result, err) - } -} diff --git a/internal/pitotql/complete.go b/internal/pitotql/complete.go index 464145e37..a15f58f28 100644 --- a/internal/pitotql/complete.go +++ b/internal/pitotql/complete.go @@ -164,12 +164,15 @@ func (c *Completer) schema(ctx context.Context, key completionKey) (*query.Schem if c.source == nil { return nil, false, false } + req := query.SchemaRequest{Window: key.window, Namespace: key.namespace, Cached: true} + if schema, err := c.source.Schema(ctx, req); err == nil && schema != nil { + return schema, true, false + } c.mu.Lock() defer c.mu.Unlock() - // Read the cache while holding the same lock that retires an in-flight - // fetch, so a stale miss cannot admit a duplicate after that fetch finishes. - schema, err := c.source.Schema(ctx, query.SchemaRequest{Window: key.window, Namespace: key.namespace, Cached: true}) - if err == nil && schema != nil { + // Recheck after locking: a fetch may have populated the cache and retired + // its in-flight entry since the first lookup missed. + if schema, err := c.source.Schema(ctx, req); err == nil && schema != nil { delete(c.failed, key) return schema, true, false } diff --git a/internal/pitotql/complete_test.go b/internal/pitotql/complete_test.go index b75d499da..7b1c18c94 100644 --- a/internal/pitotql/complete_test.go +++ b/internal/pitotql/complete_test.go @@ -755,3 +755,93 @@ func TestCompleteMarksIndexedKeys(t *testing.T) { } t.Fatalf("labels %v", completionLabels(r)) } + +func TestCompleteCacheHitDoesNotWaitForAdmissionLock(t *testing.T) { + c := warmCompleter(t) + done := make(chan CompleteResponse, 1) + c.mu.Lock() + go func() { done <- c.Complete(context.Background(), completionRequest("spans ")) }() + select { + case response := <-done: + c.mu.Unlock() + if !response.Schema.Warm { + t.Fatal("cached schema was not returned") + } + case <-time.After(time.Second): + c.mu.Unlock() + <-done + t.Fatal("cache hit waited for the admission lock") + } +} + +// A cache miss may race with a fetch publishing its result and retiring its +// in-flight entry. Admission must recheck that miss before starting more work. +type delayedCacheMiss struct { + *fakeSchemas + delay atomic.Bool + missed chan struct{} + resume chan struct{} +} + +func (s *delayedCacheMiss) Schema(ctx context.Context, req query.SchemaRequest) (*query.Schema, error) { + schema, err := s.fakeSchemas.Schema(ctx, req) + if req.Cached && schema == nil && s.delay.CompareAndSwap(true, false) { + close(s.missed) + <-s.resume + } + return schema, err +} + +func TestCompleteStaleMissDoesNotDuplicateFetch(t *testing.T) { + var calls atomic.Int32 + fetchGate := make(chan struct{}) + source := &delayedCacheMiss{ + fakeSchemas: completionSchemaFunc(func(context.Context, query.SchemaRequest) (*query.Schema, error) { + calls.Add(1) + <-fetchGate + return completionFixtureSchema(), nil + }), + missed: make(chan struct{}), resume: make(chan struct{}), + } + c := NewCompleter(source) + key := completionKey{window: "1h"} + var fetched chan struct{} + deadline := time.Now().Add(3 * time.Second) + for fetched == nil && time.Now().Before(deadline) { + c.schema(context.Background(), key) + c.mu.Lock() + fetched = c.inflight[key] + c.mu.Unlock() + if fetched == nil { + time.Sleep(time.Millisecond) + } + } + if fetched == nil { + close(fetchGate) + t.Fatal("schema fetch was not admitted") + } + source.delay.Store(true) + done := make(chan struct{}) + go func() { c.schema(context.Background(), key); close(done) }() + <-source.missed + close(fetchGate) + select { + case <-fetched: + case <-time.After(time.Second): + close(source.resume) + <-done + <-fetched + t.Fatal("cache lookup blocked fetch retirement") + } + close(source.resume) + <-done + c.mu.Lock() + pending := c.inflight[key] + c.mu.Unlock() + if pending != nil { + <-pending + } + if calls.Load() != 1 { + t.Fatalf("stale cache miss admitted %d fetches, want 1", calls.Load()) + } +} diff --git a/internal/store/control_database_name.go b/internal/store/control_database_name.go deleted file mode 100644 index 78d414e5c..000000000 --- a/internal/store/control_database_name.go +++ /dev/null @@ -1,131 +0,0 @@ -package store - -import ( - "fmt" - "log/slog" - "os" - "path/filepath" - "sort" - "strings" - - "golang.org/x/sys/unix" -) - -func previousControlDatabasePath(current string) string { - // Intentional compatibility exception: unattended upgrades must find the - // previous on-disk database instead of initializing an unclaimed instance. - return filepath.Join(filepath.Dir(current), "fanout.sqlite") -} - -// previousEnvironmentPrefix is isolated for the startup-refusal compatibility exception. -func previousEnvironmentPrefix() string { return "FANOUT_" } - -// RejectPreviousEnvironment refuses retired settings before startup can open state -// or listeners. Empty assignments count as set; values never enter diagnostics. -func RejectPreviousEnvironment(environ []string) error { - names := make(map[string]bool) - for _, assignment := range environ { - name, _, _ := strings.Cut(assignment, "=") - if strings.HasPrefix(name, previousEnvironmentPrefix()) { - names[name] = true - } - } - if len(names) == 0 { - return nil - } - ordered := make([]string, 0, len(names)) - for name := range names { - ordered = append(ordered, name) - } - sort.Strings(ordered) - return fmt.Errorf("retired environment variables: %s; rename them to PITOT_ before starting", strings.Join(ordered, ", ")) -} - -// MigrateControlDatabase preserves control state before any SQLite open can -// create a new, empty database. The service must be stopped during upgrades. -func MigrateControlDatabase(current string) error { - if filepath.Base(current) != "pitot.sqlite" || filepath.Base(filepath.Dir(current)) != "control" { - return nil - } - dir, err := os.Open(filepath.Dir(current)) - if os.IsNotExist(err) { - return nil - } - if err != nil { - return fmt.Errorf("inspect control directory: %w", err) - } - defer dir.Close() - // Serialize startup and CLI migrations without creating another on-disk name. - if err := unix.Flock(int(dir.Fd()), unix.LOCK_EX); err != nil { - return fmt.Errorf("lock control directory for database rename: %w", err) - } - defer unix.Flock(int(dir.Fd()), unix.LOCK_UN) //nolint:errcheck // Closing the descriptor also releases the lock. - - exists := func(path string) (bool, error) { - info, err := os.Lstat(path) - if os.IsNotExist(err) { - return false, nil - } - if err != nil { - return false, err - } - if !info.Mode().IsRegular() { - return false, fmt.Errorf("control database and sidecars must be regular files: %s", path) - } - return true, nil - } - previous := previousControlDatabasePath(current) - oldExists, err := exists(previous) - if err != nil { - return fmt.Errorf("inspect previous control database: %w", err) - } - newExists, err := exists(current) - if err != nil { - return fmt.Errorf("inspect current control database: %w", err) - } - if oldExists && newExists { - return fmt.Errorf("both control database names exist; stop the instance and reconcile control/ from a trusted backup before restarting") - } - var sidecars []string - for _, suffix := range []string{"-wal", "-shm", "-journal"} { - oldSidecar, err := exists(previous + suffix) - if err != nil { - return fmt.Errorf("inspect previous control sidecar: %w", err) - } - newSidecar, err := exists(current + suffix) - if err != nil { - return fmt.Errorf("inspect current control sidecar: %w", err) - } - // An interrupted move must fail closed, never open a database without its - // journal or discard a sidecar that might contain committed transactions. - if (oldSidecar && !oldExists) || (newSidecar && !newExists) { - return fmt.Errorf("orphaned or conflicting control database sidecars; stop the instance and recover the complete database file set before restarting") - } - if oldSidecar { - sidecars = append(sidecars, suffix) - } - } - if !oldExists { - return nil - } - // Rename sidecars first, then sync their directory entries before committing - // the main filename. A crash partway through leaves a refused partial state. - for _, suffix := range sidecars { - if err := os.Rename(previous+suffix, current+suffix); err != nil { - return fmt.Errorf("rename control database sidecar: %w", err) - } - } - if len(sidecars) > 0 { - if err := dir.Sync(); err != nil { - return fmt.Errorf("sync control sidecar renames: %w", err) - } - } - if err := os.Rename(previous, current); err != nil { - return fmt.Errorf("rename control database: %w", err) - } - if err := dir.Sync(); err != nil { - return fmt.Errorf("sync control database rename: %w", err) - } - slog.Info("migrated control database to the current name") - return nil -} diff --git a/internal/store/control_database_name_test.go b/internal/store/control_database_name_test.go deleted file mode 100644 index ac5753304..000000000 --- a/internal/store/control_database_name_test.go +++ /dev/null @@ -1,243 +0,0 @@ -package store - -import ( - "bytes" - "log/slog" - "os" - "path/filepath" - "strings" - "testing" -) - -func controlDatabasePaths(t *testing.T) (string, string) { - t.Helper() - dir := filepath.Join(t.TempDir(), "control") - if err := os.Mkdir(dir, 0o700); err != nil { - t.Fatal(err) - } - current := filepath.Join(dir, "pitot.sqlite") - return current, previousControlDatabasePath(current) -} - -func TestControlDatabaseUpgradePreservesExistingUsers(t *testing.T) { - current, previous := controlDatabasePaths(t) - db, err := NewSQLite(previous) - if err != nil { - t.Fatal(err) - } - if _, err := db.DB.Exec(`INSERT INTO users (id, email, role) VALUES ('existing-admin', 'admin@example.test', 'admin')`); err != nil { - t.Fatal(err) - } - if err := db.Close(); err != nil { - t.Fatal(err) - } - db, err = NewSQLite(current) - if err != nil { - t.Fatal(err) - } - defer db.Close() - var role string - if err := db.DB.QueryRow(`SELECT role FROM users WHERE id = 'existing-admin'`).Scan(&role); err != nil { - t.Fatalf("existing administrator disappeared: %v", err) - } - if role != "admin" { - t.Fatalf("role = %q", role) - } - if _, err := os.Lstat(previous); !os.IsNotExist(err) { - t.Fatalf("previous database remains: %v", err) - } -} - -func TestControlDatabaseUpgradeRefusesBothNames(t *testing.T) { - current, previous := controlDatabasePaths(t) - for _, path := range []string{current, previous} { - if err := os.WriteFile(path, []byte("preserve"), 0o600); err != nil { - t.Fatal(err) - } - } - db, err := NewSQLite(current) - if err == nil { - db.Close() - t.Fatal("opened ambiguous control database") - } - if !strings.Contains(err.Error(), "both control database names exist") { - t.Fatalf("ambiguous database error = %v", err) - } - for _, path := range []string{current, previous} { - data, err := os.ReadFile(path) - if err != nil || string(data) != "preserve" { - t.Fatalf("database modified: data=%q err=%v", data, err) - } - } -} - -func TestControlDatabaseUpgradeMovesSidecarsWithoutCopying(t *testing.T) { - current, previous := controlDatabasePaths(t) - suffixes := []string{"", "-wal", "-shm", "-journal"} - before := make(map[string]os.FileInfo) - for _, suffix := range suffixes { - path := previous + suffix - if err := os.WriteFile(path, []byte("preserve"+suffix), 0o600); err != nil { - t.Fatal(err) - } - info, err := os.Stat(path) - if err != nil { - t.Fatal(err) - } - before[suffix] = info - } - var logs bytes.Buffer - original := slog.Default() - slog.SetDefault(slog.New(slog.NewTextHandler(&logs, nil))) - t.Cleanup(func() { slog.SetDefault(original) }) - if err := MigrateControlDatabase(current); err != nil { - t.Fatal(err) - } - for _, suffix := range suffixes { - data, err := os.ReadFile(current + suffix) - if err != nil || string(data) != "preserve"+suffix { - t.Fatalf("sidecar %q: data=%q err=%v", suffix, data, err) - } - info, err := os.Stat(current + suffix) - if err != nil { - t.Fatal(err) - } - if !os.SameFile(before[suffix], info) || info.Mode().Perm() != 0o600 { - t.Fatalf("file %q was copied or permissions changed", suffix) - } - if _, err := os.Lstat(previous + suffix); !os.IsNotExist(err) { - t.Fatalf("previous file %q remains: %v", suffix, err) - } - } - if err := MigrateControlDatabase(current); err != nil { - t.Fatal(err) - } - if strings.Count(logs.String(), "migrated control database to the current name") != 1 { - t.Fatalf("migration log = %q", logs.String()) - } -} - -func TestControlDatabaseUpgradeLeavesCurrentFilesUntouched(t *testing.T) { - current, _ := controlDatabasePaths(t) - for _, suffix := range []string{"", "-wal", "-shm", "-journal"} { - if err := os.WriteFile(current+suffix, []byte("current"+suffix), 0o600); err != nil { - t.Fatal(err) - } - } - if err := MigrateControlDatabase(current); err != nil { - t.Fatal(err) - } - for _, suffix := range []string{"", "-wal", "-shm", "-journal"} { - data, err := os.ReadFile(current + suffix) - if err != nil || string(data) != "current"+suffix { - t.Fatalf("current file %q changed: data=%q err=%v", suffix, data, err) - } - } -} - -func TestControlDatabaseUpgradeRecoversCommittedWAL(t *testing.T) { - current, previous := controlDatabasePaths(t) - db, err := NewSQLite(previous) - if err != nil { - t.Fatal(err) - } - if _, err := db.DB.Exec(`PRAGMA wal_checkpoint(TRUNCATE); PRAGMA wal_autocheckpoint=0`); err != nil { - t.Fatal(err) - } - if _, err := db.DB.Exec(`INSERT INTO users (id, email, role) VALUES ('wal-admin', 'admin@example.test', 'admin')`); err != nil { - t.Fatal(err) - } - // Snapshot a quiescent writer's file set before close checkpoints the WAL. - // Restoring those bytes models an unclean shutdown without an open writer. - files := make(map[string][]byte) - for _, suffix := range []string{"", "-wal", "-shm"} { - data, err := os.ReadFile(previous + suffix) - if err != nil { - t.Fatal(err) - } - files[suffix] = data - } - if len(files["-wal"]) <= 32 { - t.Fatal("fixture has no committed WAL frames") - } - if err := db.Close(); err != nil { - t.Fatal(err) - } - for suffix, data := range files { - if err := os.WriteFile(previous+suffix, data, 0o600); err != nil { - t.Fatal(err) - } - } - db, err = NewSQLite(current) - if err != nil { - t.Fatal(err) - } - defer db.Close() - var role string - if err := db.DB.QueryRow(`SELECT role FROM users WHERE id='wal-admin'`).Scan(&role); err != nil || role != "admin" { - t.Fatalf("committed WAL user lost: role=%q err=%v", role, err) - } -} - -func TestControlDatabaseUpgradeRefusesPartialFileSets(t *testing.T) { - for _, suffix := range []string{"-wal", "-shm", "-journal"} { - for _, state := range []string{"target_sidecar", "orphan_source"} { - t.Run(suffix+"/"+state, func(t *testing.T) { - current, previous := controlDatabasePaths(t) - path := previous + suffix - if state == "target_sidecar" { - if err := os.WriteFile(previous, []byte("old"), 0o600); err != nil { - t.Fatal(err) - } - path = current + suffix - } - if err := os.WriteFile(path, []byte("pending"), 0o600); err != nil { - t.Fatal(err) - } - if db, err := NewSQLite(current); err == nil { - db.Close() - t.Fatal("opened partial migration") - } else if !strings.Contains(err.Error(), "control database sidecars") { - t.Fatal(err) - } - if _, err := os.Lstat(current); !os.IsNotExist(err) { - t.Fatalf("created current database: %v", err) - } - data, err := os.ReadFile(path) - if err != nil || string(data) != "pending" { - t.Fatalf("sidecar overwritten: data=%q err=%v", data, err) - } - }) - } - } -} - -func TestControlDatabaseUpgradeAllowsFreshInstall(t *testing.T) { - current, _ := controlDatabasePaths(t) - db, err := NewSQLite(current) - if err != nil { - t.Fatal(err) - } - defer db.Close() - var users int - if err := db.DB.QueryRow(`SELECT COUNT(*) FROM users`).Scan(&users); err != nil || users != 0 { - t.Fatalf("fresh users = %d, err=%v", users, err) - } -} - -func TestPreviousEnvironmentRefusesNamesWithoutValues(t *testing.T) { - prefix := previousEnvironmentPrefix() - err := RejectPreviousEnvironment([]string{prefix + "TOKEN=private-marker", prefix + "ADDR=", prefix + "TOKEN=another-private-marker"}) - want := "retired environment variables: " + prefix + "ADDR, " + prefix + "TOKEN; rename them to PITOT_ before starting" - if err == nil || err.Error() != want { - t.Fatalf("refusal = %v, want %q", err, want) - } -} - -func TestPreviousEnvironmentAllowsCurrentAndCaseSensitiveNames(t *testing.T) { - for _, env := range [][]string{nil, {}, {"PITOT_ADDR=:0", strings.ToLower(previousEnvironmentPrefix()) + "TOKEN=ignored", "OTHER_" + previousEnvironmentPrefix() + "TOKEN=ignored"}} { - if err := RejectPreviousEnvironment(env); err != nil { - t.Fatal(err) - } - } -} diff --git a/internal/store/store.go b/internal/store/store.go index 58d6a6653..3d84415fa 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -28,9 +28,6 @@ type SQLite struct { // NewSQLite opens (or creates) an SQLite database at dbPath and runs // embedded schema migrations via Goose. Use ":memory:" for an in-memory database. func NewSQLite(dbPath string) (*SQLite, error) { - if err := MigrateControlDatabase(dbPath); err != nil { - return nil, err - } var dsn string if dbPath == ":memory:" { dsn = fmt.Sprintf("file::memory:?_pragma=busy_timeout(%d)&_pragma=foreign_keys(1)", ControlDBBusyTimeout.Milliseconds()) diff --git a/internal/telemetry/coerce_test.go b/internal/telemetry/coerce_test.go index b2fc82eb8..83b5fd2a1 100644 --- a/internal/telemetry/coerce_test.go +++ b/internal/telemetry/coerce_test.go @@ -4,6 +4,7 @@ import ( "fmt" "math" "path/filepath" + "strconv" "testing" "github.com/labstack/pitot/internal/telemetry/semconv" @@ -92,7 +93,14 @@ func TestVerifierAgreesWithCoerce(t *testing.T) { if err != nil { t.Fatal(err) } + footer := map[string]string{} + for _, kv := range openParquetFile(t, path).Metadata().KeyValueMetadata { + footer[kv.Key] = kv.Value + } for _, key := range []semconv.Key{status, method} { + if got := footer["pitot.spill."+key.Name]; got != strconv.FormatUint(spills[key.Bit], 10) { + t.Errorf("%s footer spill count = %q, want %d", key.Name, got, spills[key.Bit]) + } if counts[key.Bit] != spills[key.Bit] { t.Errorf("%s spills %d, want %d", key.Name, counts[key.Bit], spills[key.Bit]) } diff --git a/internal/telemetry/columns.go b/internal/telemetry/columns.go index f3e3b8420..61e068e08 100644 --- a/internal/telemetry/columns.go +++ b/internal/telemetry/columns.go @@ -186,8 +186,8 @@ const metricNoRecordedValue = 1 // spillPrefix starts the footer key whose value counts the rows whose value // of a key could not be converted to its type. -// Preserve the published footer spelling so existing batches remain readable. -const spillPrefix = "fanout.spill." +// Spill counts use the Pitot footer namespace. +const spillPrefix = "pitot.spill." // Spans keep trace locality, which the trace index depends on. Logs and // metrics follow the OpenTelemetry Collector's ClickHouse exporter, with diff --git a/internal/telemetry/testdata/format5-spans-amd64.sha256 b/internal/telemetry/testdata/format5-spans-amd64.sha256 index 9092618e4..c641b2ed9 100644 --- a/internal/telemetry/testdata/format5-spans-amd64.sha256 +++ b/internal/telemetry/testdata/format5-spans-amd64.sha256 @@ -1 +1 @@ -1787ba4fb1512b19f998fc395ff08e2c6d37021b22c9e39c21cc280189043cac +96bd2af9e7617f7fecac64ddc9b31420e00176d8ed0fe349f88e807e18911d84 diff --git a/internal/telemetry/testdata/format5-spans-arm64.sha256 b/internal/telemetry/testdata/format5-spans-arm64.sha256 index ea3ff7c42..94b9ca1c7 100644 --- a/internal/telemetry/testdata/format5-spans-arm64.sha256 +++ b/internal/telemetry/testdata/format5-spans-arm64.sha256 @@ -1 +1 @@ -7032c5cf25b4862d4b3c8da8871b1eb561c414741da7403966d0240928ddeda9 +2da943432d41e5ce64e2e259be0c040c3041c3ee1591558369bed4b4f4262178 diff --git a/scripts/install.sh b/scripts/install.sh index ca6681448..52953771c 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -22,13 +22,6 @@ need() { command -v "$1" >/dev/null 2>&1 || die "$1 is required" } -# Intentional refusal exception, matching the server's previousEnvironmentPrefix. -previous_environment_prefix() { printf '%s' 'FANOUT_'; } -# NUL-delimited records keep multiline values from becoming apparent names. -# Values remain inside awk; only assignment names reach diagnostics. -previous_names=$(env -0 | tr '\n' '\001' | tr '\0' '\n' | awk -F= -v prefix="$(previous_environment_prefix)" 'index($1, prefix) == 1 { print $1 }' | sort -u) -[ -z "$previous_names" ] || die "retired environment variables: $previous_names; rename them to PITOT_ before installing" - need uname need tar need mktemp diff --git a/scripts/install.test.ts b/scripts/install.test.ts index f3d42857d..5200db08f 100644 --- a/scripts/install.test.ts +++ b/scripts/install.test.ts @@ -26,27 +26,12 @@ https://github.com/labstack/pitot/releases/download/v2026.11.4-alpha/*) cp "$INS esac `); chmodSync(join(commands, "curl"), 0o700); - const env = Object.fromEntries(Object.entries(process.env).filter(([key]) => !/^PITOT_/.test(key) && !key.startsWith(previousPrefix))); + const env = Object.fromEntries(Object.entries(process.env).filter(([key]) => !/^PITOT_/.test(key))); Object.assign(env, { PATH: `${commands}:${process.env.PATH}`, INSTALL_FIXTURES: root, - PITOT_VERSION: "v2026.11.4-alpha", PITOT_PREFIX: prefix, - [previousPrefix.toLowerCase() + "TOKEN"]: "ignored", OTHER: "private\n" + previousPrefix + "FAKE=private-continuation" }); + PITOT_VERSION: "v2026.11.4-alpha", PITOT_PREFIX: prefix }); const result = spawnSync("sh", [installer], { env, encoding: "utf8" }); expect(result.status).toBe(0); expect(readFileSync(join(prefix, "pitot"), "utf8")).toContain("fixture"); expect(result.stderr).not.toContain(prefix); } finally { rmSync(root, { recursive: true, force: true }); } }); - -const previousPrefix = readFileSync(installer, "utf8").match(/previous_environment_prefix\(\) \{ printf '%s' '([^']+)'/ )![1]; - -test("installer refuses retired variable names before downloads without exposing values", () => { - const env = { PATH: process.env.PATH, [previousPrefix + "VERSION"]: "private-version", [previousPrefix + "PREFIX"]: "", OTHER: "private\n" + previousPrefix + "FAKE=private-continuation" }; - const result = spawnSync("sh", [installer], { env, encoding: "utf8" }); - expect(result.status).not.toBe(0); - expect(result.stdout).toBe(""); - expect(result.stderr).toContain(previousPrefix + "VERSION"); - expect(result.stderr).toContain(previousPrefix + "PREFIX"); - expect(result.stderr).toContain("rename them to PITOT_"); - expect(result.stderr).not.toContain("private"); - expect(result.stderr).not.toContain("FAKE"); -}); diff --git a/scripts/query-reference-redirects.test.ts b/scripts/query-reference-redirects.test.ts deleted file mode 100644 index 6e1df0dc7..000000000 --- a/scripts/query-reference-redirects.test.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { expect, test } from "bun:test"; -import { readFileSync } from "node:fs"; - -// These published URL fixtures are intentional rename compatibility exceptions. -const previousReference = "/docs/reference/fanql"; -const currentReference = "/docs/reference/pitotql"; - -test("published query-reference URLs redirect HTML and Markdown to the current page", () => { - const redirects = readFileSync(new URL("../site/public/_redirects", import.meta.url), "utf8") - .split("\n").filter(line => line && !line.startsWith("#")).map(line => line.trim().split(/\s+/)); - expect(redirects).toContainEqual([previousReference, currentReference, "301"]); - expect(redirects).toContainEqual([previousReference + ".html", currentReference, "301"]); - expect(redirects).toContainEqual([previousReference + ".md", currentReference + ".md", "301"]); - const navigation = readFileSync(new URL("../site/astro.config.mjs", import.meta.url), "utf8"); - expect(navigation).toContain('slug: "docs/reference/pitotql"'); - expect(navigation).not.toContain(previousReference); -}); diff --git a/site/public/_redirects b/site/public/_redirects index 8ac4d7feb..120c4b8ab 100644 --- a/site/public/_redirects +++ b/site/public/_redirects @@ -1,4 +1 @@ # Published query-reference aliases; new navigation uses the current slug. -/docs/reference/fanql /docs/reference/pitotql 301 -/docs/reference/fanql.html /docs/reference/pitotql 301 -/docs/reference/fanql.md /docs/reference/pitotql.md 301 diff --git a/site/src/content/docs/docs/guides/upgrade.mdx b/site/src/content/docs/docs/guides/upgrade.mdx index 6fcfda999..5ac83254e 100644 --- a/site/src/content/docs/docs/guides/upgrade.mdx +++ b/site/src/content/docs/docs/guides/upgrade.mdx @@ -20,39 +20,22 @@ conflict with a third party's registered US trademark in cloud services. - Container images are `ghcr.io/labstack/pitot` and `docker.io/labstack/pitot`. Update deployment manifests, service names, install URLs and MCP client entries. The MCP Registry identity is `io.github.labstack/pitot`. -- The server and installer refuse to start if any variable with the exact, - case-sensitive pre-rename prefix is set, including empty assignments, - installer overrides and secret-file settings. The error lists names only. - Rename those variables to `PITOT_*` before upgrading; they no longer silently - use defaults. +- Configure the server and installer with `PITOT_*` environment variables. - The query language is now **PitotQL**. Existing structured dashboard queries, MCP `query_telemetry` arguments (`text`, `panel.query`, `panel.sql`), query keywords, Explore URL parameters and browser storage keys are unchanged. Previous query HTTP paths, MCP reference URI and error-code reader aliases - are removed. Clients must use canonical PitotQL names. Permanent website - redirects preserve existing documentation links. See - [rename compatibility](../reference/rename-compatibility/) for the cutover requirements. -- Before upgrading, stop the service and take a cold backup of the complete - data directory. On startup, Pitot automatically renames the previous control - database and its SQLite WAL, shared-memory and rollback-journal sidecars in - `control/` to `pitot.sqlite` and matching sidecar names, then syncs the directory. - No manual filename change is needed, including for unattended agent upgrades. - If both database names or an incomplete sidecar move exist, startup refuses - before opening SQLite; stop the instance and recover a consistent file set - from the cold backup before retrying. Never delete a database just to bypass - this refusal or create a fresh volume by accident. For rollback to the previous - binary, restore its complete cold backup before starting it; older binaries - do not move the database name back. -- Telemetry spill footer keys retain their persisted format-5 spelling so - existing Parquet files remain valid and byte compatibility stays unchanged. + are removed. Clients must use canonical PitotQL names. +- The control database is `control/pitot.sqlite`; startup opens only this name. +- Telemetry spill footer keys use `pitot.spill.`. Batches with previous spill + keys fail verification and must be replaced before use. - Sessions use the `pitot_session` cookie (with the `__Host-` prefix over HTTPS), so users must sign in again. Keyboard shortcut preferences reset. Saved chat activities using the previous MCP App URI no longer render; begin a new chat. Reserved diagnostic context metadata now uses `_pitot_diagnostic_context`; discard pre-rename chat history before resuming chat. -- Published SQL migrations remain unchanged. Existing ingest and OAuth secrets - and their token prefixes remain valid. The image defaults to - `/var/lib/pitot/data`: mount the existing volume there, or set `PITOT_DATA_DIR` +- Existing ingest and OAuth secrets and their token prefixes remain valid. + The image defaults to `/var/lib/pitot/data`: mount the existing volume there, or set `PITOT_DATA_DIR` to its existing mount location. - HTTP request headers and metrics use the Pitot name. Update custom clients using the product request header to `Pitot-Request` (and dashboard deletion confirmation to diff --git a/site/src/content/docs/docs/reference/rename-compatibility.mdx b/site/src/content/docs/docs/reference/rename-compatibility.mdx deleted file mode 100644 index a02548008..000000000 --- a/site/src/content/docs/docs/reference/rename-compatibility.mdx +++ /dev/null @@ -1,32 +0,0 @@ ---- -title: Pitot cutover -description: Current Pitot names and the persisted storage contracts retained during the cutover. ---- - -The current query language is **PitotQL**. New code, clients, generated references -and responses use current names. Previous query API paths, MCP resource URIs and -error-code aliases are removed. Update clients to -`/api/pitotql/`, `pitot://pitotql/reference` and `invalid_pitotql` before upgrading. -Permanent website redirects preserve published documentation links; the server -does not register old API or MCP aliases. The following spellings remain only for -website navigation, startup refusal, in-place data migration and immutable storage contracts: - -| Previous spelling | Behavior and owning boundary | -| --- | --- | -| `FANOUT_` | The server's `previousEnvironmentPrefix` beside `previousControlDatabasePath`, and the standalone installer's `previous_environment_prefix`, recognize this exact case-sensitive prefix only to refuse startup. Empty assignments also refuse; diagnostics list names only and instruct renaming to `PITOT_`. Tests exercise the same refusal. | -| `/docs/reference/fanql` (also `.html` and `.md`) | Permanent website redirects preserve existing links to the current PitotQL documentation. These are web navigation redirects, not API or MCP aliases. | -| `fanout.sqlite` | `previousControlDatabasePath` preserves users, tokens and dashboards by migrating the complete control database file set before opening SQLite. | -| Published migration comments | The two previous-name comments in `20260930000000_initial.sql` and `20261004000000_dashboard_specs.sql` remain immutable. | -| Telemetry spill footer prefix | The existing `fanout` prefix remains byte-compatible with format-5 persisted telemetry. | - -Saved dashboard specs store structured `query` objects, not a language enum or text. -MCP tool names and arguments remain `query_telemetry`, `text`, `panel.query` and -`panel.sql`. Explore URL keys (`v`, `q`, `tab`, `sql`, time and visualization), -browser storage keys, API enums and the language keywords contain no renamed -language identifier and retain their wire/storage spelling. User-authored query -text, SQL aliases, field names and dashboard titles remain data, never rewritten. -The internal ranking helper column now uses `_pitotql_below_floor`; it is transient, -not a persisted dashboard field. - -Generated browser assets use the current error codes without translating retired -ones. Historical published evidence remains unchanged. diff --git a/ui/host/e2e/support/app-lifecycle.test.ts b/ui/host/e2e/support/app-lifecycle.test.ts index 5afa5a092..aa14479a0 100644 --- a/ui/host/e2e/support/app-lifecycle.test.ts +++ b/ui/host/e2e/support/app-lifecycle.test.ts @@ -110,7 +110,7 @@ it("waits for a zombie manager to be reaped before removing ownership", async () const count = existsSync(path) ? Number(readFileSync(path, "utf8")) : 0; writeFileSync(path, String(count + 1)); if (count === 0) console.log(${JSON.stringify(`S ${instance.command} ${instance.id} ${instance.dir}`)}); - else if (count === 1) console.log("Z [node] "); + else if (count === 1) console.log(${JSON.stringify(`Z ${instance.command} ${instance.id} ${instance.dir}`)}); else { writeFileSync(${JSON.stringify(reaped)}, String(existsSync(${JSON.stringify(instance.dir)}) && existsSync(${JSON.stringify(join(`${state}.instances`, instance.id))}))); process.exit(1); @@ -174,3 +174,31 @@ it("forces an owned manager to exit after its shutdown deadline and removes its expect(await exists(instance.dir)).toBe(false); expect(await exists(`${state}.instances`)).toBe(false); }); + +it("treats an unowned zombie at a recorded PID as stopped", async () => { + const directory = await mkdtemp(join(tmpdir(), "pitot-lifecycle-test-")); + directories.push(directory); + const state = join(directory, "state.json"); + states.push(state); + const dir = join(directory, "instance"); + const bin = join(directory, "bin"); + await mkdir(dir); + await mkdir(bin); + await mkdir(`${state}.instances`); + await writeAppState(join(`${state}.instances`, "stale"), { + id: "stale", pid: process.pid, dir, command: "manager.mjs", + }); + await writeFile(join(bin, "ps"), `#!${process.execPath} + console.log("Z [unrelated] "); + `); + await chmod(join(bin, "ps"), 0o700); + const originalPath = process.env.PATH; + try { + process.env.PATH = `${bin}:${originalPath}`; + await stopAppServers(state, 100); + expect(await exists(dir)).toBe(false); + expect(await exists(`${state}.instances`)).toBe(false); + } finally { + process.env.PATH = originalPath; + } +}); diff --git a/ui/host/e2e/support/app-lifecycle.ts b/ui/host/e2e/support/app-lifecycle.ts index 1cddcf351..c41cfc5ce 100644 --- a/ui/host/e2e/support/app-lifecycle.ts +++ b/ui/host/e2e/support/app-lifecycle.ts @@ -39,7 +39,7 @@ function owns(instance: Instance, command: string) { } async function stopInstance(instance: Instance, timeout: number) { const command = await commandFor(instance.pid); - if (command) { + if (command && (!command.zombie || owns(instance, command.command))) { if (!command.zombie) { if (!owns(instance, command.command)) throw new Error(`Refusing to stop unowned process ${instance.pid}`); try { process.kill(instance.pid, "SIGTERM"); } @@ -47,7 +47,7 @@ async function stopInstance(instance: Instance, timeout: number) { } const stopped = async () => { const current = await commandFor(instance.pid); - return !current || !current.zombie && !owns(instance, current.command) ? undefined : "Seeded server is still stopping"; + return !current || !owns(instance, current.command) ? undefined : "Seeded server is still stopping"; }; try { await waitUntilReady(Date.now() + timeout, stopped, 50); } catch {