diff --git a/chaos/durability_test.go b/chaos/durability_test.go index 883b7da..1baf83c 100644 --- a/chaos/durability_test.go +++ b/chaos/durability_test.go @@ -475,6 +475,17 @@ func TestAKilledWriteOnAnyPathIsRecoveredByTheNextIndex(t *testing.T) { for _, p := range litter { os.Chtimes(p, long, long) } + // A kill between claiming a checkpoint's name and writing it + // leaves the claim empty (#380); index removes one as old as a + // temp file, for the same reason. + filepath.WalkDir(filepath.Join(vault, "sessions"), func(p string, d fs.DirEntry, err error) error { + if err == nil && !d.IsDir() { + if info, ierr := d.Info(); ierr == nil && info.Size() == 0 { + os.Chtimes(p, long, long) + } + } + return nil + }) // Recover twice: once over the existing cache, once from nothing, // which is the promise every document makes. diff --git a/cmd/logos/index.go b/cmd/logos/index.go index c8277bc..8cbe701 100644 --- a/cmd/logos/index.go +++ b/cmd/logos/index.go @@ -11,6 +11,7 @@ import ( "github.com/Coder8124/logos/internal/index" "github.com/Coder8124/logos/internal/memory" "github.com/Coder8124/logos/internal/provider" + "github.com/Coder8124/logos/internal/session" "github.com/Coder8124/logos/internal/vault" ) @@ -310,6 +311,19 @@ func indexPass(ix *index.Index, w io.Writer, p *provider.Provider, embed string) len(rep.StaleTemps)-len(removed), plural(len(rep.StaleTemps)-len(removed), ""), rmErr) } + // The same for a checkpoint or plan a kill left empty between claiming + // its name and writing it (#380): nothing in it to lose, but read as a + // checkpoint with no frontmatter until someone deleted it by hand. + gone, claimErr := session.RemoveAbandonedClaims(ix.Vault, rep.EmptyClaims) + if len(gone) > 0 { + fmt.Fprintf(w, "removed %d empty checkpoint%s left by an interrupted write: %s\n", + len(gone), plural(len(gone), ""), strings.Join(gone, ", ")) + } + if claimErr != nil { + fmt.Fprintf(os.Stderr, "· could not remove %d empty checkpoint%s left by an interrupted write: %v\n", + len(rep.EmptyClaims)-len(gone), plural(len(rep.EmptyClaims)-len(gone), ""), claimErr) + } + // A folder the walk could not enter is a folder of notes missing from // every count above; skipped, so one bad directory cannot stop the index, // but never silently. diff --git a/cmd/logos/index_claims_test.go b/cmd/logos/index_claims_test.go new file mode 100644 index 0000000..802557d --- /dev/null +++ b/cmd/logos/index_claims_test.go @@ -0,0 +1,59 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// A checkpoint or plan name is claimed with an empty O_EXCL create and filled +// by a later atomic write. A kill between the two left a zero-byte checkpoint +// that every index walked past for good (#380). Index removes one that has +// sat empty longer than any stalled write, and says so; a fresh empty one may +// be a claim whose write is still coming, and a person's empty note is not a +// claim at all. +func TestIndexRemovesACheckpointLeftEmptyByAKilledWriteAndSaysSo(t *testing.T) { + vault := t.TempDir() + t.Setenv("LOGOS_VAULT", vault) + t.Setenv("LOGOS_EMBED", "off") + old := time.Now().Add(-2 * time.Hour) + put := func(rel, body string, aged bool) string { + p := filepath.Join(vault, filepath.FromSlash(rel)) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + if aged { + os.Chtimes(p, old, old) + } + return p + } + killedCheckpoint := put("sessions/kestrel/_convos/_unattached/20261102-232711-chaos.md", "", true) + killedPlan := put("sessions/kestrel/plans/plan-20261102-232711-chaos.md", "", true) + liveClaim := put("sessions/kestrel/_convos/_unattached/20261105-101500-chaos.md", "", false) + written := put("sessions/kestrel/_convos/_unattached/20261102-101500-chaos.md", "---\ntype: checkpoint\n---\nwork\n", true) + personsNote := put("notes/20261102-empty idea.md", "", true) + + out := captureStdout(t, func() { + if err := runIndex(false); err != nil { + t.Fatalf("index: %v", err) + } + }) + for _, p := range []string{killedCheckpoint, killedPlan} { + if _, err := os.Stat(p); !os.IsNotExist(err) { + t.Errorf("%s left empty by a killed write is still there", p) + } + } + for _, p := range []string{liveClaim, written, personsNote} { + if _, err := os.Stat(p); err != nil { + t.Errorf("%s was removed: %v", p, err) + } + } + if !strings.Contains(out, "removed 2 empty checkpoints left by an interrupted write") { + t.Errorf("index did not say it removed the two empty claims:\n%s", out) + } +} diff --git a/internal/index/index.go b/internal/index/index.go index 57bbd47..41e37f7 100644 --- a/internal/index/index.go +++ b/internal/index/index.go @@ -82,6 +82,10 @@ type SyncReport struct { // and announce. Sync itself removes nothing: it is also the walk behind // setup and the MCP server, where a deletion would go unsaid. StaleTemps []string + // EmptyClaims are checkpoints and plans a killed write left empty + // (session.IsAbandonedClaim), vault-relative, for the caller to remove + // and announce, as StaleTemps are. They are not indexed. + EmptyClaims []string // Unreadable is every path the walk could not enter or stat, with why. // Skipping it is right — one bad directory must not stop the index — but // silently skipping it made a missing folder of notes look indexed. @@ -428,6 +432,14 @@ func (ix *Index) Sync() (SyncReport, error) { } return nil } + if r := ix.rel(path); session.IsClaimPath(r) { + // Named first, stat second: the stat is a syscall per file and + // only a session file can be a claim. + if info, err := d.Info(); err == nil && session.IsAbandonedClaim(r, info) { + rep.EmptyClaims = append(rep.EmptyClaims, r) + return nil + } + } if r := ix.rel(path); strings.HasPrefix(r, ".logos/") || strings.HasPrefix(r, memory.Dir+"/") || strings.HasPrefix(r, ingest.Dir+"/") { return nil } diff --git a/internal/ingest/private_e2e_test.go b/internal/ingest/private_e2e_test.go index b0592f9..91b3645 100644 --- a/internal/ingest/private_e2e_test.go +++ b/internal/ingest/private_e2e_test.go @@ -17,14 +17,17 @@ import ( // transcript read for a harvest or an auto record, an import — and each one // says it kept something out, or the person cannot tell the span was honoured. func TestAPrivateSpanIsNeverCapturedHarvestedOrImported(t *testing.T) { - const pin, pw = "bank pin is 4417", "hunter2-staging" + // The secret carries letters: a bare secret turned up in the random digits + // of t.TempDir's name, which the harvest quotes as its source (#379). + const secret = "pin-q4417z" + const pin, pw = "bank " + secret, "hunter2-staging" e, err := activity.FromHook("UserPromptSubmit", []byte(`{"session_id":"s1","cwd":"/work/shop","prompt":"my `+pin+` fix the checkout crash"}`), "shop") if err != nil { t.Fatal(err) } - if strings.Contains(e.Summary, "4417") { + if strings.Contains(e.Summary, secret) { t.Errorf("the capture line kept the private span: %q", e.Summary) } if !strings.Contains(e.Summary, "fix the checkout crash") || e.Private != 1 { @@ -53,7 +56,7 @@ func TestAPrivateSpanIsNeverCapturedHarvestedOrImported(t *testing.T) { } h := ingest.Harvest(s) - if md := h.Markdown(); strings.Contains(md, "4417") || strings.Contains(md, pw) { + if md := h.Markdown(); strings.Contains(md, secret) || strings.Contains(md, pw) { t.Errorf("the harvest kept a private span:\n%s", md) } else if !strings.Contains(md, "2 span(s) kept out") { t.Errorf("the harvest does not say what it kept out:\n%s", md) @@ -66,7 +69,7 @@ func TestAPrivateSpanIsNeverCapturedHarvestedOrImported(t *testing.T) { } filepath.Walk(vault, func(p string, fi os.FileInfo, _ error) error { if fi != nil && !fi.IsDir() { - if b, _ := os.ReadFile(p); strings.Contains(string(b), "4417") || strings.Contains(string(b), pw) { + if b, _ := os.ReadFile(p); strings.Contains(string(b), secret) || strings.Contains(string(b), pw) { t.Errorf("a private span reached the vault in %s", p) } } @@ -81,7 +84,7 @@ func TestAPrivateSpanIsNeverCapturedHarvestedOrImported(t *testing.T) { if err != nil { t.Fatal(err) } - if strings.Contains(imported.Turns[0].Text, "4417") || imported.Private != 1 { + if strings.Contains(imported.Turns[0].Text, secret) || imported.Private != 1 { t.Errorf("the import kept the private span: %q (%d counted)", imported.Turns[0].Text, imported.Private) } } diff --git a/internal/session/claims.go b/internal/session/claims.go new file mode 100644 index 0000000..7cc2713 --- /dev/null +++ b/internal/session/claims.go @@ -0,0 +1,63 @@ +package session + +import ( + "errors" + "fmt" + "os" + "path" + "path/filepath" + "strings" + + "github.com/Coder8124/logos/internal/vault" +) + +// IsClaimPath reports whether rel (vault-relative, slash-separated) names a +// checkpoint or plan: the two files reserved by an empty O_EXCL create of +// their final name and filled by a write that comes after (claimCheckpoint, +// claimPlan). The name is what makes them unique across processes, so the +// reservation cannot move to a temp name; a kill between the two steps +// leaves the empty file instead. +func IsClaimPath(rel string) bool { + if !strings.HasPrefix(rel, CheckpointDir+"/") { + return false + } + name := path.Base(rel) + return IsCheckpointFile(name) || (strings.HasPrefix(name, "plan-") && strings.HasSuffix(name, ".md")) +} + +// isAbandonedClaim is a claim still empty after vault.StaleTempAge. A live +// claim is empty for the milliseconds between the reservation and the write, +// or longer under the same stalls a temp file waits out — and removing it +// then would hand its name to a second writer. +func isAbandonedClaim(info os.FileInfo) bool { + return info.Mode().IsRegular() && info.Size() == 0 && vault.IsStaleTemp(info.ModTime()) +} + +// IsAbandonedClaim reports whether the file at rel is a claim a killed write +// left empty, for index's walk to collect. +func IsAbandonedClaim(rel string, info os.FileInfo) bool { + return IsClaimPath(rel) && isAbandonedClaim(info) +} + +// RemoveAbandonedClaims removes the empty checkpoints and plans killed writes +// left (#380). An empty one has nothing to lose, but it was read as a +// checkpoint with no frontmatter and sat in every sync client and git status +// for good. Each is checked again before removal, since the walk that found +// it may be minutes old; it keeps going past a failure, as RemoveTemps does, +// so the ones it did remove can still be said. +func RemoveAbandonedClaims(vaultDir string, rels []string) (removed []string, err error) { + var errs []error + for _, rel := range rels { + p := filepath.Join(vaultDir, filepath.FromSlash(rel)) + info, serr := os.Lstat(p) + if serr != nil || !IsAbandonedClaim(rel, info) { + continue // gone, or written since: no longer an abandoned claim + } + if rerr := os.Remove(p); rerr != nil { + errs = append(errs, fmt.Errorf("%s: %w", rel, rerr)) + continue + } + removed = append(removed, rel) + } + return removed, errors.Join(errs...) +}