feat(history): GC/compaction with active-writer and failure-path tests (slice 6/6)

docs/prompt-history.md

@@ -2,7 +2,8 @@
 
 Prompt history stores captured prompts per pi instance, can import older
 history and project session transcripts, and lets you delete prompts from the
-history selector. Compaction arrives in a later slice of the chain.
+history selector. Opted-in sessions consolidate a project's files at shutdown
+(see "Compaction" below).
 
 ## Capture is opt-in
 
@@ -45,6 +46,8 @@ Everything sits under `~/.pi/agent/history/`:
 - `projects/<hash>/<instance>.jsonl` — one append-only capture file per pi
   process.
 - `projects/<hash>/seed.jsonl` — one-time transcript import for this project.
+- `projects/<hash>/compact-<pid>-<ts>.jsonl` — older capture files merged by
+  compaction.
 - `history-global.jsonl` — imported legacy editor-history prompts.
 - `hidden.json` — deletion records (tombstones); see "Delete" below.
 
@@ -147,3 +150,45 @@ corrupt, or not an array), history is blocked with a recovery warning
 instead of resurfacing hidden prompts, transcript bootstrap waits, and
 deletes refuse to rewrite it. Recovery is explicit — restore the file or
 delete it yourself (hidden prompts may then reappear).
+
+## Compaction
+
+Compaction keeps the number of files per project small. It is housekeeping,
+**not a retention limit**: it consolidates files and never drops a visible
+prompt because of its age or of any count.
+
+It runs when an opted-in pi session shuts down, for the current project only.
+With capture off, shutdown does nothing to the store. The project is compacted
+when its directory holds **more than 50 files** or **more than 5000 entries**
+in total. Then:
+
+- The **10 newest files** stay as they are. "Newest" is the most recent entry
+  timestamp in a file (the file's modification time when it has none), so a
+  file rewritten by a delete does not jump ahead.
+- Every older file is merged, oldest first, into one new
+  `compact-<pid>-<ts>.jsonl`, which is written completely (temp file + rename)
+  before any merged file is removed. Earlier compact files are merged again
+  like any other file.
+- Never merged: `seed.jsonl` (while it exists, the transcript import does not
+  run again) and the capture file of the session that is shutting down.
+  `history-global.jsonl` sits outside the project directories and is never
+  touched.
+- Prompts with a tombstone in `hidden.json` are not copied into the compact
+  file, so they cannot reappear from it later, even after their tombstone
+  leaves the 1000-entry cache. If `hidden.json` cannot be trusted, compaction
+  is skipped.
+
+Other pi instances may still be appending to the files being merged. Each
+file is renamed to a claim name before it is read (`<name>.gc-<pid>-<ts>.jsonl`),
+so a later append by path starts a fresh file under the original name, and
+bytes written to the claimed file after it was read are moved into the
+compact file once the claim is removed.
+
+Failures never lose prompts: a file that cannot be read is left untouched,
+and if the compact file cannot be written, the claimed files stay on disk and
+are still read like any other store file. A claimed file that cannot be
+removed stays too; its prompts appear once, because the selector drops
+duplicates.
+
+Prompts leave the store only through the delete flow above, or when you remove
+files manually.

extensions/history/index.ts

@@ -5,9 +5,9 @@
 // writer and slice-2 drains, with the search input (filterPrompts +
 // forwardToSearch fallthrough), the lazy loaded window, the project<->global
 // scope toggle, the preview panel + wheel handling, the slice-4 import
-// (legacy migration + seed bootstrap inside getWriter), and the slice-5
-// modal delete (store sweep + exact tombstone). GC/compaction arrives in a
-// later slice.
+// (legacy migration + seed bootstrap inside getWriter), the slice-5
+// modal delete (store sweep + exact tombstone), and the slice-6
+// session_shutdown GC/compaction.
 //
 // Capture is OPT-IN: nothing is recorded unless
 // GENTLE_PI_HISTORY_CAPTURE=1|true|on. The selector honors the same gate:
@@ -31,6 +31,7 @@ import {
   getKeybindings,
   Input,
   matchesKey,
+  stripTerminalSequences,
   Text,
   type TUI,
   type TuiMouseEvent,
@@ -46,9 +47,11 @@ import {
   drainGlobal,
   drainProject,
   ensureRegistryEntry,
+  gcProjectDir,
   migrateLegacyStores,
   openSessionWriter,
   type SessionWriterState,
+  sessionFilePath,
   type SweepResult,
 } from "./store.ts";
 import {
@@ -148,7 +151,8 @@ export function captureEnabled(env: NodeJS.ProcessEnv = process.env): boolean {
 function sanitizeForDisplay(text: string): string {
   let out = "";
   for (let i = 0; i < text.length; i++) {
-    const cp = text.codePointAt(i)!;
+    const cp = text.codePointAt(i);
+    if (cp === undefined) break;
     if (cp === 0x0a) {
       out += "\n";
     } else if (cp === 0x09) {
@@ -229,7 +233,7 @@ class FixedRowText {
           // Truncate first so an overlong help row can never exceed width,
           // then center the truncated copy (design §C hardening).
           const truncated = truncateToWidth(this.text, width, "…");
-          const visible = truncated.replace(/\x1b\[[0-9;]*m/g, "");
+          const visible = stripTerminalSequences(truncated);
           const pad = Math.max(0, Math.floor((width - visible.length) / 2));
           return " ".repeat(pad) + truncated;
         })()
@@ -1211,6 +1215,20 @@ export default function promptHistoryExtension(
     }
   });
 
+  // Consolidate this project's store files on graceful shutdown (slice 6),
+  // only for opted-in sessions: with capture off the store is never
+  // rewritten. This instance's own capture file is never a merge candidate.
+  pi.on("session_shutdown", () => {
+    if (!captureEnabled(env)) return;
+    try {
+      gcProjectDir(root, cwd, {
+        keepFiles: [sessionFilePath(root, cwd, instanceId)],
+      });
+    } catch {
+      // GC is best-effort and never blocks shutdown
+    }
+  });
+
   // When a tool asks for user input while the history overlay is open,
   // dismiss the overlay so the tool can take over the UI.
   pi.on("tool_call", () => {

extensions/history/store.ts

@@ -5,8 +5,8 @@
 // and identity, the advisory registry, entry primitives, the per-instance
 // session writer, the scope drain/reader/query section (ordering, dedup,
 // tombstone filter, project/global drains), legacy migration, and the
-// project seed bootstrap, and scope deletes (slice 5). GC/compaction
-// arrives in a later slice. Formerly store-paths.ts + registry.ts + multi-store.ts (+ v1
+// project seed bootstrap, scope deletes (slice 5), and GC/compaction
+// (slice 6). Formerly store-paths.ts + registry.ts + multi-store.ts (+ v1
 // primitives).
 
 import { createHash } from "node:crypto";
@@ -196,8 +196,7 @@ export function parseStoreLine(raw: string): StoreEntry | null {
 }
 
 // ===========================================================================
-// Instance writer (formerly multi-store.ts; GC/compaction arrives in a
-// later slice)
+// Instance writer (formerly multi-store.ts)
 // ===========================================================================
 
 /** Mutable state of ONE pi instance's exclusive capture file. */
@@ -799,3 +798,221 @@ export function bootstrapProjectSeed(
   fs.renameSync(tmp, seed);
   return { seeded: collected.length, ran: true };
 }
+
+// ---------------------------------------------------------------------------
+// GC / compaction (design v2, slice 6)
+// ---------------------------------------------------------------------------
+
+/** Compact when a project dir holds MORE than this many store files... */
+const GC_FILE_THRESHOLD = 50;
+/** ...or MORE than this many valid entries across them. */
+const GC_LINE_THRESHOLD = 5000;
+/** The newest files (by fileSortKey) are never merged. */
+const GC_KEEP_NEWEST = 10;
+
+export interface GcResult {
+  compacted: boolean;
+  /** Store files merged into the compact file. */
+  merged: number;
+}
+
+export interface GcOptions {
+  fileThreshold?: number;
+  lineThreshold?: number;
+  keepNewest?: number;
+  /**
+   * Files that are never merge candidates — the calling instance's own
+   * capture file, which it may still append to.
+   */
+  keepFiles?: readonly string[];
+  /** Tombstone state dir (hidden.json); defaults to the store root. */
+  stateDir?: string;
+}
+
+/**
+ * Threshold check + compaction entry point (wired at session_shutdown).
+ * Compaction consolidates files: it merges the oldest store files of ONE
+ * project into a single `compact-<pid>-<ts>.jsonl` and removes the merged
+ * originals. It is not a retention limit — every visible prompt is copied;
+ * only tombstoned prompts (already deleted by the user) are dropped.
+ *
+ * Never merged: `seed.jsonl` (its presence is the bootstrap gate, so
+ * removing it would re-seed deleted prompts from transcripts), the files
+ * in `keepFiles`, and the newest `keepNewest` files. The global seed lives
+ * outside the project dir and is never touched. An untrusted hidden.json
+ * skips compaction (fail closed), and any failure leaves every original
+ * readable: GC never throws.
+ */
+export function gcProjectDir(
+  root: string,
+  cwd: string,
+  opts: GcOptions = {},
+): GcResult {
+  const none: GcResult = { compacted: false, merged: 0 };
+  try {
+    const dir = path.join(root, "projects", projectHash(cwd));
+    const files = listProjectFiles(dir).map((file) => ({
+      file,
+      entries: readFileEntries(file),
+    }));
+    if (files.length === 0) return none;
+    const totalEntries = files.reduce((sum, f) => sum + f.entries.length, 0);
+    if (
+      files.length <= (opts.fileThreshold ?? GC_FILE_THRESHOLD) &&
+      totalEntries <= (opts.lineThreshold ?? GC_LINE_THRESHOLD)
+    ) {
+      return none;
+    }
+    const hidden = readHiddenPrompts(opts.stateDir ?? root);
+    if (hidden.status === "untrusted") return none;
+
+    const excluded = new Set(
+      [seedFilePath(root, cwd), ...(opts.keepFiles ?? [])].map((file) =>
+        path.resolve(file),
+      ),
+    );
+    // Newest first by the stable ts-based key (mtime is bumped by delete
+    // rewrites); ties break by name so the pick is deterministic.
+    const candidates = files
+      .filter((f) => !excluded.has(path.resolve(f.file)))
+      .map((f) => ({ file: f.file, key: fileSortKey(f.file, f.entries) }))
+      .sort((a, b) => b.key - a.key || a.file.localeCompare(b.file));
+    const tail = candidates
+      .slice(opts.keepNewest ?? GC_KEEP_NEWEST)
+      .reverse() // oldest first: the merged output is chronological
+      .map((c) => c.file);
+    if (tail.length === 0) return none;
+    return compactFiles(dir, tail, hidden.keys);
+  } catch {
+    return none;
+  }
+}
+
+/** One merge candidate after its claim: the open descriptor + read cursor. */
+interface ClaimedFile {
+  claim: string;
+  fd: number;
+  /** Bytes consumed so far (complete lines only). */
+  consumed: number;
+}
+
+/**
+ * Keep every non-empty line except tombstoned entries; malformed lines are
+ * kept verbatim, as the delete sweep does. Each kept line ends in "\n".
+ */
+function keepVisibleLines(text: string, hidden: ReadonlySet<string>): string {
+  let out = "";
+  for (const lineText of text.split("\n")) {
+    if (lineText.length === 0) continue;
+    const parsed = parseStoreLine(lineText);
+    if (parsed && isPromptHidden(hidden, parsed.text)) continue;
+    out += `${lineText}\n`;
+  }
+  return out;
+}
+
+/**
+ * Claim a merge candidate: open it FIRST (an unreadable file is skipped
+ * untouched), then rename it to a claim name that still ends in `.jsonl`,
+ * so drains keep reading it until the compact file lands. After the
+ * rename, a live writer appending by path creates a fresh file under the
+ * original name; a write through a descriptor opened before the rename
+ * lands in the claimed inode, which the kept descriptor still reads.
+ */
+function claimFile(file: string): ClaimedFile | null {
+  let fd: number;
+  try {
+    fd = fs.openSync(file, "r");
+  } catch {
+    return null;
+  }
+  const claim = `${file}.gc-${process.pid}-${Date.now()}.jsonl`;
+  try {
+    fs.renameSync(file, claim);
+  } catch {
+    fs.closeSync(fd);
+    return null;
+  }
+  return { claim, fd, consumed: 0 };
+}
+
+/**
+ * Merge the claimed tail (oldest first) into one compact file, written
+ * atomically (tmp + rename) BEFORE any claim is removed. A failure before
+ * the compact file lands leaves every claim in place (still a readable
+ * store file); a claim that cannot be removed survives as a harmless
+ * duplicate (drains dedupe by identity). After each removal the claim's
+ * descriptor is drained once more and any late bytes are appended to the
+ * compact file (or written back under the claim name if that fails).
+ */
+function compactFiles(
+  dir: string,
+  tail: readonly string[],
+  hidden: ReadonlySet<string>,
+): GcResult {
+  const claimed = tail
+    .map(claimFile)
+    .filter((c): c is ClaimedFile => c !== null);
+  if (claimed.length === 0) return { compacted: false, merged: 0 };
+  const compact = path.join(dir, `compact-${process.pid}-${Date.now()}.jsonl`);
+  const tmp = `${compact}.tmp-${process.pid}-${Date.now()}`;
+  try {
+    let merged = "";
+    for (const c of claimed) {
+      const snapshot = readFrom(c.fd, 0);
+      c.consumed = snapshot.lastIndexOf(NEWLINE) + 1;
+      merged += keepVisibleLines(
+        snapshot.subarray(0, c.consumed).toString("utf8"),
+        hidden,
+      );
+    }
+    fs.writeFileSync(tmp, merged, "utf8");
+    fs.renameSync(tmp, compact);
+  } catch {
+    try {
+      fs.unlinkSync(tmp);
+    } catch {
+      // the tmp name never matches a *.jsonl store file
+    }
+    for (const c of claimed) fs.closeSync(c.fd);
+    return { compacted: false, merged: 0 };
+  }
+  for (const c of claimed) {
+    try {
+      fs.rmSync(c.claim);
+    } catch {
+      // the claim keeps every byte; it is merged again by a later GC
+      fs.closeSync(c.fd);
+      continue;
+    }
+    carryOver(c, compact, hidden);
+  }
+  return { compacted: true, merged: claimed.length };
+}
+
+/** Move bytes that reached a removed claim after it was read. */
+function carryOver(
+  c: ClaimedFile,
+  compact: string,
+  hidden: ReadonlySet<string>,
+): void {
+  let late: Buffer = Buffer.alloc(0);
+  try {
+    late = readFrom(c.fd, c.consumed);
+    if (late.length === 0) return;
+    // A torn last line is completed so the next append stays parseable.
+    const text = late.toString("utf8");
+    fs.appendFileSync(
+      compact,
+      keepVisibleLines(text.endsWith("\n") ? text : `${text}\n`, hidden),
+    );
+  } catch {
+    try {
+      if (late.length > 0) fs.writeFileSync(c.claim, late, { flag: "wx" });
+    } catch {
+      // nothing else can hold these bytes; the claim name is taken
+    }
+  } finally {
+    fs.closeSync(c.fd);
+  }
+}