From 9b56e0b3c575d1c8a766a152c0f2fcf4d0b12132 Mon Sep 17 00:00:00 2001 From: Carolina <26188349+carolitascl@users.noreply.github.com> Date: Fri, 18 Sep 2026 15:24:40 -0300 Subject: [PATCH 1/8] feat(history): per-instance JSONL store, project identity, storage tests Slice 1/6 of the PR #819 split (maintainer-requested review slices). - atomic-write: same-dir tmp+rename JSON writer, concurrent-instance-safe staging names, never throws - store: project identity (realpath+sha256[:16], raw-path fallback, 24-char collision re-key), project/seed/global/registry path derivations, advisory registry with fail-open reads and atomic writes, tolerant JSONL line parser, lazy per-instance session writer with command filtering - extension entry: identity constants and capture-only wiring (before_agent_start -> appendSessionCapture); migration, seeding, selector, deletion, and GC join in later slices - tests: 32 node:test cases covering storage concurrency and recovery (parallel writers, interleaved captures, burst order integrity, torn-line matrix + crash-tail recovery window, rapid same-target atomic writes with zero staging residue, two-instance registry interleaving, collision re-key, corrupt/wrong-shape fail-open) - test vectors are machine-independent: literal cwds exercise the documented raw-string fallback identically on every platform Gates: scoped history tests 32/32 green. verify-package-files and package-manifest failures are pre-existing environmental (gitignored contracts/.DS_Store; missing node_modules) and reproduce on vanilla origin/main. --- extensions/history/atomic-write.ts | 38 +++++ extensions/history/index.ts | 60 +++++++ extensions/history/store.ts | 240 +++++++++++++++++++++++++++ tests/history-atomic-write.test.ts | 57 +++++++ tests/history-multi-reader.test.ts | 203 ++++++++++++++++++++++ tests/history-registry.test.ts | 109 ++++++++++++ tests/history-session-writer.test.ts | 109 ++++++++++++ tests/history-store-paths.test.ts | 79 +++++++++ 8 files changed, 895 insertions(+) create mode 100644 extensions/history/atomic-write.ts create mode 100644 extensions/history/index.ts create mode 100644 extensions/history/store.ts create mode 100644 tests/history-atomic-write.test.ts create mode 100644 tests/history-multi-reader.test.ts create mode 100644 tests/history-registry.test.ts create mode 100644 tests/history-session-writer.test.ts create mode 100644 tests/history-store-paths.test.ts diff --git a/extensions/history/atomic-write.ts b/extensions/history/atomic-write.ts new file mode 100644 index 000000000..cc6ae8147 --- /dev/null +++ b/extensions/history/atomic-write.ts @@ -0,0 +1,38 @@ +import path from "node:path"; +// SPDX-FileCopyrightText: 2026 ExoPro. Inspired by @jasonish/pi-prompt-history +// SPDX-License-Identifier: MIT + +import fs from "node:fs"; + +/** + * Shared atomic JSON writer (design §D3): serialize to a `.tmp` file in the + * SAME directory as the target, then renameSync it into place — a same-dir + * rename is atomic on POSIX/APFS, so readers see the old or the new file, + * never a partial write. Any error returns false and never throws. + * + * No fsync: both consumers treat lost writes as derived cache (a lost index + * rebuilds on the next open; a lost tombstone resurfaces a prompt the user + * can re-delete), so the per-write fsync cost is not justified — the crash + * window is documented, not fixed. The staging name is unique per write + * (`.tmp--`, the same convention as the store.ts writers): the + * state dir is shared across concurrent pi instances, so a fixed + * `${filePath}.tmp` would let two writers clobber the same staging file + * (torn target JSON, spurious rename failures). A failed write unlinks its + * staging file, so orphaned `.tmp` files do not accumulate. + */ +export function writeJsonAtomic(filePath: string, value: unknown): boolean { + const tmpPath = `${filePath}.tmp-${process.pid}-${Date.now()}`; + try { + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(tmpPath, JSON.stringify(value), "utf8"); + fs.renameSync(tmpPath, filePath); + return true; + } catch { + try { + fs.unlinkSync(tmpPath); + } catch { + // staging file never created or already renamed + } + return false; + } +} diff --git a/extensions/history/index.ts b/extensions/history/index.ts new file mode 100644 index 000000000..26616733f --- /dev/null +++ b/extensions/history/index.ts @@ -0,0 +1,60 @@ +// SPDX-FileCopyrightText: 2026 ExoPro. Inspired by @jasonish/pi-prompt-history +// SPDX-License-Identifier: MIT + +// Prompt-history extension entry (slice 1): identity constants, the +// per-instance writer lifecycle, and the before_agent_start capture +// handler. Selector UI, shortcut/command, scope drains, legacy migration +// and seed bootstrap, and GC arrive in later slices. + +import { randomUUID } from "node:crypto"; +import { homedir } from "node:os"; +import { join } from "node:path"; +import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; +import { + appendSessionCapture, + ensureRegistryEntry, + openSessionWriter, + type SessionWriterState, +} from "./store.ts"; + +// v2 multi-concurrency store root (design: tmp/multi-concurrency-design.md). +const PI_HISTORY_ROOT = join(homedir(), ".pi", "agent", "history"); +const AGENT_DIR = join(homedir(), ".pi", "agent"); +const CURRENT_CWD = process.cwd(); +// Instance identity: one exclusive capture file per pi process. +const INSTANCE_ID = randomUUID(); + +let writerState: SessionWriterState | null = null; + +/** + * One-time init per extension load: register the project in the advisory + * registry, then open this instance's exclusive capture file. Legacy + * migration and seed bootstrap join this init order in a later slice. + */ +function getWriter(): SessionWriterState { + if (!writerState) { + try { + ensureRegistryEntry(PI_HISTORY_ROOT, CURRENT_CWD); + } catch { + // registry is advisory + } + writerState = openSessionWriter(PI_HISTORY_ROOT, CURRENT_CWD, INSTANCE_ID); + } + return writerState; +} + +export default function promptHistoryExtension(pi: ExtensionAPI) { + // One writer per extension load; see getWriter() for the init order. + + // Persist every delivered user prompt (write-through, append-only JSONL). + // The local ExtensionAPI stub types handler args as unknown; narrow here. + pi.on("before_agent_start", (...args: unknown[]) => { + try { + const event = args[0] as { prompt?: string } | undefined; + appendSessionCapture(getWriter(), event?.prompt ?? "", Date.now()); + } catch { + // A capture failure must never break the agent loop or unregister + // the handler - swallow and keep the next prompt capturable. + } + }); +} diff --git a/extensions/history/store.ts b/extensions/history/store.ts new file mode 100644 index 000000000..8631707b9 --- /dev/null +++ b/extensions/history/store.ts @@ -0,0 +1,240 @@ +// SPDX-FileCopyrightText: 2026 ExoPro. Inspired by @jasonish/pi-prompt-history +// SPDX-License-Identifier: MIT + +// Consolidated multi-concurrency store (v2), slice 1: project paths and +// identity, the advisory registry, entry primitives, and the per-instance +// session writer. Scope drains/deletes, legacy migration and seed +// bootstrap, and GC/compaction arrive in later slices. +// Formerly store-paths.ts + registry.ts + multi-store.ts (+ v1 primitives). + +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; + +// =========================================================================== +// Paths (formerly store-paths.ts) +// =========================================================================== + +/** + * Project identity for the multi-concurrency store (design v2). + * + * The cwd is canonicalized through realpath — the same resolution pi's + * session-manager applies — so symlinked or differently-spelled paths to one + * project merge into a single identity. A failed resolution (deleted cwd) + * falls back to hashing the raw string: identity degrades, never throws. + */ +export function projectHash(cwd: string): string { + let canonical = cwd; + try { + canonical = fs.realpathSync(cwd); + } catch { + // fall back to the raw path + } + return createHash("sha256").update(canonical).digest("hex").slice(0, 16); +} + +/** The project's directory under the store root. */ +export function projectDir(root: string, cwd: string): string { + return path.join(root, "projects", projectHash(cwd)); +} + +/** The capture file owned by one pi instance (per session/process). */ +export function sessionFilePath( + root: string, + cwd: string, + instanceId: string, +): string { + return path.join(projectDir(root, cwd), `${instanceId}.jsonl`); +} + +/** The rebuildable bootstrap output for a project. */ +export function seedFilePath(root: string, cwd: string): string { + return path.join(projectDir(root, cwd), "seed.jsonl"); +} + +/** The one-time legacy/global seed (never GC'd). */ +export function globalSeedPath(root: string): string { + return path.join(root, "history-global.jsonl"); +} + +/** Advisory hash → cwd map for display labels. */ +export function registryPath(root: string): string { + return path.join(root, "registry.json"); +} + +// =========================================================================== +// Registry (formerly registry.ts) +// =========================================================================== + +export interface RegistryEntryResult { + hash: string; + created: boolean; +} + +type RegistryData = Record; + +function readRegistry(root: string): RegistryData { + try { + const raw = fs.readFileSync(registryPath(root), "utf8"); + const parsed: unknown = JSON.parse(raw); + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + return {}; + } + const out: RegistryData = {}; + for (const [key, value] of Object.entries( + parsed as Record, + )) { + if (typeof value === "string") out[key] = value; + } + return out; + } catch { + return {}; + } +} + +function writeRegistryAtomic(root: string, data: RegistryData): void { + const target = registryPath(root); + const tmp = `${target}.tmp-${process.pid}-${Date.now()}`; + fs.mkdirSync(root, { recursive: true }); + fs.writeFileSync(tmp, JSON.stringify(data, null, 2) + "\n", "utf8"); + fs.renameSync(tmp, target); +} + +/** + * Ensure the advisory registry maps this project's hash to its cwd. + * Idempotent: an existing identical entry writes nothing. A hash mapped to a + * DIFFERENT cwd is a (practically unreachable) collision — the entry is + * re-keyed at 24 hash chars so both identities coexist. + */ +export function ensureRegistryEntry( + root: string, + cwd: string, +): RegistryEntryResult { + const hash = projectHash(cwd); + const data = readRegistry(root); + if (data[hash] === cwd) return { hash, created: false }; + if (data[hash] !== undefined) { + // Collision: re-key the EXISTING occupant at 24 hash chars so both + // identities coexist; the incoming cwd keeps the short hash — the + // key shape projectDir/sessionFilePath/drains derive. + const existing = data[hash]; + data[projectHashLong(existing)] = existing; + data[hash] = cwd; + writeRegistryAtomic(root, data); + return { hash, created: true }; + } + data[hash] = cwd; + writeRegistryAtomic(root, data); + return { hash, created: true }; +} + +function projectHashLong(cwd: string): string { + // Reuse the same canonicalization as projectHash but keep 24 chars. + let canonical = cwd; + try { + canonical = fs.realpathSync(cwd); + } catch { + // fall back to the raw path + } + return createHash("sha256").update(canonical).digest("hex").slice(0, 24); +} + +// =========================================================================== +// Entry primitives (from v1 history-store.ts) +// =========================================================================== + +/** One line of `editor-history.jsonl`. */ +export interface StoreEntry { + /** Schema version; 1 when absent in the source line. */ + v: number; + text: string; + /** Capture epoch-ms; optional, line order is authoritative for recency. */ + ts?: number; +} + +/** + * Parse one JSONL line. Returns null for malformed lines (bad JSON, + * non-string or whitespace-only text) so callers can skip them; a torn + * last line from a crash is handled the same way. + */ +export function parseStoreLine(raw: string): StoreEntry | null { + if (raw.length === 0) return null; + try { + const value: unknown = JSON.parse(raw); + if (!value || typeof value !== "object") return null; + const record = value as { v?: unknown; text?: unknown; ts?: unknown }; + if (typeof record.text !== "string") return null; + if (record.text.trim().length === 0) return null; + const entry: StoreEntry = { v: 1, text: record.text }; + if (typeof record.v === "number" && Number.isFinite(record.v)) { + entry.v = record.v; + } + if (typeof record.ts === "number" && Number.isFinite(record.ts)) { + entry.ts = record.ts; + } + return entry; + } catch { + return null; + } +} + +// =========================================================================== +// Instance writer (formerly multi-store.ts; scope drains/deletes and GC +// arrive in later slices) +// =========================================================================== + +/** Mutable state of ONE pi instance's exclusive capture file. */ +export interface SessionWriterState { + filePath: string; + /** Logical line count of this instance's file. */ + lineCount: number; +} + +/** Command-like prompts (`/name ...`) are UI commands, not prompts. */ +function isLikelyCommand(text: string): boolean { + return /^\/[A-Za-z]/.test(text.trim()); +} + +function serializeEntry(entry: StoreEntry): string { + const out: { v: number; text: string; ts?: number } = { + v: entry.v, + text: entry.text, + }; + if (entry.ts !== undefined) out.ts = entry.ts; + return JSON.stringify(out); +} + +/** + * Open the writer for this pi instance. The file is created LAZILY by the + * first capture — starting pi must not litter empty files. Only this + * instance ever appends here (design v2: zero shared writes). + */ +export function openSessionWriter( + root: string, + cwd: string, + instanceId: string, +): SessionWriterState { + return { + filePath: sessionFilePath(root, cwd, instanceId), + lineCount: 0, + }; +} + +/** + * Append one prompt line to the instance's own file (write-through). + * Skips empty/whitespace-only and command-like prompts. + */ +export function appendSessionCapture( + state: SessionWriterState, + text: string, + ts?: number, +): void { + if (typeof text !== "string" || text.trim().length === 0) return; + if (isLikelyCommand(text)) return; + + const entry: StoreEntry = { v: 1, text }; + if (ts !== undefined) entry.ts = ts; + fs.mkdirSync(path.dirname(state.filePath), { recursive: true }); + fs.appendFileSync(state.filePath, serializeEntry(entry) + "\n", "utf8"); + state.lineCount += 1; +} diff --git a/tests/history-atomic-write.test.ts b/tests/history-atomic-write.test.ts new file mode 100644 index 000000000..5448398e1 --- /dev/null +++ b/tests/history-atomic-write.test.ts @@ -0,0 +1,57 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { writeJsonAtomic } from "../extensions/history/atomic-write.ts"; + +// Shared atomic writer (design §D3): tmp+rename in the TARGET's directory. +// Fixtures live under the OS temp dir — never the workspace tmp. Failure +// paths exercise the catch branch: false return, no throw, and the staging +// file unlinked so `.tmp-*` files never accumulate beside the target. + +function makeRoot(): string { + return fs.mkdtempSync(path.join(os.tmpdir(), "atomic-write-")); +} + +function tmpLeftovers(dir: string): string[] { + return fs.readdirSync(dir).filter((f) => f.includes(".tmp-")); +} + +test("success: missing parent dirs are created recursively and the JSON parses back", () => { + const root = makeRoot(); + const target = path.join(root, "deep", "nested", "state.json"); + const value = { key: "value", nested: { n: 1 } }; + assert.equal(writeJsonAtomic(target, value), true); + assert.deepEqual(JSON.parse(fs.readFileSync(target, "utf8")), value); + assert.deepEqual(tmpLeftovers(path.dirname(target)), []); +}); + +test("a parent chain holding a regular file (ENOTDIR) returns false without throwing", () => { + const root = makeRoot(); + const blocker = path.join(root, "blocker"); + fs.writeFileSync(blocker, "regular file", "utf8"); + const target = path.join(blocker, "child", "state.json"); + assert.equal(writeJsonAtomic(target, { a: 1 }), false); + assert.equal(fs.existsSync(target), false); +}); + +test("an existing directory as the target fails the rename: false and no leftover staging file", () => { + const root = makeRoot(); + const target = path.join(root, "state.json"); + fs.mkdirSync(target, { recursive: true }); + assert.equal(writeJsonAtomic(target, { a: 1 }), false); + // The catch branch unlinked its staging file — no `.tmp-*` accumulation. + assert.deepEqual(tmpLeftovers(root), []); + // The directory itself is untouched. + assert.equal(fs.statSync(target).isDirectory(), true); +}); + +test("overwrite of an existing target replaces the content", () => { + const root = makeRoot(); + const target = path.join(root, "state.json"); + assert.equal(writeJsonAtomic(target, { v: 1 }), true); + assert.equal(writeJsonAtomic(target, { v: 2 }), true); + assert.deepEqual(JSON.parse(fs.readFileSync(target, "utf8")), { v: 2 }); + assert.deepEqual(tmpLeftovers(root), []); +}); diff --git a/tests/history-multi-reader.test.ts b/tests/history-multi-reader.test.ts new file mode 100644 index 000000000..c82ebcbc7 --- /dev/null +++ b/tests/history-multi-reader.test.ts @@ -0,0 +1,203 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { writeJsonAtomic } from "../extensions/history/atomic-write.ts"; +import { + appendSessionCapture, + ensureRegistryEntry, + openSessionWriter, + parseStoreLine, + projectDir, + projectHash, + registryPath, + sessionFilePath, +} from "../extensions/history/store.ts"; + +// Slice-1 concurrency/recovery coverage. The dev-suite multi-reader drain +// scenarios are re-expressed against the slice-1 surface (per-instance +// writers, parseStoreLine, atomic writes): parallel writers on one project +// dir, torn-line tolerance, and same-target atomic-write collisions. The +// drain/read ordering scenarios themselves arrive with the slice-2 reader. + +const PROJECT_A = "/pi-history-test/project-a"; +const PROJECT_B = "/pi-history-test/project-b"; + +function makeRoot(): string { + return fs.mkdtempSync(path.join(os.tmpdir(), "pi-history-multi-reader-")); +} + +function storedTexts(file: string): string[] { + return fs + .readFileSync(file, "utf8") + .split("\n") + .filter((l) => l.trim().length > 0) + .map((l) => (JSON.parse(l) as { text: string }).text); +} + +// --- parallel writers --- + +test("growth from a concurrent instance is visible on the next read", () => { + const root = makeRoot(); + const a = openSessionWriter(root, PROJECT_A, "inst-a"); + appendSessionCapture(a, "first"); + const dirA = projectDir(root, PROJECT_A); + assert.deepEqual(fs.readdirSync(dirA), ["inst-a.jsonl"]); + + // A second pi instance grows the SAME project dir through its OWN file; + // neither writer reads or rewrites the other's bytes. + const b = openSessionWriter(root, PROJECT_A, "inst-b"); + appendSessionCapture(b, "from-other-instance"); + + assert.deepEqual(fs.readdirSync(dirA).sort(), [ + "inst-a.jsonl", + "inst-b.jsonl", + ]); + assert.deepEqual(storedTexts(sessionFilePath(root, PROJECT_A, "inst-a")), [ + "first", + ]); + assert.deepEqual(storedTexts(sessionFilePath(root, PROJECT_A, "inst-b")), [ + "from-other-instance", + ]); + assert.equal(a.lineCount, 1); + assert.equal(b.lineCount, 1); +}); + +test("interleaved captures from multiple writers never clobber each other", () => { + const root = makeRoot(); + const writers = [ + openSessionWriter(root, PROJECT_A, "w0"), + openSessionWriter(root, PROJECT_A, "w1"), + openSessionWriter(root, PROJECT_A, "w2"), + ]; + for (let i = 0; i < 10; i++) { + for (let w = 0; w < writers.length; w++) { + appendSessionCapture(writers[w], `w${w}-line-${i}`); + } + } + const dir = projectDir(root, PROJECT_A); + assert.deepEqual(fs.readdirSync(dir).sort(), [ + "w0.jsonl", + "w1.jsonl", + "w2.jsonl", + ]); + for (let w = 0; w < writers.length; w++) { + assert.equal(writers[w].lineCount, 10); + assert.deepEqual( + storedTexts(writers[w].filePath), + Array.from({ length: 10 }, (_, i) => `w${w}-line-${i}`), + ); + } +}); + +test("a high-volume burst on one writer keeps every line, in order", () => { + const root = makeRoot(); + const writer = openSessionWriter(root, PROJECT_A, "burst"); + const expected: string[] = []; + for (let i = 0; i < 100; i++) { + const text = `burst-${i}`; + expected.push(text); + appendSessionCapture(writer, text, 1000 + i); + } + assert.equal(writer.lineCount, 100); + const lines = fs.readFileSync(writer.filePath, "utf8").trim().split("\n"); + assert.equal(lines.length, 100); + const parsed = lines.map((l) => JSON.parse(l) as { text: string; ts: number }); + assert.deepEqual( + parsed.map((e) => e.text), + expected, + ); + assert.equal(parsed[42].ts, 1042); +}); + +// --- torn-line / crash recovery --- + +test("torn and malformed lines parse to null (crash garbage never resurfaces)", () => { + // A torn final line (process died mid-write) is a truncated JSON doc. + const torn = JSON.stringify({ v: 1, text: "survivor" }).slice(0, 12); + const malformed: string[] = [ + "", + "{torn", + torn, + "not json at all", + JSON.stringify([]), + JSON.stringify("scalar"), + JSON.stringify(null), + JSON.stringify({ v: 1 }), + JSON.stringify({ text: 42 }), + JSON.stringify({ text: " " }), + ]; + for (const line of malformed) { + assert.equal(parseStoreLine(line), null, JSON.stringify(line)); + } + // Valid lines keep parsing: absent v defaults to 1, ts/v flow through. + assert.deepEqual(parseStoreLine(JSON.stringify({ v: 1, text: "survivor" })), { + v: 1, + text: "survivor", + }); + assert.deepEqual(parseStoreLine(JSON.stringify({ text: "y" })), { + v: 1, + text: "y", + }); + assert.deepEqual(parseStoreLine(JSON.stringify({ v: 2, text: "x", ts: 7 })), { + v: 2, + text: "x", + ts: 7, + }); +}); + +test("a torn final line is tolerated: skipped by readers, later appends continue", () => { + const root = makeRoot(); + const writer = openSessionWriter(root, PROJECT_A, "torn"); + appendSessionCapture(writer, "before-crash"); + // Crash mid-write: a partial line lands WITHOUT its trailing newline. + fs.appendFileSync(writer.filePath, `{"v":1,"text":"tor`, "utf8"); + // parseStoreLine skips the torn tail instead of throwing... + assert.equal(parseStoreLine('{"v":1,"text":"tor'), null); + // ...and the instance keeps capturing. The first append after a + // newline-less torn tail merges with the fragment (one accepted lost + // entry — the same crash window the design documents for lost writes); + // the next full line parses cleanly again. + appendSessionCapture(writer, "after-crash"); + appendSessionCapture(writer, "after-crash-2"); + assert.equal(writer.lineCount, 3); + const lines = fs.readFileSync(writer.filePath, "utf8").trim().split("\n"); + assert.equal(lines.length, 3); + assert.equal((JSON.parse(lines[0]) as { text: string }).text, "before-crash"); + assert.equal(parseStoreLine(lines[1]), null); // torn fragment + merged entry + assert.equal( + (JSON.parse(lines[2]) as { text: string }).text, + "after-crash-2", + ); +}); + +// --- atomic-write collisions --- + +test("rapid same-target atomic writes leave one valid document and no staging files", () => { + const root = makeRoot(); + const target = path.join(root, "shared-state.json"); + for (let i = 0; i < 25; i++) { + assert.equal(writeJsonAtomic(target, { writer: i }), true); + } + const final = JSON.parse(fs.readFileSync(target, "utf8")) as { + writer: number; + }; + assert.ok(final.writer >= 0 && final.writer <= 24); + const leftovers = fs.readdirSync(root).filter((f) => f.includes(".tmp-")); + assert.deepEqual(leftovers, []); +}); + +test("interleaved registry updates from two instances keep both entries", () => { + const root = makeRoot(); + for (let i = 0; i < 3; i++) { + ensureRegistryEntry(root, PROJECT_A); + ensureRegistryEntry(root, PROJECT_B); + } + const raw = JSON.parse( + fs.readFileSync(registryPath(root), "utf8"), + ) as Record; + assert.equal(Object.keys(raw).length, 2); + assert.equal(raw[projectHash(PROJECT_A)], PROJECT_A); + assert.equal(raw[projectHash(PROJECT_B)], PROJECT_B); +}); diff --git a/tests/history-registry.test.ts b/tests/history-registry.test.ts new file mode 100644 index 000000000..c985972a1 --- /dev/null +++ b/tests/history-registry.test.ts @@ -0,0 +1,109 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { + ensureRegistryEntry, + projectHash, + registryPath, +} from "../extensions/history/store.ts"; + +// Slice-1 port note: the dev suite asserted lookups through the dead +// `lookupCwd` export, which slice 1 drops. Every lookup assertion is +// re-expressed against the persisted registry.json content. + +function makeRoot(): string { + return fs.mkdtempSync(path.join(os.tmpdir(), "pi-history-registry-")); +} + +function readRegistryFile(root: string): Record { + return JSON.parse( + fs.readFileSync(registryPath(root), "utf8"), + ) as Record; +} + +test("creates the registry with the first entry (idempotent)", () => { + const root = makeRoot(); + const result = ensureRegistryEntry(root, "/pi-history-test/project-a"); + assert.deepEqual(result, { hash: "4be15ec687e9df85", created: true }); + ensureRegistryEntry(root, "/pi-history-test/project-a"); + assert.deepEqual(readRegistryFile(root), { + "4be15ec687e9df85": "/pi-history-test/project-a", + }); +}); + +test("second project appends without touching the first", () => { + const root = makeRoot(); + ensureRegistryEntry(root, "/pi-history-test/project-a"); + const b = ensureRegistryEntry(root, "/pi-history-test/project-b"); + assert.equal(b.created, true); + const raw = readRegistryFile(root); + assert.equal(Object.keys(raw).length, 2); + assert.equal(raw[b.hash], "/pi-history-test/project-b"); +}); + +test("registry.json maps known hashes and omits unknown ones", () => { + const root = makeRoot(); + const { hash } = ensureRegistryEntry(root, "/pi-history-test/project-a"); + const raw = readRegistryFile(root); + assert.equal(raw[hash], "/pi-history-test/project-a"); + assert.equal(raw["0000000000000000"], undefined); + // A fresh root has no registry file until its first entry lands. + assert.equal(fs.existsSync(registryPath(makeRoot())), false); +}); + +test("corrupt registry json is treated as empty and rebuilt on next entry", () => { + const root = makeRoot(); + fs.writeFileSync(registryPath(root), "{not-json", "utf8"); + const result = ensureRegistryEntry(root, "/pi-history-test/project-a"); + assert.equal(result.created, true); + // The corrupt content was discarded (fail-open to empty), so the rebuilt + // registry contains exactly the new entry and nothing else. + assert.deepEqual(readRegistryFile(root), { + "4be15ec687e9df85": "/pi-history-test/project-a", + }); +}); + +test("no leftover tmp files after writes", () => { + const root = makeRoot(); + ensureRegistryEntry(root, "/a"); + ensureRegistryEntry(root, "/b"); + const leftovers = fs.readdirSync(root).filter((f) => f.includes(".tmp-")); + assert.deepEqual(leftovers, []); +}); + +test("hash collision re-keys the existing occupant; the new cwd keeps the short hash", () => { + const root = makeRoot(); + const cwd = "/pi-history-test/project-a"; + const hash = projectHash(cwd); + // Simulate a collision: the short hash is pre-mapped to a different cwd. + fs.mkdirSync(root, { recursive: true }); + fs.writeFileSync( + registryPath(root), + JSON.stringify({ [hash]: "/some/other/project" }), + "utf8", + ); + const result = ensureRegistryEntry(root, cwd); + assert.deepEqual(result, { hash, created: true }); + const raw = readRegistryFile(root); + assert.equal(raw[hash], cwd); + const longKeys = Object.keys(raw).filter((k) => k.length === 24); + assert.equal(longKeys.length, 1); + assert.equal(raw[longKeys[0]], "/some/other/project"); +}); + +test("wrong-shaped registry (array / scalar / null) fails open and is rebuilt on the next entry", () => { + // Valid JSON, wrong shape: the readRegistry shape guard treats each as an + // empty registry, and the next entry rebuilds a valid object-mapped + // registry around itself. + const shapes: unknown[] = [["an", "array"], "scalar-string", null]; + const cwd = "/pi-history-test/project-a"; + for (const shape of shapes) { + const root = makeRoot(); + fs.writeFileSync(registryPath(root), JSON.stringify(shape), "utf8"); + const result = ensureRegistryEntry(root, cwd); + assert.deepEqual(result, { hash: projectHash(cwd), created: true }); + assert.deepEqual(readRegistryFile(root), { [projectHash(cwd)]: cwd }); + } +}); diff --git a/tests/history-session-writer.test.ts b/tests/history-session-writer.test.ts new file mode 100644 index 000000000..8601cea14 --- /dev/null +++ b/tests/history-session-writer.test.ts @@ -0,0 +1,109 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { + appendSessionCapture, + openSessionWriter, + projectHash, + sessionFilePath, +} from "../extensions/history/store.ts"; +import promptHistoryExtension from "../extensions/history/index.ts"; + +function makeRoot(): string { + return fs.mkdtempSync(path.join(os.tmpdir(), "pi-history-writer-")); +} + +const CWD = "/pi-history-test/project-a"; + +function fileTexts(file: string): string[] { + return fs + .readFileSync(file, "utf8") + .split("\n") + .filter((l) => l.trim().length > 0) + .map((l) => (JSON.parse(l) as { text: string }).text); +} + +function openWriterForTest(root: string, instanceId: string) { + return openSessionWriter(root, CWD, instanceId); +} + +test("no file is created until the first capture", () => { + const root = makeRoot(); + const state = openWriterForTest(root, "sess-1"); + const file = sessionFilePath(root, CWD, "sess-1"); + assert.equal(fs.existsSync(file), false); + assert.equal(state.lineCount, 0); +}); + +test("first capture lazily creates the file and appends one line", () => { + const root = makeRoot(); + const state = openWriterForTest(root, "sess-1"); + appendSessionCapture(state, "hello world", 1234); + const file = sessionFilePath(root, CWD, "sess-1"); + assert.equal(fs.existsSync(file), true); + const lines = fs.readFileSync(file, "utf8").trim().split("\n"); + assert.equal(lines.length, 1); + const parsed = JSON.parse(lines[0]); + assert.equal(parsed.text, "hello world"); + assert.equal(parsed.ts, 1234); + assert.equal(parsed.v, 1); + assert.equal(state.lineCount, 1); +}); + +test("captures append in order; count tracks", () => { + const root = makeRoot(); + const state = openWriterForTest(root, "sess-2"); + appendSessionCapture(state, "one"); + appendSessionCapture(state, "two"); + appendSessionCapture(state, "three"); + assert.deepEqual(fileTexts(sessionFilePath(root, CWD, "sess-2")), [ + "one", + "two", + "three", + ]); + assert.equal(state.lineCount, 3); +}); + +test("command-like and empty captures are skipped", () => { + const root = makeRoot(); + const state = openWriterForTest(root, "sess-3"); + appendSessionCapture(state, "/compact"); + appendSessionCapture(state, " "); + appendSessionCapture(state, ""); + appendSessionCapture(state, "kept"); + assert.deepEqual(fileTexts(sessionFilePath(root, CWD, "sess-3")), ["kept"]); + assert.equal(state.lineCount, 1); +}); + +test("two writers own separate files in the same project dir", () => { + const root = makeRoot(); + const a = openWriterForTest(root, "inst-a"); + const b = openWriterForTest(root, "inst-b"); + appendSessionCapture(a, "from-a"); + appendSessionCapture(b, "from-b"); + const dir = path.join(root, "projects", projectHash(CWD)); + const files = fs.readdirSync(dir).sort(); + assert.deepEqual(files, ["inst-a.jsonl", "inst-b.jsonl"]); +}); + +test("the slice-1 extension entry registers only the capture handler", () => { + // Module load must stay side-effect free (importing index.ts parses the + // whole slice-1 graph without touching the real ~/.pi store root), and + // slice 1 wires exactly one handler: before_agent_start. + const registered: Array<[string, unknown]> = []; + const pi = { + on: (event: string, handler: unknown) => { + registered.push([event, handler]); + }, + }; + promptHistoryExtension(pi as never); + assert.deepEqual( + registered.map(([event]) => event), + ["before_agent_start"], + ); + // The handler is callable but is NEVER invoked here: a real invocation + // would run getWriter() against the user's real ~/.pi/agent/history. + assert.equal(typeof registered[0][1], "function"); +}); diff --git a/tests/history-store-paths.test.ts b/tests/history-store-paths.test.ts new file mode 100644 index 000000000..a6e5be45a --- /dev/null +++ b/tests/history-store-paths.test.ts @@ -0,0 +1,79 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { + globalSeedPath, + projectDir, + projectHash, + registryPath, + seedFilePath, + sessionFilePath, +} from "../extensions/history/store.ts"; + +const ROOT = path.join(os.tmpdir(), "pi-history-test-root"); + +test("projectHash returns 16 lowercase hex chars", () => { + const hash = projectHash("/pi-history-test/project-a"); + assert.match(hash, /^[0-9a-f]{16}$/); +}); + +test("known vector: stable hash for a fixed path", () => { + // The literal exists on no machine, so every platform exercises the + // documented raw-string fallback: sha256(literal), first 16 hex chars. + assert.equal( + projectHash("/pi-history-test/project-a"), + "4be15ec687e9df85", + ); +}); + +test("distinct paths produce distinct hashes", () => { + assert.notEqual( + projectHash("/pi-history-test/project-a"), + projectHash("/pi-history-test/project-b"), + ); +}); + +test("symlinked cwd resolves to the same hash as its target", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "paths-sym-")); + const target = path.join(dir, "real-project"); + fs.mkdirSync(target); + const link = path.join(dir, "link-project"); + fs.symlinkSync(target, link); + assert.equal(projectHash(link), projectHash(target)); +}); + +test("trailing slash does not change the identity", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "paths-slash-")); + assert.equal(projectHash(dir), projectHash(`${dir}/`)); +}); + +test("nonexistent path falls back to hashing the raw string (no throw)", () => { + const missing = path.join(os.tmpdir(), "paths-missing-does-not-exist"); + const hash = projectHash(missing); + assert.match(hash, /^[0-9a-f]{16}$/); +}); + +test("path derivations compose under the root", () => { + const cwd = "/pi-history-test/project-a"; + const hash = projectHash(cwd); + assert.equal(projectDir(ROOT, cwd), path.join(ROOT, "projects", hash)); + assert.equal( + sessionFilePath(ROOT, cwd, "abc-123"), + path.join(ROOT, "projects", hash, "abc-123.jsonl"), + ); + assert.equal( + seedFilePath(ROOT, cwd), + path.join(ROOT, "projects", hash, "seed.jsonl"), + ); + assert.equal(globalSeedPath(ROOT), path.join(ROOT, "history-global.jsonl")); + assert.equal(registryPath(ROOT), path.join(ROOT, "registry.json")); +}); + +test("two cwds map to sibling project dirs", () => { + const a = projectDir(ROOT, "/pi-history-test/project-a"); + const b = projectDir(ROOT, "/pi-history-test/project-b"); + assert.notEqual(a, b); + assert.equal(path.dirname(a), path.dirname(b)); +}); From 2b90751e8ff7cb97a8c0e4eba2348a95cbbaab33 Mon Sep 17 00:00:00 2001 From: Carolina <26188349+carolitascl@users.noreply.github.com> Date: Fri, 18 Sep 2026 18:37:15 -0300 Subject: [PATCH 2/8] fix(history): stable registry collision mappings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review fix (CodeRabbit #5160388228): ensureRegistryEntry now searches the registry for an existing mapping of the incoming cwd before the collision branch, returning the existing short or long key unchanged. Previously, re-entering a cwd that an earlier collision had re-keyed to 24 chars re-triggered the collision and flipped the other occupant's key every time — collision assignments were not stable. Adds a stability test: the re-keyed cwd keeps its long key, the short-hash holder keeps its key, and the registry bytes do not change across re-entries. Note: the atomic-write staging-name race CodeRabbit reported in the original commit was already hardened on this branch (unique .tmp-- staging + unlink-on-failure); no further change. --- extensions/history/store.ts | 5 +++++ tests/history-registry.test.ts | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/extensions/history/store.ts b/extensions/history/store.ts index 8631707b9..f3723f6ad 100644 --- a/extensions/history/store.ts +++ b/extensions/history/store.ts @@ -113,6 +113,11 @@ export function ensureRegistryEntry( const hash = projectHash(cwd); const data = readRegistry(root); if (data[hash] === cwd) return { hash, created: false }; + // An earlier collision may have re-keyed THIS cwd to a long key. + // Return the existing mapping unchanged so collision assignments stay + // stable across calls instead of flipping the other occupant's key. + const existingKey = Object.keys(data).find((k) => data[k] === cwd); + if (existingKey !== undefined) return { hash: existingKey, created: false }; if (data[hash] !== undefined) { // Collision: re-key the EXISTING occupant at 24 hash chars so both // identities coexist; the incoming cwd keeps the short hash — the diff --git a/tests/history-registry.test.ts b/tests/history-registry.test.ts index c985972a1..23235f8cd 100644 --- a/tests/history-registry.test.ts +++ b/tests/history-registry.test.ts @@ -1,5 +1,6 @@ import { test } from "node:test"; import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -93,6 +94,39 @@ test("hash collision re-keys the existing occupant; the new cwd keeps the short assert.equal(raw[longKeys[0]], "/some/other/project"); }); +test("a re-keyed cwd keeps its long key on later calls (stable collision mappings)", () => { + // projectHashLong is private: derive the documented 24-char key here — + // the literals never exist, so canonicalization falls back to the raw + // string on every platform. + const longKey = (cwd: string) => + createHash("sha256").update(cwd).digest("hex").slice(0, 24); + const root = makeRoot(); + const a = "/pi-history-test/registry-collide-a"; + const b = "/pi-history-test/registry-collide-b"; + // Simulate the collision: b's short hash is pre-mapped to a different + // cwd, so entering b re-keys that occupant to a 24-char key. + const shortHash = projectHash(b); + fs.mkdirSync(root, { recursive: true }); + fs.writeFileSync( + registryPath(root), + JSON.stringify({ [shortHash]: a }), + "utf8", + ); + ensureRegistryEntry(root, b); // collision: a re-keyed to 24 chars + const before = readRegistryFile(root); + // Re-entering the re-keyed cwd must return its EXISTING long key and + // leave the other occupant's short-hash mapping untouched. + const again = ensureRegistryEntry(root, a); + assert.equal(again.created, false); + assert.equal(again.hash, longKey(a)); + const after = readRegistryFile(root); + assert.deepEqual(after, before); + // And re-entering the short-hash holder keeps the short key. + const holder = ensureRegistryEntry(root, b); + assert.equal(holder.hash, projectHash(b)); + assert.deepEqual(readRegistryFile(root), before); +}); + test("wrong-shaped registry (array / scalar / null) fails open and is rebuilt on the next entry", () => { // Valid JSON, wrong shape: the readRegistry shape guard treats each as an // empty registry, and the next entry rebuilds a valid object-mapped From a5ff13d64dae7181ea7ce68ec578d322ef40a4ae Mon Sep 17 00:00:00 2001 From: Carolina <26188349+carolitascl@users.noreply.github.com> Date: Fri, 18 Sep 2026 15:46:43 -0300 Subject: [PATCH 3/8] feat(history): read, ordering, deduplication, and project/global query APIs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Slice 2/6 of the PR #819 split (maintainer-requested review slices). - store: reader/query section — file listing with mtime resolution, drain ordering (newest entry ts, mtime fallback; stable under atomic rewrites), dedup + tombstone filter + cap drain, project scope drain (hash dir) and global scope drain (all project dirs, legacy global seed last); dead generator fileEntriesBackward (zero callers) dropped - hide-prompts: tombstone file contract (fail-open reader, atomic sorted writer, shared dedup key) — lands here because the drain APIs filter hidden prompts via the optional stateDir parameter; slice 5 delivers deletion semantics on top - selector-helpers (new): entry/dedup-key normalization, keep-first read-time dedup, records shaping with provenance, result filter with MAX_RESULTS cap; windowing/nav helpers follow in slice 3 - tests: 21 new node:test cases (cumulative 53/53): drain ordering across mixed mtimes, hidden-prompt filtering incl. corrupt hidden.json fail-open, dedup key normalization, cap at exactly 10000, hide/write contract incl. ENOTDIR failure; portable CWD literals throughout (no machine-specific paths) Gates: cumulative scoped history tests 53/53 green (slice-1 set unchanged). Known pre-existing environmental gate failures unchanged (contracts/.DS_Store; missing node_modules for package-manifest). --- extensions/history/hide-prompts.ts | 74 +++++++++++ extensions/history/selector-helpers.ts | 105 +++++++++++++++ extensions/history/store.ts | 170 ++++++++++++++++++++++++- tests/history-dedupe-entries.test.ts | 123 ++++++++++++++++++ tests/history-drain-hidden.test.ts | 54 ++++++++ tests/history-drain-order.test.ts | 86 +++++++++++++ tests/history-hide-prompts.test.ts | 99 ++++++++++++++ tests/history-max-results-cap.test.ts | 76 +++++++++++ 8 files changed, 781 insertions(+), 6 deletions(-) create mode 100644 extensions/history/hide-prompts.ts create mode 100644 extensions/history/selector-helpers.ts create mode 100644 tests/history-dedupe-entries.test.ts create mode 100644 tests/history-drain-hidden.test.ts create mode 100644 tests/history-drain-order.test.ts create mode 100644 tests/history-hide-prompts.test.ts create mode 100644 tests/history-max-results-cap.test.ts diff --git a/extensions/history/hide-prompts.ts b/extensions/history/hide-prompts.ts new file mode 100644 index 000000000..9cffd6954 --- /dev/null +++ b/extensions/history/hide-prompts.ts @@ -0,0 +1,74 @@ +// SPDX-FileCopyrightText: 2026 ExoPro. Inspired by @jasonish/pi-prompt-history +// SPDX-License-Identifier: MIT + +import fs from "node:fs"; +import path from "node:path"; +import { writeJsonAtomic } from "./atomic-write.ts"; +import { promptDedupKey } from "./selector-helpers.ts"; + +/** Name of the tombstone file inside the injected state dir (spec C4). */ +const HIDE_FILE_NAME = "hidden.json"; + +/** + * Result of one tombstone write (spec C4): `written` on a successful atomic + * write, or an error object carrying a short, toast-suitable reason. Never + * throws. + */ +export type HideResult = + | { status: "written" } + | { status: "error"; message: string }; + +/** + * Load the tombstone key set from `stateDir/hidden.json` — the READ half of + * the hide-file contract (spec C4). Fail-open: a missing, unreadable, + * corrupt, or wrong-shaped file is an EMPTY set and the call never throws; + * a corrupt file is rewritten clean by the next hide (the WRITE half, + * `hidePrompt`, lands in WU4). Keys are `promptDedupKey` strings written by + * `hidePrompt`; foreign values are ignored, never trusted. + */ +export function loadHiddenPrompts(stateDir: string): Set { + let raw: string; + try { + raw = fs.readFileSync(path.join(stateDir, HIDE_FILE_NAME), "utf8"); + } catch { + return new Set(); // missing or unreadable → empty tombstones + } + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + return new Set(); // corrupt bytes → fail-open empty + } + const keys = new Set(); + if (!Array.isArray(parsed)) return keys; // wrong shape → fail-open empty + for (const item of parsed) { + if (typeof item === "string" && item !== "") keys.add(item); + } + return keys; +} + +/** + * Write the tombstone key for `text` into `stateDir/hidden.json` — the + * WRITE half of the hide-file contract (spec C4). The key is the shared + * `promptDedupKey` (byte-match normative with the merge filter — never a + * re-implementation); the set compacts on write and persists as a SORTED + * array via the shared atomic tmp+rename writer. Fail-open both ways: a + * corrupt or missing file reads as empty (this clean rewrite IS the + * recovery — the corrupt contents are untrustworthy by definition) and any + * write failure returns an error object for the delete-flow toast; the + * call never throws. + */ +export function hidePrompt(stateDir: string, text: string): HideResult { + const keys = loadHiddenPrompts(stateDir); + keys.add(promptDedupKey(text)); + const written = writeJsonAtomic( + path.join(stateDir, HIDE_FILE_NAME), + [...keys].sort(), + ); + return written + ? { status: "written" } + : { + status: "error", + message: "Could not write the hide file; the prompt may reappear.", + }; +} diff --git a/extensions/history/selector-helpers.ts b/extensions/history/selector-helpers.ts new file mode 100644 index 000000000..c5257bafe --- /dev/null +++ b/extensions/history/selector-helpers.ts @@ -0,0 +1,105 @@ +export interface PromptRecord { + text: string; + searchText: string; + /** + * Provenance (spec C3): set on records built from PromptEntry inputs; + * ABSENT on records built from bare strings so the pinned deepEqual + * record shape ({text, searchText}) stays byte-compatible (design §I). + */ + source?: PromptSource; + /** Session records only: the resolved ms-epoch ordering timestamp. */ + ts?: number; +} + +/** Provenance of a prompt record or merge-loader entry (spec C3). */ +export type PromptSource = "editor" | "session"; + +/** + * Merge-loader currency (pre-dedup, spec C3): one editor-store or + * session-derived prompt. Session entries carry the resolved ms-epoch `ts`; + * editor entries do not (block ordering at the seam, proposal R8). + */ +export interface PromptEntry { + text: string; + source: PromptSource; + ts?: number; +} + +export function buildPromptRecords( + entries: ReadonlyArray, +): PromptRecord[] { + return entries.map((entry): PromptRecord => { + if (typeof entry === "string") { + // Bare string input keeps the EXACT Change 2 runtime shape — the + // pinned deepEqual records carry only {text, searchText}. + return { text: entry, searchText: entry.toLowerCase() }; + } + const record: PromptRecord = { + text: entry.text, + searchText: entry.text.toLowerCase(), + source: entry.source, + }; + if (entry.ts !== undefined) { + record.ts = entry.ts; + } + return record; + }); +} + +/** + * Normalization key for read-time dedup (spec C3): byte-matches the + * APPLIED patch key in nav/patches/editor.cjs (:480-:586) — whitespace + * runs collapse, then trim, then a 120-char prefix slice, then lowercase. + * The literal is the single-backslash applied-patch form; the raw patch + * file stores \\s+ only because its code sits inside a template literal. + * Shared by contract (spec C4): hide-prompts tombstone keys and the + * merge-history session-half tombstone filter MUST byte-match this key. + */ +export function promptDedupKey(entry: string): string { + return entry.replace(/\s+/g, " ").trim().slice(0, 120).toLowerCase(); +} + +/** + * Read-time dedup pass (spec C3): keep-first over input order (file order + * is newest-first, mirroring the patch's keep-first dedup-on-save), and + * empty-key entries (empty or whitespace-only) are skipped — excluded from + * the output and never usable as collision keys. Removes ONLY duplicate- + * normalized entries; no snapshot cap (filterPrompts still caps output). + * Generic over string | PromptEntry (WU3): over the COMBINED merged input + * the keep-first rule makes the editor copy win at the seam — objects pass + * through with provenance intact; no new dedup logic exists anywhere. + */ +export function dedupePromptEntries( + entries: readonly T[], +): T[] { + const seen = new Set(); + const deduped: T[] = []; + for (const entry of entries) { + const key = + typeof entry === "string" + ? promptDedupKey(entry) + : promptDedupKey(entry.text); + if (key !== "" && !seen.has(key)) { + seen.add(key); + deduped.push(entry); + } + } + return deduped; +} + +const MAX_RESULTS = 10000; + +export function filterPrompts( + records: PromptRecord[], + query: string, +): PromptRecord[] { + const trimmed = query.trim(); + if (!trimmed) return records.slice(0, MAX_RESULTS); + + const tokens = trimmed.toLowerCase().split(/\s+/).filter(Boolean); + const filtered = records.filter((record) => { + return tokens.every((token) => record.searchText.includes(token)); + }); + + return filtered.slice(0, MAX_RESULTS); +} diff --git a/extensions/history/store.ts b/extensions/history/store.ts index f3723f6ad..bf2a41c16 100644 --- a/extensions/history/store.ts +++ b/extensions/history/store.ts @@ -1,15 +1,17 @@ // SPDX-FileCopyrightText: 2026 ExoPro. Inspired by @jasonish/pi-prompt-history // SPDX-License-Identifier: MIT -// Consolidated multi-concurrency store (v2), slice 1: project paths and -// identity, the advisory registry, entry primitives, and the per-instance -// session writer. Scope drains/deletes, legacy migration and seed -// bootstrap, and GC/compaction arrive in later slices. +// Consolidated multi-concurrency store (v2), slices 1+2: project paths and +// identity, the advisory registry, entry primitives, the per-instance +// session writer, and the scope drain/reader/query section (ordering, +// dedup, tombstone filter, project/global drains). Legacy migration and +// seed bootstrap, scope deletes, and GC/compaction arrive in later slices. // Formerly store-paths.ts + registry.ts + multi-store.ts (+ v1 primitives). import { createHash } from "node:crypto"; import fs from "node:fs"; import path from "node:path"; +import { loadHiddenPrompts } from "./hide-prompts.ts"; // =========================================================================== // Paths (formerly store-paths.ts) @@ -184,8 +186,8 @@ export function parseStoreLine(raw: string): StoreEntry | null { } // =========================================================================== -// Instance writer (formerly multi-store.ts; scope drains/deletes and GC -// arrive in later slices) +// Instance writer (formerly multi-store.ts; scope deletes and GC arrive +// in later slices) // =========================================================================== /** Mutable state of ONE pi instance's exclusive capture file. */ @@ -243,3 +245,159 @@ export function appendSessionCapture( fs.appendFileSync(state.filePath, serializeEntry(entry) + "\n", "utf8"); state.lineCount += 1; } + + +// --------------------------------------------------------------------------- +// Multi-file reader (design v2: k-way backward merge) +// --------------------------------------------------------------------------- + +/** UI-level prompt identity: whitespace-collapsed, case-insensitive. */ +function promptKey(text: string): string { + return text.replace(/\s+/g, " ").trim().toLowerCase(); +} + +function fileMtimeMs(file: string): number { + try { + return fs.statSync(file).mtimeMs; + } catch { + return 0; + } +} + +function listProjectFiles(dir: string): string[] { + let entries: fs.Dirent[]; + try { + entries = fs.readdirSync(dir, { withFileTypes: true }); + } catch { + return []; + } + return entries + .filter((e) => e.isFile() && e.name.endsWith(".jsonl")) + .map((e) => path.join(dir, e.name)) + .sort((a, b) => fileMtimeMs(b) - fileMtimeMs(a)); +} + +/** Read one file's valid entries (chronological). */ +function readFileEntries(file: string): StoreEntry[] { + let raw = ""; + try { + raw = fs.readFileSync(file, "utf8"); + } catch { + return []; + } + const entries: StoreEntry[] = []; + for (const lineText of raw.split("\n")) { + const parsed = parseStoreLine(lineText); + if (parsed) entries.push(parsed); + } + return entries; +} + +/** + * Sort key = the newest entry ts in the file (fallback: file mtime). + * ts-based keys are STABLE under atomic rewrites (deletes/compaction + * bump mtime, which used to reshuffle the drain order). + */ +function fileSortKey(file: string, entries: StoreEntry[]): number { + let maxTs = 0; + for (const entry of entries) { + if (entry.ts !== undefined && entry.ts > maxTs) maxTs = entry.ts; + } + return maxTs > 0 ? maxTs : fileMtimeMs(file); +} + +/** Tombstone key - byte-compatible with hide-prompts' promptDedupKey. */ +function promptDedupKeyOf(text: string): string { + return text.replace(/\s+/g, " ").trim().slice(0, 120).toLowerCase(); +} + +/** + * Sequential backward drain over PRE-SORTED files: each file fully, + * newest-line-first, deduped by UI-level identity, capped at `limit`. + */ +function drainFiles( + files: string[], + limit: number, + hidden: Set = new Set(), +): string[] { + const seen = new Set(); + const out: string[] = []; + for (const file of files) { + const entries = readFileEntries(file); + for (let i = entries.length - 1; i >= 0; i--) { + const key = promptKey(entries[i].text); + if (seen.has(key)) continue; + if (hidden.size > 0 && hidden.has(promptDedupKeyOf(entries[i].text))) { + continue; + } + seen.add(key); + out.push(entries[i].text); + if (out.length >= limit) return out; + } + } + return out; +} + +/** Sort files for draining: ts-keyed, newest first, empty files dropped. */ +function sortFilesForDrain(files: string[]): string[] { + return files + .map((file) => ({ file, entries: readFileEntries(file) })) + .filter((f) => f.entries.length > 0) + .sort( + (a, b) => + fileSortKey(b.file, b.entries) - fileSortKey(a.file, a.entries), + ) + .map((f) => f.file); +} + +/** + * Drain the PROJECT scope: all .jsonl files in the project dir (seed.jsonl + * included), mtime-newest-first, deduped, capped at `limit` (default 1000). + */ +export function drainProject( + root: string, + cwd: string, + limit: number = 1000, + stateDir?: string, +): string[] { + return drainFiles( + sortFilesForDrain(listProjectFiles(path.join(root, "projects", projectHash(cwd)))), + limit, + stateDir ? loadHiddenPrompts(stateDir) : new Set(), + ); +} + +/** + * Drain the GLOBAL scope: the legacy global seed (newest single source) + * plus every project dir's files, mtime-newest-first, deduped, capped. + */ +export function drainGlobal( + root: string, + limit: number = 1000, + stateDir?: string, +): string[] { + const files: string[] = []; + const globalSeed = globalSeedPath(root); + + let projectDirs: fs.Dirent[]; + try { + projectDirs = fs.readdirSync(path.join(root, "projects"), { + withFileTypes: true, + }); + } catch { + projectDirs = []; + } + for (const dirEntry of projectDirs) { + if (!dirEntry.isDirectory()) continue; + files.push( + ...listProjectFiles(path.join(root, "projects", dirEntry.name)), + ); + } + const sorted = sortFilesForDrain(files); + if (fs.existsSync(globalSeed)) sorted.push(globalSeed); // legacy last + return drainFiles( + sorted, + limit, + stateDir ? loadHiddenPrompts(stateDir) : new Set(), + ); +} diff --git a/tests/history-dedupe-entries.test.ts b/tests/history-dedupe-entries.test.ts new file mode 100644 index 000000000..f929880e9 --- /dev/null +++ b/tests/history-dedupe-entries.test.ts @@ -0,0 +1,123 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { dedupePromptEntries } from "../extensions/history/selector-helpers.ts"; + +// AC-L5-1..AC-L5-5 — read-time dedup pass (spec C3, design §D5). +// +// The normalization key MUST byte-match the APPLIED patch form from +// nav/patches/editor.cjs :480–:586: +// +// entry.replace(/\s+/g, " ").trim().slice(0, 120).toLowerCase() +// +// The raw patch file stores `\\s+` because the replacement code sits inside +// a template literal; the code that actually runs in the editor contains +// `/\s+/g`. An implementation copying the double-backslash form would build +// a regex matching a literal backslash: whitespace variants would stop +// collapsing (T1 fails) and empty-key entries would leak through (T2 fails). +// +// The dev suite's T3 source-parse pins (dedupePromptEntries wired between +// drainForScope and buildPromptRecords inside openHistorySelector) cover the +// slice-3 selector wiring in extensions/history/index.ts and port with that +// slice — index.ts stays at its slice-1 surface here. + +// T1 — AC-L5-1: keep-first over newest-first input order (file order). + +test("keep-first: [A, B, A″] where A″ normalizes equal to A yields [A, B] (AC-L5-1)", () => { + const a = "deploy the API"; + const b = "write the tests"; + const aDoublePrime = "deploy the API"; + assert.deepEqual(dedupePromptEntries([a, b, aDoublePrime]), [a, b]); +}); + +// T1 — AC-L5-2: normalization groups each collapse to their first entry. + +test("normalization group: internal whitespace runs collapse to the first entry (AC-L5-2)", () => { + const first = "run the build now"; + assert.deepEqual( + dedupePromptEntries([ + first, + "run the build now", + "run the build now ", + " run the build now", + ]), + [first], + ); +}); + +test("normalization group: tabs and newlines collapse to the first entry (AC-L5-2)", () => { + const first = "run the build now"; + assert.deepEqual( + dedupePromptEntries([ + first, + "run\tthe\tbuild\tnow", + "run\nthe\nbuild\nnow", + "run \t the \n build now", + ]), + [first], + ); +}); + +test("normalization group: letter case collapses to the first entry (AC-L5-2)", () => { + const first = "Run The Build NOW"; + assert.deepEqual( + dedupePromptEntries([first, "run the build now", "RUN THE BUILD NOW"]), + [first], + ); +}); + +// T1 — AC-L5-2: 120-char normalized prefix collisions collapse (accepted +// patch-mirror semantics, NOT the P5 dedup-key fix). + +test("entries sharing the normalized 120-char prefix but differing later collapse (AC-L5-2)", () => { + const prefix = "x".repeat(120); + const first = `${prefix} tail one`; + const second = `${prefix} tail two`; + assert.deepEqual(dedupePromptEntries([first, second]), [first]); +}); + +test("entries differing within the first 120 normalized chars stay distinct (AC-L5-2)", () => { + const a = `${"x".repeat(119)}a ${"y".repeat(10)}`; + const b = `${"x".repeat(119)}b ${"y".repeat(10)}`; + assert.deepEqual(dedupePromptEntries([a, b]), [a, b]); +}); + +// T2 — AC-L5-3: empty-key entries (empty string, whitespace-only) are +// skipped: excluded from the output, never usable as collision keys, real +// entries pass through. + +test("empty-key entries are excluded from the output while real entries pass through (AC-L5-3)", () => { + const real = "a real prompt"; + assert.deepEqual(dedupePromptEntries(["", " ", "\t\n ", real, "\t"]), [ + real, + ]); +}); + +test("whitespace-only entries never shadow real entries as collision keys (AC-L5-3)", () => { + const first = "another real prompt"; + assert.deepEqual(dedupePromptEntries(["", " ", first]), [first]); +}); + +// T2 — AC-L5-5: no truncation beyond duplicate removal (no snapshot cap). + +test("output length equals input length minus duplicate-normalized entries (AC-L5-5)", () => { + const entries = [ + "alpha", + "alpha", // duplicate of 0 + "beta", + " ALPHA ", // duplicate of 0 (case + whitespace) + "beta\t", // duplicate of 2 + "gamma", + ]; + assert.equal(dedupePromptEntries(entries).length, 3); +}); + +test("no snapshot cap: every unique entry is kept past MAX_RESULTS (AC-L5-5)", () => { + const entries: string[] = []; + for (let i = 0; i < 1200; i++) { + entries.push(`unique prompt number ${i}`); + } + const deduped = dedupePromptEntries(entries); + assert.equal(deduped.length, 1200); + assert.equal(deduped[0], entries[0]); + assert.equal(deduped[1199], entries[1199]); +}); diff --git a/tests/history-drain-hidden.test.ts b/tests/history-drain-hidden.test.ts new file mode 100644 index 000000000..90c479758 --- /dev/null +++ b/tests/history-drain-hidden.test.ts @@ -0,0 +1,54 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { + drainGlobal, + drainProject, + globalSeedPath, + projectHash, +} from "../extensions/history/store.ts"; + +// Portable project identity: a never-existing literal. projectHash falls +// back to hashing the raw string when realpath fails, so the identity is +// deterministic on every machine (no machine-specific absolute paths). + +const CWD = "/pi-history-test/drain-hidden-project"; + +function write(file: string, texts: string[], ts = 100): void { + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync( + file, + `${texts.map((t) => JSON.stringify({ v: 1, text: t, ts })).join("\n")}\n`, + "utf8", + ); +} + +test("drains skip tombstoned prompts in seeds and session files", () => { + const base = fs.mkdtempSync(path.join(os.tmpdir(), "hid-")); + const root = path.join(base, "h"); + const stateDir = path.join(base, "state"); + fs.mkdirSync(stateDir, { recursive: true }); + fs.writeFileSync( + path.join(stateDir, "hidden.json"), + JSON.stringify(["deleted from seed", "deleted from session"]), + "utf8", + ); + const dir = path.join(root, "projects", projectHash(CWD)); + write(path.join(dir, "seed.jsonl"), ["keep", "deleted from seed"], 100); + write(path.join(dir, "s1.jsonl"), ["also keep", "deleted from session"], 200); + write(globalSeedPath(root), ["deleted from seed", "legacy keep"], 50); + + assert.deepEqual(drainProject(root, CWD, 1000, stateDir), [ + "also keep", + "keep", + ]); + assert.deepEqual(drainGlobal(root, 1000, stateDir), [ + "also keep", + "keep", + "legacy keep", + ]); + // Without a stateDir the filter is off (raw drain semantics). + assert.equal(drainProject(root, CWD).includes("deleted from seed"), true); +}); diff --git a/tests/history-drain-order.test.ts b/tests/history-drain-order.test.ts new file mode 100644 index 000000000..86cfddca9 --- /dev/null +++ b/tests/history-drain-order.test.ts @@ -0,0 +1,86 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { + drainGlobal, + drainProject, + globalSeedPath, + projectHash, +} from "../extensions/history/store.ts"; + +// Portable project identity: a never-existing literal. projectHash falls +// back to hashing the raw string when realpath fails, so the identity is +// deterministic on every machine (no machine-specific absolute paths). + +const CWD = "/pi-history-test/drain-order-project"; + +function writeTs(file: string, texts: string[], ts: number): void { + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync( + file, + `${texts.map((t) => JSON.stringify({ v: 1, text: t, ts })).join("\n")}\n`, + "utf8", + ); +} + +test("atomic rewrite (delete) does not reshuffle the drain order", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "ord-")); + const dir = path.join(root, "projects", projectHash(CWD)); + writeTs(path.join(dir, "old.jsonl"), ["a-old"], 100); + writeTs(path.join(dir, "new.jsonl"), ["z-new"], 200); + assert.deepEqual(drainProject(root, CWD), ["z-new", "a-old"]); + // Slice 5 ports deleteFromProject; its observable effect on the drain is + // simulated directly here: an atomic rewrite of the affected file that + // empties it — the mtime jumps to NOW, and the drain order must not move. + fs.writeFileSync(path.join(dir, "old.jsonl"), "", "utf8"); + fs.utimesSync(path.join(dir, "old.jsonl"), new Date(), new Date()); + assert.deepEqual(drainProject(root, CWD), ["z-new"]); + // Re-add with an OLD ts via direct write: still ordered by ts, not mtime. + writeTs(path.join(dir, "old2.jsonl"), ["b-old"], 150); + fs.utimesSync( + path.join(dir, "old2.jsonl"), + new Date(Date.now() + 99999), + new Date(Date.now() + 99999), + ); + assert.deepEqual(drainProject(root, CWD), ["z-new", "b-old"]); +}); + +test("global drain puts the legacy seed last regardless of its fresh mtime", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "seed-")); + const dir = path.join(root, "projects", projectHash(CWD)); + writeTs(path.join(dir, "s.jsonl"), ["fresh"], 200); + const seed = globalSeedPath(root); + writeTs(seed, ["legacy-1", "legacy-2"], 10); + fs.utimesSync(seed, new Date(Date.now() + 5000), new Date(Date.now() + 5000)); + assert.deepEqual(drainGlobal(root), ["fresh", "legacy-2", "legacy-1"]); +}); + +test( + "an unreadable store file is skipped; the rest drain in the expected order", + { skip: process.getuid?.() === 0 }, + () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "ord-sealed-")); + const dir = path.join(root, "projects", projectHash(CWD)); + writeTs(path.join(dir, "old.jsonl"), ["a-old"], 100); + const sealed = path.join(dir, "sealed.jsonl"); + writeTs(sealed, ["sealed-never"], 150); + writeTs(path.join(dir, "new.jsonl"), ["z-new"], 200); + // The sealed file's fresh mtime would sort it FIRST if it were readable — + // its absence from the drain is caused by the unreadable skip alone. + fs.utimesSync( + sealed, + new Date(Date.now() + 99999), + new Date(Date.now() + 99999), + ); + fs.chmodSync(sealed, 0o000); + try { + // An unreadable file reads as zero entries and drops out of the drain; + // the readable files keep their ts order. No throw. + assert.deepEqual(drainProject(root, CWD), ["z-new", "a-old"]); + } finally { + fs.chmodSync(sealed, 0o644); // restore before cleanup + } + }, +); diff --git a/tests/history-hide-prompts.test.ts b/tests/history-hide-prompts.test.ts new file mode 100644 index 000000000..03c054863 --- /dev/null +++ b/tests/history-hide-prompts.test.ts @@ -0,0 +1,99 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { hidePrompt, loadHiddenPrompts } from "../extensions/history/hide-prompts.ts"; +import { promptDedupKey } from "../extensions/history/selector-helpers.ts"; + +// Unit WU4 — tombstone write half + read half (spec C4, design §D6). fs-only +// coverage. The dev suite's deleteCurrent source-parse pins (T27/T28) and +// the deletionActionsFor planner pins cover the slice-3 selector branch and +// the slice-5 delete flow; they port with those slices. + +function makeStateDir(name: string): string { + return fs.mkdtempSync(path.join(os.tmpdir(), `hide-prompts-${name}-`)); +} + +function readHideFile(stateDir: string) { + return JSON.parse( + fs.readFileSync(path.join(stateDir, "hidden.json"), "utf8"), + ); +} + +// T24 — AC-S4-1: hide-key fidelity. Tombstone keys must byte-match the +// Change 2 dedup key for the same text — same imported helper, never a +// re-implementation: the stored file content is compared against +// promptDedupKey's own output with strict equality. +test("T24 (AC-S4-1): hide keys byte-match promptDedupKey across whitespace, case, and >120-char groups", () => { + const stateDir = makeStateDir("t24"); + // Three normalization groups: internal whitespace runs (space + tab), + // letter case, and a text longer than the 120-char key prefix. + const texts = [ + "fix\t the build", + "Deploy THE api", + `${"pad ".repeat(40)}tail beyond one hundred twenty chars`, + ]; + for (const text of texts) { + assert.deepEqual(hidePrompt(stateDir, text), { status: "written" }); + } + const stored = readHideFile(stateDir); + assert.ok(Array.isArray(stored), "hidden.json must hold a JSON array"); + // Byte-match: the file holds EXACTLY the shared helper's output, sorted. + assert.deepEqual(stored, texts.map((text) => promptDedupKey(text)).sort()); + // The loaded set agrees. + const loaded = loadHiddenPrompts(stateDir); + for (const key of stored) { + assert.ok(loaded.has(key)); + } +}); + +// T25 — AC-S4-2: hide persistence and tolerance. Two deletes of the same +// text compact to ONE key; a missing hide file reads as an empty set; reads +// never throw. +test("T25 (AC-S4-2): duplicate hides compact to one key; a missing file reads as empty; reads never throw", () => { + const stateDir = makeStateDir("t25"); + // Missing file: empty set, no throw (before any write exists). + assert.equal(loadHiddenPrompts(stateDir).size, 0); + // Two deletes of the same text — variants differing by case + whitespace + // runs normalize onto the same key. + assert.deepEqual(hidePrompt(stateDir, "Same Text"), { status: "written" }); + assert.deepEqual(hidePrompt(stateDir, "same text"), { status: "written" }); + const stored = readHideFile(stateDir); + assert.deepEqual(stored, [promptDedupKey("same text")]); + const loaded = loadHiddenPrompts(stateDir); + assert.equal(loaded.size, 1); + assert.ok(loaded.has(promptDedupKey("same text"))); +}); + +// T26 — AC-S4-5: corrupt hidden.json is fail-open (READ half) AND the next +// hide rewrites the file clean as a sorted compact array — the rewrite half +// is the recovery path. +test("T26 (AC-S4-5): corrupt hidden.json loads as empty and the next hide rewrites it clean", () => { + const stateDir = makeStateDir("t26"); + fs.writeFileSync( + path.join(stateDir, "hidden.json"), + "{corrupt bytes", + "utf8", + ); + assert.equal(loadHiddenPrompts(stateDir).size, 0); + assert.deepEqual(hidePrompt(stateDir, "beta prompt"), { status: "written" }); + // The rewrite landed: clean JSON holding exactly the new key. + assert.deepEqual(readHideFile(stateDir), [promptDedupKey("beta prompt")]); + assert.equal(loadHiddenPrompts(stateDir).size, 1); +}); + +// WU4c — write-failure path (AC-S4-2 triangulation): a state dir that cannot +// be created (its parent is a regular file) makes the atomic write return +// false, and hidePrompt maps that to the toast-suitable error object — +// never a throw. +test("hide write failure returns the exact error shape for the delete-flow toast", () => { + const base = makeStateDir("fail"); + const blocker = path.join(base, "blocker"); + fs.writeFileSync(blocker, "regular file", "utf8"); + const stateDir = path.join(blocker, "sealed"); // parent is a file → ENOTDIR + assert.deepEqual(hidePrompt(stateDir, "kept prompt"), { + status: "error", + message: "Could not write the hide file; the prompt may reappear.", + }); +}); diff --git a/tests/history-max-results-cap.test.ts b/tests/history-max-results-cap.test.ts new file mode 100644 index 000000000..fb319c8ec --- /dev/null +++ b/tests/history-max-results-cap.test.ts @@ -0,0 +1,76 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { filterPrompts } from "../extensions/history/selector-helpers.ts"; + +/** + * WU5 tests (AC-S5-1, AC-S5-2): the MAX_RESULTS raise 1000 → 10000 is an + * OUTPUT cap only — filterPrompts caps both of its slice sites; the load + * path never snapshots. Pure import (no pi-tui graph) plus a source-parse + * pin on the selector-helpers slice sites. + * + * The dev suite's openHistorySelector body pin (no slice() on the load + * path) covers the slice-3 selector wiring in extensions/history/index.ts + * and ports with that slice — index.ts stays at its slice-1 surface here. + */ + +interface CapRecord { + text: string; + searchText: string; +} + +function record(text: string): CapRecord { + return { text, searchText: text.toLowerCase() }; +} + +// T29 — AC-S5-1: the cap value. More than 10,000 records → exactly 10,000 at +// the empty-query slice AND at a filtered-query slice. NEW pin — no existing +// test pins the old 1000 literal (design §D9; zero existing-test edits +// repo-wide, so this file ADDS the pin instead of editing one). + +test("T29 (AC-S5-1): empty-query slice caps at the raised 10000", () => { + const records: CapRecord[] = []; + for (let i = 0; i < 10500; i++) { + records.push(record(`unique prompt number ${i}`)); + } + const result = filterPrompts(records, ""); + assert.equal(result.length, 10000); +}); + +test("T29 (AC-S5-1): filtered-query slice caps at the raised 10000", () => { + const records: CapRecord[] = []; + // 10,500 matches for the query token plus non-matching padding rows: the + // FILTERED set alone is above the cap, so the filtered slice site is the + // one being exercised here. + for (let i = 0; i < 10500; i++) { + records.push(record(`match me ${i}`)); + } + records.push(record("unrelated row one")); + records.push(record("unrelated row two")); + const result = filterPrompts(records, "match"); + assert.ok(result.length > 1000, "the filtered set must exceed the old cap"); + assert.equal(result.length, 10000); + assert.ok(result.every((r) => r.searchText.includes("match"))); +}); + +// T30 — AC-S5-2: output-cap-only semantics (source-parse). Selector-helpers +// reads the constant at exactly the two sanctioned filterPrompts slice +// sites — no other cap exists in the helper module. + +const helperSource = fs.readFileSync( + fileURLToPath( + new URL("../extensions/history/selector-helpers.ts", import.meta.url), + ), + "utf8", +); + +test("T30 (AC-S5-2): filterPrompts hosts exactly the two sanctioned cap slice sites", () => { + const sliceSites = helperSource.split("slice(0, MAX_RESULTS)").length - 1; + assert.equal( + sliceSites, + 2, + "filterPrompts hosts exactly the two sanctioned cap slice sites", + ); +}); From 73c55ff99193804815c80058b77b5c3e57195c55 Mon Sep 17 00:00:00 2001 From: Carolina <26188349+carolitascl@users.noreply.github.com> Date: Fri, 18 Sep 2026 18:38:16 -0300 Subject: [PATCH 4/8] docs(history): correct drainGlobal seed-ordering docblock Review fix (Copilot suppressed comment, store.ts): the docblock claimed the legacy global seed is the "newest single source", but the code deliberately appends it after sorting (`// legacy last`) so per-project entries win recency and keep-first dedup. Document the actual, intended behavior instead of changing it: migrated legacy history is the least specific source. --- extensions/history/store.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/extensions/history/store.ts b/extensions/history/store.ts index bf2a41c16..fb470515c 100644 --- a/extensions/history/store.ts +++ b/extensions/history/store.ts @@ -368,8 +368,10 @@ export function drainProject( } /** - * Drain the GLOBAL scope: the legacy global seed (newest single source) - * plus every project dir's files, mtime-newest-first, deduped, capped. + * Drain the GLOBAL scope: every project dir's files, mtime-newest-first, + * deduped, capped — with the legacy global seed appended LAST (deliberate: + * it is the least specific, migrated source, so per-project entries win + * recency and keep-first dedup favors them). */ export function drainGlobal( root: string, From 84c1232361ca221e4b77b7aa8d2d0ebb6ddf3097 Mon Sep 17 00:00:00 2001 From: Carolina <26188349+carolitascl@users.noreply.github.com> Date: Thu, 24 Sep 2026 14:47:37 -0300 Subject: [PATCH 5/8] fix(history): make prompt capture opt-in and document the store Review follow-up on the slice-01 PR: the before_agent_start handler recorded delivered prompts by default while the deletion UI is still unshipped, so an intermediate release could accumulate sensitive prompts with no removal path. - Capture is now strictly opt-in via GENTLE_PI_HISTORY_CAPTURE=1|true|on (default off); the switch doubles as the disable path, is checked per prompt, and a disabled session writes nothing - no registry entry, no files. - promptHistoryExtension takes injectable deps (env/root/cwd/ instanceId/now) with one writer closure per extension load. - New tests: strict opt-in matrix, default-off inertness, opted-in capture, disable-leaves-existing-files. - docs/prompt-history.md documents the switch, storage locations, permissions/readers, and disable/removal semantics; the README docs table gains a pointer. --- README.md | 1 + docs/prompt-history.md | 61 ++++++++++++++++++++++ extensions/history/index.ts | 75 +++++++++++++++++++--------- tests/history-session-writer.test.ts | 61 +++++++++++++++++++++- 4 files changed, 174 insertions(+), 24 deletions(-) create mode 100644 docs/prompt-history.md diff --git a/README.md b/README.md index 9e20d72f4..0207c389c 100644 --- a/README.md +++ b/README.md @@ -880,6 +880,7 @@ To opt out: | `docs/skill-style-guide.md` | Normative style guide used by the packaged skill creation/improvement skills. | | `docs/native-authority-architecture.md` | Post-U8 ownership boundary, reproducible slimming metrics, Windows evidence, exact #191 seam, and the `review-integration/v1`→`v2` migration status, including the "compact-v2" naming disambiguation. | | `docs/review-integration.md` | Negotiated provider/consumer contract and the current Gentle Pi adoption boundary. | +| `docs/prompt-history.md` | Prompt-history slice 1: opt-in capture switch, storage layout, readers, and disable/removal semantics. | ## Development diff --git a/docs/prompt-history.md b/docs/prompt-history.md new file mode 100644 index 000000000..1fa8eba4f --- /dev/null +++ b/docs/prompt-history.md @@ -0,0 +1,61 @@ +# Prompt history + +Slice 1 of the prompt-history extension (#819 split) ships the storage layer only: +a per-instance JSONL capture store, project identity, and the read/write +primitives later slices build on. The selector UI, deletion/scope drains, and GC +arrive in later slices of the chain. + +## Capture is opt-in + +Recording is **off by default**. Delivered prompts can contain secrets, and the +deletion UI is not shipped yet, so nothing is stored unless you explicitly opt in: + +```bash +GENTLE_PI_HISTORY_CAPTURE=1 pi +``` + +- Enabled by `1`, `true`, or `on` (case-insensitive). Unset, empty, or any other + value means **off** — the same switch is the disable path. +- The check runs per prompt: unsetting the switch (or setting it to `0`) stops + new captures immediately, no pi restart needed. +- With capture off the extension is inert: no registry entry, no files, and + prompts are never written. + +## Where the files live + +Everything sits under `~/.pi/agent/history/`: + +- `registry.json` — advisory map of project hash → cwd, used for display + labels. +- `projects//.jsonl` — one append-only capture file per pi + process. + +`` is the first 16 hex chars of the SHA-256 of the canonicalized project +cwd; `` is a per-process UUID. Each line is one delivered prompt: + +```json +{"v":1,"text":"the prompt as delivered","ts":1700000000000} +``` + +UI command-like prompts (`/name ...`) and empty lines are never stored. Later +slices add the rebuildable `seed.jsonl`, scope drains/deletes, and GC. + +## Who can read them + +The store is plain JSONL on your local disk, not encrypted. Files are created by +the pi process with default umask permissions (typically `0644` files inside +`0755` directories), so any process running as your OS user can read them, and +other local accounts can too wherever they can traverse your home directory. +Treat the store as sensitive: it holds your prompts verbatim. + +## What disabling capture does + +Turning the switch off only stops **new** captures. Nothing is deleted: files +already written — and the registry entry — stay on disk until you remove them or +the deletion UI ships. To erase the store manually while capture is off (or pi +is not running): + +```bash +rm -rf ~/.pi/agent/history # whole store +rm -rf ~/.pi/agent/history/projects/ # one project (see registry.json) +``` diff --git a/extensions/history/index.ts b/extensions/history/index.ts index 26616733f..74dab5942 100644 --- a/extensions/history/index.ts +++ b/extensions/history/index.ts @@ -5,6 +5,12 @@ // per-instance writer lifecycle, and the before_agent_start capture // handler. Selector UI, shortcut/command, scope drains, legacy migration // and seed bootstrap, and GC arrive in later slices. +// +// Capture is OPT-IN while the deletion/privacy behavior is unshipped: +// nothing is recorded unless GENTLE_PI_HISTORY_CAPTURE=1|true|on. With the +// switch off the handler is a no-op — no registry entry, no files, and +// prompts are never written. Unsetting the switch only stops NEW captures; +// files already written stay on disk (docs/prompt-history.md). import { randomUUID } from "node:crypto"; import { homedir } from "node:os"; @@ -19,39 +25,62 @@ import { // v2 multi-concurrency store root (design: tmp/multi-concurrency-design.md). const PI_HISTORY_ROOT = join(homedir(), ".pi", "agent", "history"); -const AGENT_DIR = join(homedir(), ".pi", "agent"); -const CURRENT_CWD = process.cwd(); -// Instance identity: one exclusive capture file per pi process. -const INSTANCE_ID = randomUUID(); -let writerState: SessionWriterState | null = null; +export interface HistoryDeps { + env?: NodeJS.ProcessEnv; + root?: string; + cwd?: string; + instanceId?: string; + now?: () => number; +} /** - * One-time init per extension load: register the project in the advisory - * registry, then open this instance's exclusive capture file. Legacy - * migration and seed bootstrap join this init order in a later slice. + * Strict opt-in: capture stays off unless GENTLE_PI_HISTORY_CAPTURE is + * explicitly 1, true, or on (case-insensitive). The same switch is the + * disable path — unsetting it stops new captures; files already on disk + * are left untouched until the deletion tooling lands. */ -function getWriter(): SessionWriterState { - if (!writerState) { - try { - ensureRegistryEntry(PI_HISTORY_ROOT, CURRENT_CWD); - } catch { - // registry is advisory - } - writerState = openSessionWriter(PI_HISTORY_ROOT, CURRENT_CWD, INSTANCE_ID); - } - return writerState; +export function captureEnabled(env: NodeJS.ProcessEnv = process.env): boolean { + const value = env.GENTLE_PI_HISTORY_CAPTURE?.trim().toLowerCase(); + return value === "1" || value === "true" || value === "on"; } -export default function promptHistoryExtension(pi: ExtensionAPI) { - // One writer per extension load; see getWriter() for the init order. +export default function promptHistoryExtension( + pi: ExtensionAPI, + deps: HistoryDeps = {}, +): void { + const env = deps.env ?? process.env; + const root = deps.root ?? PI_HISTORY_ROOT; + const cwd = deps.cwd ?? process.cwd(); + const instanceId = deps.instanceId ?? randomUUID(); + const now = deps.now ?? Date.now; + let writerState: SessionWriterState | null = null; + + /** + * One-time init per extension load: register the project in the advisory + * registry, then open this instance's exclusive capture file. Legacy + * migration and seed bootstrap join this init order in a later slice. + */ + const getWriter = (): SessionWriterState => { + if (!writerState) { + try { + ensureRegistryEntry(root, cwd); + } catch { + // registry is advisory + } + writerState = openSessionWriter(root, cwd, instanceId); + } + return writerState; + }; - // Persist every delivered user prompt (write-through, append-only JSONL). - // The local ExtensionAPI stub types handler args as unknown; narrow here. + // Persist every delivered user prompt (write-through, append-only JSONL), + // but only for opted-in sessions — see captureEnabled(). The local + // ExtensionAPI stub types handler args as unknown; narrow here. pi.on("before_agent_start", (...args: unknown[]) => { + if (!captureEnabled(env)) return; try { const event = args[0] as { prompt?: string } | undefined; - appendSessionCapture(getWriter(), event?.prompt ?? "", Date.now()); + appendSessionCapture(getWriter(), event?.prompt ?? "", now()); } catch { // A capture failure must never break the agent loop or unregister // the handler - swallow and keep the next prompt capturable. diff --git a/tests/history-session-writer.test.ts b/tests/history-session-writer.test.ts index 8601cea14..4128b7eef 100644 --- a/tests/history-session-writer.test.ts +++ b/tests/history-session-writer.test.ts @@ -9,7 +9,7 @@ import { projectHash, sessionFilePath, } from "../extensions/history/store.ts"; -import promptHistoryExtension from "../extensions/history/index.ts"; +import promptHistoryExtension, { captureEnabled } from "../extensions/history/index.ts"; function makeRoot(): string { return fs.mkdtempSync(path.join(os.tmpdir(), "pi-history-writer-")); @@ -29,6 +29,24 @@ function openWriterForTest(root: string, instanceId: string) { return openSessionWriter(root, CWD, instanceId); } +/** Load the extension against a temp root and return the capture handler. */ +function captureHandlerWith(env: NodeJS.ProcessEnv, root: string) { + const registered: Array<[string, unknown]> = []; + const pi = { + on: (event: string, handler: unknown) => { + registered.push([event, handler]); + }, + }; + promptHistoryExtension(pi as never, { + env, + root, + cwd: CWD, + instanceId: "inst-entry", + now: () => 1700000000000, + }); + return registered[0][1] as (event: unknown) => void; +} + test("no file is created until the first capture", () => { const root = makeRoot(); const state = openWriterForTest(root, "sess-1"); @@ -107,3 +125,44 @@ test("the slice-1 extension entry registers only the capture handler", () => { // would run getWriter() against the user's real ~/.pi/agent/history. assert.equal(typeof registered[0][1], "function"); }); + +test("captureEnabled is a strict opt-in", () => { + assert.equal(captureEnabled({}), false); + assert.equal(captureEnabled({ GENTLE_PI_HISTORY_CAPTURE: "0" }), false); + assert.equal(captureEnabled({ GENTLE_PI_HISTORY_CAPTURE: "false" }), false); + assert.equal(captureEnabled({ GENTLE_PI_HISTORY_CAPTURE: "off" }), false); + assert.equal(captureEnabled({ GENTLE_PI_HISTORY_CAPTURE: "yes" }), false); + assert.equal(captureEnabled({ GENTLE_PI_HISTORY_CAPTURE: " 1 " }), true); + assert.equal(captureEnabled({ GENTLE_PI_HISTORY_CAPTURE: "TRUE" }), true); + assert.equal(captureEnabled({ GENTLE_PI_HISTORY_CAPTURE: "On" }), true); +}); + +test("the capture handler is a no-op unless the user opts in", () => { + const root = makeRoot(); + const handler = captureHandlerWith({}, root); + handler({ prompt: "sensitive prompt" }); + handler({ prompt: "another one" }); + // Nothing at all: no capture file, no project dir, no registry entry. + assert.deepEqual(fs.readdirSync(root), []); +}); + +test("an opted-in session captures delivered prompts", () => { + const root = makeRoot(); + const handler = captureHandlerWith({ GENTLE_PI_HISTORY_CAPTURE: "1" }, root); + handler({ prompt: "hello store" }); + assert.deepEqual(fileTexts(sessionFilePath(root, CWD, "inst-entry")), [ + "hello store", + ]); +}); + +test("disabling capture stops new lines and leaves existing files alone", () => { + const root = makeRoot(); + const env: NodeJS.ProcessEnv = { GENTLE_PI_HISTORY_CAPTURE: "true" }; + const handler = captureHandlerWith(env, root); + handler({ prompt: "kept" }); + const file = sessionFilePath(root, CWD, "inst-entry"); + assert.equal(fs.existsSync(file), true); + delete env.GENTLE_PI_HISTORY_CAPTURE; + handler({ prompt: "never written" }); + assert.deepEqual(fileTexts(file), ["kept"]); +}); From 5501d12d1ddc39e7efed2c6f5908b23364d6fcef Mon Sep 17 00:00:00 2001 From: Carolina <26188349+carolitascl@users.noreply.github.com> Date: Thu, 24 Sep 2026 15:34:51 -0300 Subject: [PATCH 6/8] fix(history): fail closed on untrustworthy hidden.json tombstones A corrupt or unreadable hide file previously loaded as an empty hidden set (fail open), resurfacing prompts the user may have hidden because they contain secrets. The next hide also rewrote the file clean, silently clearing the incident. - readHiddenPrompts replaces loadHiddenPrompts: ENOENT stays trusted-empty (nothing ever hidden); any other read error, JSON parse failure, or non-array shape is untrusted (unreadable/corrupt/ malformed) and carries a recovery message naming hidden.json - hidePrompt refuses to write over an untrusted file: recovery is the explicit delete-or-restore of hidden.json, never a silent rewrite - drainProject/drainGlobal return DrainResult: untrusted tombstones block the drain (status "blocked", no prompts field) so the future selector UI must surface the warning; no stateDir keeps raw drain semantics Tests: rewrite T26 to pin the refusal + byte-unchanged file + manual unlink recovery; add malformed-shape, junk-item tolerance, and chmod 000 unreadable cases; drains pin the blocked shape (no prompts field) and the missing-file-stays-ok case. --- extensions/history/hide-prompts.ts | 83 ++++++++++--- extensions/history/store.ts | 53 ++++++-- tests/history-drain-hidden.test.ts | 62 +++++++++- tests/history-drain-order.test.ts | 22 +++- tests/history-hide-prompts.test.ts | 187 +++++++++++++++++++++++------ 5 files changed, 330 insertions(+), 77 deletions(-) diff --git a/extensions/history/hide-prompts.ts b/extensions/history/hide-prompts.ts index 9cffd6954..6d91a57d6 100644 --- a/extensions/history/hide-prompts.ts +++ b/extensions/history/hide-prompts.ts @@ -9,6 +9,14 @@ import { promptDedupKey } from "./selector-helpers.ts"; /** Name of the tombstone file inside the injected state dir (spec C4). */ const HIDE_FILE_NAME = "hidden.json"; +/** + * Shared recovery warning for a file that exists but cannot be trusted + * (spec C4, fail-closed READ half): toast-suitable, names hidden.json, and + * gives the user the explicit restore-or-delete choice. + */ +const RECOVERY_MESSAGE = + "The prompt-history hide list (hidden.json) is corrupt or unreadable. History is blocked until you restore the file or delete it (hidden prompts may then reappear)."; + /** * Result of one tombstone write (spec C4): `written` on a successful atomic * write, or an error object carrying a short, toast-suitable reason. Never @@ -19,32 +27,64 @@ export type HideResult = | { status: "error"; message: string }; /** - * Load the tombstone key set from `stateDir/hidden.json` — the READ half of - * the hide-file contract (spec C4). Fail-open: a missing, unreadable, - * corrupt, or wrong-shaped file is an EMPTY set and the call never throws; - * a corrupt file is rewritten clean by the next hide (the WRITE half, - * `hidePrompt`, lands in WU4). Keys are `promptDedupKey` strings written by - * `hidePrompt`; foreign values are ignored, never trusted. + * Result of one tombstone read (spec C4): `trusted` keys when the file is + * missing or holds a valid array, or `untrusted` when the file exists but + * cannot be trusted. History reads FAIL CLOSED on `untrusted`: callers must + * block the drain instead of emptying the tombstone set, because hidden + * prompts may contain secrets an empty set would resurface. */ -export function loadHiddenPrompts(stateDir: string): Set { +export type HiddenRead = + | { status: "trusted"; keys: Set } + | { + status: "untrusted"; + reason: "unreadable" | "corrupt" | "malformed"; + message: string; + }; + +/** + * Read the tombstone key set from `stateDir/hidden.json` — the READ half of + * the hide-file contract (spec C4). Fail-closed for history: a file that + * exists but is unreadable, corrupt, or wrong-shaped returns `untrusted` + * with the recovery warning so callers block the drain; it never degrades + * to an empty trusted set. A MISSING file — before any deletion — is the + * safe empty case and reads `trusted` with no keys. A valid array is + * trusted; junk items inside it are ignored, never trusted. Keys are + * `promptDedupKey` strings written by `hidePrompt`; the call never throws. + */ +export function readHiddenPrompts(stateDir: string): HiddenRead { let raw: string; try { raw = fs.readFileSync(path.join(stateDir, HIDE_FILE_NAME), "utf8"); - } catch { - return new Set(); // missing or unreadable → empty tombstones + } catch (error) { + const code = (error as { code?: unknown } | null | undefined)?.code; + if (code === "ENOENT") { + // Missing before any deletion: the safe empty tombstone set. + return { status: "trusted", keys: new Set() }; + } + return { + status: "untrusted", + reason: "unreadable", + message: RECOVERY_MESSAGE, + }; } let parsed: unknown; try { parsed = JSON.parse(raw); } catch { - return new Set(); // corrupt bytes → fail-open empty + return { status: "untrusted", reason: "corrupt", message: RECOVERY_MESSAGE }; } const keys = new Set(); - if (!Array.isArray(parsed)) return keys; // wrong shape → fail-open empty + if (!Array.isArray(parsed)) { + return { + status: "untrusted", + reason: "malformed", + message: RECOVERY_MESSAGE, + }; + } for (const item of parsed) { if (typeof item === "string" && item !== "") keys.add(item); } - return keys; + return { status: "trusted", keys }; } /** @@ -52,18 +92,23 @@ export function loadHiddenPrompts(stateDir: string): Set { * WRITE half of the hide-file contract (spec C4). The key is the shared * `promptDedupKey` (byte-match normative with the merge filter — never a * re-implementation); the set compacts on write and persists as a SORTED - * array via the shared atomic tmp+rename writer. Fail-open both ways: a - * corrupt or missing file reads as empty (this clean rewrite IS the - * recovery — the corrupt contents are untrustworthy by definition) and any - * write failure returns an error object for the delete-flow toast; the + * array via the shared atomic tmp+rename writer. An untrusted existing file + * is never silently reset (a clean rewrite would clear the blocked state + * one hide later): hidePrompt refuses with the recovery warning until the + * user restores or deletes the file. A missing file is the clean baseline; + * any write failure returns an error object for the delete-flow toast; the * call never throws. */ export function hidePrompt(stateDir: string, text: string): HideResult { - const keys = loadHiddenPrompts(stateDir); - keys.add(promptDedupKey(text)); + const read = readHiddenPrompts(stateDir); + if (read.status === "untrusted") { + // Refuse without writing: never reset the untrusted state silently. + return { status: "error", message: read.message }; + } + read.keys.add(promptDedupKey(text)); const written = writeJsonAtomic( path.join(stateDir, HIDE_FILE_NAME), - [...keys].sort(), + [...read.keys].sort(), ); return written ? { status: "written" } diff --git a/extensions/history/store.ts b/extensions/history/store.ts index fb470515c..b63b07719 100644 --- a/extensions/history/store.ts +++ b/extensions/history/store.ts @@ -11,7 +11,7 @@ import { createHash } from "node:crypto"; import fs from "node:fs"; import path from "node:path"; -import { loadHiddenPrompts } from "./hide-prompts.ts"; +import { readHiddenPrompts } from "./hide-prompts.ts"; // =========================================================================== // Paths (formerly store-paths.ts) @@ -350,20 +350,53 @@ function sortFilesForDrain(files: string[]): string[] { .map((f) => f.file); } +/** + * Result of a scope drain: `ok` with the drained prompts, or `blocked` + * when the tombstone file is untrusted (fail-closed READ half). The + * blocked shape carries NO prompts field, so a caller cannot accidentally + * render prompts that may include hidden ones. + */ +export type DrainResult = + | { status: "ok"; prompts: string[] } + | { status: "blocked"; message: string }; + +/** + * Shared drain tail: without a `stateDir` the raw drain semantics hold (no + * filter). With one, the tombstone filter applies and fails CLOSED: an + * untrusted hidden.json (unreadable, corrupt, wrong shape) blocks the + * whole drain with the recovery message instead of resurfacing hidden + * prompts; a missing file is the safe empty tombstone set and drains + * normally. + */ +function drainWithHidden( + files: string[], + limit: number, + stateDir?: string, +): DrainResult { + if (!stateDir) return { status: "ok", prompts: drainFiles(files, limit) }; + const read = readHiddenPrompts(stateDir); + if (read.status === "untrusted") { + return { status: "blocked", message: read.message }; + } + return { status: "ok", prompts: drainFiles(files, limit, read.keys) }; +} + /** * Drain the PROJECT scope: all .jsonl files in the project dir (seed.jsonl * included), mtime-newest-first, deduped, capped at `limit` (default 1000). + * With a `stateDir`, the tombstone filter applies and fails closed: an + * untrusted hidden.json blocks the drain (see DrainResult). */ export function drainProject( root: string, cwd: string, limit: number = 1000, stateDir?: string, -): string[] { - return drainFiles( +): DrainResult { + return drainWithHidden( sortFilesForDrain(listProjectFiles(path.join(root, "projects", projectHash(cwd)))), limit, - stateDir ? loadHiddenPrompts(stateDir) : new Set(), + stateDir, ); } @@ -371,13 +404,15 @@ export function drainProject( * Drain the GLOBAL scope: every project dir's files, mtime-newest-first, * deduped, capped — with the legacy global seed appended LAST (deliberate: * it is the least specific, migrated source, so per-project entries win - * recency and keep-first dedup favors them). + * recency and keep-first dedup favors them). With a `stateDir`, the + * tombstone filter applies and fails closed: an untrusted hidden.json + * blocks the drain (see DrainResult). */ export function drainGlobal( root: string, limit: number = 1000, stateDir?: string, -): string[] { +): DrainResult { const files: string[] = []; const globalSeed = globalSeedPath(root); @@ -397,9 +432,5 @@ export function drainGlobal( } const sorted = sortFilesForDrain(files); if (fs.existsSync(globalSeed)) sorted.push(globalSeed); // legacy last - return drainFiles( - sorted, - limit, - stateDir ? loadHiddenPrompts(stateDir) : new Set(), - ); + return drainWithHidden(sorted, limit, stateDir); } diff --git a/tests/history-drain-hidden.test.ts b/tests/history-drain-hidden.test.ts index 90c479758..73a686b86 100644 --- a/tests/history-drain-hidden.test.ts +++ b/tests/history-drain-hidden.test.ts @@ -8,6 +8,7 @@ import { drainProject, globalSeedPath, projectHash, + type DrainResult, } from "../extensions/history/store.ts"; // Portable project identity: a never-existing literal. projectHash falls @@ -25,6 +26,14 @@ function write(file: string, texts: string[], ts = 100): void { ); } +// Unwrap the ok shape. Drains FAIL CLOSED: the blocked variant carries no +// prompts field at all (asserted in the blocked test below). +function okPrompts(result: DrainResult): string[] { + assert.equal(result.status, "ok"); + if (result.status !== "ok") throw new Error("unreachable"); + return result.prompts; +} + test("drains skip tombstoned prompts in seeds and session files", () => { const base = fs.mkdtempSync(path.join(os.tmpdir(), "hid-")); const root = path.join(base, "h"); @@ -40,15 +49,62 @@ test("drains skip tombstoned prompts in seeds and session files", () => { write(path.join(dir, "s1.jsonl"), ["also keep", "deleted from session"], 200); write(globalSeedPath(root), ["deleted from seed", "legacy keep"], 50); - assert.deepEqual(drainProject(root, CWD, 1000, stateDir), [ + assert.deepEqual(okPrompts(drainProject(root, CWD, 1000, stateDir)), [ "also keep", "keep", ]); - assert.deepEqual(drainGlobal(root, 1000, stateDir), [ + assert.deepEqual(okPrompts(drainGlobal(root, 1000, stateDir)), [ "also keep", "keep", "legacy keep", ]); // Without a stateDir the filter is off (raw drain semantics). - assert.equal(drainProject(root, CWD).includes("deleted from seed"), true); + assert.equal( + okPrompts(drainProject(root, CWD)).includes("deleted from seed"), + true, + ); +}); + +// Fail-closed seam: an untrusted hidden.json BLOCKS both drains with the +// recovery message and no prompts field; a stateDir whose hidden.json is +// MISSING stays the safe empty-tombstones case (the full expected prompts). +test("corrupt hidden.json blocks both drains with no prompts field; a missing file drains normally", () => { + const base = fs.mkdtempSync(path.join(os.tmpdir(), "hid-blocked-")); + const root = path.join(base, "h"); + const stateDir = path.join(base, "state"); + fs.mkdirSync(stateDir, { recursive: true }); + fs.writeFileSync( + path.join(stateDir, "hidden.json"), + "{corrupt bytes", + "utf8", + ); + const dir = path.join(root, "projects", projectHash(CWD)); + write(path.join(dir, "seed.jsonl"), ["secret prompt", "keeper"], 100); + + for (const result of [ + drainProject(root, CWD, 1000, stateDir), + drainGlobal(root, 1000, stateDir), + ]) { + assert.equal(result.status, "blocked"); + if (result.status !== "blocked") throw new Error("unreachable"); + assert.ok(result.message.includes("hidden.json")); + // The blocked shape carries no prompts to render. + assert.equal("prompts" in result, false); + } + + // Missing file: safe empty tombstones — the full drain comes back. + const missingBase = fs.mkdtempSync(path.join(os.tmpdir(), "hid-missing-")); + const missingRoot = path.join(missingBase, "h"); + const missingState = path.join(missingBase, "state"); + fs.mkdirSync(missingState, { recursive: true }); + const missingDir = path.join(missingRoot, "projects", projectHash(CWD)); + write(path.join(missingDir, "seed.jsonl"), ["kept", "shown"], 100); + assert.deepEqual( + okPrompts(drainProject(missingRoot, CWD, 1000, missingState)), + ["shown", "kept"], + ); + assert.deepEqual(okPrompts(drainGlobal(missingRoot, 1000, missingState)), [ + "shown", + "kept", + ]); }); diff --git a/tests/history-drain-order.test.ts b/tests/history-drain-order.test.ts index 86cfddca9..be2f8fdfc 100644 --- a/tests/history-drain-order.test.ts +++ b/tests/history-drain-order.test.ts @@ -8,6 +8,7 @@ import { drainProject, globalSeedPath, projectHash, + type DrainResult, } from "../extensions/history/store.ts"; // Portable project identity: a never-existing literal. projectHash falls @@ -25,18 +26,25 @@ function writeTs(file: string, texts: string[], ts: number): void { ); } +// Mechanical unwrap of the ok shape (drains can also return blocked). +function okPrompts(result: DrainResult): string[] { + assert.equal(result.status, "ok"); + if (result.status !== "ok") throw new Error("unreachable"); + return result.prompts; +} + test("atomic rewrite (delete) does not reshuffle the drain order", () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), "ord-")); const dir = path.join(root, "projects", projectHash(CWD)); writeTs(path.join(dir, "old.jsonl"), ["a-old"], 100); writeTs(path.join(dir, "new.jsonl"), ["z-new"], 200); - assert.deepEqual(drainProject(root, CWD), ["z-new", "a-old"]); + assert.deepEqual(okPrompts(drainProject(root, CWD)), ["z-new", "a-old"]); // Slice 5 ports deleteFromProject; its observable effect on the drain is // simulated directly here: an atomic rewrite of the affected file that // empties it — the mtime jumps to NOW, and the drain order must not move. fs.writeFileSync(path.join(dir, "old.jsonl"), "", "utf8"); fs.utimesSync(path.join(dir, "old.jsonl"), new Date(), new Date()); - assert.deepEqual(drainProject(root, CWD), ["z-new"]); + assert.deepEqual(okPrompts(drainProject(root, CWD)), ["z-new"]); // Re-add with an OLD ts via direct write: still ordered by ts, not mtime. writeTs(path.join(dir, "old2.jsonl"), ["b-old"], 150); fs.utimesSync( @@ -44,7 +52,7 @@ test("atomic rewrite (delete) does not reshuffle the drain order", () => { new Date(Date.now() + 99999), new Date(Date.now() + 99999), ); - assert.deepEqual(drainProject(root, CWD), ["z-new", "b-old"]); + assert.deepEqual(okPrompts(drainProject(root, CWD)), ["z-new", "b-old"]); }); test("global drain puts the legacy seed last regardless of its fresh mtime", () => { @@ -54,7 +62,11 @@ test("global drain puts the legacy seed last regardless of its fresh mtime", () const seed = globalSeedPath(root); writeTs(seed, ["legacy-1", "legacy-2"], 10); fs.utimesSync(seed, new Date(Date.now() + 5000), new Date(Date.now() + 5000)); - assert.deepEqual(drainGlobal(root), ["fresh", "legacy-2", "legacy-1"]); + assert.deepEqual(okPrompts(drainGlobal(root)), [ + "fresh", + "legacy-2", + "legacy-1", + ]); }); test( @@ -78,7 +90,7 @@ test( try { // An unreadable file reads as zero entries and drops out of the drain; // the readable files keep their ts order. No throw. - assert.deepEqual(drainProject(root, CWD), ["z-new", "a-old"]); + assert.deepEqual(okPrompts(drainProject(root, CWD)), ["z-new", "a-old"]); } finally { fs.chmodSync(sealed, 0o644); // restore before cleanup } diff --git a/tests/history-hide-prompts.test.ts b/tests/history-hide-prompts.test.ts index 03c054863..ccd1f8ea8 100644 --- a/tests/history-hide-prompts.test.ts +++ b/tests/history-hide-prompts.test.ts @@ -3,13 +3,20 @@ import assert from "node:assert/strict"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { hidePrompt, loadHiddenPrompts } from "../extensions/history/hide-prompts.ts"; +import { + hidePrompt, + readHiddenPrompts, +} from "../extensions/history/hide-prompts.ts"; import { promptDedupKey } from "../extensions/history/selector-helpers.ts"; // Unit WU4 — tombstone write half + read half (spec C4, design §D6). fs-only -// coverage. The dev suite's deleteCurrent source-parse pins (T27/T28) and -// the deletionActionsFor planner pins cover the slice-3 selector branch and -// the slice-5 delete flow; they port with those slices. +// coverage. The READ half FAILS CLOSED for history: a file that exists but +// cannot be trusted (unreadable, corrupt, wrong shape) reads `untrusted` +// with a recovery warning instead of an empty tombstone set, and the WRITE +// half refuses without a silent rewrite. The dev suite's deleteCurrent +// source-parse pins (T27/T28) and the deletionActionsFor planner pins cover +// the slice-3 selector branch and the slice-5 delete flow; they port with +// those slices. function makeStateDir(name: string): string { return fs.mkdtempSync(path.join(os.tmpdir(), `hide-prompts-${name}-`)); @@ -21,6 +28,29 @@ function readHideFile(stateDir: string) { ); } +/** Assert an untrusted read of the expected reason; returns its message. */ +function assertUntrusted( + stateDir: string, + reason: "unreadable" | "corrupt" | "malformed", +): string { + const read = readHiddenPrompts(stateDir); + assert.equal(read.status, "untrusted"); + if (read.status !== "untrusted") throw new Error("unreachable"); + assert.equal(read.reason, reason); + // The recovery warning names the file and offers restore-or-delete. + assert.ok(read.message.includes("hidden.json")); + assert.ok(/restore|delete/.test(read.message)); + return read.message; +} + +/** Assert a trusted read and return its key set. */ +function trustedKeys(stateDir: string): Set { + const read = readHiddenPrompts(stateDir); + assert.equal(read.status, "trusted"); + if (read.status !== "trusted") throw new Error("unreachable"); + return read.keys; +} + // T24 — AC-S4-1: hide-key fidelity. Tombstone keys must byte-match the // Change 2 dedup key for the same text — same imported helper, never a // re-implementation: the stored file content is compared against @@ -41,59 +71,138 @@ test("T24 (AC-S4-1): hide keys byte-match promptDedupKey across whitespace, case assert.ok(Array.isArray(stored), "hidden.json must hold a JSON array"); // Byte-match: the file holds EXACTLY the shared helper's output, sorted. assert.deepEqual(stored, texts.map((text) => promptDedupKey(text)).sort()); - // The loaded set agrees. - const loaded = loadHiddenPrompts(stateDir); + // The read half agrees. + const keys = trustedKeys(stateDir); + assert.equal(keys.size, stored.length); for (const key of stored) { - assert.ok(loaded.has(key)); + assert.ok(keys.has(key)); } }); // T25 — AC-S4-2: hide persistence and tolerance. Two deletes of the same -// text compact to ONE key; a missing hide file reads as an empty set; reads -// never throw. -test("T25 (AC-S4-2): duplicate hides compact to one key; a missing file reads as empty; reads never throw", () => { +// text compact to ONE key; a MISSING hide file (before any deletion) is the +// safe empty case — trusted with no keys; reads never throw. +test("T25 (AC-S4-2): duplicate hides compact to one key; a missing file reads trusted-empty; reads never throw", () => { const stateDir = makeStateDir("t25"); - // Missing file: empty set, no throw (before any write exists). - assert.equal(loadHiddenPrompts(stateDir).size, 0); + // Missing file: trusted empty tombstones (before any write exists). + assert.equal(trustedKeys(stateDir).size, 0); // Two deletes of the same text — variants differing by case + whitespace // runs normalize onto the same key. assert.deepEqual(hidePrompt(stateDir, "Same Text"), { status: "written" }); assert.deepEqual(hidePrompt(stateDir, "same text"), { status: "written" }); const stored = readHideFile(stateDir); assert.deepEqual(stored, [promptDedupKey("same text")]); - const loaded = loadHiddenPrompts(stateDir); - assert.equal(loaded.size, 1); - assert.ok(loaded.has(promptDedupKey("same text"))); + const keys = trustedKeys(stateDir); + assert.equal(keys.size, 1); + assert.ok(keys.has(promptDedupKey("same text"))); }); -// T26 — AC-S4-5: corrupt hidden.json is fail-open (READ half) AND the next -// hide rewrites the file clean as a sorted compact array — the rewrite half -// is the recovery path. -test("T26 (AC-S4-5): corrupt hidden.json loads as empty and the next hide rewrites it clean", () => { +// T26 — AC-S4-5: corrupt hidden.json FAILS CLOSED for history reads. The +// READ half reports untrusted (corrupt) so callers block history instead of +// resurfacing hidden prompts; the WRITE half refuses WITHOUT a silent clean +// rewrite (the old fail-open behavior cleared the blocked state one hide +// later). Recovery is manual — restore or delete the file; after deletion +// the next hide succeeds and reads are trusted again. +test("T26 (AC-S4-5): corrupt hidden.json reads untrusted; hide refuses without rewriting; deleting the file recovers", () => { const stateDir = makeStateDir("t26"); - fs.writeFileSync( - path.join(stateDir, "hidden.json"), - "{corrupt bytes", - "utf8", - ); - assert.equal(loadHiddenPrompts(stateDir).size, 0); + const hidePath = path.join(stateDir, "hidden.json"); + fs.writeFileSync(hidePath, "{corrupt bytes", "utf8"); + // READ half: untrusted/corrupt — never an empty trusted set. + const message = assertUntrusted(stateDir, "corrupt"); + // WRITE half: refused, and the corrupt bytes are UNCHANGED — the blocked + // state is never silently reset (no-silent-rewrite pin). + const before = fs.readFileSync(hidePath, "utf8"); + assert.deepEqual(hidePrompt(stateDir, "beta prompt"), { + status: "error", + message, + }); + assert.equal(fs.readFileSync(hidePath, "utf8"), before); + // Manual recovery: delete the file; the next hide succeeds and reads are + // trusted with exactly the new key. + fs.unlinkSync(hidePath); assert.deepEqual(hidePrompt(stateDir, "beta prompt"), { status: "written" }); - // The rewrite landed: clean JSON holding exactly the new key. - assert.deepEqual(readHideFile(stateDir), [promptDedupKey("beta prompt")]); - assert.equal(loadHiddenPrompts(stateDir).size, 1); + const keys = trustedKeys(stateDir); + assert.equal(keys.size, 1); + assert.ok(keys.has(promptDedupKey("beta prompt"))); }); -// WU4c — write-failure path (AC-S4-2 triangulation): a state dir that cannot -// be created (its parent is a regular file) makes the atomic write return -// false, and hidePrompt maps that to the toast-suitable error object — -// never a throw. -test("hide write failure returns the exact error shape for the delete-flow toast", () => { - const base = makeStateDir("fail"); - const blocker = path.join(base, "blocker"); - fs.writeFileSync(blocker, "regular file", "utf8"); - const stateDir = path.join(blocker, "sealed"); // parent is a file → ENOTDIR - assert.deepEqual(hidePrompt(stateDir, "kept prompt"), { +// Wrong-shaped file: valid JSON that is not an array fails closed too (both +// halves), while junk items inside a VALID array are ignored and the real +// keys stay trusted. +test("malformed hidden.json fails closed for reads and writes; junk items in a valid array are ignored", () => { + const malformed = makeStateDir("malformed"); + const hidePath = path.join(malformed, "hidden.json"); + for (const shape of ["{}", JSON.stringify({ keys: [] })]) { + fs.writeFileSync(hidePath, shape, "utf8"); + assertUntrusted(malformed, "malformed"); + } + // The write half refuses the malformed file as well. + const message = assertUntrusted(malformed, "malformed"); + assert.deepEqual(hidePrompt(malformed, "kept prompt"), { status: "error", - message: "Could not write the hide file; the prompt may reappear.", + message, }); + + // Junk items are ignored, never trusted; real keys survive. + const junk = makeStateDir("junk"); + fs.writeFileSync( + path.join(junk, "hidden.json"), + JSON.stringify([42, "", "real-key", null]), + "utf8", + ); + const keys = trustedKeys(junk); + assert.equal(keys.size, 1); + assert.ok(keys.has("real-key")); }); + +// Unreadable file: a hidden.json that cannot be read at all fails closed +// for reads, and the write half refuses too. Skipped as root, where chmod +// 000 does not block reads; permissions are restored in finally. +test( + "an unreadable hidden.json fails closed for reads and refuses writes", + { skip: process.getuid?.() === 0 }, + () => { + const stateDir = makeStateDir("sealed"); + const hidePath = path.join(stateDir, "hidden.json"); + fs.writeFileSync(hidePath, '["kept-key"]', "utf8"); + fs.chmodSync(hidePath, 0o000); + try { + const message = assertUntrusted(stateDir, "unreadable"); + assert.deepEqual(hidePrompt(stateDir, "beta prompt"), { + status: "error", + message, + }); + } finally { + fs.chmodSync(hidePath, 0o644); // restore before cleanup + } + }, +); + +// WU4c — write-failure path (AC-S4-2 triangulation): a trusted read whose +// atomic write fails makes hidePrompt return the toast-suitable error +// object — never a throw. The state dir is made non-writable while the +// existing hidden.json stays readable (a state dir whose PATH is blocked +// by a regular file is now the untrusted-refusal case instead — the READ +// half fails closed before any write). Skipped as root, where chmod-based +// write blocking does not apply; permissions are restored in finally. +test( + "hide write failure returns the exact error shape for the delete-flow toast", + { skip: process.getuid?.() === 0 }, + () => { + const stateDir = makeStateDir("fail"); + fs.writeFileSync( + path.join(stateDir, "hidden.json"), + '["kept-key"]', + "utf8", + ); + fs.chmodSync(stateDir, 0o555); // read+execute, no write → EACCES on tmp + try { + assert.deepEqual(hidePrompt(stateDir, "kept prompt"), { + status: "error", + message: "Could not write the hide file; the prompt may reappear.", + }); + } finally { + fs.chmodSync(stateDir, 0o700); // restore before cleanup + } + }, +); From 892da55c51d7eb56d5e9186b329fe9f95de98a81 Mon Sep 17 00:00:00 2001 From: Carolina <26188349+carolitascl@users.noreply.github.com> Date: Thu, 24 Sep 2026 17:18:44 -0300 Subject: [PATCH 7/8] chore(readme): remove README delta from history slice The history slice branches must not touch README.md: the docs table lives in main and evolves independently of the extension slices. The opt-in capture documentation stays in docs/prompt-history.md; the README pointer row introduced by the capture-gate commit is dropped and README.md is restored to upstream/main verbatim. --- README.md | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index df59b272e..a8a5ee727 100644 --- a/README.md +++ b/README.md @@ -86,8 +86,6 @@ A bare terminal answers "what is the agent doing?" only with scrollback. gentle- ### el Gentleman — Think before you build -Diagram of el Gentleman turning human intent into clarified scope, a smallest workflow choice, evidence, and a human delivery decision - Say what you need once, then keep moving. el Gentleman helps turn intent into clear scope, a sensible next step, and evidence people can review — without making every task feel like a process meeting. **[Docs →](docs/readme-reference.md#organic-driven-development)** @@ -156,9 +154,7 @@ Model, effort, and who does what should be choices, not accidents. Named profile ### Command palette — Every command, one keystroke away -Command palette with a search field and grouped entries: Configuration, Session, Diagnostics, SDD, and Skills - -Extension commands are only useful if you can find them. `alt+k` opens a curated, grouped palette — Configuration, Session, Diagnostics, SDD, and Skills — searchable by label, command name, or description, showing entries only when they are actually registered. +Extension commands are only useful if you can find them. `alt+k` opens a curated, grouped palette — Configuration, Session, Diagnostics, and Skills — searchable by label, command name, or description, showing entries only when they are actually registered. **[Docs →](docs/gentle-shell.md#command-palette)** @@ -278,13 +274,12 @@ Start with the product-facing destination, then move into the operational refere | Destination | Purpose | | --- | --- | | [gentle-shell reference](docs/gentle-shell.md) | Workspace layout, changes, usage, agents, and todo interactions. | -| [ODD workflow](docs/readme-reference.md#organic-driven-development) · [Technical reference](docs/readme-reference.md) | Everyday work and recovery, optional SDD/OpenSpec, installation, configuration, commands, and contributor detail. | +| [ODD workflow](docs/readme-reference.md#organic-driven-development) · [Technical reference](docs/readme-reference.md) | Everyday work and recovery, installation, configuration, commands, and contributor detail. | | [Review integration](docs/review-integration.md) | The provider/consumer boundary for native review. | | [Native authority architecture](docs/native-authority-architecture.md) | Ownership boundaries and review architecture. | | [Telemetry](docs/telemetry.md) | Approved fields and source limitations. | | [Delegated verification](docs/delegated-verification.md) | Practical verification guidance. | | [Skill style guide](docs/skill-style-guide.md) | The package skill contract. | -| [Prompt history](docs/prompt-history.md) | Opt-in capture switch, storage layout and readability, and disable/removal semantics. |

Back to top ↑

From aae37cb3d22e2cabe11f83719e32d8d65fb433ef Mon Sep 17 00:00:00 2001 From: Carolina <26188349+carolitascl@users.noreply.github.com> Date: Thu, 24 Sep 2026 17:19:47 -0300 Subject: [PATCH 8/8] chore(readme): remove README delta from history slice The history slice branches must not touch README.md: the docs table lives in main and evolves independently of the extension slices. The opt-in capture documentation stays in docs/prompt-history.md; the README pointer row introduced by the capture-gate commit is dropped and README.md is restored to upstream/main verbatim. --- README.md | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index df59b272e..a8a5ee727 100644 --- a/README.md +++ b/README.md @@ -86,8 +86,6 @@ A bare terminal answers "what is the agent doing?" only with scrollback. gentle- ### el Gentleman — Think before you build -Diagram of el Gentleman turning human intent into clarified scope, a smallest workflow choice, evidence, and a human delivery decision - Say what you need once, then keep moving. el Gentleman helps turn intent into clear scope, a sensible next step, and evidence people can review — without making every task feel like a process meeting. **[Docs →](docs/readme-reference.md#organic-driven-development)** @@ -156,9 +154,7 @@ Model, effort, and who does what should be choices, not accidents. Named profile ### Command palette — Every command, one keystroke away -Command palette with a search field and grouped entries: Configuration, Session, Diagnostics, SDD, and Skills - -Extension commands are only useful if you can find them. `alt+k` opens a curated, grouped palette — Configuration, Session, Diagnostics, SDD, and Skills — searchable by label, command name, or description, showing entries only when they are actually registered. +Extension commands are only useful if you can find them. `alt+k` opens a curated, grouped palette — Configuration, Session, Diagnostics, and Skills — searchable by label, command name, or description, showing entries only when they are actually registered. **[Docs →](docs/gentle-shell.md#command-palette)** @@ -278,13 +274,12 @@ Start with the product-facing destination, then move into the operational refere | Destination | Purpose | | --- | --- | | [gentle-shell reference](docs/gentle-shell.md) | Workspace layout, changes, usage, agents, and todo interactions. | -| [ODD workflow](docs/readme-reference.md#organic-driven-development) · [Technical reference](docs/readme-reference.md) | Everyday work and recovery, optional SDD/OpenSpec, installation, configuration, commands, and contributor detail. | +| [ODD workflow](docs/readme-reference.md#organic-driven-development) · [Technical reference](docs/readme-reference.md) | Everyday work and recovery, installation, configuration, commands, and contributor detail. | | [Review integration](docs/review-integration.md) | The provider/consumer boundary for native review. | | [Native authority architecture](docs/native-authority-architecture.md) | Ownership boundaries and review architecture. | | [Telemetry](docs/telemetry.md) | Approved fields and source limitations. | | [Delegated verification](docs/delegated-verification.md) | Practical verification guidance. | | [Skill style guide](docs/skill-style-guide.md) | The package skill contract. | -| [Prompt history](docs/prompt-history.md) | Opt-in capture switch, storage layout and readability, and disable/removal semantics. |

Back to top ↑