diff --git a/.changeset/import-hash-check.md b/.changeset/import-hash-check.md new file mode 100644 index 0000000..62a1648 --- /dev/null +++ b/.changeset/import-hash-check.md @@ -0,0 +1,7 @@ +--- +"lossless-core": patch +"pi-lossless": patch +"opencode-lossless": patch +--- + +Verify file bytes and sha256 against the linked message text before inserting file rows on import. A repeated entry id with different text, or a file row whose text differs from the stored message, also refuses the import without writing. Existing rows stay untouched. diff --git a/.changeset/ingest-watermark-signal.md b/.changeset/ingest-watermark-signal.md new file mode 100644 index 0000000..bc993b4 --- /dev/null +++ b/.changeset/ingest-watermark-signal.md @@ -0,0 +1,6 @@ +--- +"pi-lossless": patch +"lossless-core": patch +--- + +Signal a watermark miss on the ingest path. A full rescan now writes one `ingest-rescan` metric row and, in interactive sessions, one UI warning. Empty entries stay silent and ingest behavior is unchanged. diff --git a/.changeset/mechanical-noisy-tier.md b/.changeset/mechanical-noisy-tier.md new file mode 100644 index 0000000..54e1469 --- /dev/null +++ b/.changeset/mechanical-noisy-tier.md @@ -0,0 +1,7 @@ +--- +"lossless-core": patch +"pi-lossless": patch +"opencode-lossless": patch +--- + +Add a mechanical tier for noisy tools. Name tools in `mechanicalTools` in summarizer-config, `/lcm` config, or the OpenCode plugin options. Their spans compact into deterministic `mechanical` leaves with zero summarizer spend at compaction time. Every message stays stored verbatim and covered, so the integrity coverage check passes, and heal re-derives a skipped leaf from provenance on demand. There is no ingest filtering. diff --git a/.changeset/orphan-sidecar-report.md b/.changeset/orphan-sidecar-report.md new file mode 100644 index 0000000..83bdf4d --- /dev/null +++ b/.changeset/orphan-sidecar-report.md @@ -0,0 +1,7 @@ +--- +"lossless-core": patch +"pi-lossless": patch +"opencode-lossless": patch +--- + +Report orphan sidecars in doctor. `integritySnapshot` now lists sidecar files and stale `wal` and `shm` files with no live session (`.sqlite` by default, `.db` for Pi). `checkIntegrity` reports them as a read-only `orphan-sidecar` finding through `lcm doctor` and the diagnose bundle. Report only. Deletion stays manual. diff --git a/.changeset/recall-search-upgrades.md b/.changeset/recall-search-upgrades.md new file mode 100644 index 0000000..89af75f --- /dev/null +++ b/.changeset/recall-search-upgrades.md @@ -0,0 +1,7 @@ +--- +"lossless-core": patch +"pi-lossless": patch +"opencode-lossless": patch +--- + +Upgrade recall search. `lcm_grep` takes `role`, `date_start`/`date_end`, and `tool_name` filters. Role and time are SQL predicates on the stored columns with a covering index. Tool scope is an FTS rewrite against `[tool:Name]` markers with no new table or index. `lcm_expand` takes an `anchor` entry id that centers the listing window with edge clamping. The recall manifest and the retrieval brief order by stop-word stripped query-word overlap over summary headers. A read-only files view lists each linked file once with an incomplete-coverage flag and a `large-inline` mark. No migration and no new writer. diff --git a/.changeset/static-sidecar-dump.md b/.changeset/static-sidecar-dump.md new file mode 100644 index 0000000..cca4ac1 --- /dev/null +++ b/.changeset/static-sidecar-dump.md @@ -0,0 +1,6 @@ +--- +"pi-lossless": patch +"opencode-lossless": patch +--- + +Add a read-only static sidecar dump. `/lcm dump` and `/lossless-dump` open the session sidecar `queryOnly` and reuse the recall views and metric rows. The dump redacts displayed seams, labels removed rows, and reports old generations as named skips. Output is capped at 16 KB. diff --git a/packages/core/src/compaction-engine.ts b/packages/core/src/compaction-engine.ts index c234d2c..8e2f416 100644 --- a/packages/core/src/compaction-engine.ts +++ b/packages/core/src/compaction-engine.ts @@ -24,9 +24,15 @@ import { summarizeTerse, summarizeWithEscalation, deterministicTruncate, + LEVEL3_TOKENS, type EscalationResult, type LlmComplete, } from "./summarize.ts"; +import { + isMechanicalToolMessage, + mechanicalToolNameOf, + normalizeMechanicalTools, +} from "./util/summarizer-config.ts"; const LEAF_CHUNK_TOKENS = 3000; /** Deepest condensed node the DAG may grow. With MAX_DAG_FANOUT children this @@ -49,6 +55,8 @@ export interface CompactionInput { targetTokens?: number; /** Where the returned summary must reach back to; defaults to the span start. */ frontierFrom?: string; + /** Read at compaction time only; ingest stores every message verbatim. */ + mechanicalTools?: readonly string[]; } export type CompactionOutcome = @@ -58,6 +66,10 @@ export type CompactionOutcome = leafSummaryIds: number[]; condensedSummaryIds: number[]; ingested: number; + /** Leaves stored without a model call on the configured mechanical path. */ + mechanicalLeaves?: number; + /** Condensed nodes built without a model call from mechanical children. */ + mechanicalCondensed?: number; } /** No text, and why. A span another live run claimed, a span an existing node * already covers, and a span the store holds no row for are all states a pass @@ -87,6 +99,8 @@ export async function runCompaction( session?: string; maxDepth?: number; stage?: CompactionStage; + /** Empty keeps the LLM path. */ + mechanicalTools?: readonly string[]; }, ): Promise { const targetTokens = input.targetTokens ?? 1500; @@ -114,13 +128,142 @@ export async function runCompaction( const leafIds: number[] = []; const claimedLeaves: PlainSummary[] = []; + let mechanicalLeafCount = 0; + let mechanicalCondensedCount = 0; + const mechanicalTools = normalizeMechanicalTools([ + ...(input.mechanicalTools ?? []), + ...(opts?.mechanicalTools ?? []), + ]); + const uncoveredAll = store.uncoveredMessagesInSpan(bounds.firstEntryId, bounds.lastEntryId); + const payloadByEntry = new Map(input.span.map((e) => [e.entryId, e.payload])); + const mechanicalIds = mechanicalMessageIds(uncoveredAll, mechanicalTools, payloadByEntry); + const maxChunks = opts?.maxChunks ?? 24; + // The split below multiplies ranges past the cap (alternating noisy and + // ordinary messages split one chunk into one range each), so the cap applies + // again after it: over the cap adjacent ranges merge and share one attempt. const chunks = mergeToLimit( - chunkContiguous( - store.uncoveredMessagesInSpan(bounds.firstEntryId, bounds.lastEntryId), - leafChunkBudget, + mergeToLimit(chunkContiguous(uncoveredAll, leafChunkBudget), maxChunks).flatMap((chunk) => + splitByMechanical(chunk, mechanicalIds), ), - opts?.maxChunks ?? 24, + maxChunks, ); + // Same redacted source the ladder would fall back to, stored without a call; + // provenance keeps rows covered for heal. + async function storeMechanicalLeaf( + remaining: StoredMessage[], + chunkHead: StoredMessage, + chunkTail: StoredMessage, + ): Promise { + const started = Date.now(); + const fileRefs = store.filesForMessages(remaining.map((m) => m.id)); + const fileByMessage = new Map(fileRefs.map((r) => [r.messageId, r])); + const source = remaining.map((m) => { + const ref = fileByMessage.get(m.id); + return { role: m.role, text: ref ? redact(filePreviewText(ref)) : redact(m.text) }; + }); + const sourceChars = source.reduce((n, m) => n + m.text.length, 0); + const plain = source.map((m) => `[${m.role}]\n${m.text}`).join("\n\n"); + const truncated = deterministicTruncate(plain, LEVEL3_TOKENS); + const stored = `${TRUNCATED_SUMMARY_PREFIX}\n${redact(truncated)}`; + const tokens = estimateTokens(stored); + const claimed = store.uncoveredMessagesInSpan(chunkHead.entryId, chunkTail.entryId); + const ours = + claimed.length === remaining.length && claimed.every((m, i) => m.id === remaining[i]?.id); + if (!ours) { + remaining.length = 0; + remaining.push(...claimed); + if (remaining.length === 0) return; + if (!remaining.every((m) => mechanicalIds.has(m.id))) return; + await storeMechanicalLeaf(remaining, remaining[0]!, remaining[remaining.length - 1]!); + return; + } + const inserted = store.insertSummary({ + kind: "leaf", + text: stored, + tokens, + depth: 0, + quality: "mechanical", + firstEntryId: chunkHead.entryId, + lastEntryId: chunkTail.entryId, + messageIds: remaining.map((m) => m.id), + fileIds: [...new Set(fileRefs.map((r) => r.fileId))], + }); + const durationMs = Date.now() - started; + if (!inserted.created) { + reportSkippedExisting( + inserted.id, + inserted.existingTokens, + session, + "leaf", + 0, + [chunkHead.entryId, chunkTail.entryId], + tokens, + durationMs, + emit, + ); + if (inserted.foreignLive) { + claimedLeaves.push({ + id: inserted.id, + depth: 0, + firstEntryId: chunkHead.entryId, + lastEntryId: chunkTail.entryId, + text: inserted.existingText, + }); + } + if ( + inserted.own && + replaceIfSmaller( + store, + inserted.id, + inserted.existingTokens, + stored, + tokens, + "mechanical", + session, + "leaf", + started, + emit, + ) + ) { + leafIds.push(inserted.id); + mechanicalLeafCount++; + reportFallbackStored( + "leaf", + 3, + session, + [chunkHead.entryId, chunkTail.entryId], + sourceChars, + inserted.id, + emit, + ); + } + return; + } + reportFallbackStored( + "leaf", + 3, + session, + [chunkHead.entryId, chunkTail.entryId], + sourceChars, + inserted.id, + emit, + ); + leafIds.push(inserted.id); + mechanicalLeafCount++; + emit({ + event: "lcm", + kind: "summary-stored", + stage: "leaf", + level: 3, + quality: "mechanical", + depth: 0, + tokens, + summaryId: inserted.id, + session, + span: [chunkHead.entryId, chunkTail.entryId], + durationMs, + }); + } // Another writer can claim part of a chunk between the snapshot above and the // insert below, so the range is re-asked after the model answers and checked // with no await before the insert, leaving no gap for another claim. @@ -137,8 +280,26 @@ export async function runCompaction( // and the clip metric stays quiet. const refs = store.filesForMessages(remaining.map((m) => m.id)); const refByMessage = new Map(refs.map((r) => [r.messageId, r])); + if ( + mechanicalTools.length > 0 && + remaining.length > 0 && + remaining.every((m) => mechanicalIds.has(m.id)) + ) { + await storeMechanicalLeaf(remaining, chunkHead, chunkTail); + return; + } let sourceClipped = false; + let mixedSkipped = false; + // A post-split re-cap can join noisy and ordinary runs, so a mixed + // range keeps full provenance but prompts only ordinary messages. + // Contiguous spans stay intact for the span index and the frontier. + // Skipped noisy bodies count as clipped source, so the leaf stays + // healable and never reads as clean. const items = remaining.flatMap((m) => { + if (mechanicalIds.has(m.id)) { + mixedSkipped = true; + return []; + } const ref = refByMessage.get(m.id); // The summarizer never reads a stored body unmasked. The mask runs on // the whole body rather than on the clipped copy, so a value that @@ -175,7 +336,7 @@ export async function runCompaction( } const stored = storedSummaryText(text, level, redact); const tokens = estimateTokens(stored); - const quality = leafQuality(level, sourceClipped); + const quality = leafQuality(level, sourceClipped || mixedSkipped); const inserted = store.insertSummary({ kind: "leaf", text: stored, @@ -316,15 +477,25 @@ export async function runCompaction( if (!groupHead || !groupTail) break; const depth = groupHead.depth + 1; const started = Date.now(); - const answer = await summarizeWithEscalation( - group.map((n) => ({ role: "summary", text: n.text })), - targetTokens, - llm, - session, - ); - const { text, level } = answer; const span: [string, string] = [groupHead.firstEntryId, groupTail.lastEntryId]; const sourceChars = group.reduce((n, s) => n + s.text.length, 0); + // No clean source in mechanical-only groups, so truncate without a call; + // heal repairs leaves first, as with any tainted ancestor. + const mechanicalGroup = mechanicalTools.length > 0 && group.every((n) => isMechanicalNode(n)); + const answer = mechanicalGroup + ? null + : await summarizeWithEscalation( + group.map((n) => ({ role: "summary", text: n.text })), + targetTokens, + llm, + session, + ); + const { text, level } = + answer ?? + (() => { + const plain = group.map((n) => `[summary]\n${n.text}`).join("\n\n"); + return { text: deterministicTruncate(plain, LEVEL3_TOKENS), level: 3 as const }; + })(); const stored = storedSummaryText(text, level, redact); const tokens = estimateTokens(stored); const quality = condensedQuality(level, group); @@ -340,7 +511,7 @@ export async function runCompaction( // A parent inherits its children's handles, which is how a top-level // node answers "which files does this cover" without expanding. fileIds: store.fileIdsForSummaries(group.map((n) => n.id)), - ...(answer.thoroughText === undefined ? {} : { thoroughText: answer.thoroughText }), + ...(answer?.thoroughText === undefined ? {} : { thoroughText: answer.thoroughText }), }); const durationMs = Date.now() - started; if (!inserted.created) { @@ -373,6 +544,7 @@ export async function runCompaction( ) ) { condensedIds.push(inserted.id); + if (mechanicalGroup) mechanicalCondensedCount++; reportFallbackStored("condensed", level, session, span, sourceChars, inserted.id, emit); } frontier = store.frontier(from, bounds.lastEntryId); @@ -380,8 +552,9 @@ export async function runCompaction( continue; } condensedIds.push(inserted.id); + if (mechanicalGroup) mechanicalCondensedCount++; reportFallbackStored("condensed", level, session, span, sourceChars, inserted.id, emit); - if (tokens > targetTokens || answer.retried) { + if (!mechanicalGroup && answer && (tokens > targetTokens || answer.retried)) { reportOversize( "condensed", depth, @@ -417,7 +590,10 @@ export async function runCompaction( // overshoot counts, which is the same twice-the-target bar the final render // clips at, so the reduction hits exactly the nodes that would be cut. const lone = frontier.length === 1 ? frontier[0] : undefined; - if (lone && frontierTok > targetTokens * 2) { + // A lone mechanical node under an opted-in list already stores the + // truncate, so skip the re-summarize call. + const loneMechanical = lone !== undefined && mechanicalTools.length > 0 && isMechanicalNode(lone); + if (lone && frontierTok > targetTokens * 2 && !loneMechanical) { const beforeTokens = lone.tokens; const started = Date.now(); const answer = await summarizeTerse( @@ -533,6 +709,8 @@ export async function runCompaction( leafSummaryIds: leafIds, condensedSummaryIds: condensedIds, ingested: ingestStats.inserted, + mechanicalLeaves: mechanicalLeafCount, + mechanicalCondensed: mechanicalCondensedCount, }; } @@ -608,6 +786,8 @@ function replaceIfSmaller( return true; } +/** A leaf that omitted listed noisy bodies is clipped, like a length clip: + * the model did not see the full source, but heal reads the full rows. */ function leafQuality(level: SummaryLevel, sourceClipped: boolean): SummaryQuality { if (level === 3) return "mechanical"; return sourceClipped ? "clipped" : "clean"; @@ -772,3 +952,69 @@ function adjacentRuns(frontier: readonly SummaryNode[]): SummaryNode[][] { } return runs; } + +/** Mechanical by verdict, or by the stored truncate marker old rows keep. */ +function isMechanicalNode(node: SummaryNode): boolean { + return node.quality === "mechanical" || node.text.startsWith(TRUNCATED_SUMMARY_PREFIX); +} + +/** A direct tool-name hit, or an unnamed toolResult right after a listed row, + * which is how Pi stores a result whose own text names no tool. A result that + * names another tool closes the scope, the same rule toolScopeMessageIds uses, + * so only a result with no tool of its own follows the listed row. */ +function mechanicalMessageIds( + messages: readonly StoredMessage[], + tools: readonly string[], + payloadByEntry?: ReadonlyMap, +): Set { + const ids = new Set(); + if (tools.length === 0) return ids; + let prevDirectListed = false; + for (const m of messages) { + const payload = payloadByEntry?.get(m.entryId) ?? m.payload; + const direct = isMechanicalToolMessage({ text: m.text, payload, role: m.role }, tools); + if (direct) { + ids.add(m.id); + prevDirectListed = true; + continue; + } + if (m.role === "toolResult" && prevDirectListed) { + const own = mechanicalToolNameOf({ text: m.text, payload, role: m.role }); + if (own !== undefined) { + prevDirectListed = false; + continue; + } + ids.add(m.id); + prevDirectListed = false; + continue; + } + prevDirectListed = false; + } + return ids; +} + +/** Split a chunk so a noisy result isolates from the prose around it. */ +function splitByMechanical( + chunk: StoredMessage[], + mechanicalIds: ReadonlySet, +): StoredMessage[][] { + if (chunk.length === 0 || mechanicalIds.size === 0) return chunk.length === 0 ? [] : [chunk]; + const runs: StoredMessage[][] = []; + let current: StoredMessage[] = []; + let currentFlag: boolean | undefined; + for (const m of chunk) { + const flag = mechanicalIds.has(m.id); + if (current.length === 0) { + current = [m]; + currentFlag = flag; + } else if (flag === currentFlag) { + current.push(m); + } else { + runs.push(current); + current = [m]; + currentFlag = flag; + } + } + if (current.length > 0) runs.push(current); + return runs; +} diff --git a/packages/core/src/import.ts b/packages/core/src/import.ts index d12a2f1..312db83 100644 --- a/packages/core/src/import.ts +++ b/packages/core/src/import.ts @@ -1,3 +1,5 @@ +import { createHash } from "node:crypto"; + import { EXPORT_VERSION, type ExportFileRow, @@ -6,7 +8,7 @@ import { type ExportSummaryRow, } from "./export.ts"; import { FILE_KINDS, isFileKind, type FileDescriptor } from "./files.ts"; -import { ingestEntries } from "./ingest.ts"; +import { ingestEntries, type IngestStats } from "./ingest.ts"; import { isSummaryQuality, normalizeRole, @@ -217,6 +219,73 @@ const REFUSED = (reason: string): ImportResult => ({ refused: reason, }); +/** A repeated entry id must carry the same text every time. Ingest keeps the + * first row and skips the rest, so a conflict has no single reading to check + * a file against. Refuse before any write. */ +function duplicateEntryProblem(rows: readonly ExportRow[]): string | undefined { + const seen = new Map(); + for (const r of rows) { + if (r.rowType !== "message") continue; + const prev = seen.get(r.entryId); + if (prev === undefined) { + seen.set(r.entryId, r.text); + } else if (prev !== r.text) { + return `message entry ${r.entryId} repeats with different text`; + } + } + return undefined; +} + +/** Runs before any write, so a corrupt file refuses without a partial import. + * Rows with no linked entries carry nothing to check. The first row wins, + * which is what ingest keeps when an entry id repeats. */ +function fileHashProblem(rows: readonly ExportRow[]): string | undefined { + const textByEntry = new Map(); + for (const r of rows) { + if (r.rowType !== "message") continue; + if (!textByEntry.has(r.entryId)) textByEntry.set(r.entryId, r.text); + } + for (const f of rows) { + if (f.rowType !== "file") continue; + for (const entryId of f.entryIds) { + const text = textByEntry.get(entryId); + if (text === undefined) continue; + if (text.length !== f.bytes) { + return `file ${f.fileId} claims ${f.bytes} chars but entry ${entryId} holds ${text.length}`; + } + const actual = createHash("sha256").update(text).digest("hex"); + if (actual !== f.sha256) { + return `file ${f.fileId} carries sha256 ${f.sha256} but entry ${entryId} hashes to ${actual}`; + } + } + } + return undefined; +} + +/** A file row links to the stored message for its entry id, and ingest skips an + * entry id the target already holds. When the target holds different text, the + * file would link to bytes its hash was not checked against, so refuse before + * any write and leave the stored row alone. */ +function existingEntryProblem(store: LcmStore, rows: readonly ExportRow[]): string | undefined { + const textByEntry = new Map(); + for (const r of rows) { + if (r.rowType !== "message") continue; + if (!textByEntry.has(r.entryId)) textByEntry.set(r.entryId, r.text); + } + for (const f of rows) { + if (f.rowType !== "file") continue; + for (const entryId of f.entryIds) { + const supplied = textByEntry.get(entryId); + if (supplied === undefined) continue; + const existing = store.messageByEntryId(entryId); + if (existing !== undefined && existing.text !== supplied) { + return `file ${f.fileId} names entry ${entryId} whose text in this store differs from the file`; + } + } + } + return undefined; +} + function fileDescriptor(f: ExportFileRow): FileDescriptor { return { fileId: f.fileId, @@ -245,33 +314,55 @@ export function importTranscript(store: LcmStore, rows: readonly ExportRow[]): I const problem = provenanceProblem(rows); if (problem) return REFUSED(problem); - const exported = new Set(messages.map((m) => m.entryId)); - const foreign = store.allMessages().filter((m) => !exported.has(m.entryId)); - if (foreign.length > 0) { - return REFUSED( - `this store already holds ${foreign.length} message(s) the file does not name; import into an empty store instead`, - ); - } + const duplicateProblem = duplicateEntryProblem(rows); + if (duplicateProblem) return REFUSED(duplicateProblem); - const ingested = ingestEntries( - store, - messages.map((m) => ({ - entryId: m.entryId, - role: m.role, - text: m.text, - timestamp: m.timestamp, - ...(m.payload === undefined ? {} : { payload: m.payload }), - })), - ); - const idByEntry = new Map(store.allMessages().map((m) => [m.entryId, m.id])); - for (const f of rows) { - if (f.rowType !== "file") continue; - store.insertFile(fileDescriptor(f), f.entryIds[0] ?? ""); - for (const entryId of f.entryIds) { - const messageId = idByEntry.get(entryId); - if (messageId !== undefined) store.linkMessageFile(messageId, f.fileId); + const hashProblem = fileHashProblem(rows); + if (hashProblem) return REFUSED(hashProblem); + + // The stored-text check and everything it protects share one write + // transaction. BEGIN IMMEDIATE takes the write lock before the first read, + // so a second connection cannot insert a same-entry row with different text + // between the check and the file links; INSERT OR IGNORE then keeps the + // checked text. Same-process writers cannot interleave this synchronous + // function in any case. + const prepared = store.transaction< + ImportResult | { ingested: IngestStats; idByEntry: Map } + >(() => { + const clashProblem = existingEntryProblem(store, rows); + if (clashProblem) return REFUSED(clashProblem); + + const exported = new Set(messages.map((m) => m.entryId)); + const foreign = store.allMessages().filter((m) => !exported.has(m.entryId)); + if (foreign.length > 0) { + return REFUSED( + `this store already holds ${foreign.length} message(s) the file does not name; import into an empty store instead`, + ); } - } + + const ingested = ingestEntries( + store, + messages.map((m) => ({ + entryId: m.entryId, + role: m.role, + text: m.text, + timestamp: m.timestamp, + ...(m.payload === undefined ? {} : { payload: m.payload }), + })), + ); + const idByEntry = new Map(store.allMessages().map((m) => [m.entryId, m.id])); + for (const f of rows) { + if (f.rowType !== "file") continue; + store.insertFile(fileDescriptor(f), f.entryIds[0] ?? ""); + for (const entryId of f.entryIds) { + const messageId = idByEntry.get(entryId); + if (messageId !== undefined) store.linkMessageFile(messageId, f.fileId); + } + } + return { ingested, idByEntry }; + }); + if (!("ingested" in prepared)) return prepared; + const { ingested, idByEntry } = prepared; const idByOrdinal = new Map(); let summariesInserted = 0; let summariesSkipped = 0; diff --git a/packages/core/src/integrity.ts b/packages/core/src/integrity.ts index 6a2d156..da2fa58 100644 --- a/packages/core/src/integrity.ts +++ b/packages/core/src/integrity.ts @@ -1,4 +1,9 @@ -import { SCHEMA_VERSION, type IntegritySnapshot, type LcmStore } from "./store.ts"; +import { + SCHEMA_VERSION, + type IntegritySnapshot, + type IntegritySnapshotOptions, + type LcmStore, +} from "./store.ts"; const SAMPLE = 10; @@ -11,6 +16,7 @@ export type FindingKind = | "foreign-key" | "schema-version" | "dead-run" + | "orphan-sidecar" | "coverage"; export interface Finding { @@ -159,6 +165,20 @@ const CHECKS: { readonly [K in FindingKind]: Check } = { .map((d) => `${d.first}..${d.last} (${d.kind}) x${d.count}`), }, + // Orphan sidecars are work to clear, not a broken invariant; deletion stays manual. + // "Live" means a handle this process holds open: cross-session recall still + // reads an unopened sidecar on demand, so the finding names files, not gone sessions. + "orphan-sidecar": (snap) => + snap.orphanSidecars.length === 0 + ? undefined + : { + kind: "orphan-sidecar", + severity: "info", + count: snap.orphanSidecars.length, + detail: `${snap.orphanSidecars.length} sidecar(s) not open in this process. Deletion stays manual.`, + sample: snap.orphanSidecars.slice(0, SAMPLE), + }, + "foreign-key": (snap) => snap.foreignKeyViolations === 0 ? undefined @@ -195,8 +215,8 @@ const CHECKS: { readonly [K in FindingKind]: Check } = { }, }; -export function checkIntegrity(store: LcmStore): IntegrityReport { - const snap = store.integritySnapshot(); +export function checkIntegrity(store: LcmStore, opts?: IntegritySnapshotOptions): IntegrityReport { + const snap = store.integritySnapshot(opts); const findings: Finding[] = []; for (const check of Object.values(CHECKS)) { const finding = check(snap); diff --git a/packages/core/src/metrics.ts b/packages/core/src/metrics.ts index 9c1bceb..f6d9774 100644 --- a/packages/core/src/metrics.ts +++ b/packages/core/src/metrics.ts @@ -633,6 +633,13 @@ export type BranchRemovedRow = LcmRowBase & { restored: number; }; +/** A watermark miss forced a full rescan. Empty deltas never write one. */ +export type IngestRescanRow = LcmRowBase & { + kind: "ingest-rescan"; + watermark: string; + entries: number; +}; + export type FileExternalizedRow = LcmRowBase & { kind: "file-externalized"; entryId: string; @@ -747,6 +754,7 @@ export type LcmMetricRecord = | GcDeletedRow | StoreMigratedRow | BranchRemovedRow + | IngestRescanRow | FileExternalizedRow | LargeInlineRow | RecallRow diff --git a/packages/core/src/recall-view.ts b/packages/core/src/recall-view.ts index 69e1e0d..5083173 100644 --- a/packages/core/src/recall-view.ts +++ b/packages/core/src/recall-view.ts @@ -9,6 +9,7 @@ import { type PatternScanOutcome, } from "./pattern-scan.ts"; import type { LcmStore, StoredMessage, SummaryNode } from "./store.ts"; +import { mechanicalToolNameOf } from "./util/summarizer-config.ts"; /** * Every view masks stored text with the injected `redact` before it is cut down @@ -26,6 +27,105 @@ export type ViewResult = | { ok: true; text: string; details: Record } | { ok: false; text: string }; +const LCM_ROLES = ["user", "assistant", "toolResult", "custom"] as const; + +type GrepFilter = { + role?: (typeof LCM_ROLES)[number]; + dateStart?: number; + dateEnd?: number; + toolName?: string; +}; + +function grepFilterProblem(filter: GrepFilter): string | undefined { + if (filter.role !== undefined && !LCM_ROLES.includes(filter.role)) { + return `Unknown role ${JSON.stringify(filter.role)}. Pass one of ${LCM_ROLES.join(", ")}.`; + } + for (const key of ["dateStart", "dateEnd"] as const) { + const value = filter[key]; + if (value !== undefined && (!Number.isFinite(value) || value < 0)) { + return `${key} must be a non-negative timestamp number.`; + } + } + if (filter.toolName !== undefined && filter.toolName.trim().length === 0) { + return "toolName must be a non-empty tool name."; + } + return undefined; +} + +/** The `[tool:Name]` marker in front of a tool call's arguments, so a scoped + * search needs no table or index. */ +export function toolMarker(toolName: string): string { + return `[tool:${toolName}]`; +} + +/** Quote a string as one FTS5 phrase. Embedded quotes double; JSON.stringify + * emits backslash escapes which FTS5 rejects. */ +export function ftsPhrase(text: string): string { + return `"${text.replace(/"/g, '""')}"`; +} + +/** Rewrite an FTS5 query to its tool-scoped form. The marker passes as a quoted + * phrase, which the FTS5 tokenizer splits the same way as the stored text, so + * prose with the same adjacent words (for example "which tool read the file") + * can also match. Anchoring to the row start would miss mixed blocks because + * entryToText appends blocks in order, so the scoped search keeps this broader + * match and callers read the hit to confirm the tool. */ +export function withToolScope(query: string, toolName: string): string { + return `(${query}) AND ${ftsPhrase(toolMarker(toolName))}`; +} + +/** Row ids a tool scope covers: the call rows that name the tool plus the + * toolResult rows that follow them. Pi stores a call and its result in + * separate rows, so a same-row `query AND marker` FTS misses a word found + * only in the result. A trailing run of toolResult rows after a marker row + * belongs to that call batch; any other role ends the run, as does a + * toolResult row that names a different tool. Bracket text is checked + * directly so one row naming two tools anchors both, and + * mechanicalToolNameOf covers the `tool ` and payload forms. User and + * custom rows never open a scope, so quoting a marker in chat pulls in no + * results; a mixed assistant block still opens through its literal marker. */ +export function toolScopeMessageIds( + store: LcmStore, + toolName: string, + opts?: { includeRemoved?: boolean; withinMessageIds?: ReadonlySet }, +): Set { + const want = toolName.trim().toLowerCase(); + const out = new Set(); + if (want.length === 0) return out; + const scope = opts?.withinMessageIds; + const includeRemoved = opts?.includeRemoved === true; + const visible = store + .allMessages() + .filter( + (m) => + (includeRemoved || m.removedAt === undefined) && (scope === undefined || scope.has(m.id)), + ); + let open = false; + for (const row of visible) { + const namesTool = + row.role !== "user" && + row.role !== "custom" && + (row.text.toLowerCase().includes(`[tool:${want}]`) || + mechanicalToolNameOf({ text: row.text, payload: row.payload, role: row.role }) === want); + if (namesTool) { + out.add(row.id); + open = true; + continue; + } + if (row.role === "toolResult" && open) { + const other = mechanicalToolNameOf({ text: row.text, payload: row.payload }) ?? ""; + if ((other.length > 0 && other !== want) || /\[tool:/i.test(row.text)) { + open = false; + continue; + } + out.add(row.id); + continue; + } + if (row.role !== "toolResult") open = false; + } + return out; +} + export function grepView( store: LcmStore, redact: (text: string) => string, @@ -43,6 +143,11 @@ export function grepView( now?: () => number; /** Tool the continuation marker names. Defaults to `lcm_expand_query`. */ expandTool?: string; + role?: GrepFilter["role"]; + dateStart?: number; + dateEnd?: number; + /** Query searches only; with a pattern it is refused. */ + toolName?: string; }, ): ViewResult { let withinMessageIds: Set | undefined; @@ -52,6 +157,19 @@ export function grepView( } withinMessageIds = new Set(store.coveredMessageIds(params.summaryId)); } + const filterProblem = grepFilterProblem({ + ...(params.role === undefined ? {} : { role: params.role }), + ...(params.dateStart === undefined ? {} : { dateStart: params.dateStart }), + ...(params.dateEnd === undefined ? {} : { dateEnd: params.dateEnd }), + ...(params.toolName === undefined ? {} : { toolName: params.toolName }), + }); + if (filterProblem !== undefined) return { ok: false, text: filterProblem }; + const grepOpts = { + withinMessageIds, + ...(params.role === undefined ? {} : { role: params.role }), + ...(params.dateStart === undefined ? {} : { dateStart: params.dateStart }), + ...(params.dateEnd === undefined ? {} : { dateEnd: params.dateEnd }), + }; const limit = params.limit ?? GREP_PAGE_HITS; const offset = Math.max(0, params.offset ?? 0); const includeRemoved = params.includeRemoved === true; @@ -60,6 +178,9 @@ export function grepView( return { ok: false, text: "Pass query or pattern, not both." }; } if (params.pattern !== undefined) { + if (params.toolName !== undefined) { + return { ok: false, text: "toolName scopes a query search. Pass query rather than pattern." }; + } let afterRowId = 0; if (params.after !== undefined) { const row = store.messageByEntryId(params.after); @@ -71,11 +192,11 @@ export function grepView( caseSensitive: params.caseSensitive === true, offset, limit, - withinMessageIds, includeRemoved, afterRowId, now: params.now ?? Date.now, expandTool, + ...grepOpts, }); } if (params.after !== undefined) { @@ -84,20 +205,35 @@ export function grepView( if (params.query === undefined) { return { ok: false, text: "Pass a query (FTS5) or a pattern (regular expression)." }; } - let pattern = params.query; + if (params.toolName !== undefined) { + return toolScopedView(store, redact, { + query: params.query, + toolName: params.toolName, + limit, + offset, + includeRemoved, + expandTool, + withinMessageIds, + ...(params.role === undefined ? {} : { role: params.role }), + ...(params.dateStart === undefined ? {} : { dateStart: params.dateStart }), + ...(params.dateEnd === undefined ? {} : { dateEnd: params.dateEnd }), + }); + } + const scoped = params.query; + let pattern = scoped; let hits: ReturnType | undefined; // FTS5 rejects some syntax (unbalanced quotes, stray operators) with an // error, so the same query is retried as a quoted phrase. const search = (query: string, removed: boolean, off: number) => { try { - return store.grep(query, { limit, offset: off, withinMessageIds, includeRemoved: removed }); + return store.grep(query, { limit, offset: off, includeRemoved: removed, ...grepOpts }); } catch { return undefined; } }; hits = search(pattern, includeRemoved, offset); if (hits === undefined) { - pattern = JSON.stringify(params.query); + pattern = ftsPhrase(params.query); hits = search(pattern, includeRemoved, offset); } if (hits === undefined) { @@ -125,7 +261,7 @@ export function grepView( }; } let budget = RESULT_BUDGET_CHARS; - const total = store.grepCount(pattern, { withinMessageIds, includeRemoved }); + const total = store.grepCount(pattern, { includeRemoved, ...grepOpts }); const lines: string[] = [ total === undefined || total <= limit ? `Found ${hits.length} match(es):` @@ -159,6 +295,155 @@ export function grepView( }; } +/** A scoped FTS search that stays complete. `store.grep` applies its SQL + * LIMIT before it intersects the id set, so one bounded call can miss rows + * that rank past the window. The unscoped count bounds the rank order, so one + * fetch of that size holds every in-scope match and the caller pages the + * filtered set. Rank order is preserved. Throws on invalid FTS5 syntax, like + * `store.grep`, so callers keep their quoted-phrase fallback. */ +export function scopedGrepAll( + store: LcmStore, + pattern: string, + opts: { + includeRemoved: boolean; + role?: GrepFilter["role"]; + dateStart?: number; + dateEnd?: number; + scoped: ReadonlySet; + }, +): ReturnType { + if (opts.scoped.size === 0) return []; + const unscoped = { + includeRemoved: opts.includeRemoved, + ...(opts.role === undefined ? {} : { role: opts.role }), + ...(opts.dateStart === undefined ? {} : { dateStart: opts.dateStart }), + ...(opts.dateEnd === undefined ? {} : { dateEnd: opts.dateEnd }), + }; + const totalRaw = store.grepCount(pattern, unscoped); + if (totalRaw === 0) return []; + return store.grep(pattern, { + limit: totalRaw ?? opts.scoped.size, + offset: 0, + ...unscoped, + withinMessageIds: opts.scoped, + }); +} + +/** A tool-scoped query search: the FTS query runs inside the rows the tool + * owns (its call rows plus the toolResult rows after them), so a word found + * only in a Pi result still matches. Paging and totals come from that row + * set, and the removed-branch probe reads the same set with removed rows. */ +function toolScopedView( + store: LcmStore, + redact: (text: string) => string, + params: { + query: string; + toolName: string; + limit: number; + offset: number; + includeRemoved: boolean; + expandTool: string; + withinMessageIds?: ReadonlySet; + role?: GrepFilter["role"]; + dateStart?: number; + dateEnd?: number; + }, +): ViewResult { + const { query, toolName, limit, offset, includeRemoved, expandTool } = params; + const roleOpts = { + ...(params.role === undefined ? {} : { role: params.role }), + ...(params.dateStart === undefined ? {} : { dateStart: params.dateStart }), + ...(params.dateEnd === undefined ? {} : { dateEnd: params.dateEnd }), + }; + const visibleIds = toolScopeMessageIds(store, toolName, { + includeRemoved, + withinMessageIds: params.withinMessageIds, + }); + let pattern = query; + const runAll = (text: string, ids: ReadonlySet, removed: boolean) => + scopedGrepAll(store, text, { includeRemoved: removed, ...roleOpts, scoped: ids }); + let all: ReturnType | undefined; + try { + all = runAll(pattern, visibleIds, includeRemoved); + } catch { + all = undefined; + } + if (all === undefined) { + pattern = ftsPhrase(query); + try { + all = runAll(pattern, visibleIds, includeRemoved); + } catch { + all = undefined; + } + } + if (all === undefined) { + return { + ok: false, + text: `Invalid search query ${JSON.stringify(query)}; try simpler keywords.`, + }; + } + const total = all.length; + const hits = all.slice(offset, offset + limit); + if (hits.length === 0) { + if (!includeRemoved) { + const allIds = toolScopeMessageIds(store, toolName, { + includeRemoved: true, + withinMessageIds: params.withinMessageIds, + }); + let hidden: ReturnType = []; + try { + hidden = runAll(pattern, allIds, true); + } catch { + hidden = []; + } + if (hidden.slice(offset, offset + limit).length > 0) { + return { + ok: true, + text: `Match(es) for ${JSON.stringify(query)} exist on a branch that left the active path; pass include_removed to read them.`, + details: { hits: 0, removed: hidden.slice(offset, offset + limit).length }, + }; + } + } + return { + ok: true, + text: `No matches for ${JSON.stringify(query)} in session history.`, + details: { hits: 0 }, + }; + } + let budget = RESULT_BUDGET_CHARS; + const lines: string[] = [ + total <= limit + ? `Found ${hits.length} match(es):` + : `Match(es) ${offset + 1}..${offset + hits.length} of ${total}:`, + ]; + for (const h of hits) { + if (budget <= 0) { + lines.push(`… ${hits.length - lines.length + 1} more (narrow your query)`); + break; + } + const pointer = h.coveringSummaryId === undefined ? "" : ` #${h.coveringSummaryId}`; + const removed = h.removed === true ? " [removed]" : ""; + const flat = redact(h.text.replace(/\s+/g, " ")); + const dropped = flat.length - MAX_LINE_CHARS; + const shown = + dropped > 0 + ? `${flat.slice(0, MAX_LINE_CHARS)} [lcm:more ${dropped} chars; read them with ${expandTool}({entry_id:"${h.entryId}"})]` + : flat; + const line = `[${h.entryId}] (${h.role})${pointer}${removed} ${shown}`; + budget -= line.length; + lines.push(line); + } + const seen = offset + hits.length; + if (seen < total) { + lines.push(`… ${total - seen} more; pass offset ${seen} for the next page.`); + } + return { + ok: true, + text: lines.join("\n"), + details: { hits: hits.length, offset, total }, + }; +} + function patternView( store: LcmStore, redact: (text: string) => string, @@ -172,6 +457,9 @@ function patternView( afterRowId: number; now: () => number; expandTool: string; + role?: GrepFilter["role"]; + dateStart?: number; + dateEnd?: number; }, ): ViewResult { const compiled = compilePattern(params.pattern, params.caseSensitive); @@ -257,6 +545,9 @@ function scanPaged( includeRemoved: boolean; afterRowId: number; now: () => number; + role?: GrepFilter["role"]; + dateStart?: number; + dateEnd?: number; }, deadlineAt: number, ): PatternScanOutcome { @@ -269,11 +560,15 @@ function scanPaged( stopped: "end", }; let skipped = 0; + const pageOpts = { + withinMessageIds: params.withinMessageIds, + includeRemoved: params.includeRemoved, + ...(params.role === undefined ? {} : { role: params.role }), + ...(params.dateStart === undefined ? {} : { dateStart: params.dateStart }), + ...(params.dateEnd === undefined ? {} : { dateEnd: params.dateEnd }), + }; for (;;) { - const page = store.messagePage(cursor, PATTERN_PAGE_ROWS, { - withinMessageIds: params.withinMessageIds, - includeRemoved: params.includeRemoved, - }); + const page = store.messagePage(cursor, PATTERN_PAGE_ROWS, pageOpts); if (page.length === 0) break; cursor = page[page.length - 1]!.rowid; const part = scanRows(page, { @@ -317,6 +612,7 @@ export function describeView( const costs = nodeCosts(store, node, children); const lines = [ `Summary #${node.id} (${node.kind}, depth ${node.depth}, ${node.tokens} tokens)`, + `Quality: ${node.quality}`, `Span: ${node.firstEntryId} .. ${node.lastEntryId}`, `Costs: source ${costs.source} tokens across ${costs.messages} messages; subtree ${costs.subtree} tokens`, ...((node.thoroughText ?? "") === "" @@ -334,7 +630,13 @@ export function describeView( return { ok: true, text: lines.join("\n"), - details: { id: node.id, kind: node.kind, depth: node.depth, tokens: node.tokens }, + details: { + id: node.id, + kind: node.kind, + depth: node.depth, + tokens: node.tokens, + quality: node.quality, + }, }; } @@ -394,11 +696,19 @@ function readWindow( }; } -/** List the summary's messages, or read one of them as a window of characters. - * A message read without a summary names an entry directly, which is how an - * entry that no leaf covers yet is read at all. */ +/** List a summary's messages, or read one as a window. Without a summary an + * entry id reads directly; with `anchor` and offset 0 the window centers on + * that entry, clamped at the span edges. A nonzero offset with `anchor` is an + * absolute message index, so the next page starts from the reported offset. */ export type ExpandParams = - | { mode: "list"; id: number; offset: number; limit: number; maxChars: number } + | { + mode: "list"; + id: number; + offset: number; + limit: number; + maxChars: number; + anchor?: string; + } | { mode: "message"; id?: number; entryId: string; charOffset: number; maxChars: number }; export function expandView( @@ -410,8 +720,38 @@ export function expandView( const node = store.getSummary(params.id); if (!node) return { ok: false, text: `No summary with id ${params.id}.` }; const messages = store.messagesByIds(store.coveredMessageIds(params.id)); - const offset = Math.max(0, params.offset); - const slice = messages.slice(offset, offset + Math.max(0, params.limit)); + const limit = Math.max(0, params.limit); + let offset = Math.max(0, params.offset); + let anchorIndex: number | undefined; + if (params.anchor !== undefined) { + // Resolve the anchor through the span, not the covered list, so a typo + // and an entry outside the summary read differently. + const bounds = store.spanBounds([node.firstEntryId, node.lastEntryId]); + const spanMessages = + bounds === null ? [] : store.messagesInSpan(bounds.firstEntryId, bounds.lastEntryId); + const at = spanMessages.findIndex((m) => m.entryId === params.anchor); + if (at === -1) return { ok: false, text: `No stored message with entry id ${params.anchor}.` }; + const coveredAt = messages.findIndex((m) => m.entryId === params.anchor); + if (coveredAt === -1) { + return { + ok: false, + text: `Message ${params.anchor} is not covered by summary #${params.id}.`, + }; + } + anchorIndex = coveredAt; + // With offset 0 the window centers on the anchor; a nonzero offset is an + // absolute message index so a caller paging from the reported offset moves + // forward without overlap. + if (Math.max(0, params.offset) === 0) { + offset = Math.max( + 0, + Math.min(coveredAt - Math.floor(limit / 2), Math.max(0, messages.length - limit)), + ); + } else { + offset = Math.max(0, params.offset); + } + } + const slice = messages.slice(offset, offset + limit); const parts: string[] = []; for (const m of slice) { parts.push( @@ -419,26 +759,142 @@ export function expandView( ); } const hasMore = offset + slice.length < messages.length; - const listed = `Summary #${params.id} → ${messages.length} original messages (showing ${slice.length} from ${offset})${hasMore ? ", use offset to page" : ""}`; + const listed = `Summary #${params.id} → ${messages.length} original messages (showing ${slice.length} from ${offset})${anchorIndex !== undefined ? ` anchored at ${params.anchor}` : ""}${hasMore ? ", use offset to page" : ""}`; return { ok: true, text: `${listed}\n\n${parts.join("\n\n")}`, - details: { total: messages.length, offset }, + details: { + total: messages.length, + offset, + ...(anchorIndex === undefined ? {} : { anchor: params.anchor, anchorIndex }), + }, }; } -export function historyManifest(store: LcmStore, limit: number): string { +/** Tie-break for display order only, not a ranking model. */ +const OVERLAP_STOP_WORDS = new Set( + "a,an,and,are,as,at,be,but,by,for,from,had,has,have,how,i,in,is,it,its,of,on,or,that,the,their,then,there,these,they,this,to,was,were,what,when,where,which,who,with,you,your".split( + ",", + ), +); + +function queryWords(query: string): Set { + const words = query + .toLowerCase() + .split(/[^a-z0-9]+/) + .filter((w) => w.length > 2 && !OVERLAP_STOP_WORDS.has(w)); + return new Set(words); +} + +/** Sort headers the caller already holds, so FTS plus regex stay the recall + * path and no score is stored. Ties keep input order. */ +export function orderSummariesByOverlap( + query: string, + nodes: readonly T[], +): T[] { + const words = queryWords(query); + if (words.size === 0) return [...nodes]; + return nodes + .map((node, index) => { + const haystack = new Set(node.text.toLowerCase().split(/[^a-z0-9]+/)); + let score = 0; + for (const w of words) if (haystack.has(w)) score++; + return { node, index, score }; + }) + .sort((a, b) => b.score - a.score || a.index - b.index) + .map((r) => r.node); +} + +/** At or above this length with no linked file a body reads as `large-inline`, + * matching the ingest default, so view and writer agree. */ +export const LARGE_INLINE_CHARS = 32_000; + +/** A read-only files manifest, built at read time, so nothing is stored and + * no model is called. */ +export function filesView( + store: LcmStore, + redact: (text: string) => string, + opts?: { summaryId?: number }, +): ViewResult { + let firstEntryId: string; + let lastEntryId: string; + let messageIds: number[]; + if (opts?.summaryId !== undefined) { + const node = store.getSummary(opts.summaryId); + if (!node) return { ok: false, text: `No summary with id ${opts.summaryId}.` }; + firstEntryId = node.firstEntryId; + lastEntryId = node.lastEntryId; + messageIds = store.coveredMessageIds(node.id); + } else { + const span = store.entrySpan(); + if (!span) return { ok: true, text: "No history is stored yet.", details: { files: 0 } }; + firstEntryId = span.first; + lastEntryId = span.last; + messageIds = store.allMessages().map((m) => m.id); + } + const linked = store.filesForMessages(messageIds); + const seen = new Map(); + for (const f of linked) if (!seen.has(f.fileId)) seen.set(f.fileId, f); + const files = [...seen.values()].sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0)); + const withFiles = new Set(linked.map((f) => f.messageId)); + const uncovered = store.uncoveredMessagesInSpan(firstEntryId, lastEntryId); + const bodies = store.messagesInSpan(firstEntryId, lastEntryId); + const largeInline = bodies.filter( + (m) => m.text.length >= LARGE_INLINE_CHARS && !withFiles.has(m.id), + ); + const lines = [ + `Files linked in ${firstEntryId} .. ${lastEntryId}: ${files.length} file(s), ${uncovered.length} uncovered message(s).`, + ]; + for (const f of files) { + const structure = redact(f.preview.split("\n")[0] ?? ""); + lines.push(`${f.path} (${f.fileId}, ${f.bytes} chars, ${f.kind}; ${structure})`); + } + if (uncovered.length > 0) { + lines.push( + `[lcm:incomplete-coverage ${uncovered.length} message(s) in this span are covered by no summary]`, + ); + } + for (const m of largeInline) { + lines.push(`[lcm:large-inline ${m.entryId} holds ${m.text.length} chars with no linked file]`); + } + if (files.length === 0 && uncovered.length === 0 && largeInline.length === 0) { + lines.push("No files are linked in this span."); + } + return { + ok: true, + text: lines.join("\n"), + details: { + files: files.length, + uncovered: uncovered.length, + largeInline: largeInline.map((m) => m.entryId), + ...(uncovered.length > 0 ? { incompleteCoverage: true } : {}), + }, + }; +} + +export function historyManifest(store: LcmStore, limit: number, query?: string): string { const span = store.entrySpan(); if (!span) return "No history is stored yet."; const frontier = store.frontier(span.first, span.last); if (frontier.length === 0) { return `${store.messageCount()} messages, ${span.first} .. ${span.last}; no summaries yet, so lcm_grep searches the raw messages.`; } - const shown = frontier.slice(-limit).reverse(); - const omitted = frontier.length - shown.length; + const queryOrdered = query !== undefined && query.trim().length > 0; + // Frontier reads oldest first; score a newest-first copy so ties and + // wordless queries keep the newest summaries in a limited manifest. + const ordered = queryOrdered + ? orderSummariesByOverlap(query ?? "", [...frontier].reverse()).slice(0, limit) + : frontier.slice(-limit).reverse(); + const omitted = frontier.length - ordered.length; + const omittedNote = + omitted <= 0 + ? "" + : queryOrdered + ? ` (${omitted} lower-overlap not listed)` + : ` (${omitted} older not listed)`; return [ - `${frontier.length} top-level summaries cover messages ${span.first} .. ${span.last}${omitted > 0 ? ` (${omitted} older not listed)` : ""}:`, - ...shown.map((n) => costLine(store, n)), + `${frontier.length} top-level summaries cover messages ${span.first} .. ${span.last}${omittedNote}:`, + ...ordered.map((n) => costLine(store, n)), ].join("\n"); } diff --git a/packages/core/src/report.ts b/packages/core/src/report.ts index 69d91a3..ec2bab9 100644 --- a/packages/core/src/report.ts +++ b/packages/core/src/report.ts @@ -105,6 +105,7 @@ const NUMBER_FIELDS = [ "failed", "surfaceTokens", "turns", + "entries", ] as const; const STRING_FIELDS = [ "event", @@ -136,6 +137,7 @@ const STRING_FIELDS = [ "quality", "qualityBefore", "qualityAfter", + "watermark", ] as const; const BOOLEAN_FIELDS = [ "committed", diff --git a/packages/core/src/retrieval.ts b/packages/core/src/retrieval.ts index f2f83bd..a38d3a6 100644 --- a/packages/core/src/retrieval.ts +++ b/packages/core/src/retrieval.ts @@ -79,10 +79,15 @@ const LOOP_TOOLS: ReaderTool[] = [ { name: "lcm_expand", description: - "Read the original messages a summary covers, or one named entry. Without entry_id, list `limit` messages from message index `offset`. With entry_id, read that one message from `char_offset` for up to `max_chars` characters; continue from the char_offset the result reports. `id` is the summary to list, or to check that the entry is covered; omit it to read an entry that no summary covers yet.", + "Read the original messages a summary covers, or one named entry. Without entry_id, list `limit` messages from message index `offset`, or centered on `anchor` when it names an entry id. With entry_id, read that one message from `char_offset` for up to `max_chars` characters; continue from the char_offset the result reports. `id` is the summary to list, or to check that the entry is covered; omit it to read an entry that no summary covers yet.", parameters: Type.Object({ id: Type.Optional(Type.Integer({ description: "Summary id, the N in [lcm:summary #N ...]" })), entry_id: Type.Optional(Type.String({ description: "Read this one message by entry id" })), + anchor: Type.Optional( + Type.String({ + description: "Center the listing window on this entry id (with id, without entry_id)", + }), + ), offset: Type.Optional( Type.Integer({ minimum: 0, description: "Message index to start listing from" }), ), @@ -212,7 +217,7 @@ function brief(store: LcmStore, request: RetrievalRequest): string { ]; const address = addressedLine(store, request.scope); if (address !== undefined) lines.push("", address); - lines.push("", historyManifest(store, MANIFEST_NODES)); + lines.push("", historyManifest(store, MANIFEST_NODES, request.query)); return lines.join("\n"); } @@ -296,6 +301,7 @@ function renderTool( case "lcm_expand": { const id = asInt(args["id"]); const entryId = asString(args["entry_id"]); + const anchor = asString(args["anchor"]); const maxChars = Math.min(asInt(args["max_chars"]) ?? MAX_EXPAND_CHARS, MAX_EXPAND_CHARS); const offset = asInt(args["offset"]) ?? 0; if (entryId === undefined) { @@ -307,6 +313,7 @@ function renderTool( offset, limit: asInt(args["limit"]) ?? 25, maxChars, + ...(anchor === undefined ? {} : { anchor }), }); } return expandView(store, redact, { diff --git a/packages/core/src/sessions.ts b/packages/core/src/sessions.ts index 18f39bf..8971e02 100644 --- a/packages/core/src/sessions.ts +++ b/packages/core/src/sessions.ts @@ -4,6 +4,7 @@ import { join } from "node:path"; import { Type, type Static } from "typebox"; import { Compile } from "typebox/compile"; +import { ftsPhrase, scopedGrepAll, toolScopeMessageIds } from "./recall-view.ts"; import { hashSessionPath } from "./retention.ts"; import { LcmStore, StaleStoreError, type GrepHit } from "./store.ts"; @@ -212,6 +213,10 @@ export function searchSessions( deadlineMs?: number; perSession?: number; now?: () => number; + role?: "user" | "assistant" | "toolResult" | "custom"; + dateStart?: number; + dateEnd?: number; + toolName?: string; }, ): SessionSearchResult { const now = opts?.now ?? Date.now; @@ -243,7 +248,12 @@ export function searchSessions( } const store = opened.store; try { - const found = searchStore(store, query, Math.min(perSession, limit - hits.length)); + const found = searchStore(store, query, Math.min(perSession, limit - hits.length), { + ...(opts?.role === undefined ? {} : { role: opts.role }), + ...(opts?.dateStart === undefined ? {} : { dateStart: opts.dateStart }), + ...(opts?.dateEnd === undefined ? {} : { dateEnd: opts.dateEnd }), + ...(opts?.toolName === undefined ? {} : { toolName: opts.toolName }), + }); scanned++; for (const hit of found) { hits.push({ @@ -331,14 +341,45 @@ export function sessionsForScope(opts: { }); } -function searchStore(store: LcmStore, query: string, limit: number): GrepHit[] { - try { - return store.grep(query, { limit }); - } catch { +function searchStore( + store: LcmStore, + query: string, + limit: number, + filters?: { + role?: "user" | "assistant" | "toolResult" | "custom"; + dateStart?: number; + dateEnd?: number; + toolName?: string; + }, +): GrepHit[] { + const grepFilters = { + ...(filters?.role === undefined ? {} : { role: filters.role }), + ...(filters?.dateStart === undefined ? {} : { dateStart: filters.dateStart }), + ...(filters?.dateEnd === undefined ? {} : { dateEnd: filters.dateEnd }), + }; + if (filters?.toolName === undefined) { try { - return store.grep(JSON.stringify(query), { limit }); + return store.grep(query, { limit, ...grepFilters }); } catch { - return []; + try { + return store.grep(ftsPhrase(query), { limit, ...grepFilters }); + } catch { + return []; + } } } + const scopedIds = toolScopeMessageIds(store, filters.toolName, {}); + const searchIn = (ids: ReadonlySet, text: string) => { + try { + return scopedGrepAll(store, text, { + includeRemoved: false, + ...grepFilters, + scoped: ids, + }).slice(0, limit); + } catch { + return undefined; + } + }; + const hits = searchIn(scopedIds, query) ?? searchIn(scopedIds, ftsPhrase(query)); + return hits ?? []; } diff --git a/packages/core/src/store.ts b/packages/core/src/store.ts index ed2f1cf..b69d0ad 100644 --- a/packages/core/src/store.ts +++ b/packages/core/src/store.ts @@ -1,5 +1,14 @@ -import { chmodSync, closeSync, copyFileSync, existsSync, mkdirSync, openSync } from "node:fs"; -import { dirname } from "node:path"; +import { + chmodSync, + closeSync, + copyFileSync, + existsSync, + mkdirSync, + openSync, + readdirSync, + statSync, +} from "node:fs"; +import { dirname, join, resolve } from "node:path"; import { DatabaseSync, type StatementSync } from "node:sqlite"; import { estimateTokens } from "./estimate-tokens.ts"; @@ -104,6 +113,57 @@ export interface IntegritySnapshot { foreignKeyViolations: number; schemaVersion: number; uncoveredMessages: number; + /** Full paths, sorted, so doctor can name what a manual drop would remove. */ + orphanSidecars: string[]; +} + +export interface IntegritySnapshotOptions { + /** Paths with an open store in this process; every other sidecar file in + * `sidecarDir` reports as unopened, even when its session still exists. */ + liveSidecarPaths?: readonly string[]; + sidecarDir?: string; + sidecarExtension?: string; +} + +/** Files in `dir` with no open store in this process; missing dirs and non-files read as empty. */ +function findOrphanSidecars( + dir: string | undefined, + livePaths: readonly string[], + extension: string, +): string[] { + if (dir === undefined) return []; + let names: string[]; + try { + names = readdirSync(dir); + } catch { + return []; + } + const live = new Set(livePaths.map((p) => resolve(p))); + const out: string[] = []; + for (const name of names) { + const full = join(dir, name); + if (name.endsWith(extension)) { + if (live.has(resolve(full))) continue; + try { + if (!statSync(full).isFile()) continue; + } catch { + continue; + } + out.push(full); + continue; + } + if (name.endsWith(`${extension}-wal`) || name.endsWith(`${extension}-shm`)) { + const base = full.slice(0, -4); + if (live.has(resolve(base))) continue; + try { + if (!statSync(full).isFile()) continue; + } catch { + continue; + } + out.push(full); + } + } + return out.sort((a, b) => (a < b ? -1 : a > b ? 1 : 0)); } /** A fresh row, or the row that already owns the span; `existingTokens` comes @@ -408,6 +468,17 @@ function createSpanIndex(db: DatabaseSync): void { db.exec(CREATE_SPAN_INDEX); } +/** One covering index for the recall role and time filters. It lives outside + * `SCHEMA` on purpose: old stores gain it on their next writer open without a + * generation bump, and a read-only open never needs it because the predicates + * also run without it. */ +const CREATE_ROLE_TIME_INDEX = + "CREATE INDEX IF NOT EXISTS idx_messages_role_timestamp ON messages(role, timestamp)"; + +function createRoleTimeIndex(db: DatabaseSync): void { + db.exec(CREATE_ROLE_TIME_INDEX); +} + function reason(error: unknown): string { return error instanceof Error ? error.message : String(error); } @@ -564,6 +635,9 @@ export class LcmStore { try { createSpanIndex(this.db); } catch {} + try { + createRoleTimeIndex(this.db); + } catch {} } this.generation = readVersion(this.db); const prep = (sql: string): StatementSync => this.db.prepare(sql); @@ -1191,17 +1265,25 @@ export class LcmStore { offset?: number; withinMessageIds?: ReadonlySet; includeRemoved?: boolean; + role?: LcmRole; + dateStart?: number; + dateEnd?: number; }, ): GrepHit[] { const limit = opts?.limit ?? 20; const offset = Math.max(0, opts?.offset ?? 0); const scope = opts?.withinMessageIds; + if (opts?.role !== undefined && !isLcmRole(opts.role)) return []; const want = limit + offset; - const rows = this.stmts.grep.all( + const { sql, args } = grepSql( pattern, - opts?.includeRemoved === true ? 1 : 0, + opts?.includeRemoved === true, + opts?.role, + opts?.dateStart, + opts?.dateEnd, scope ? Math.max(want, scope.size + want) : want, - ) as Array<{ + ); + const rows = this.db.prepare(sql).all(...args) as Array<{ id: number; entry_id: string; role: string; @@ -1232,7 +1314,13 @@ export class LcmStore { grepCount( pattern: string, - opts?: { withinMessageIds?: ReadonlySet; includeRemoved?: boolean }, + opts?: { + withinMessageIds?: ReadonlySet; + includeRemoved?: boolean; + role?: LcmRole; + dateStart?: number; + dateEnd?: number; + }, ): number | undefined { const scope = opts?.withinMessageIds; if (scope) { @@ -1240,12 +1328,21 @@ export class LcmStore { limit: scope.size, withinMessageIds: scope, includeRemoved: opts?.includeRemoved === true, + ...(opts?.role === undefined ? {} : { role: opts.role }), + ...(opts?.dateStart === undefined ? {} : { dateStart: opts.dateStart }), + ...(opts?.dateEnd === undefined ? {} : { dateEnd: opts.dateEnd }), }).length; } + if (opts?.role !== undefined && !isLcmRole(opts.role)) return 0; try { - const row = this.stmts.grepCount.get(pattern, opts?.includeRemoved === true ? 1 : 0) as - | { c: number } - | undefined; + const { sql, args } = grepCountSql( + pattern, + opts?.includeRemoved === true, + opts?.role, + opts?.dateStart, + opts?.dateEnd, + ); + const row = this.db.prepare(sql).get(...args) as { c: number } | undefined; if (!row) return undefined; return Number(row.c); } catch { @@ -1256,15 +1353,26 @@ export class LcmStore { messagePage( afterRowId: number, limit: number, - opts?: { withinMessageIds?: ReadonlySet; includeRemoved?: boolean }, + opts?: { + withinMessageIds?: ReadonlySet; + includeRemoved?: boolean; + role?: LcmRole; + dateStart?: number; + dateEnd?: number; + }, ): ScannedRow[] { + if (opts?.role !== undefined && !isLcmRole(opts.role)) return []; const scope = opts?.withinMessageIds; const want = scope ? Math.max(limit, scope.size + limit) : limit; - const rows = this.stmts.messagePage.all( + const { sql, args } = messagePageSql( afterRowId, - opts?.includeRemoved === true ? 1 : 0, + opts?.includeRemoved === true, + opts?.role, + opts?.dateStart, + opts?.dateEnd, want, - ) as Array<{ + ); + const rows = this.db.prepare(sql).all(...args) as Array<{ id: number; entry_id: string; role: string; @@ -1369,7 +1477,7 @@ export class LcmStore { return out; } - integritySnapshot(): IntegritySnapshot { + integritySnapshot(opts?: IntegritySnapshotOptions): IntegritySnapshot { let ftsOk = true; // Structural damage is the one signal an external-content FTS5 table // reports: counts and rowid scans read the content table instead. @@ -1428,6 +1536,11 @@ export class LcmStore { foreignKeyViolations: this.db.prepare("PRAGMA foreign_key_check").all().length, schemaVersion: readVersion(this.db), uncoveredMessages: num((this.stmts.uncoveredMessages.get(null) as { c: number }).c), + orphanSidecars: findOrphanSidecars( + opts?.sidecarDir ?? (this.dbPath === ":memory:" ? undefined : dirname(this.dbPath)), + opts?.liveSidecarPaths ?? (this.dbPath === ":memory:" ? [] : [this.dbPath]), + opts?.sidecarExtension ?? ".sqlite", + ), }; } @@ -1728,5 +1841,101 @@ function rToSummary(r: Record): SummaryNode { }; } +function isLcmRole(value: unknown): value is LcmRole { + return value === "user" || value === "assistant" || value === "toolResult" || value === "custom"; +} + +/** Empty means no filter; the FTS `MATCH` stays the rank and access path either way. */ +function recallPredicates( + role: LcmRole | undefined, + dateStart: number | undefined, + dateEnd: number | undefined, +): { sql: string; args: Array } { + const parts: string[] = []; + const args: Array = []; + if (role !== undefined) { + parts.push("m.role = ?"); + args.push(role); + } + if (dateStart !== undefined) { + parts.push("m.timestamp >= ?"); + args.push(dateStart); + } + if (dateEnd !== undefined) { + parts.push("m.timestamp <= ?"); + args.push(dateEnd); + } + return { sql: parts.length === 0 ? "" : ` AND ${parts.join(" AND ")}`, args }; +} + +function grepSql( + pattern: string, + includeRemoved: boolean, + role: LcmRole | undefined, + dateStart: number | undefined, + dateEnd: number | undefined, + limit: number, +): { sql: string; args: Array } { + const extra = recallPredicates(role, dateStart, dateEnd); + return { + sql: + `SELECT m.id, m.entry_id, m.role, m.text, m.removed_at` + + ` FROM messages_fts f JOIN messages m ON m.id = f.rowid` + + ` WHERE messages_fts MATCH ?` + + ` AND (? = 1 OR m.removed_at IS NULL)` + + `${extra.sql} ORDER BY rank LIMIT ?`, + args: [pattern, includeRemoved ? 1 : 0, ...extra.args, limit], + }; +} + +function grepCountSql( + pattern: string, + includeRemoved: boolean, + role: LcmRole | undefined, + dateStart: number | undefined, + dateEnd: number | undefined, +): { sql: string; args: Array } { + const extra = recallPredicates(role, dateStart, dateEnd); + return { + sql: + `SELECT COUNT(*) c FROM messages_fts f JOIN messages m ON m.id = f.rowid` + + ` WHERE messages_fts MATCH ?` + + ` AND (? = 1 OR m.removed_at IS NULL)` + + `${extra.sql}`, + args: [pattern, includeRemoved ? 1 : 0, ...extra.args], + }; +} + +function messagePageSql( + afterRowId: number, + includeRemoved: boolean, + role: LcmRole | undefined, + dateStart: number | undefined, + dateEnd: number | undefined, + limit: number, +): { sql: string; args: Array } { + const parts: string[] = ["m.rowid > ?"]; + const args: Array = [afterRowId]; + if (!includeRemoved) parts.push("m.removed_at IS NULL"); + if (role !== undefined) { + parts.push("m.role = ?"); + args.push(role); + } + if (dateStart !== undefined) { + parts.push("m.timestamp >= ?"); + args.push(dateStart); + } + if (dateEnd !== undefined) { + parts.push("m.timestamp <= ?"); + args.push(dateEnd); + } + return { + sql: + `SELECT m.id, m.entry_id, m.role, m.text, m.removed_at FROM messages m` + + ` WHERE ${parts.join(" AND ")} ORDER BY m.rowid ASC LIMIT ?`, + args: [...args, limit], + }; +} + const CREATE_SPAN_INDEX = "CREATE UNIQUE INDEX IF NOT EXISTS idx_summaries_span ON summaries(first_entry_id, last_entry_id, kind)"; diff --git a/packages/core/src/summarize.ts b/packages/core/src/summarize.ts index db5eee5..70e1be7 100644 --- a/packages/core/src/summarize.ts +++ b/packages/core/src/summarize.ts @@ -170,7 +170,8 @@ async function tryLevel( } } -const LEVEL3_TOKENS = 512; +/** Same budget the ladder falls back to, so heal sees one shape. */ +export const LEVEL3_TOKENS = 512; export { TRUNCATED_SUMMARY_PREFIX } from "./store.ts"; diff --git a/packages/core/src/util/recall-tools.ts b/packages/core/src/util/recall-tools.ts index e640f10..5f0eeb6 100644 --- a/packages/core/src/util/recall-tools.ts +++ b/packages/core/src/util/recall-tools.ts @@ -11,9 +11,10 @@ export const RECALL_SCOPES = ["session", "sessions", "all_sessions"] as const; /** One of `RECALL_SCOPES`; `sessions` and `all_sessions` are equivalent and both read past sidecars. */ export type RecallScope = (typeof RECALL_SCOPES)[number]; -/** What `lcm_expand` reads: a summary listing by id, or one message by entry id with an optional summary check. */ +/** What `lcm_expand` reads: a summary listing by id (optionally centered on an + * anchor entry), or one message by entry id with an optional summary check. */ export type Address = - | { readonly kind: "summary"; readonly id: number } + | { readonly kind: "summary"; readonly id: number; readonly anchor?: string } | { readonly kind: "entry"; readonly entryId: string; readonly id?: number }; /** Past-search outcome: redacted text plus hit, scanned, and skipped session counts. */ @@ -45,6 +46,7 @@ export interface RecallToolDeps { query: string, scope: Exclude, limit: unknown, + filters?: GrepFilterArgs, ) => RecallPastSearchResult; readonly openPast?: (session: string, pastSession: string) => OpenPastResult; readonly pastSearchUnreachableText?: string; @@ -65,6 +67,16 @@ export function grepAnswer(deps: RecallToolDeps, callerSessionID: string, input: const scope = rawScope === undefined ? "session" : asScope(rawScope); const query = typeof args.query === "string" ? args.query : undefined; const queryChars = query?.length ?? 0; + const filters = readGrepFilters(args); + if (typeof filters === "string") { + recordRecall(deps, session, { + tool: "lcm_grep", + outcome: "error", + queryChars, + durationMs: Date.now() - started, + }); + return filters; + } if (scope === undefined) { recordRecall(deps, session, { tool: "lcm_grep", @@ -96,7 +108,7 @@ export function grepAnswer(deps: RecallToolDeps, callerSessionID: string, input: return "LCM: the reader is session-scoped, so it cannot search past sessions."; } if (scope !== "session") { - return grepPastSessions(deps, session, query, scope, started, args.limit); + return grepPastSessions(deps, session, query, scope, started, args.limit, filters); } const store = deps.storeFor(callerSessionID); if (!store) { @@ -112,6 +124,7 @@ export function grepAnswer(deps: RecallToolDeps, callerSessionID: string, input: const view = grepView(store, deps.redact, { query, ...(typeof args.limit === "number" ? { limit: args.limit } : {}), + ...filters, expandTool: "lcm_expand", }); const hits = view.ok && typeof view.details.hits === "number" ? view.details.hits : 0; @@ -125,6 +138,48 @@ export function grepAnswer(deps: RecallToolDeps, callerSessionID: string, input: return view.text; } +/** Optional `lcm_grep` filters, shared by the session search and the past-session + * scan. Unknown roles and bad timestamps are caller errors; an empty tool name + * is too, while an unknown-but-well-formed tool name simply matches nothing. */ +export interface GrepFilterArgs { + role?: "user" | "assistant" | "toolResult" | "custom"; + dateStart?: number; + dateEnd?: number; + toolName?: string; +} + +function readGrepFilters(args: Record): GrepFilterArgs | string { + const out: GrepFilterArgs = {}; + if (args.role !== undefined) { + if ( + args.role !== "user" && + args.role !== "assistant" && + args.role !== "toolResult" && + args.role !== "custom" + ) { + return "lcm_grep role must be user, assistant, toolResult, or custom."; + } + out.role = args.role; + } + for (const key of ["date_start", "date_end"] as const) { + const value = args[key]; + if (value === undefined) continue; + if (typeof value !== "number" || !Number.isFinite(value) || value < 0) { + return `lcm_grep ${key} must be a non-negative timestamp number.`; + } + if (key === "date_start") out.dateStart = value; + else out.dateEnd = value; + } + const toolKey = args.tool_name ?? args.toolName; + if (toolKey !== undefined) { + if (typeof toolKey !== "string" || toolKey.trim().length === 0) { + return "lcm_grep tool_name must be a non-empty tool name."; + } + out.toolName = toolKey; + } + return out; +} + function grepPastSessions( deps: RecallToolDeps, session: string, @@ -132,6 +187,7 @@ function grepPastSessions( scope: Exclude, started: number, limit: unknown, + filters?: GrepFilterArgs, ): string { if (!deps.searchPast) { recordRecall(deps, session, { @@ -143,7 +199,7 @@ function grepPastSessions( }); return deps.pastSearchUnreachableText ?? DEFAULT_PAST_SEARCH_UNREACHABLE; } - const result = deps.searchPast(session, query, scope, limit); + const result = deps.searchPast(session, query, scope, limit, filters); recordRecall(deps, session, { tool: "lcm_grep", outcome: result.hits === 0 ? "miss" : "hit", @@ -285,16 +341,26 @@ function expandPastSession( } } -/** Parse `id`/`entry_id` args into an `Address`: error text when malformed, undefined when neither is present. */ +/** Parse `id`/`entry_id`/`anchor` args into an `Address`: error text when malformed, undefined when neither is present. */ export function readAddress(args: Record): Address | string | undefined { const hasId = args.id !== undefined; const hasEntry = args.entry_id !== undefined; + const hasAnchor = args.anchor !== undefined; if (hasId && (typeof args.id !== "number" || !Number.isInteger(args.id) || args.id <= 0)) { return "Nothing was read: id must be a positive integer."; } if (hasEntry && (typeof args.entry_id !== "string" || args.entry_id.trim() === "")) { return "Nothing was read: entry_id must be a non-empty string."; } + if (hasAnchor && (typeof args.anchor !== "string" || args.anchor.trim() === "")) { + return "Nothing was read: anchor must be a non-empty entry id."; + } + if (hasAnchor && !hasId) { + return "Nothing was read: anchor centers a summary listing, so pass id with it."; + } + if (hasAnchor && hasEntry) { + return "Nothing was read: pass anchor or entry_id, not both."; + } if (hasEntry) { const entryId = args.entry_id as string; return { @@ -303,14 +369,24 @@ export function readAddress(args: Record): Address | string | u ...(typeof args.id === "number" ? { id: args.id } : {}), }; } - if (typeof args.id === "number") return { kind: "summary", id: args.id }; + if (typeof args.id === "number") { + return { + kind: "summary", + id: args.id, + ...(typeof args.anchor === "string" ? { anchor: args.anchor } : {}), + }; + } return undefined; } /** Refusal text when an address names no stored summary or message, else undefined. */ export function addressProblem(store: LcmStore, address: Address): string | undefined { if (address.kind === "summary") { - return store.getSummary(address.id) ? undefined : `No summary with id ${address.id}.`; + if (!store.getSummary(address.id)) return `No summary with id ${address.id}.`; + if (address.anchor !== undefined && !store.hasEntry(address.anchor)) { + return `No stored message with entry id ${address.anchor}.`; + } + return undefined; } if (!store.hasEntry(address.entryId)) { return `No stored message with entry id ${address.entryId}.`; @@ -344,6 +420,7 @@ export function readAddressed( offset: typeof args.offset === "number" ? args.offset : 0, limit: typeof args.limit === "number" ? args.limit : 25, maxChars, + ...(address.anchor === undefined ? {} : { anchor: address.anchor }), }); } @@ -386,6 +463,7 @@ function recordExpand( durationMs: Date.now() - started, ...(typeof args.id === "number" ? { summaryId: args.id } : {}), ...(typeof args.entry_id === "string" ? { entryId: args.entry_id } : {}), + ...(typeof args.anchor === "string" ? { anchor: args.anchor } : {}), }); } diff --git a/packages/core/src/util/summarizer-config.ts b/packages/core/src/util/summarizer-config.ts index 6ac64c5..e9dadac 100644 --- a/packages/core/src/util/summarizer-config.ts +++ b/packages/core/src/util/summarizer-config.ts @@ -24,6 +24,115 @@ export interface SummarizerOptions { readonly summarizerTokensPerPass?: number; /** Tokens one on-demand heal run may spend across all of its calls. */ readonly summarizerTokensPerHeal?: number; + /** Fires at compaction time only; ingest still stores every message verbatim. */ + readonly mechanicalTools?: readonly string[]; +} + +/** Empty default: callers opt in and existing passes keep their spend. */ +export const DEFAULT_MECHANICAL_TOOLS: readonly string[] = []; + +export function normalizeMechanicalTools(tools: readonly string[] | undefined): string[] { + if (!tools) return []; + const seen = new Set(); + const out: string[] = []; + for (const entry of tools) { + const name = entry.trim().toLowerCase(); + if (name.length === 0 || seen.has(name)) continue; + seen.add(name); + out.push(name); + } + return out; +} + +/** The tool name a message carries, if any. Reads both hosts: `tool ` + * (OpenCode), `[tool:]` (core), `tool-call: `, plus a JSON + * payload that names the call. Lowercased, else undefined. + * + * A marker only counts at a block start: `entryToText` joins content blocks + * with newlines, so a genuine call marker opens a line while prose quoting one + * sits mid-line. The caller may pass the stored role: `user` and `custom` rows + * never carry calls, and an `assistant` row mixing prose with a call keeps the + * prose on the summarizer path (tool-only rows still skip). */ +export function mechanicalToolNameOf(message: { + text: string; + payload?: string; + role?: string; +}): string | undefined { + if (message.role === "user" || message.role === "custom") return undefined; + if ( + message.role === "assistant" && + message.text.trim().length > 0 && + !isToolOnlyText(message.text) + ) { + return undefined; + } + const fromText = toolNameOfText(message.text, message.role); + if (fromText !== undefined) return fromText; + return toolNameOfPayload(message.payload); +} + +function toolNameOfText(text: string, role?: string): string | undefined { + const head = text.match(/^\s*tool\s+([A-Za-z0-9_:\-/.]+)/i); + if (head?.[1]) return head[1].toLowerCase(); + const lineMarker = /^\s*(?:\[tool:([^\]]+)\]|tool[_-]?call:\s*([A-Za-z0-9_:\-/.]+))/i; + if (role === "toolResult") { + // A result body may quote another tool's marker further down, so only the + // opening line can name the tool that produced it. + const first = text.split("\n").find((line) => line.trim().length > 0) ?? ""; + const hit = first.match(lineMarker); + const name = (hit?.[1] ?? hit?.[2])?.trim().toLowerCase(); + return name && name.length > 0 ? name : undefined; + } + for (const line of text.split("\n")) { + const hit = line.match(lineMarker); + const name = (hit?.[1] ?? hit?.[2])?.trim().toLowerCase(); + if (name && name.length > 0) return name; + } + return undefined; +} + +/** Every non-blank line opens a tool-call block, the shape `entryToText` + * renders for a message holding calls and nothing else. */ +function isToolOnlyText(text: string): boolean { + const lines = text.split("\n").filter((line) => line.trim().length > 0); + if (lines.length === 0) return false; + return lines.every((line) => + /^\s*(?:tool\s+[A-Za-z0-9_:\-/.]+|\[tool:[^\]]+\]|tool[_-]?call:\s*[A-Za-z0-9_:\-/.]+)/i.test( + line, + ), + ); +} + +function toolNameOfPayload(payload: string | undefined): string | undefined { + if (!payload) return undefined; + let parsed: unknown; + try { + parsed = JSON.parse(payload); + } catch { + return undefined; + } + const blocks = Array.isArray(parsed) ? parsed : [parsed]; + for (const block of blocks) { + if (!block || typeof block !== "object") continue; + const record = block as Record; + for (const key of ["name", "tool", "toolName"]) { + const value = record[key]; + if (typeof value === "string" && value.trim().length > 0) return value.trim().toLowerCase(); + } + } + return undefined; +} + +/** Normalized here, so callers may pass raw config values. */ +export function isMechanicalToolMessage( + message: { text: string; payload?: string; role?: string }, + tools: readonly string[], +): boolean { + if (tools.length === 0) return false; + const name = mechanicalToolNameOf(message); + if (name === undefined) return false; + const allowed = tools.some((entry) => entry.trim().toLowerCase() === name); + return allowed; } /** The engine config for these options. An unresolved budget keeps the core diff --git a/packages/core/test/files.test.ts b/packages/core/test/files.test.ts index ab2eb93..d9911d9 100644 --- a/packages/core/test/files.test.ts +++ b/packages/core/test/files.test.ts @@ -6,7 +6,7 @@ import { DatabaseSync } from "node:sqlite"; import { test } from "vite-plus/test"; -import { EXPORT_VERSION, exportTranscript } from "../src/export.ts"; +import { EXPORT_VERSION, exportTranscript, type ExportRow } from "../src/export.ts"; import { describeFile, filePreviewText, kindOf } from "../src/files.ts"; import { importTranscript, parseExport } from "../src/import.ts"; import { ingestEntries } from "../src/ingest.ts"; @@ -360,3 +360,97 @@ test("import: a file row naming an entry the file does not hold is refused", () assert.equal(target.stats().files, 0); target.close(); }); + +test("export/import/export: a file store round-trips byte-identical", () => { + const source = new LcmStore(":memory:"); + const body = jsonBody(400); + ingestEntries( + source, + [ + { + entryId: "e0", + role: "toolResult", + text: body, + timestamp: 1, + fileHint: { path: "/x/big.json" }, + }, + ], + POLICY, + ); + const first = exportTranscript(source).toJSONL(); + const target = new LcmStore(":memory:"); + const parsed = parseExport(first); + assert.ok("rows" in parsed); + const result = importTranscript(target, (parsed as { rows: never[] }).rows); + assert.equal(result.refused, undefined); + const second = exportTranscript(target).toJSONL(); + assert.equal(second, first); + target.close(); + source.close(); +}); + +test("import: a file row whose hash does not match its message is refused without writing", () => { + const source = new LcmStore(":memory:"); + const body = jsonBody(400); + ingestEntries( + source, + [ + { + entryId: "e0", + role: "toolResult", + text: body, + timestamp: 1, + fileHint: { path: "/x/big.json" }, + }, + ], + POLICY, + ); + const parsed = parseExport(exportTranscript(source).toJSONL()); + assert.ok("rows" in parsed); + const tampered: ExportRow[] = (parsed as { rows: ExportRow[] }).rows.map((r) => + r.rowType === "file" ? { ...r, sha256: "0".repeat(64) } : r, + ); + const target = new LcmStore(":memory:"); + const result = importTranscript(target, tampered); + assert.match(result.refused ?? "", /sha256/); + assert.equal(target.messageCount(), 0); + assert.equal(target.stats().files, 0); + target.close(); + source.close(); +}); + +test("import: a file row whose length does not match its message is refused, existing rows untouched", () => { + const source = new LcmStore(":memory:"); + const body = jsonBody(400); + ingestEntries( + source, + [ + { + entryId: "e0", + role: "toolResult", + text: body, + timestamp: 1, + fileHint: { path: "/x/big.json" }, + }, + ], + POLICY, + ); + const text = exportTranscript(source).toJSONL(); + const parsed = parseExport(text); + assert.ok("rows" in parsed); + const target = new LcmStore(":memory:"); + const first = importTranscript(target, (parsed as { rows: never[] }).rows); + assert.equal(first.refused, undefined); + assert.equal(target.stats().files, 1); + const before = target.allFiles()[0]!; + const tampered: ExportRow[] = (parsed as { rows: ExportRow[] }).rows.map((r) => + r.rowType === "file" ? { ...r, bytes: before.bytes + 1 } : r, + ); + const result = importTranscript(target, tampered); + assert.match(result.refused ?? "", /chars/); + assert.equal(target.stats().files, 1); + assert.equal(target.allFiles()[0]!.sha256, before.sha256); + assert.equal(target.allFiles()[0]!.bytes, before.bytes); + target.close(); + source.close(); +}); diff --git a/packages/core/test/heal.test.ts b/packages/core/test/heal.test.ts index 03a83c3..c14760d 100644 --- a/packages/core/test/heal.test.ts +++ b/packages/core/test/heal.test.ts @@ -873,3 +873,36 @@ test("heal: a leaf source uses the file handle and mask a compaction leaf would" store.close(); } }); + +test("heal: a mixed clipped leaf is repaired from its full provenance", async () => { + const store = new LcmStore(":memory:"); + try { + ingestEntries(store, [ + { entryId: "e0", role: "user", text: `ORDINARY-0 ${"p".repeat(50)}`, timestamp: 0 }, + { + entryId: "e1", + role: "toolResult", + text: `tool noisy-read {}\nNOISY-BODY ${"z".repeat(200)}`, + timestamp: 1, + }, + ]); + const messages = store.messagesInSpan("e0", "e1"); + const leaf = insertLeaf(store, messages, "ordinary only", undefined, "clipped"); + const calls: string[] = []; + const llm = scripted( + (user) => (user.includes("NOISY-BODY") ? "healed full" : "missing noisy"), + calls, + ); + + const outcome = await runHeal(store, { targetTokens: 100 }, llm); + + assert.deepEqual(outcome.repaired, [leaf]); + assert.equal(calls.length, 1); + assert.ok(calls[0]!.includes("ORDINARY-0")); + assert.ok(calls[0]!.includes("NOISY-BODY"), "heal includes noisy text on explicit repair"); + assert.equal(summary(store, leaf).quality, "clean"); + assert.equal(summary(store, leaf).text, "healed full"); + } finally { + store.close(); + } +}); diff --git a/packages/core/test/import.test.ts b/packages/core/test/import.test.ts index 048cf34..fa7fe8f 100644 --- a/packages/core/test/import.test.ts +++ b/packages/core/test/import.test.ts @@ -1,8 +1,14 @@ import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; import { test } from "vite-plus/test"; -import { EXPORT_VERSION, exportTranscript, type ExportSummaryRow } from "../src/export.ts"; +import { + EXPORT_VERSION, + exportTranscript, + type ExportRow, + type ExportSummaryRow, +} from "../src/export.ts"; import { importTranscript, parseExport } from "../src/import.ts"; import { ingestEntries } from "../src/ingest.ts"; import { LcmStore, TRUNCATED_SUMMARY_PREFIX } from "../src/store.ts"; @@ -239,6 +245,165 @@ test("import: provenance naming a child the file does not hold is refused", () = target.close(); }); +test("import: a repeated entry id with different text is refused before any write", () => { + const first = "first body for e00"; + const second = "second body for e00, different"; + const rows: ExportRow[] = [ + { + v: EXPORT_VERSION, + rowType: "message", + entryId: "e00", + role: "user", + text: first, + tokens: 1, + timestamp: 1, + }, + { + v: EXPORT_VERSION, + rowType: "message", + entryId: "e00", + role: "user", + text: second, + tokens: 1, + timestamp: 2, + }, + { + v: EXPORT_VERSION, + rowType: "file", + fileId: "0123456789abcdef", + path: "/x/a.txt", + bytes: second.length, + sha256: createHash("sha256").update(second).digest("hex"), + kind: "text", + preview: "preview", + entryIds: ["e00"], + }, + ]; + const target = new LcmStore(":memory:"); + const result = importTranscript(target, rows); + assert.match(result.refused ?? "", /repeats with different text/); + assert.equal(target.messageCount(), 0, "the conflicting rows write nothing"); + assert.equal(target.stats().files, 0); + target.close(); +}); + +test("import: a file row clashing with stored text for the same entry id is refused", () => { + const stored = "stored body for e00"; + const supplied = "supplied different body for e00"; + const target = new LcmStore(":memory:"); + ingestEntries(target, [{ entryId: "e00", role: "user", text: stored, timestamp: 1 }]); + const rows: ExportRow[] = [ + { + v: EXPORT_VERSION, + rowType: "message", + entryId: "e00", + role: "user", + text: supplied, + tokens: 1, + timestamp: 1, + }, + { + v: EXPORT_VERSION, + rowType: "file", + fileId: "0123456789abcdef", + path: "/x/a.txt", + bytes: supplied.length, + sha256: createHash("sha256").update(supplied).digest("hex"), + kind: "text", + preview: "preview", + entryIds: ["e00"], + }, + ]; + const result = importTranscript(target, rows); + assert.match(result.refused ?? "", /differs from the file/); + assert.equal(target.messageCount(), 1, "the stored row stays alone"); + assert.equal(target.messageByEntryId("e00")?.text, stored, "the stored text is untouched"); + assert.equal(target.stats().files, 0, "the file does not link to the wrong text"); + target.close(); +}); + +test("import: the stored-text check, message inserts, and file links share one transaction", () => { + const body = "body for e00"; + const rows: ExportRow[] = [ + { + v: EXPORT_VERSION, + rowType: "message", + entryId: "e00", + role: "user", + text: body, + tokens: 1, + timestamp: 1, + }, + { + v: EXPORT_VERSION, + rowType: "file", + fileId: "0123456789abcdef", + path: "/x/a.txt", + bytes: body.length, + sha256: createHash("sha256").update(body).digest("hex"), + kind: "text", + preview: "preview", + entryIds: ["e00"], + }, + ]; + const target = new LcmStore(":memory:"); + // A second connection cannot slip a same-entry row between the check and + // the links while this handle holds BEGIN IMMEDIATE, so the test records + // the boundary instead of racing a real second connection. + const events: string[] = []; + const runInner = target.transaction.bind(target); + target.transaction = (fn: () => T): T => { + events.push("begin"); + try { + const value = runInner(fn); + events.push("commit"); + return value; + } catch (error) { + events.push("rollback"); + throw error; + } + }; + const readRow = target.messageByEntryId.bind(target); + target.messageByEntryId = (entryId: string) => { + events.push(`check:${entryId}`); + return readRow(entryId); + }; + const linkRow = target.linkMessageFile.bind(target); + target.linkMessageFile = (messageId: number, fileId: string) => { + events.push(`link:${messageId}:${fileId}`); + linkRow(messageId, fileId); + }; + + const result = importTranscript(target, rows); + assert.equal(result.refused, undefined); + assert.equal(result.messagesInserted, 1); + const begin = events.indexOf("begin"); + const commit = events.indexOf("commit"); + assert.ok( + begin >= 0 && commit > begin, + `one transaction wraps the import, saw ${events.join(",")}`, + ); + assert.equal(events.filter((e) => e === "begin").length, 1, "the import commits once"); + const inside = events.slice(begin, commit + 1); + assert.ok( + inside.some((e) => e.startsWith("check:")), + "the stored-text check runs inside the transaction", + ); + assert.ok( + inside.some((e) => e.startsWith("link:")), + "the file links land before the commit", + ); + assert.ok( + !events.slice(0, begin).some((e) => e.startsWith("check:") || e.startsWith("link:")), + "no check or link runs before the transaction", + ); + assert.ok( + !events.slice(commit + 1).some((e) => e.startsWith("check:") || e.startsWith("link:")), + "no check or link runs after the commit", + ); + target.close(); +}); + test("import: a role outside the stored four folds on the way in", () => { const source = seeded(); const rows = ( diff --git a/packages/core/test/integrity.test.ts b/packages/core/test/integrity.test.ts index e6b4c2d..9c5a136 100644 --- a/packages/core/test/integrity.test.ts +++ b/packages/core/test/integrity.test.ts @@ -1,5 +1,5 @@ import assert from "node:assert/strict"; -import { mkdtempSync, rmSync } from "node:fs"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { DatabaseSync } from "node:sqlite"; @@ -52,12 +52,12 @@ function messagesOnly(): { dir: string; path: string; store: LcmStore } { return { dir, path, store }; } -test("integrity: a healthy store reports nine checks and no violation", () => { +test("integrity: a healthy store reports ten checks and no violation", () => { const { store } = fixture(); const report = checkIntegrity(store); assert.equal( report.checks, - 9, + 10, "one per FindingKind; the check table is exhaustive at compile time", ); assert.deepEqual(report.findings, [], "a leaf covers every message"); @@ -289,3 +289,59 @@ test("integrity: repair leaves a reversed span alone, because the fix needs the assert.equal(f.store.allSummaries().length, 2, "the row is retained and reported"); f.store.close(); }); + +test("integrity: an orphan sidecar reports the finding with its path", () => { + const dir = mkdtempSync(join(tmpdir(), "lcm-orphan-")); + const livePath = join(dir, "live.sqlite"); + const live = new LcmStore(livePath); + const orphanPath = join(dir, "orphan.sqlite"); + const orphan = new LcmStore(orphanPath); + orphan.close(); + writeFileSync(`${orphanPath}-wal`, "stale write-ahead log"); + writeFileSync(`${orphanPath}-shm`, "stale shared memory"); + try { + const snap = live.integritySnapshot(); + assert.deepEqual(snap.orphanSidecars, [orphanPath, `${orphanPath}-shm`, `${orphanPath}-wal`]); + const report = checkIntegrity(live); + const finding = report.findings.find((x) => x.kind === "orphan-sidecar"); + assert.ok(finding, JSON.stringify(report.findings)); + assert.equal(finding.severity, "info", "a file with no session is work, not a violation"); + assert.equal(finding.count, 3); + assert.ok( + finding.detail.includes("not open in this process"), + `the finding names unopened files, not gone sessions: ${finding.detail}`, + ); + assert.ok( + !finding.detail.includes("no live session"), + `the finding must not claim the session is gone: ${finding.detail}`, + ); + assert.ok(finding.sample.includes(orphanPath), finding.sample.join(", ")); + assert.ok( + !finding.sample.some((p) => p === livePath || p === `${livePath}-wal`), + "the live session never reports", + ); + } finally { + live.close(); + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("integrity: a live session sidecar never reports", () => { + const dir = mkdtempSync(join(tmpdir(), "lcm-orphan-")); + const firstPath = join(dir, "first.sqlite"); + const secondPath = join(dir, "second.sqlite"); + const first = new LcmStore(firstPath); + const second = new LcmStore(secondPath); + try { + const report = checkIntegrity(first, { liveSidecarPaths: [firstPath, secondPath] }); + assert.deepEqual( + report.findings.filter((x) => x.kind === "orphan-sidecar"), + [], + JSON.stringify(report.findings), + ); + } finally { + first.close(); + second.close(); + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/core/test/mechanical-tier.test.ts b/packages/core/test/mechanical-tier.test.ts new file mode 100644 index 0000000..bd00d49 --- /dev/null +++ b/packages/core/test/mechanical-tier.test.ts @@ -0,0 +1,578 @@ +import assert from "node:assert/strict"; + +import { test } from "vite-plus/test"; + +import { runCompaction } from "../src/compaction-engine.ts"; +import { estimateTokens } from "../src/estimate-tokens.ts"; +import { runHeal } from "../src/heal.ts"; +import { ingestEntries } from "../src/ingest.ts"; +import { checkIntegrity } from "../src/integrity.ts"; +import { describeView } from "../src/recall-view.ts"; +import { LcmStore, TRUNCATED_SUMMARY_PREFIX } from "../src/store.ts"; +import type { LlmComplete } from "../src/summarize.ts"; +import { + isMechanicalToolMessage, + mechanicalToolNameOf, + normalizeMechanicalTools, +} from "../src/util/summarizer-config.ts"; +import { storedPass } from "./support/pass-outcome.ts"; + +function countingLlm(reply: () => string, calls: string[]): LlmComplete { + return Object.assign( + async (_system: string, user: string, _max: number): Promise => { + calls.push(user); + return reply(); + }, + { metrics: () => {} }, + ); +} + +test("mechanical config: names normalize and match the stored renderings", () => { + assert.deepEqual(normalizeMechanicalTools([" Read ", "read", "", "BASH"]), ["read", "bash"]); + assert.deepEqual(normalizeMechanicalTools(undefined), []); + assert.equal(mechanicalToolNameOf({ text: `tool read {"path":"/x"}\nbody` }), "read"); + assert.equal(mechanicalToolNameOf({ text: `[tool:bash] {"cmd":"ls"}` }), "bash"); + assert.equal(mechanicalToolNameOf({ text: `tool-call: grep pattern` }), "grep"); + assert.equal(mechanicalToolNameOf({ text: "plain prose" }), undefined); + assert.equal(isMechanicalToolMessage({ text: `tool read {"p":1}\nbody` }, ["read"]), true); + assert.equal(isMechanicalToolMessage({ text: `tool read {"p":1}\nbody` }, ["bash"]), false); + assert.equal(isMechanicalToolMessage({ text: "plain" }, ["read"]), false); + assert.equal(isMechanicalToolMessage({ text: `tool read {}\nbody` }, []), false); +}); + +test("mechanical tier: a noisy span compacts with zero summarizer calls", async () => { + const store = new LcmStore(":memory:"); + try { + const span = [ + { + entryId: "e0", + role: "toolResult" as const, + text: `tool noisy-read {"path":"/a"}\n${"x".repeat(2000)}`, + }, + { + entryId: "e1", + role: "toolResult" as const, + text: `tool noisy-read {"path":"/b"}\n${"y".repeat(2000)}`, + }, + ]; + const calls: string[] = []; + const outcome = await storedPass( + store, + { span, targetTokens: 1500, mechanicalTools: ["noisy-read"] }, + countingLlm(() => "must not run", calls), + ); + assert.equal(calls.length, 0, "the skip path makes no model call"); + assert.equal(outcome.leafSummaryIds.length, 1); + const leaf = store.getSummary(outcome.leafSummaryIds[0]!); + assert.ok(leaf); + assert.equal(leaf.quality, "mechanical"); + assert.ok(leaf.text.startsWith(`${TRUNCATED_SUMMARY_PREFIX}\n`), leaf.text.slice(0, 40)); + assert.deepEqual(store.uncoveredMessagesInSpan("e0", "e1"), []); + const report = checkIntegrity(store); + assert.ok( + !report.findings.some((f) => f.kind === "coverage"), + `coverage stays complete: ${JSON.stringify(report.findings)}`, + ); + assert.equal(store.stats().messages, 2, "ingest keeps every message"); + } finally { + store.close(); + } +}); + +test("mechanical tier: mixed spans isolate noisy leaves and still call once for prose", async () => { + const store = new LcmStore(":memory:"); + try { + const span = [ + { entryId: "e0", role: "user" as const, text: `decide the plan ${"x".repeat(400)}` }, + { + entryId: "e1", + role: "toolResult" as const, + text: `tool noisy-read {}\n${"z".repeat(3000)}`, + }, + ]; + const calls: string[] = []; + const outcome = await storedPass( + store, + { span, targetTokens: 1500, mechanicalTools: ["noisy-read"] }, + countingLlm(() => "prose summary", calls), + { leafChunkTokens: 100_000 }, + ); + assert.equal(calls.length, 1, "only the prose chunk pays for a call"); + assert.ok(calls[0]!.includes("decide the plan")); + assert.ok(!calls[0]!.includes("noisy-read"), "the noisy body never reaches the model"); + const leaves = outcome.leafSummaryIds.map((id) => store.getSummary(id)!); + assert.equal(leaves.length, 2); + assert.deepEqual( + leaves.map((l) => [l.firstEntryId, l.quality]), + [ + ["e0", "clean"], + ["e1", "mechanical"], + ], + ); + assert.deepEqual(store.uncoveredMessagesInSpan("e0", "e1"), []); + } finally { + store.close(); + } +}); + +test("mechanical tier: quoted markers are not tool traffic", () => { + assert.equal( + isMechanicalToolMessage({ text: "I saw [tool:noisy] in the docs", role: "user" }, ["noisy"]), + false, + ); + assert.equal( + isMechanicalToolMessage( + { text: "note: the log said tool-call: noisy mid-line", role: "assistant" }, + ["noisy"], + ), + false, + ); + assert.equal( + isMechanicalToolMessage( + { text: 'Why this is safe.\n[tool:noisy] {"path":"/a"}', role: "assistant" }, + ["noisy"], + ), + false, + ); + assert.equal( + isMechanicalToolMessage({ text: '[tool:noisy] {"path":"/a"}', role: "assistant" }, ["noisy"]), + true, + ); +}); + +test("mechanical tier: mixed prose plus noisy call stays on the summarizer path", async () => { + const store = new LcmStore(":memory:"); + try { + const prose = "Here is why the migration is safe"; + const span = [ + { + entryId: "e0", + role: "assistant" as const, + text: `${prose}.\n[tool:noisy-read] {"path":"/a"}`, + }, + ]; + const calls: string[] = []; + const outcome = await storedPass( + store, + { span, targetTokens: 1500, mechanicalTools: ["noisy-read"] }, + countingLlm(() => "prose summary", calls), + ); + assert.equal(calls.length, 1, "mixed prose falls back to the model path"); + assert.ok(calls[0]!.includes(prose)); + const leaf = store.getSummary(outcome.leafSummaryIds[0]!); + assert.ok(leaf); + assert.equal(leaf.quality, "clean"); + assert.deepEqual(store.uncoveredMessagesInSpan("e0", "e0"), []); + } finally { + store.close(); + } +}); + +test("mechanical tier: alternating spans keep leaf work within maxChunks", async () => { + const store = new LcmStore(":memory:"); + try { + const span = Array.from({ length: 20 }, (_, i) => + i % 2 === 0 + ? { + entryId: `e${i}`, + role: "user" as const, + text: `ordinary question ${i} ${"p".repeat(50)}`, + } + : { + entryId: `e${i}`, + role: "toolResult" as const, + text: `tool noisy-read {"n":${i}}}\n${"z".repeat(200)}`, + }, + ); + const calls: string[] = []; + const outcome = await storedPass( + store, + { span, targetTokens: 1500, mechanicalTools: ["noisy-read"] }, + countingLlm(() => "summary", calls), + { leafChunkTokens: 100_000, maxChunks: 4 }, + ); + assert.ok(calls.length <= 4, `leaf attempts stay capped, got ${calls.length}`); + assert.ok(outcome.leafSummaryIds.length <= 4); + assert.deepEqual(store.uncoveredMessagesInSpan("e0", "e19"), []); + } finally { + store.close(); + } +}); + +test("mechanical tier: alternating over-cap spans never prompt noisy bodies", async () => { + const store = new LcmStore(":memory:"); + try { + const span = Array.from({ length: 20 }, (_, i) => + i % 2 === 0 + ? { + entryId: `e${i}`, + role: "user" as const, + text: `ORDINARY-${i} ${"p".repeat(50)}`, + } + : { + entryId: `e${i}`, + role: "toolResult" as const, + text: `tool noisy-read {"n":${i}}\nNOISY-BODY-${i} ${"z".repeat(200)}`, + }, + ); + const calls: string[] = []; + const outcome = await storedPass( + store, + { span, targetTokens: 1500, mechanicalTools: ["noisy-read"] }, + countingLlm(() => "summary", calls), + { leafChunkTokens: 100_000, maxChunks: 4 }, + ); + assert.ok(calls.length >= 1 && calls.length <= 4, `bounded attempts, got ${calls.length}`); + assert.ok(outcome.leafSummaryIds.length <= 4); + for (const prompt of calls) { + assert.ok(!prompt.includes("NOISY-BODY"), `noisy body leaked: ${prompt.slice(0, 120)}`); + assert.ok(!prompt.includes("noisy-read"), `noisy marker leaked: ${prompt.slice(0, 120)}`); + } + const joined = calls.join("\n"); + for (let i = 0; i < 20; i += 2) { + assert.ok(joined.includes(`ORDINARY-${i}`), `ordinary ${i} reaches a prompt`); + } + const allIds = store + .allMessages() + .map((m) => m.id) + .sort((a, b) => a - b); + const covered: number[] = []; + for (const id of outcome.leafSummaryIds) covered.push(...store.summarySources(id).messages); + assert.deepEqual( + [...covered].sort((a, b) => a - b), + allIds, + ); + assert.equal(new Set(covered).size, allIds.length, "no double summary"); + assert.deepEqual(store.uncoveredMessagesInSpan("e0", "e19"), []); + const frontier = store.frontier("e0", "e19"); + assert.deepEqual( + frontier.map((n) => n.id), + outcome.leafSummaryIds, + ); + assert.deepEqual( + frontier.map((n) => n.firstEntryId), + ["e0", "e5", "e10", "e15"], + ); + } finally { + store.close(); + } +}); + +test("mechanical tier: a Pi result follows its call onto the skip path", async () => { + const store = new LcmStore(":memory:"); + try { + const span = [ + { entryId: "e0", role: "assistant" as const, text: `[tool:noisy] {"path":"/a"}` }, + { entryId: "e1", role: "toolResult" as const, text: `large output ${"w".repeat(2000)}` }, + ]; + const calls: string[] = []; + const outcome = await storedPass( + store, + { span, targetTokens: 1500, mechanicalTools: ["noisy"] }, + countingLlm(() => "must not run", calls), + ); + assert.equal(calls.length, 0); + assert.equal(outcome.leafSummaryIds.length, 1); + assert.equal(store.getSummary(outcome.leafSummaryIds[0]!)?.quality, "mechanical"); + } finally { + store.close(); + } +}); + +test("mechanical tier: heal restores a skipped leaf from provenance on demand", async () => { + const store = new LcmStore(":memory:"); + try { + const span = [ + { + entryId: "e0", + role: "toolResult" as const, + text: `tool noisy-read {}\nSOURCE-BODY ${"q".repeat(500)}`, + }, + ]; + const calls: string[] = []; + const compacted = await storedPass( + store, + { span, targetTokens: 1500, mechanicalTools: ["noisy-read"] }, + countingLlm(() => "must not run", calls), + ); + const leafId = compacted.leafSummaryIds[0]!; + assert.equal(store.getSummary(leafId)?.quality, "mechanical"); + const healCalls: string[] = []; + const healLlm: LlmComplete = Object.assign( + async (_system: string, user: string, _max: number): Promise => { + healCalls.push(user); + return "healed summary"; + }, + { metrics: () => {} }, + ); + const healed = await runHeal(store, { targetTokens: 100 }, healLlm); + assert.deepEqual(healed.repaired, [leafId]); + assert.equal(healCalls.length, 1); + assert.ok(healCalls[0]!.includes("SOURCE-BODY")); + const after = store.getSummary(leafId)!; + assert.equal(after.text, "healed summary"); + assert.equal(after.quality, "clean"); + assert.equal(after.tokens, estimateTokens("healed summary")); + } finally { + store.close(); + } +}); + +test("mechanical tier: without the list the same span pays for a call", async () => { + const store = new LcmStore(":memory:"); + try { + const span = [ + { + entryId: "e0", + role: "toolResult" as const, + text: `tool noisy-read {}\n${"x".repeat(500)}`, + }, + ]; + const calls: string[] = []; + await storedPass( + store, + { span, targetTokens: 1500 }, + countingLlm(() => "summary", calls), + ); + assert.equal(calls.length, 1); + } finally { + store.close(); + } +}); + +test("mechanical tier: a lone fallback node without the list still pays for the terse path", async () => { + const store = new LcmStore(":memory:"); + try { + ingestEntries( + store, + Array.from({ length: 6 }, (_, i) => ({ + entryId: `e${i}`, + role: "user" as const, + text: `message ${i}`, + timestamp: i + 1, + })), + ); + const messageIds = store.uncoveredMessagesInSpan("e0", "e5").map((m) => m.id); + const leaf = store.insertSummary({ + kind: "leaf", + text: "leaf text", + tokens: 4, + depth: 0, + firstEntryId: "e0", + lastEntryId: "e5", + messageIds, + }).id; + // A level-3 fallback from before the tier existed: mechanical by verdict + // and by the stored prefix, with no opted-in tool behind it. + const node = store.insertSummary({ + kind: "condensed", + text: `${TRUNCATED_SUMMARY_PREFIX}\n${"long ".repeat(2000)}`, + tokens: 2500, + depth: 1, + quality: "mechanical", + firstEntryId: "e0", + lastEntryId: "e5", + childSummaryIds: [leaf], + }).id; + const terse = "terse ".repeat(40).trim(); + const calls: string[] = []; + const span = Array.from({ length: 6 }, (_, i) => ({ + entryId: `e${i}`, + role: "user" as const, + text: `message ${i}`, + })); + const outcome = await storedPass( + store, + { span, targetTokens: 500 }, + countingLlm(() => terse, calls), + { leafChunkTokens: 100_000 }, + ); + assert.equal(calls.length, 1, "the empty list keeps the terse re-summarize"); + assert.equal(store.getSummary(node)?.text, terse); + assert.deepEqual(outcome.condensedSummaryIds, []); + } finally { + store.close(); + } +}); + +test("mechanical tier: opts carry the list when the input does not", async () => { + const store = new LcmStore(":memory:"); + try { + const span = [ + { + entryId: "e0", + role: "toolResult" as const, + text: `tool noisy-read {}\n${"x".repeat(500)}`, + }, + ]; + const calls: string[] = []; + const outcome = await runCompaction( + store, + { span, targetTokens: 1500 }, + countingLlm(() => "must not run", calls), + { mechanicalTools: ["noisy-read"] }, + ); + assert.equal(outcome.kind, "stored"); + assert.equal(calls.length, 0); + } finally { + store.close(); + } +}); + +test("mechanical tier: merged mixed leaves stay clipped and heal restores detail", async () => { + const store = new LcmStore(":memory:"); + try { + const span = Array.from({ length: 20 }, (_, i) => + i % 2 === 0 + ? { + entryId: `e${i}`, + role: "user" as const, + text: `ORDINARY-${i} ${"p".repeat(50)}`, + } + : { + entryId: `e${i}`, + role: "toolResult" as const, + text: `tool noisy-read {"n":${i}}\nNOISY-BODY-${i} ${"z".repeat(200)}`, + }, + ); + const calls: string[] = []; + const outcome = await storedPass( + store, + { span, targetTokens: 1500, mechanicalTools: ["noisy-read"] }, + countingLlm(() => "summary", calls), + { leafChunkTokens: 100_000, maxChunks: 4 }, + ); + assert.ok(calls.length >= 1 && calls.length <= 4, `bounded attempts, got ${calls.length}`); + for (const prompt of calls) { + assert.ok(!prompt.includes("NOISY-BODY"), "noisy body never reaches a model call"); + assert.ok(!prompt.includes("noisy-read"), "noisy marker never reaches a model call"); + } + const joined = calls.join("\n"); + for (let i = 0; i < 20; i += 2) { + assert.ok(joined.includes(`ORDINARY-${i}`), `ordinary ${i} reaches a prompt`); + } + const leaves = outcome.leafSummaryIds.map((id) => store.getSummary(id)!); + assert.ok(leaves.length <= 4); + for (const leaf of leaves) { + assert.equal(leaf.quality, "clipped", "a mixed leaf never reads as clean"); + } + const allIds = store + .allMessages() + .map((m) => m.id) + .sort((a, b) => a - b); + const covered: number[] = []; + for (const id of outcome.leafSummaryIds) covered.push(...store.summarySources(id).messages); + assert.deepEqual( + [...covered].sort((a, b) => a - b), + allIds, + "every message stays in exactly one leaf", + ); + assert.equal(new Set(covered).size, allIds.length, "no double summary"); + assert.deepEqual(store.uncoveredMessagesInSpan("e0", "e19"), []); + const view = describeView(store, (t) => t, leaves[0]!.id); + assert.ok(view.ok); + if (!view.ok) return; + assert.ok(view.text.includes("Quality: clipped"), view.text); + assert.equal(view.details["quality"], "clipped"); + const healCalls: string[] = []; + const healLlm: LlmComplete = Object.assign( + async (_system: string, user: string, _max: number): Promise => { + healCalls.push(user); + return "healed summary"; + }, + { metrics: () => {} }, + ); + const healed = await runHeal(store, { targetTokens: 100 }, healLlm); + assert.deepEqual( + [...healed.repaired].sort((a, b) => a - b), + [...outcome.leafSummaryIds].sort((a, b) => a - b), + ); + assert.ok(healCalls.length >= 1); + assert.ok( + healCalls.join("\n").includes("NOISY-BODY"), + "heal reads the full rows including noisy text", + ); + for (const id of outcome.leafSummaryIds) { + assert.equal(store.getSummary(id)?.quality, "clean"); + } + } finally { + store.close(); + } +}); + +test("mechanical tier: a result naming another tool stays on the prose path", async () => { + const store = new LcmStore(":memory:"); + try { + const span = [ + { + entryId: "e0", + role: "toolResult" as const, + text: `tool noisy-read {"n":1}\nNOISY-BODY ${"x".repeat(500)}`, + }, + { + entryId: "e1", + role: "toolResult" as const, + text: `tool bash {"cmd":"ls"}\nBASH-BODY ${"y".repeat(500)}`, + }, + ]; + const calls: string[] = []; + const outcome = await storedPass( + store, + { span, targetTokens: 1500, mechanicalTools: ["noisy-read"] }, + countingLlm(() => "prose summary", calls), + { leafChunkTokens: 100_000 }, + ); + assert.equal(calls.length, 1, "only the unlisted tool pays for a call"); + assert.ok(calls[0]!.includes("BASH-BODY"), "the unlisted result reaches the model"); + assert.ok(!calls[0]!.includes("NOISY-BODY"), "listed noisy body never reaches a model call"); + const leaves = outcome.leafSummaryIds.map((id) => store.getSummary(id)!); + assert.equal(leaves.length, 2); + assert.deepEqual( + leaves.map((l) => [l.firstEntryId, l.quality]), + [ + ["e0", "mechanical"], + ["e1", "clean"], + ], + ); + const covered: number[] = []; + for (const id of outcome.leafSummaryIds) covered.push(...store.summarySources(id).messages); + const allIds = store + .allMessages() + .map((m) => m.id) + .sort((a, b) => a - b); + assert.deepEqual( + [...covered].sort((a, b) => a - b), + allIds, + ); + assert.equal(new Set(covered).size, allIds.length, "no double summary"); + assert.deepEqual(store.uncoveredMessagesInSpan("e0", "e1"), []); + } finally { + store.close(); + } +}); + +test("mechanical tier: an unnamed result after a listed result stays mechanical", async () => { + const store = new LcmStore(":memory:"); + try { + const span = [ + { + entryId: "e0", + role: "toolResult" as const, + text: `tool noisy-read {"n":1}\nNOISY-BODY ${"x".repeat(2000)}`, + }, + { entryId: "e1", role: "toolResult" as const, text: `plain output ${"w".repeat(2000)}` }, + ]; + const calls: string[] = []; + const outcome = await storedPass( + store, + { span, targetTokens: 1500, mechanicalTools: ["noisy-read"] }, + countingLlm(() => "must not run", calls), + ); + assert.equal(calls.length, 0, "the unnamed follow-up needs no model call"); + assert.equal(outcome.leafSummaryIds.length, 1); + assert.equal(store.getSummary(outcome.leafSummaryIds[0]!)?.quality, "mechanical"); + assert.deepEqual(store.uncoveredMessagesInSpan("e0", "e1"), []); + } finally { + store.close(); + } +}); diff --git a/packages/core/test/recall-upgrades.test.ts b/packages/core/test/recall-upgrades.test.ts new file mode 100644 index 0000000..5b2dbd6 --- /dev/null +++ b/packages/core/test/recall-upgrades.test.ts @@ -0,0 +1,764 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { test } from "vite-plus/test"; + +import { ingestEntries } from "../src/ingest.ts"; +import { + expandView, + filesView, + ftsPhrase, + grepView, + historyManifest, + LARGE_INLINE_CHARS, + orderSummariesByOverlap, + toolMarker, + toolScopeMessageIds, + withToolScope, +} from "../src/recall-view.ts"; +import { redactSecrets } from "../src/redact.ts"; +import { searchSessions } from "../src/sessions.ts"; +import { LcmStore } from "../src/store.ts"; + +function seedRoleTime(): LcmStore { + const store = new LcmStore(":memory:"); + ingestEntries( + store, + Array.from({ length: 6 }, (_, i) => ({ + entryId: `e${String(i).padStart(2, "0")}`, + role: (i % 2 === 0 ? "user" : "assistant") as "user" | "assistant", + text: `filterword message ${i} about the engine`, + timestamp: 1000 * (i + 1), + })), + ); + const ids = store.messagesInSpan("e00", "e05").map((m) => m.id); + store.insertSummary({ + kind: "leaf", + text: "leaf over the filterword span", + tokens: 4, + depth: 0, + firstEntryId: "e00", + lastEntryId: "e05", + messageIds: ids, + }); + return store; +} + +function hitIds(view: ReturnType): string[] { + assert.equal(view.ok, true); + return view.text + .split("\n") + .slice(1) + .filter((l) => l.startsWith("[")) + .map((l) => l.slice(1, l.indexOf("]"))); +} + +test("grepView: a role filter equals the unfiltered result plus the filter", () => { + const store = seedRoleTime(); + try { + const all = hitIds(grepView(store, redactSecrets, { query: "filterword" })); + assert.deepEqual(all, ["e00", "e01", "e02", "e03", "e04", "e05"]); + const users = hitIds(grepView(store, redactSecrets, { query: "filterword", role: "user" })); + assert.deepEqual(users, ["e00", "e02", "e04"]); + const asst = hitIds(grepView(store, redactSecrets, { query: "filterword", role: "assistant" })); + assert.deepEqual(asst, ["e01", "e03", "e05"]); + } finally { + store.close(); + } +}); + +test("grepView: a date range keeps its window and an empty range returns empty", () => { + const store = seedRoleTime(); + try { + const window = hitIds( + grepView(store, redactSecrets, { query: "filterword", dateStart: 2000, dateEnd: 4000 }), + ); + assert.deepEqual(window, ["e01", "e02", "e03"]); + const empty = grepView(store, redactSecrets, { + query: "filterword", + dateStart: 9000, + dateEnd: 1000, + }); + assert.equal(empty.ok, true); + assert.match(empty.text, /No matches/); + assert.deepEqual(empty.details, { hits: 0 }); + } finally { + store.close(); + } +}); + +test("grepView: the pattern path respects role and date filters", () => { + const store = seedRoleTime(); + try { + const users = grepView(store, redactSecrets, { pattern: "filterword", role: "user" }); + assert.equal(users.ok, true); + for (const want of ["e00 (user)", "e02 (user)", "e04 (user)"]) + assert.ok(users.text.includes(`[${want}]`), users.text); + assert.ok(!users.text.includes("e01 (assistant)"), users.text); + const window = grepView(store, redactSecrets, { + pattern: "filterword", + dateStart: 2000, + dateEnd: 2000, + }); + assert.equal(window.ok, true); + assert.ok(window.text.includes("[e01 (assistant)]"), window.text); + assert.ok(!window.text.includes("e02 (user)"), window.text); + } finally { + store.close(); + } +}); + +test("grepView: an unknown role is refused and bad timestamps are refused", () => { + const store = seedRoleTime(); + try { + const bad = grepView(store, redactSecrets, { + query: "filterword", + role: "owner" as "user", + }); + assert.equal(bad.ok, false); + const time = grepView(store, redactSecrets, { query: "filterword", dateStart: -1 }); + assert.equal(time.ok, false); + } finally { + store.close(); + } +}); + +function seedTools(): LcmStore { + const store = new LcmStore(":memory:"); + ingestEntries(store, [ + { + entryId: "t00", + role: "toolResult", + text: '[tool:read] {"path": "/a"} cache key rot13', + timestamp: 1, + }, + { + entryId: "t01", + role: "toolResult", + text: '[tool:write] {"path": "/b"} cache key rot13', + timestamp: 2, + }, + { entryId: "t02", role: "user", text: "cache key rot13 in plain prose", timestamp: 3 }, + { entryId: "t03", role: "assistant", text: "cache key rot13 answered", timestamp: 4 }, + ]); + return store; +} + +test("grepView: a tool scope equals the unfiltered query plus the tool filter", () => { + const store = seedTools(); + try { + const scoped = hitIds(grepView(store, redactSecrets, { query: "cache", toolName: "read" })); + assert.deepEqual(scoped, ["t00"]); + const write = hitIds(grepView(store, redactSecrets, { query: "cache", toolName: "write" })); + assert.deepEqual(write, ["t01"]); + assert.ok(!scoped.includes("t02") && !scoped.includes("t03")); + // An unknown tool name returns empty, not an error. + const unknown = grepView(store, redactSecrets, { query: "cache", toolName: "nosuchtool" }); + assert.equal(unknown.ok, true); + assert.match(unknown.text, /No matches/); + } finally { + store.close(); + } +}); + +test("grepView: toolName with a pattern is refused, not silently ignored", () => { + const store = seedTools(); + try { + const view = grepView(store, redactSecrets, { pattern: "cache", toolName: "read" }); + assert.equal(view.ok, false); + } finally { + store.close(); + } +}); + +test("orderSummariesByOverlap: query-word overlap orders headers deterministically", () => { + const nodes = [ + { text: "unrelated weather report" }, + { text: "the compaction engine keeps history" }, + { text: "engine room compaction notes" }, + { text: "compaction engine" }, + ]; + const ordered = orderSummariesByOverlap("compaction engine repair", nodes); + assert.deepEqual( + ordered.map((n) => n.text), + [ + "the compaction engine keeps history", + "engine room compaction notes", + "compaction engine", + "unrelated weather report", + ], + ); + // Stop words carry no weight. + assert.deepEqual(orderSummariesByOverlap("the and of", nodes), nodes); + const tied = orderSummariesByOverlap("compaction", [nodes[1]!, nodes[3]!]); + assert.deepEqual(tied, [nodes[1], nodes[3]]); +}); + +test("historyManifest: a query seeds overlap order without changing the set", () => { + const store = new LcmStore(":memory:"); + try { + ingestEntries(store, [ + { entryId: "m00", role: "user", text: "compaction engine keeps history", timestamp: 1 }, + { entryId: "m01", role: "user", text: "weather report", timestamp: 2 }, + { entryId: "m02", role: "user", text: "weather report", timestamp: 3 }, + ]); + for (const span of ["m00", "m01", "m02"]) { + const ids = store.messagesInSpan(span, span).map((m) => m.id); + store.insertSummary({ + kind: "leaf", + text: span === "m00" ? "compaction engine keeps history" : "weather report", + tokens: 2, + depth: 0, + firstEntryId: span, + lastEntryId: span, + messageIds: ids, + }); + } + const plain = historyManifest(store, 20); + const seeded = historyManifest(store, 20, "compaction engine"); + const idsOf = (text: string) => + [...text.matchAll(/#(\d+)/g)] + .flatMap((m) => (m[1] === undefined ? [] : [m[1]])) + .sort((a, b) => (a < b ? -1 : 1)); + assert.deepEqual(idsOf(plain), idsOf(seeded)); + assert.ok(plain.indexOf("m02 .. m02") < plain.indexOf("m00 .. m00"), plain); + assert.ok(seeded.indexOf("m00 .. m00") < seeded.indexOf("m02 .. m02"), seeded); + } finally { + store.close(); + } +}); + +test("filesView: lists each linked file once, flags gaps, marks large-inline", () => { + const store = new LcmStore(":memory:"); + try { + ingestEntries(store, [ + { entryId: "f00", role: "toolResult", text: "small body one", timestamp: 1 }, + { entryId: "f01", role: "toolResult", text: "small body two", timestamp: 2 }, + { entryId: "f02", role: "user", text: "plain prose with no file", timestamp: 3 }, + { entryId: "f03", role: "toolResult", text: `x`.repeat(LARGE_INLINE_CHARS), timestamp: 4 }, + ]); + const byEntry = new Map(store.allMessages().map((m) => [m.entryId, m.id])); + store.insertFile( + { + fileId: "file-a", + path: "/a.txt", + bytes: 10, + sha256: "s", + kind: "text", + preview: "preview a", + }, + "f00", + ); + store.linkMessageFile(byEntry.get("f00")!, "file-a"); + store.linkMessageFile(byEntry.get("f01")!, "file-a"); + const covered = [byEntry.get("f00")!, byEntry.get("f01")!]; + store.insertSummary({ + kind: "leaf", + text: "leaf over f00 f01", + tokens: 2, + depth: 0, + firstEntryId: "f00", + lastEntryId: "f01", + messageIds: covered, + }); + const view = filesView(store, redactSecrets); + assert.equal(view.ok, true); + assert.ok(view.text.includes("/a.txt (file-a"), view.text); + assert.equal(view.text.match(/\/a\.txt/g)?.length, 1, view.text); + assert.ok(view.text.includes("incomplete-coverage"), view.text); + assert.deepEqual(view.details, { + files: 1, + uncovered: 2, + largeInline: ["f03"], + incompleteCoverage: true, + }); + assert.ok(view.text.includes("large-inline"), view.text); + } finally { + store.close(); + } +}); + +test("filesView: a fully covered span with no files reports no gaps", () => { + const store = new LcmStore(":memory:"); + try { + ingestEntries(store, [{ entryId: "g00", role: "user", text: "hello", timestamp: 1 }]); + const ids = store.messagesInSpan("g00", "g00").map((m) => m.id); + store.insertSummary({ + kind: "leaf", + text: "leaf", + tokens: 1, + depth: 0, + firstEntryId: "g00", + lastEntryId: "g00", + messageIds: ids, + }); + const view = filesView(store, redactSecrets); + assert.equal(view.ok, true); + assert.ok(!view.text.includes("incomplete-coverage"), view.text); + } finally { + store.close(); + } +}); + +function seedSpan(n = 10): { store: LcmStore; leaf: number } { + const store = new LcmStore(":memory:"); + ingestEntries( + store, + Array.from({ length: n }, (_, i) => ({ + entryId: `e${String(i).padStart(2, "0")}`, + role: "user" as const, + text: `message ${i}`, + timestamp: i, + })), + ); + const leaf = store.insertSummary({ + kind: "leaf", + text: "leaf over the span", + tokens: 2, + depth: 0, + firstEntryId: "e00", + lastEntryId: `e${String(n - 1).padStart(2, "0")}`, + messageIds: store.messagesInSpan("e00", `e${String(n - 1).padStart(2, "0")}`).map((m) => m.id), + }).id; + return { store, leaf }; +} + +function listedIds(view: ReturnType): string[] { + assert.equal(view.ok, true); + return view.text + .split("\n") + .filter((l) => l.startsWith("[e")) + .map((l) => l.slice(1, l.indexOf("]"))); +} + +test("expandView: a mid-span anchor returns a window containing it with neighbors", () => { + const { store, leaf } = seedSpan(); + try { + const view = expandView(store, redactSecrets, { + mode: "list", + id: leaf, + offset: 0, + limit: 5, + maxChars: 4000, + anchor: "e05", + }); + assert.deepEqual(listedIds(view), ["e03", "e04", "e05", "e06", "e07"]); + } finally { + store.close(); + } +}); + +test("expandView: span-edge anchors clamp instead of running past the ends", () => { + const { store, leaf } = seedSpan(); + try { + const head = expandView(store, redactSecrets, { + mode: "list", + id: leaf, + offset: 0, + limit: 5, + maxChars: 4000, + anchor: "e00", + }); + assert.deepEqual(listedIds(head), ["e00", "e01", "e02", "e03", "e04"]); + const tail = expandView(store, redactSecrets, { + mode: "list", + id: leaf, + offset: 0, + limit: 5, + maxChars: 4000, + anchor: "e09", + }); + assert.deepEqual(listedIds(tail), ["e05", "e06", "e07", "e08", "e09"]); + } finally { + store.close(); + } +}); + +test("expandView: an unknown anchor returns a typed empty", () => { + const { store, leaf } = seedSpan(); + try { + const view = expandView(store, redactSecrets, { + mode: "list", + id: leaf, + offset: 0, + limit: 5, + maxChars: 4000, + anchor: "e99", + }); + assert.equal(view.ok, false); + assert.match(view.text, /No stored message with entry id e99/); + } finally { + store.close(); + } +}); + +test("grepView: a tool name with a quote builds a valid FTS5 phrase", () => { + assert.equal(ftsPhrase('[tool:a"b]'), '"[tool:a""b]"'); + assert.equal(withToolScope("cache", 'a"b'), '(cache) AND "[tool:a""b]"'); + assert.equal(withToolScope("q", "t"), `(q) AND ${ftsPhrase(toolMarker("t"))}`); + const store = seedTools(); + try { + const view = grepView(store, redactSecrets, { query: "cache", toolName: 'a"b' }); + assert.equal(view.ok, true); + assert.match(view.text, /No matches/); + } finally { + store.close(); + } +}); + +test("searchSessions: a quoted tool name shares the helper and stays valid", () => { + const dir = mkdtempSync(join(tmpdir(), "lcm-session-quote-")); + const storePath = join(dir, "store.db"); + const setup = new LcmStore(storePath); + try { + ingestEntries(setup, [ + { + entryId: "t00", + role: "toolResult", + text: '[tool:read] {"path": "/a"} cache key', + timestamp: 1, + }, + ]); + } finally { + setup.close(); + } + try { + const result = searchSessions( + [ + { + hash: "h", + sessionFile: join(dir, "s.jsonl"), + storePath, + cwd: "/x", + sessionId: "s", + when: 1, + }, + ], + "cache", + { toolName: 'a"b' }, + ); + assert.deepEqual(result.hits, []); + assert.equal(result.scanned, 1); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("expandView: an anchor with a nonzero offset pages as an absolute index", () => { + const { store, leaf } = seedSpan(); + try { + const first = expandView(store, redactSecrets, { + mode: "list", + id: leaf, + offset: 0, + limit: 5, + maxChars: 4000, + anchor: "e05", + }); + assert.deepEqual(listedIds(first), ["e03", "e04", "e05", "e06", "e07"]); + assert.equal(first.ok === true && first.details.offset, 3); + const second = expandView(store, redactSecrets, { + mode: "list", + id: leaf, + offset: 8, + limit: 5, + maxChars: 4000, + anchor: "e05", + }); + assert.deepEqual(listedIds(second), ["e08", "e09"]); + assert.equal(second.ok === true && second.details.offset, 8); + assert.deepEqual( + listedIds(second).filter((id) => listedIds(first).includes(id)), + [], + ); + } finally { + store.close(); + } +}); + +function seedTiedManifest(): LcmStore { + const store = new LcmStore(":memory:"); + ingestEntries(store, [ + { entryId: "m00", role: "user", text: "weather report", timestamp: 1 }, + { entryId: "m01", role: "user", text: "weather report", timestamp: 2 }, + { entryId: "m02", role: "user", text: "weather report", timestamp: 3 }, + { entryId: "m03", role: "user", text: "weather report", timestamp: 4 }, + ]); + for (const span of ["m00", "m01", "m02", "m03"]) { + const ids = store.messagesInSpan(span, span).map((m) => m.id); + store.insertSummary({ + kind: "leaf", + text: "weather report", + tokens: 2, + depth: 0, + firstEntryId: span, + lastEntryId: span, + messageIds: ids, + }); + } + return store; +} + +test("historyManifest: ties and wordless queries keep the newest summaries", () => { + const store = seedTiedManifest(); + try { + const tied = historyManifest(store, 2, "weather"); + assert.ok(tied.includes("m03 .. m03") && tied.includes("m02 .. m02"), tied); + assert.ok(!tied.includes("m00 .. m00"), tied); + const wordless = historyManifest(store, 2, "the and of"); + assert.ok(wordless.includes("m03 .. m03") && wordless.includes("m02 .. m02"), wordless); + assert.ok(!wordless.includes("m00 .. m00"), wordless); + } finally { + store.close(); + } +}); + +test("historyManifest: the omitted label names overlap order or age", () => { + const store = new LcmStore(":memory:"); + try { + ingestEntries(store, [ + { entryId: "m00", role: "user", text: "compaction engine keeps history", timestamp: 1 }, + { entryId: "m01", role: "user", text: "weather report", timestamp: 2 }, + { entryId: "m02", role: "user", text: "weather report", timestamp: 3 }, + ]); + for (const span of ["m00", "m01", "m02"]) { + const ids = store.messagesInSpan(span, span).map((m) => m.id); + store.insertSummary({ + kind: "leaf", + text: span === "m00" ? "compaction engine keeps history" : "weather report", + tokens: 2, + depth: 0, + firstEntryId: span, + lastEntryId: span, + messageIds: ids, + }); + } + const plain = historyManifest(store, 1); + assert.match(plain, /\(2 older not listed\)/); + const seeded = historyManifest(store, 1, "compaction engine"); + assert.match(seeded, /\(2 lower-overlap not listed\)/); + assert.ok(!seeded.includes("older not listed"), seeded); + } finally { + store.close(); + } +}); + +function seedPiCallResult(): LcmStore { + const store = new LcmStore(":memory:"); + ingestEntries(store, [ + { entryId: "p00", role: "assistant", text: '[tool:read] {"path": "/a"}', timestamp: 1 }, + { + entryId: "p01", + role: "toolResult", + text: "file body holds uniqueword123 nowhere else", + timestamp: 2, + }, + { entryId: "p02", role: "user", text: "unrelated chatter", timestamp: 3 }, + ]); + return store; +} + +test("grepView: a tool scope finds a Pi result row after its call row", () => { + const store = seedPiCallResult(); + try { + const scoped = hitIds( + grepView(store, redactSecrets, { query: "uniqueword123", toolName: "read" }), + ); + assert.deepEqual(scoped, ["p01"]); + const wrongTool = grepView(store, redactSecrets, { + query: "uniqueword123", + toolName: "write", + }); + assert.equal(wrongTool.ok, true); + assert.match(wrongTool.text, /No matches/); + const asResult = hitIds( + grepView(store, redactSecrets, { + query: "uniqueword123", + toolName: "read", + role: "toolResult", + }), + ); + assert.deepEqual(asResult, ["p01"]); + } finally { + store.close(); + } +}); + +test("grepView: a tool scope keeps fetching until the filtered page is complete", () => { + const store = new LcmStore(":memory:"); + try { + ingestEntries(store, [ + ...Array.from({ length: 22 }, (_, i) => ({ + entryId: `u${String(i).padStart(2, "0")}`, + role: "user" as const, + text: Array(10).fill("targetword").join(" "), + timestamp: i + 1, + })), + { + entryId: "c00", + role: "assistant" as const, + text: '[tool:read] {"path": "/a"}', + timestamp: 100, + }, + { + entryId: "r00", + role: "toolResult" as const, + text: `targetword ${"filler prose ".repeat(20)}`, + timestamp: 101, + }, + { + entryId: "r01", + role: "toolResult" as const, + text: `targetword ${"filler prose ".repeat(20)}`, + timestamp: 102, + }, + ]); + const view = grepView(store, redactSecrets, { + query: "targetword", + toolName: "read", + limit: 20, + }); + assert.equal(view.ok, true); + assert.deepEqual(view.details, { hits: 2, offset: 0, total: 2 }); + assert.ok(view.text.includes("[r00]") && view.text.includes("[r01]"), view.text); + } finally { + store.close(); + } +}); + +test("searchSessions: a tool scope finds a Pi result row after its call row", () => { + const dir = mkdtempSync(join(tmpdir(), "lcm-session-pi-")); + const storePath = join(dir, "store.db"); + const setup = new LcmStore(storePath); + try { + ingestEntries(setup, [ + { entryId: "p00", role: "assistant", text: '[tool:read] {"path": "/a"}', timestamp: 1 }, + { + entryId: "p01", + role: "toolResult", + text: "file body holds uniqueword123 nowhere else", + timestamp: 2, + }, + ]); + } finally { + setup.close(); + } + try { + const result = searchSessions( + [ + { + hash: "h", + sessionFile: join(dir, "s.jsonl"), + storePath, + cwd: "/x", + sessionId: "s", + when: 1, + }, + ], + "uniqueword123", + { toolName: "read" }, + ); + assert.equal(result.scanned, 1); + assert.deepEqual( + result.hits.map((h) => h.entryId), + ["p01"], + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("toolScope: a user-quoted marker opens no scope but mixed assistant still matches", () => { + const store = new LcmStore(":memory:"); + try { + ingestEntries(store, [ + { entryId: "u00", role: "user", text: 'quoting "[tool:read]" in prose', timestamp: 1 }, + { entryId: "r00", role: "toolResult", text: "followword body here", timestamp: 2 }, + { entryId: "c00", role: "custom", text: 'quoting "[tool:read]" in prose', timestamp: 3 }, + { entryId: "r01", role: "toolResult", text: "customfollowword body here", timestamp: 4 }, + { + entryId: "a00", + role: "assistant", + text: 'checking [tool:read] {"path": "/a"} plus prose', + timestamp: 5, + }, + { entryId: "r02", role: "toolResult", text: "mixedword body here", timestamp: 6 }, + ]); + const byEntry = new Map(store.allMessages().map((m) => [m.entryId, m.id])); + const scoped = toolScopeMessageIds(store, "read"); + assert.ok(!scoped.has(byEntry.get("u00")!), "user marker row opens no scope"); + assert.ok(!scoped.has(byEntry.get("r00")!), "result after a user marker stays out"); + assert.ok(!scoped.has(byEntry.get("c00")!), "custom marker row opens no scope"); + assert.ok(!scoped.has(byEntry.get("r01")!), "result after a custom marker stays out"); + assert.ok(scoped.has(byEntry.get("a00")!), "mixed assistant marker still opens"); + assert.ok(scoped.has(byEntry.get("r02")!), "result after mixed assistant stays in"); + const missed = grepView(store, redactSecrets, { query: "followword", toolName: "read" }); + assert.equal(missed.ok, true); + assert.match(missed.text, /No matches/); + const kept = hitIds(grepView(store, redactSecrets, { query: "mixedword", toolName: "read" })); + assert.deepEqual(kept, ["r02"]); + } finally { + store.close(); + } +}); + +test("grepView: an unbalanced-quote query falls back to a phrase search", () => { + const store = new LcmStore(":memory:"); + try { + ingestEntries(store, [ + { entryId: "p00", role: "assistant", text: '[tool:read] {"path": "/a"}', timestamp: 1 }, + { entryId: "p01", role: "toolResult", text: "hello world cache here", timestamp: 2 }, + ]); + const plain = grepView(store, redactSecrets, { query: 'hello"' }); + assert.equal(plain.ok, true); + assert.ok(plain.text.includes("[p01]"), plain.text); + const scoped = grepView(store, redactSecrets, { query: 'hello"', toolName: "read" }); + assert.equal(scoped.ok, true); + assert.ok(scoped.text.includes("[p01]"), scoped.text); + const miss = grepView(store, redactSecrets, { query: 'foo"', toolName: "read" }); + assert.equal(miss.ok, true); + assert.match(miss.text, /No matches/); + } finally { + store.close(); + } +}); + +test("searchSessions: an unbalanced-quote query runs a phrase search", () => { + const dir = mkdtempSync(join(tmpdir(), "lcm-session-phrase-")); + const storePath = join(dir, "store.db"); + const setup = new LcmStore(storePath); + try { + ingestEntries(setup, [ + { entryId: "p00", role: "assistant", text: '[tool:read] {"path": "/a"}', timestamp: 1 }, + { entryId: "p01", role: "toolResult", text: "hello world cache here", timestamp: 2 }, + ]); + } finally { + setup.close(); + } + const ref = { + hash: "h", + sessionFile: join(dir, "s.jsonl"), + storePath, + cwd: "/x", + sessionId: "s", + when: 1, + }; + try { + const plain = searchSessions([ref], 'hello"'); + assert.equal(plain.scanned, 1); + assert.deepEqual( + plain.hits.map((h) => h.entryId), + ["p01"], + ); + const scoped = searchSessions([ref], 'hello"', { toolName: "read" }); + assert.equal(scoped.scanned, 1); + assert.deepEqual( + scoped.hits.map((h) => h.entryId), + ["p01"], + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/opencode/README.md b/packages/opencode/README.md index 4414684..6f631a2 100644 --- a/packages/opencode/README.md +++ b/packages/opencode/README.md @@ -27,7 +27,7 @@ The equivalent entry in `opencode.json`: ## Quick start Zero config required: passive mirroring starts immediately, and swaps begin -once context crosses the swap point. Ten commands, all `lossless-*`: +once context crosses the swap point. Eleven commands, all `lossless-*`: ```sh /lossless-status # message and compaction summary counts per live sidecar @@ -40,11 +40,14 @@ once context crosses the swap point. Ten commands, all `lossless-*`: /lossless-diagnose # redacted, pasteable diagnostics block; --save writes it at mode 0600 /lossless-migrate # upgrade older-schema sidecars in place (dry-run by default) /lossless-gc # list old sidecars, or export and delete them (never the current session) +/lossless-dump # read-only dump of this session's sidecar; --include-removed keeps removed rows, --save writes mode 0600 ``` The model gets four recall tools: `lcm_grep` (search this session, `scope` -reaches past sessions), `lcm_describe` (inspect one summary without expanding -it), `lcm_expand` (read the originals a summary covers), and +reaches past sessions, `role`, `date_start`, `date_end`, and `tool_name` +filter rows), `lcm_describe` (inspect one summary without expanding +it), `lcm_expand` (read the originals a summary covers, `anchor` centers +the listing on one entry), and `lcm_expand_query` (a bounded child session reads and answers, so the originals never enter the parent's context). diff --git a/packages/opencode/skills/opencode-lossless/SKILL.md b/packages/opencode/skills/opencode-lossless/SKILL.md index c88aa79..dc9de9d 100644 --- a/packages/opencode/skills/opencode-lossless/SKILL.md +++ b/packages/opencode/skills/opencode-lossless/SKILL.md @@ -1,6 +1,6 @@ --- name: opencode-lossless -description: "Use when an OpenCode v2 session has opencode-lossless installed: read compacted history with lcm_grep, lcm_describe, lcm_expand, or lcm_expand_query; run /lossless-status, /lossless-report, /lossless-pass, /lossless-heal, /lossless-backup, /lossless-export, /lossless-import, /lossless-diagnose, /lossless-migrate, or /lossless-gc; or tune the swap point." +description: "Use when an OpenCode v2 session has opencode-lossless installed: read compacted history with lcm_grep, lcm_describe, lcm_expand, or lcm_expand_query; run /lossless-status, /lossless-report, /lossless-pass, /lossless-heal, /lossless-backup, /lossless-export, /lossless-import, /lossless-diagnose, /lossless-migrate, /lossless-gc, or /lossless-dump; or tune the swap point." license: MIT metadata: type: composition @@ -40,13 +40,14 @@ Equivalent `opencode.json` entry: - `/lossless-diagnose` — redacted, pasteable diagnostics block; `--save` writes it at mode 0600. - `/lossless-migrate` — upgrade older-schema sidecars in place (dry-run by default). - `/lossless-gc` — list old sidecars, or export and delete them (never the current session). +- `/lossless-dump` prints a read-only dump of the session sidecar. `--include-removed` keeps removed-branch rows and `--save` writes the output to a file at mode 0600. ## Recall tools -- `lcm_grep` — full-text search over this session; `scope` reaches past sessions. -- `lcm_describe` — inspect one summary (kind, costs, children) without expanding it. -- `lcm_expand` — read the verbatim originals a summary covers. -- `lcm_expand_query` — a bounded child session reads and answers, so originals never enter the parent context. +- `lcm_grep` searches full text over this session, and `scope` reaches past sessions. `role` keeps only rows stored with that role, `date_start` and `date_end` bound rows by timestamp, and `tool_name` keeps only tool-result rows for that tool. +- `lcm_describe` inspects one summary (kind, costs, children) without expanding it. +- `lcm_expand` reads the verbatim originals a summary covers, and `anchor` centers the listing on one entry. +- `lcm_expand_query` runs a bounded child session that reads and answers, so originals never enter the parent context. ## Config essentials diff --git a/packages/opencode/src/compaction.ts b/packages/opencode/src/compaction.ts index fd1e311..730ec55 100644 --- a/packages/opencode/src/compaction.ts +++ b/packages/opencode/src/compaction.ts @@ -180,7 +180,12 @@ export function createCheckpointWriter(deps: CheckpointDeps): CheckpointWriter { targetTokens: deps.options.summaryTokens, stage: "blocking", ...(previousText === undefined ? {} : { previousSummary: previousText }), - summarizer: summarizerConfig(deps.options), + summarizer: { + ...summarizerConfig(deps.options), + ...(deps.options.mechanicalTools === undefined + ? {} + : { mechanicalTools: deps.options.mechanicalTools }), + }, }, ); } catch (error) { diff --git a/packages/opencode/src/context.ts b/packages/opencode/src/context.ts index 26422dd..d81573a 100644 --- a/packages/opencode/src/context.ts +++ b/packages/opencode/src/context.ts @@ -199,7 +199,12 @@ export function createContextProjector(deps: ProjectorDeps): Projector { targetTokens: deps.options.summaryTokens, stage: "async", ...(previousSummary === undefined ? {} : { previousSummary }), - summarizer: summarizerConfig(deps.options), + summarizer: { + ...summarizerConfig(deps.options), + ...(deps.options.mechanicalTools === undefined + ? {} + : { mechanicalTools: deps.options.mechanicalTools }), + }, }, ) .then(() => undefined) diff --git a/packages/opencode/src/cross-session.ts b/packages/opencode/src/cross-session.ts index 1f71e2e..3c27dbd 100644 --- a/packages/opencode/src/cross-session.ts +++ b/packages/opencode/src/cross-session.ts @@ -53,7 +53,14 @@ export function searchPastSessions( deps: PastSearchDeps, callingSessionId: string | undefined, query: string, - opts: { limit?: number; now?: () => number } = {}, + opts: { + limit?: number; + now?: () => number; + role?: "user" | "assistant" | "toolResult" | "custom"; + dateStart?: number; + dateEnd?: number; + toolName?: string; + } = {}, ): PastSearchResult { const now = opts.now ?? deps.now ?? Date.now; const started = now(); @@ -89,6 +96,10 @@ export function searchPastSessions( const view = grepView(store, deps.redact, { query, limit: Math.min(CROSS_SESSION_PER_STORE_HITS, hitLimit - hits), + ...(opts.role === undefined ? {} : { role: opts.role }), + ...(opts.dateStart === undefined ? {} : { dateStart: opts.dateStart }), + ...(opts.dateEnd === undefined ? {} : { dateEnd: opts.dateEnd }), + ...(opts.toolName === undefined ? {} : { toolName: opts.toolName }), expandTool: "lcm_expand", }); if (!view.ok) { diff --git a/packages/opencode/src/lifecycle.ts b/packages/opencode/src/lifecycle.ts index a588fe8..40ddd1b 100644 --- a/packages/opencode/src/lifecycle.ts +++ b/packages/opencode/src/lifecycle.ts @@ -9,12 +9,14 @@ import { chmodSync, existsSync, readFileSync, + renameSync, statSync, + unlinkSync, writeFileSync, type Stats, } from "node:fs"; import { homedir } from "node:os"; -import { basename, join, resolve } from "node:path"; +import { basename, dirname, join, resolve } from "node:path"; import { LcmStore, @@ -40,6 +42,7 @@ import { import type { CommandDefinition, ModelRef } from "./host.ts"; import { readMetricRows } from "./metrics.ts"; +import { dumpSidecar } from "./sidecar-dump.ts"; import type { SidecarRef, Sidecars } from "./sidecars.ts"; import { summarizerConfig, type SummarizerOptions } from "./summarizer.ts"; @@ -113,6 +116,44 @@ function chmodFile(path: string): void { } catch {} } +function writeDumpTarget(target: string, text: string): void { + // Unique temp with exclusive creation, then an atomic rename: a planted + // predictable path is never touched and the destination swaps at once. + const dir = dirname(target); + const base = basename(target); + let tmp = ""; + for (let attempt = 0; attempt < 5; attempt++) { + const candidate = join( + dir, + `.${base}.${process.pid}.${Date.now()}.${Math.floor(Math.random() * 1e9)}.tmp`, + ); + try { + writeFileSync(candidate, text, { encoding: "utf8", mode: 0o600, flag: "wx" }); + tmp = candidate; + break; + } catch (error) { + if ((error as { code?: string })?.code === "EEXIST") continue; + // A write can create the private file and then fail (disk full), so + // remove the candidate before surfacing the error. EEXIST keeps its + // continue above and never unlinks a pre-existing collision. + try { + unlinkSync(candidate); + } catch {} + throw error; + } + } + if (tmp === "") throw new Error(`LCM dump: cannot create temp file beside ${target}.`); + try { + chmodFile(tmp); + renameSync(tmp, target); + } catch (error) { + try { + unlinkSync(tmp); + } catch {} + throw error; + } +} + function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null; } @@ -242,7 +283,14 @@ export function diagnoseCommand(deps: LifecycleDeps): CommandDefinition { } } const stats = store.stats(); - const integrity = checkIntegrity(store); + const livePaths = deps.sidecars.ids().map((id) => deps.sidecars.pathFor(id)); + const currentPath = deps.sidecars.pathFor(input.sessionID); + if (!livePaths.includes(currentPath)) livePaths.push(currentPath); + const integrity = checkIntegrity(store, { + sidecarDir: deps.sidecars.dir, + liveSidecarPaths: livePaths, + sidecarExtension: deps.sidecars.extension, + }); // The bundle is a diagnostic, not a probe: a host model lookup that // fails says less, it does not fail the command. let model: ModelRef | undefined; @@ -604,6 +652,64 @@ export function healCommand(deps: LifecycleDeps): CommandDefinition { }; } +/** Read-only dump of this session's sidecar plus its metrics, opened with + * `queryOnly` so a dump never mutates the store. */ +export function dumpCommand(deps: LifecycleDeps): CommandDefinition { + const redact = makeRedactConfigAdapter(deps.options); + return { + name: "lossless-dump", + description: + "Print a read-only dump of this session's sidecar. It holds history, sampled summaries and messages, and metrics.", + execute: async (input) => { + const args = argsOf(input.prompt.text); + const includeRemoved = args.includes("--include-removed"); + if (args.includes("--save") && valueOf(args, "--save") === undefined) { + await deps.prompt( + input.sessionID, + "LCM dump --save: name the file to write, e.g. `/lossless-dump --save lossless-dump.txt`.", + ); + return; + } + const save = valueOf(args, "--save"); + const query = args.filter((arg) => !arg.startsWith("--") && arg !== save).join(" "); + const result = dumpSidecar({ + path: deps.sidecars.pathFor(input.sessionID), + sessionId: input.sessionID, + redact, + ...(includeRemoved ? { includeRemoved: true as const } : {}), + metricRows: readMetricRows(deps.sidecars.dir).filter( + (row): row is Record => isRecord(row), + ), + ...(query.length > 0 ? { sampleQuery: query } : {}), + }); + if (result.status === "skipped") { + await deps.prompt( + input.sessionID, + `LCM dump skipped: ${result.reason} (${result.detail}).`, + ); + return; + } + if (save !== undefined) { + try { + const target = resolve(save); + writeDumpTarget(target, `${result.text}\n`); + await deps.prompt( + input.sessionID, + `LCM dump: wrote ${Buffer.byteLength(result.text, "utf8")} bytes to ${target} (mode 0600).`, + ); + } catch (error) { + await deps.prompt( + input.sessionID, + `LCM dump failed: ${error instanceof Error ? error.message : String(error)}`, + ); + } + return; + } + await deps.prompt(input.sessionID, result.text); + }, + }; +} + /** The compact result a command prompt carries. No summary or message text * leaves the store; the counts are the whole answer. */ export function healReport(outcome: HealOutcome): string { diff --git a/packages/opencode/src/pass.ts b/packages/opencode/src/pass.ts index 925f8e6..2da3391 100644 --- a/packages/opencode/src/pass.ts +++ b/packages/opencode/src/pass.ts @@ -132,15 +132,24 @@ export async function runLosslessPass( ...(opts.targetTokens === undefined ? {} : { targetTokens: opts.targetTokens }), }, llm, - // The pass stage is the engine's own vocabulary, so a direct pass labels - // its completer rows and its engine rows alike. - { redact: makeRedactConfigAdapter({}), session: sessionID, stage }, + { + // The pass stage is the engine's own vocabulary, so a direct pass labels + // its completer rows and its engine rows alike. + redact: makeRedactConfigAdapter({}), + session: sessionID, + stage, + ...(opts.summarizer?.mechanicalTools === undefined + ? {} + : { mechanicalTools: opts.summarizer.mechanicalTools }), + }, ); if (outcome.kind === "stored") { - // A stored outcome with no model answer is the ladder's mechanical - // truncate. Committing it would report success for a summarizer that - // never worked, so a manual pass abandons it and says so. - if (tally.answers === 0) { + // A stored outcome with no model answer is usually the ladder's + // mechanical truncate. Committing it would report success for a + // summarizer that never worked, so a pass abandons it and says so. + // A span whose every node came from the configured mechanical path + // needs no model call, so it commits without an answer. + if (tally.answers === 0 && !isAllMechanical(outcome)) { throw new Error("lcm: the summarizer did not answer, so nothing was stored"); } store.commitRun(token); @@ -172,6 +181,20 @@ export async function runLosslessPass( } } +/** Every stored node came from the configured mechanical path, so zero + * model answers is the expected shape, not a failed summarizer. Level-3 + * fallback leaves carry mechanical quality but are not counted in the + * engine totals, so they still throw. */ +function isAllMechanical(outcome: CompactionOutcome): boolean { + if (outcome.kind !== "stored") return false; + const leaves = outcome.leafSummaryIds.length; + const condensed = outcome.condensedSummaryIds.length; + if (leaves + condensed === 0) return false; + return ( + leaves === (outcome.mechanicalLeaves ?? 0) && condensed === (outcome.mechanicalCondensed ?? 0) + ); +} + function messageOf(error: unknown): string { return (error instanceof Error ? error.message : String(error)).slice(0, 300); } diff --git a/packages/opencode/src/plugin.ts b/packages/opencode/src/plugin.ts index 8937805..4fc445c 100644 --- a/packages/opencode/src/plugin.ts +++ b/packages/opencode/src/plugin.ts @@ -23,6 +23,7 @@ import { compactionMark, mirror, mirrorBusEvent, toolEntry, userEntry } from "./ import { backupCommand, diagnoseCommand, + dumpCommand, exportCommand, gcCommand, healCommand, @@ -75,6 +76,10 @@ export interface OpencodeLosslessOptions { * A dry run spends nothing. * @default 120000 */ readonly summarizerTokensPerHeal?: number; + /** Tool names whose spans compact without a model call. Empty keeps the + * LLM path. + * @default [] */ + readonly mechanicalTools?: readonly string[]; /** The occupancy share at which a request swaps in the projection. The * engine clamps it to 0.9 of the window. * @default 0.7 */ @@ -262,6 +267,9 @@ export async function setupOpencodeLossless( ...summarizerConfig(options), summarizerTokensPerHeal: options.summarizerTokensPerHeal, ...(options.zones === undefined ? {} : { zones: options.zones }), + ...(options.mechanicalTools === undefined + ? {} + : { mechanicalTools: options.mechanicalTools }), }, version: pluginLabel, host: () => "opencode v2", @@ -335,6 +343,7 @@ export async function setupOpencodeLossless( exportCommand, importCommand, diagnoseCommand, + dumpCommand, migrateCommand, gcCommand, healCommand, @@ -387,7 +396,12 @@ export async function setupOpencodeLossless( { targetTokens: options.summaryTokens, stage: "direct", - summarizer: summarizerConfig(options), + summarizer: { + ...summarizerConfig(options), + ...(options.mechanicalTools === undefined + ? {} + : { mechanicalTools: options.mechanicalTools }), + }, }, ); }, @@ -502,6 +516,7 @@ export function readOptions(raw: unknown): RuntimeOptions { const recutAtRatio = readRatio(record.recutAtRatio) ?? DEFAULT_RECUT_RATIO; const zones = readZones(record.zones); const healSourceTokens = readHealSourceTokens(record.healSourceTokens); + const mechanicalTools = readMechanicalTools(record.mechanicalTools); return { enabled: typeof record.enabled === "boolean" ? record.enabled : true, sidecarDir: typeof record.sidecarDir === "string" ? record.sidecarDir : DEFAULT_SIDECAR_DIR, @@ -522,6 +537,7 @@ export function readOptions(raw: unknown): RuntimeOptions { ...(summarizer === undefined ? {} : { summarizer }), ...(smartZone === undefined ? {} : { smartZone }), ...(zones === undefined ? {} : { zones }), + ...(mechanicalTools === undefined ? {} : { mechanicalTools }), }; } @@ -532,6 +548,25 @@ function readSummarizer(raw: unknown): string | string[] | undefined { return chain.length === 0 ? undefined : chain; } +/** Tool names that skip the model. One name or a list; blanks and repeats + * drop, and an empty result keeps the LLM path. */ +function readMechanicalTools(raw: unknown): string[] | undefined { + const list = typeof raw === "string" ? [raw] : Array.isArray(raw) ? raw : undefined; + if (list === undefined) return undefined; + const seen = new Set(); + const out: string[] = []; + for (const entry of list) { + if (typeof entry !== "string") continue; + const name = entry.trim(); + if (name.length === 0) continue; + const key = name.toLowerCase(); + if (seen.has(key)) continue; + seen.add(key); + out.push(name); + } + return out.length === 0 ? undefined : out; +} + function positiveNumber(raw: unknown): number | undefined { return typeof raw === "number" && Number.isFinite(raw) && raw > 0 ? raw : undefined; } diff --git a/packages/opencode/src/sidecar-dump.ts b/packages/opencode/src/sidecar-dump.ts new file mode 100644 index 0000000..b374582 --- /dev/null +++ b/packages/opencode/src/sidecar-dump.ts @@ -0,0 +1,279 @@ +import { existsSync } from "node:fs"; +import { basename } from "node:path"; + +import { + describeView, + expandView, + grepView, + historyManifest, + LcmStore, + StaleStoreError, +} from "lossless-core"; + +/** Max bytes one dump prints; callers copy the file when they need more. */ +export const DUMP_MAX_BYTES = 16_384; +/** Summaries described in full; the header still counts the rest. */ +export const DUMP_MAX_SUMMARIES = 10; +/** Metric rows kept from the tail of the session's rows. */ +export const DUMP_MAX_METRIC_ROWS = 20; +/** Top-level summaries the manifest names. */ +export const DUMP_MAX_HISTORY = 20; +/** Messages read as windows from the first summary. */ +export const DUMP_MAX_SAMPLE_MESSAGES = 3; +/** Chars one sampled message window holds. */ +export const DUMP_SAMPLE_CHARS = 400; +/** Bytes kept from each described summary; the tail names the cut. */ +export const DUMP_MAX_SUMMARY_BYTES = 1_200; + +/** Why a sidecar could not be dumped. Each reason is a count the caller + * reports, never an error it raises. */ +export type DumpSkipReason = "missing" | "unreadable" | "older-generation"; + +export type SidecarDumpResult = + | { status: "dumped"; text: string } + | { status: "skipped"; reason: DumpSkipReason; detail: string }; + +export interface SidecarDumpOptions { + /** Sidecar file to open read-only. Never created when absent. */ + readonly path: string; + /** Session id naming the dump and filtering metric rows. */ + readonly sessionId: string; + /** Masks secret-shaped text on every displayed seam. */ + readonly redact: (text: string) => string; + /** Include matches from a branch that left the active path. */ + readonly includeRemoved?: boolean; + /** Metric rows for this sidecar directory, oldest first. */ + readonly metricRows?: readonly Record[]; + /** Optional FTS5 query sampled through grepView. */ + readonly sampleQuery?: string; +} + +/** Read-only dump of one session sidecar plus its metrics. Opens the file + * with `queryOnly`, so SQLite refuses writes, and closes it before returning. + * An older generation reports a named skip instead of migrating. */ +export function dumpSidecar(options: SidecarDumpOptions): SidecarDumpResult { + if (!existsSync(options.path)) { + return { + status: "skipped", + reason: "missing", + detail: `no sidecar at ${basename(options.path)}`, + }; + } + let store: LcmStore | undefined; + try { + store = new LcmStore(options.path, { queryOnly: true }); + } catch (error) { + if (error instanceof StaleStoreError) { + return { + status: "skipped", + reason: "older-generation", + detail: + `generation ${error.generation} is older than this build. ` + + `Run migrate first. No rows were changed.`, + }; + } + return { + status: "skipped", + reason: "unreadable", + detail: error instanceof Error ? error.message.slice(0, 200) : String(error), + }; + } + try { + return { status: "dumped", text: buildDump(store, options) }; + } catch (error) { + return { + status: "skipped", + reason: "unreadable", + detail: error instanceof Error ? error.message.slice(0, 200) : String(error), + }; + } finally { + try { + store.close(); + } catch {} + } +} + +function buildDump(store: LcmStore, options: SidecarDumpOptions): string { + const includeRemoved = options.includeRemoved === true; + const stats = store.stats(); + const removed = stats.removedMessages > 0 ? ` (${stats.removedMessages} removed, retained)` : ""; + const head = [ + `LCM sidecar dump: ${options.sessionId}`, + `store: ${basename(options.path)}, generation ${store.generation}, ` + + `${stats.messages} messages${removed}, ${stats.summaries} summaries, ` + + `${(stats.dbBytes / 1024).toFixed(0)} KB`, + `history:`, + historyManifest(store, DUMP_MAX_HISTORY), + ]; + const summaries = store.allSummaries().slice(0, DUMP_MAX_SUMMARIES); + const totalSummaries = store.allSummaries().length; + const sections: string[] = [...head]; + if (summaries.length > 0) { + sections.push(`summaries: ${summaries.length} shown of ${totalSummaries}`); + for (const summary of summaries) { + const view = describeView(store, options.redact, summary.id); + sections.push(view.ok ? capSummaryText(view.text) : `Summary #${summary.id}: unreadable.`); + } + const first = summaries[0]!; + const coveredIds = store.coveredMessageIds(first.id); + const coveredMsgs = store.messagesByIds(coveredIds); + const coveredIdSet = new Set(coveredMsgs.map((m) => m.entryId)); + const removedIds = new Set( + coveredMsgs.filter((m) => m.removedAt !== undefined).map((m) => m.entryId), + ); + const visibleMsgs = includeRemoved + ? coveredMsgs + : coveredMsgs.filter((m) => m.removedAt === undefined); + const list = expandView(store, options.redact, { + mode: "list", + id: first.id, + offset: 0, + limit: DUMP_MAX_SAMPLE_MESSAGES, + maxChars: DUMP_SAMPLE_CHARS, + }); + sections.push( + `messages (summary #${first.id}):`, + list.ok + ? shapeSampleList( + list.text, + first.id, + coveredIdSet, + removedIds, + includeRemoved, + visibleMsgs.length, + ) + : "No messages.", + ); + for (const message of visibleMsgs.slice(0, DUMP_MAX_SAMPLE_MESSAGES)) { + const window = expandView(store, options.redact, { + mode: "message", + entryId: message.entryId, + charOffset: 0, + maxChars: DUMP_SAMPLE_CHARS, + }); + sections.push(window.ok ? window.text : `No stored message ${message.entryId}.`); + } + } else { + sections.push("summaries: none stored yet."); + } + if (options.sampleQuery !== undefined && options.sampleQuery.length > 0) { + const found = grepView(store, options.redact, { + query: options.sampleQuery, + limit: DUMP_MAX_SAMPLE_MESSAGES, + includeRemoved, + }); + sections.push( + `search ${JSON.stringify(options.sampleQuery)}${includeRemoved ? " (including removed)" : ""}:`, + found.text, + ); + } + const rows = (options.metricRows ?? []).filter((row) => row.session === options.sessionId); + const shown = rows.slice(Math.max(0, rows.length - DUMP_MAX_METRIC_ROWS)); + sections.push(`metrics: ${rows.length} row(s) for this session, ${shown.length} shown`); + for (const row of shown) sections.push(` ${JSON.stringify(sanitizeRow(row, options.redact))}`); + sections.push( + `omitted: summary text past ${DUMP_MAX_SUMMARY_BYTES} bytes each, message bodies past ${DUMP_SAMPLE_CHARS} chars per sample, ` + + `summaries past ${DUMP_MAX_SUMMARIES}, metric rows past ${DUMP_MAX_METRIC_ROWS}. ` + + `Removed rows carry [removed]. Read them with include_removed.`, + ); + const text = options.redact(sections.join("\n")); + if (Buffer.byteLength(text, "utf8") <= DUMP_MAX_BYTES) return text; + return clip(text); +} + +/** Reshape the sampled list for the dump: removed rows stay out by default + * and carry a [removed] label when --include-removed keeps them. Untouched + * when no covered row left the active path. */ +function shapeSampleList( + text: string, + summaryId: number, + coveredIds: ReadonlySet, + removedIds: ReadonlySet, + includeRemoved: boolean, + visibleTotal: number, +): string { + if (removedIds.size === 0) return text; + const groups: { id: string | undefined; chunks: string[] }[] = []; + for (const chunk of text.split("\n\n").slice(1)) { + const at = chunk.indexOf("\n"); + const first = at === -1 ? chunk : chunk.slice(0, at); + const id = /^\[([^\]]+)\] \((?:user|assistant|toolResult|custom)\)$/.exec(first)?.[1]; + if (id !== undefined && coveredIds.has(id)) groups.push({ id, chunks: [chunk] }); + else if (groups.length > 0) groups[groups.length - 1]!.chunks.push(chunk); + else groups.push({ id: undefined, chunks: [chunk] }); + } + const kept: string[] = []; + let shown = 0; + for (const group of groups) { + const removed = group.id !== undefined && removedIds.has(group.id); + if (removed && !includeRemoved) continue; + if (group.id !== undefined) shown++; + kept.push( + group.chunks + .map((chunk, index) => (index === 0 && removed ? labelRemoved(chunk) : chunk)) + .join("\n\n"), + ); + } + const head = + `Summary #${summaryId} → ${visibleTotal} original messages ` + + `(showing ${shown} from 0)${visibleTotal > shown ? ", use offset to page" : ""}`; + return [head, ...kept].join("\n\n"); +} + +/** Mark one kept list row as off the active path, grepView style. */ +function labelRemoved(chunk: string): string { + const at = chunk.indexOf("\n"); + return at === -1 ? `${chunk} [removed]` : `${chunk.slice(0, at)} [removed]${chunk.slice(at)}`; +} + +/** Keep one described summary bounded so large summaries cannot clip the tail. */ +function capSummaryText(text: string): string { + if (Buffer.byteLength(text, "utf8") <= DUMP_MAX_SUMMARY_BYTES) return text; + let used = 0; + let end = 0; + for (const chunk of text) { + const cost = Buffer.byteLength(chunk, "utf8"); + if (used + cost > DUMP_MAX_SUMMARY_BYTES) break; + used += cost; + end += chunk.length; + } + const more = Buffer.byteLength(text, "utf8") - used; + return `${text.slice(0, end)}\n[lcm:more ${more} bytes]`; +} + +/** Redact string seams in one metric row; keys and stub pointers stay intact. */ +function sanitizeRow( + row: Record, + redact: (text: string) => string, +): Record { + const out: Record = {}; + for (const [key, value] of Object.entries(row)) out[key] = sanitizeValue(value, redact); + return out; +} + +function sanitizeValue(value: unknown, redact: (text: string) => string): unknown { + if (typeof value === "string") return redact(value); + if (Array.isArray(value)) return value.map((item) => sanitizeValue(item, redact)); + if (typeof value === "object" && value !== null) { + const out: Record = {}; + for (const [key, item] of Object.entries(value as Record)) { + out[key] = sanitizeValue(item, redact); + } + return out; + } + return value; +} + +function clip(text: string): string { + const marker = `... truncated at the ${DUMP_MAX_BYTES} byte limit`; + const budget = DUMP_MAX_BYTES - Buffer.byteLength(marker, "utf8") - 1; + let kept = ""; + let used = 0; + for (const line of text.split("\n")) { + const cost = Buffer.byteLength(line, "utf8") + 1; + if (used + cost > budget) break; + kept += (kept.length === 0 ? "" : "\n") + line; + used += cost; + } + return `${kept}\n${marker}`; +} diff --git a/packages/opencode/src/tools.ts b/packages/opencode/src/tools.ts index d0487a2..9115dc0 100644 --- a/packages/opencode/src/tools.ts +++ b/packages/opencode/src/tools.ts @@ -52,6 +52,17 @@ export function registerRecallTools(editor: ToolEditor, deps: RecallDeps): void description: "FTS5 query: quoted phrases, AND/OR, prefix*", }, limit: { type: "integer", description: "Hits to return" }, + role: { + type: "string", + enum: ["user", "assistant", "toolResult", "custom"], + description: "Keep only rows stored with this role", + }, + date_start: { type: "number", description: "Keep only rows at or after this timestamp" }, + date_end: { type: "number", description: "Keep only rows at or before this timestamp" }, + tool_name: { + type: "string", + description: "Keep only tool-result rows for this tool. It matches [tool:Name] markers.", + }, scope: { type: "string", enum: [...RECALL_SCOPES], @@ -84,12 +95,16 @@ export function registerRecallTools(editor: ToolEditor, deps: RecallDeps): void name: "lcm_expand", options: { codemode: false }, description: - "Read the original messages a summary covers, or one named entry. Without entry_id, list `limit` messages from message index `offset`. With entry_id, read that one message from `char_offset` for up to `max_chars` characters; continue from the char_offset the result reports. `id` is the summary to list, or to check that the entry is covered; omit it to read an entry that no summary covers yet. Pass `session` with the id from an [lcm:session ] hit to read that past session's sidecar; the text is labelled as a past session's history and the read is read-only.", + "Read the original messages a summary covers, or one named entry. Without entry_id, list `limit` messages from message index `offset`, or centered on `anchor` when it names an entry id. With entry_id, read that one message from `char_offset` for up to `max_chars` characters; continue from the char_offset the result reports. `id` is the summary to list, or to check that the entry is covered; omit it to read an entry that no summary covers yet. Pass `session` with the id from an [lcm:session ] hit to read that past session's sidecar; the text is labelled as a past session's history and the read is read-only.", input: { type: "object", properties: { id: { type: "integer", description: "Summary id, the N in [lcm:summary #N ...]" }, entry_id: { type: "string", description: "Read this one message by entry id" }, + anchor: { + type: "string", + description: "Center the listing window on this entry id (with id, without entry_id)", + }, session: { type: "string", description: @@ -215,13 +230,14 @@ function toCoreDeps(deps: RecallDeps): CoreRecallDeps { record: deps.record, sessionOf: deps.sessionOf, isReader: deps.isReader, - searchPast: (session, query, scope, limit) => { - const result = searchPastSessions( - { past, redact: deps.redact }, - session, - query, - typeof limit === "number" ? { limit } : {}, - ); + searchPast: (session, query, scope, limit, filters) => { + const result = searchPastSessions({ past, redact: deps.redact }, session, query, { + ...(typeof limit === "number" ? { limit } : {}), + ...(filters?.role === undefined ? {} : { role: filters.role }), + ...(filters?.dateStart === undefined ? {} : { dateStart: filters.dateStart }), + ...(filters?.dateEnd === undefined ? {} : { dateEnd: filters.dateEnd }), + ...(filters?.toolName === undefined ? {} : { toolName: filters.toolName }), + }); return { text: result.text, hits: result.hits, diff --git a/packages/opencode/test/lifecycle.test.ts b/packages/opencode/test/lifecycle.test.ts index e5f671b..b3b4a6c 100644 --- a/packages/opencode/test/lifecycle.test.ts +++ b/packages/opencode/test/lifecycle.test.ts @@ -155,6 +155,51 @@ describe("lifecycle commands", () => { } }); + it("diagnoses an orphan sidecar with its path and never reports a live one", async () => { + const h = harness(); + try { + seed(h.sidecars, SESSION, "the current session"); + seed(h.sidecars, "ses_live", "the live session"); + seedDetached(h.sidecars, "ses_old", "the old session"); + const oldPath = h.sidecars.pathFor("ses_old"); + const livePath = h.sidecars.pathFor("ses_live"); + const text = await run(diagnoseCommand, h, "/lossless-diagnose"); + expect(text).toContain("orphan-sidecar"); + expect(text).toContain("ses_old"); + expect(text).toContain(oldPath); + expect(text).not.toContain(livePath); + } finally { + h.sidecars.close(); + } + }); + + it("after a restart names a retained but unopened sidecar as unopened, not sessionless", async () => { + const h = harness(); + try { + seed(h.sidecars, SESSION, "the current session"); + seedDetached(h.sidecars, "ses_retained", "the retained session"); + // Restart: drop every open handle, then diagnose through a fresh cache + // that holds only the diagnosing session open. + h.sidecars.close(); + const fresh = new Sidecars(h.dir); + try { + const text = await run( + diagnoseCommand, + { ...h, deps: { ...h.deps, sidecars: fresh } }, + "/lossless-diagnose", + ); + expect(text).toContain("orphan-sidecar"); + expect(text).toContain("ses_retained"); + expect(text).toContain("not open in this process"); + expect(text).not.toContain("no live session"); + } finally { + fresh.close(); + } + } finally { + h.sidecars.close(); + } + }); + it("reports a current schema as nothing to migrate", async () => { const h = harness(); try { diff --git a/packages/opencode/test/pass.test.ts b/packages/opencode/test/pass.test.ts index 2c81cd0..efcd94d 100644 --- a/packages/opencode/test/pass.test.ts +++ b/packages/opencode/test/pass.test.ts @@ -239,6 +239,105 @@ describe("runLosslessPass", () => { } }); + it("forwards mechanicalTools so a listed tool skips the model", async () => { + const session = "ses_pass_mechanical"; + const path = sidecarPath(); + const store = new LcmStore(path); + try { + const prose = `decide the plan ${"x".repeat(400)}`; + const entries: AlignedEntry[] = [ + { entryId: `${session}:prompt:msg_1`, role: "user", text: prose }, + { + entryId: `${session}:tool:msg_2:call_1`, + role: "toolResult", + text: `tool noisy-read {"path":"/a"}\n${"z".repeat(3000)}`, + }, + ]; + const prompts: string[] = []; + const ctx = ctxWith({ + text: async (input: { prompt: string }) => { + prompts.push(input.prompt); + return { text: "prose summary" }; + }, + }); + const outcome = await runLosslessPass( + { store: () => store, ctx, model: MODEL }, + session, + entries, + { targetTokens: 1500, summarizer: { mechanicalTools: ["noisy-read"] } }, + ); + expect(outcome.kind).toBe("stored"); + if (outcome.kind !== "stored") return; + expect(prompts).toHaveLength(1); + expect(prompts[0]).toContain("decide the plan"); + expect(prompts[0]).not.toContain("noisy-read"); + const leaves = outcome.leafSummaryIds.map((id) => store.getSummary(id)!); + expect(leaves).toHaveLength(2); + expect(leaves.map((leaf) => leaf.quality).sort()).toEqual(["clean", "mechanical"]); + } finally { + store.close(); + } + }); + + it("commits a mechanical-only span without a model answer", async () => { + const session = "ses_pass_mechanical_only"; + const path = sidecarPath(); + const store = new LcmStore(path); + try { + const entries: AlignedEntry[] = [ + { + entryId: `${session}:tool:msg_1:call_1`, + role: "toolResult", + text: `tool noisy-read {"path":"/a"}\n${"z".repeat(2000)}`, + }, + { + entryId: `${session}:tool:msg_2:call_2`, + role: "toolResult", + text: `tool noisy-read {"path":"/b"}\n${"z".repeat(2000)}`, + }, + ]; + const ctx = ctxWith({ + text: () => Promise.reject(new Error("model must not be called")), + }); + const outcome = await runLosslessPass( + { store: () => store, ctx, model: MODEL }, + session, + entries, + { targetTokens: 1500, summarizer: { mechanicalTools: ["noisy-read"] } }, + ); + expect(outcome.kind).toBe("stored"); + if (outcome.kind !== "stored") return; + expect(outcome.leafSummaryIds.length).toBeGreaterThan(0); + expect(outcome.mechanicalLeaves).toBe(outcome.leafSummaryIds.length); + expect(outcome.mechanicalCondensed ?? 0).toBe(outcome.condensedSummaryIds.length); + for (const id of outcome.leafSummaryIds) { + expect(store.getSummary(id)?.quality).toBe("mechanical"); + } + expect(countRows(path, "status = 'committed'")).toBeGreaterThan(0); + expect(countRows(path, "status = 'pending'")).toBe(0); + } finally { + store.close(); + } + }); + + it("still throws when a prose span fails with mechanicalTools configured", async () => { + const session = "ses_pass_mechanical_prose_fail"; + const { store, path, entries } = seeded(session); + try { + const ctx = ctxWith({ text: () => Promise.reject(new Error("provider down")) }); + await expect( + runLosslessPass({ store: () => store, ctx, model: MODEL }, session, entries, { + targetTokens: 200, + summarizer: { mechanicalTools: ["noisy-read"] }, + }), + ).rejects.toThrow(/summarizer did not answer/); + expect(countRows(path, "status = 'committed'")).toBe(0); + expect(countRows(path, "status = 'pending'")).toBe(0); + } finally { + store.close(); + } + }); + it("returns a no-entries outcome without opening a run", async () => { const { store } = seeded(); try { diff --git a/packages/opencode/test/plugin.test.ts b/packages/opencode/test/plugin.test.ts index c869b38..4cc2c0a 100644 --- a/packages/opencode/test/plugin.test.ts +++ b/packages/opencode/test/plugin.test.ts @@ -122,6 +122,21 @@ describe("setupOpencodeLossless", () => { } }); + it("resolves mechanicalTools and keeps the default empty on invalid values", () => { + expect( + readOptions({ mechanicalTools: ["noisy-read", " noisy-read ", ""] }).mechanicalTools, + ).toEqual(["noisy-read"]); + expect(readOptions({ mechanicalTools: "noisy-read" }).mechanicalTools).toEqual(["noisy-read"]); + expect(readOptions({}).mechanicalTools).toBeUndefined(); + for (const raw of [ + { mechanicalTools: 42 }, + { mechanicalTools: [""] }, + { mechanicalTools: [] }, + ]) { + expect(readOptions(raw).mechanicalTools).toBeUndefined(); + } + }); + it("mounts the mirror, the recall tools, and the status command", async () => { const host = fakeHost({ sidecarDir: tempDir(), assembly: false }); const cleanup = await setupOpencodeLossless(host.ctx, host.options); @@ -141,6 +156,7 @@ describe("setupOpencodeLossless", () => { "lossless-export", "lossless-import", "lossless-diagnose", + "lossless-dump", "lossless-migrate", "lossless-gc", "lossless-heal", @@ -278,6 +294,7 @@ describe("setupOpencodeLossless", () => { "lossless-export", "lossless-import", "lossless-diagnose", + "lossless-dump", "lossless-migrate", "lossless-gc", "lossless-heal", @@ -296,6 +313,7 @@ describe("setupOpencodeLossless", () => { "lossless-export", "lossless-import", "lossless-diagnose", + "lossless-dump", "lossless-migrate", "lossless-gc", "lossless-heal", diff --git a/packages/opencode/test/sidecar-dump.test.ts b/packages/opencode/test/sidecar-dump.test.ts new file mode 100644 index 0000000..6089be5 --- /dev/null +++ b/packages/opencode/test/sidecar-dump.test.ts @@ -0,0 +1,523 @@ +import { + chmodSync, + existsSync, + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { DatabaseSync } from "node:sqlite"; + +import { ingestEntries, LcmStore, redactSecrets } from "lossless-core"; +import { describe, expect, it } from "vite-plus/test"; + +import type { CommandInvocation } from "../src/host.ts"; +import { dumpCommand, type LifecycleDeps } from "../src/lifecycle.ts"; +import { DUMP_MAX_BYTES, dumpSidecar } from "../src/sidecar-dump.ts"; +import { Sidecars } from "../src/sidecars.ts"; + +const SESSION = "ses_dump_test"; +const SECRET = "sk-ant-abcdefghijklmnopqrstuvwxyz0123456789abcd"; + +function tempDir(): string { + return mkdtempSync(join(tmpdir(), "opencode-lossless-dump-")); +} + +function seedStore(path: string): void { + const store = new LcmStore(path); + try { + ingestEntries(store, [ + { entryId: "m1", role: "user", text: `first note about harbors ${SECRET}`, timestamp: 1 }, + { entryId: "m2", role: "assistant", text: "second note about harbors", timestamp: 2 }, + ]); + const first = store.messageByEntryId("m1"); + const second = store.messageByEntryId("m2"); + if (first && second) { + store.insertSummary({ + kind: "leaf", + text: "harbor summary", + tokens: 5, + depth: 0, + firstEntryId: "m1", + lastEntryId: "m2", + messageIds: [first.id, second.id], + }); + store.setRemoved(["m2"], Date.now()); + } + } finally { + store.close(); + } +} + +function countMessages(path: string): number { + const store = new LcmStore(path, { queryOnly: true }); + try { + return store.messageCount(); + } finally { + store.close(); + } +} + +describe("sidecar dump", () => { + it("dumps a live session read-only with redaction and removed labels", () => { + const dir = tempDir(); + try { + const path = join(dir, "live.sqlite"); + seedStore(path); + const before = countMessages(path); + const filesBefore = readdirSync(dir).sort(); + const result = dumpSidecar({ + path, + sessionId: SESSION, + redact: redactSecrets, + includeRemoved: true, + metricRows: [{ session: SESSION, kind: "context-decision", note: SECRET }], + sampleQuery: "harbors", + }); + expect(result.status).toBe("dumped"); + if (result.status !== "dumped") return; + expect(result.text).toContain("LCM sidecar dump"); + expect(result.text).not.toContain(SECRET); + expect(result.text).toContain("[REDACTED]"); + expect(result.text).toContain("[removed]"); + expect(countMessages(path)).toBe(before); + const filesAfter = readdirSync(dir).sort(); + expect(filesAfter.filter((f) => f.endsWith("-journal"))).toEqual([]); + expect(filesAfter).toEqual(filesBefore); + expect(existsSync(`${path}-journal`)).toBe(false); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("reports an old generation as a named skip instead of crashing", () => { + const dir = tempDir(); + try { + const path = join(dir, "old.sqlite"); + seedStore(path); + const raw = new DatabaseSync(path); + try { + raw.exec("PRAGMA user_version = 1"); + } finally { + raw.close(); + } + const result = dumpSidecar({ path, sessionId: SESSION, redact: redactSecrets }); + expect(result.status).toBe("skipped"); + if (result.status !== "skipped") return; + expect(result.reason).toBe("older-generation"); + expect(result.detail.length).toBeGreaterThan(0); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("reports a missing sidecar as a named skip", () => { + const dir = tempDir(); + try { + const result = dumpSidecar({ + path: join(dir, "absent.sqlite"), + sessionId: SESSION, + redact: redactSecrets, + }); + expect(result.status).toBe("skipped"); + if (result.status !== "skipped") return; + expect(result.reason).toBe("missing"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("stays within its byte cap", () => { + const dir = tempDir(); + try { + const path = join(dir, "big.sqlite"); + const store = new LcmStore(path); + try { + ingestEntries( + store, + Array.from({ length: 30 }, (_, i) => ({ + entryId: `e${i}`, + role: "user" as const, + text: `harbor note ${i} ${"filler ".repeat(200)}`, + timestamp: i, + })), + ); + } finally { + store.close(); + } + const result = dumpSidecar({ + path, + sessionId: SESSION, + redact: redactSecrets, + metricRows: Array.from({ length: 60 }, (_, i) => ({ + session: SESSION, + kind: "usage", + input: i, + note: `row ${i} ${"padding ".repeat(200)}`, + })), + }); + expect(result.status).toBe("dumped"); + if (result.status !== "dumped") return; + expect(Buffer.byteLength(result.text, "utf8")).toBeLessThanOrEqual(DUMP_MAX_BYTES); + expect(result.text).toContain("truncated at the"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("caps per-summary text so metrics and footer survive", () => { + const dir = tempDir(); + try { + const path = join(dir, "capped.sqlite"); + const store = new LcmStore(path); + try { + ingestEntries( + store, + Array.from({ length: 10 }, (_, i) => ({ + entryId: `c${i}`, + role: "user" as const, + text: `note ${i}`, + timestamp: i, + })), + ); + const ids = Array.from( + { length: 10 }, + (_, i) => store.messageByEntryId(`c${i}`)?.id, + ).filter((v): v is number => v !== undefined); + for (let s = 0; s < 5; s++) { + store.insertSummary({ + kind: "leaf", + text: `summary ${s}`, + tokens: 1500, + depth: 0, + firstEntryId: `c${s * 2}`, + lastEntryId: `c${s * 2 + 1}`, + messageIds: [ids[s * 2]!, ids[s * 2 + 1]!], + thoroughText: `thorough ${s} ${"x".repeat(6000)}`, + }); + } + } finally { + store.close(); + } + const marker = "cap-marker-note"; + const result = dumpSidecar({ + path, + sessionId: SESSION, + redact: redactSecrets, + metricRows: [{ session: SESSION, kind: "usage", note: marker }], + sampleQuery: "note", + }); + expect(result.status).toBe("dumped"); + if (result.status !== "dumped") return; + expect(result.text).toContain("[lcm:more"); + expect(result.text).toContain(marker); + expect(result.text).toContain("metrics:"); + expect(result.text).toContain("omitted:"); + expect(result.text).not.toContain("no absolute paths"); + expect(result.text).not.toContain("truncated at the"); + expect(Buffer.byteLength(result.text, "utf8")).toBeLessThanOrEqual(DUMP_MAX_BYTES); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("caps CJK summaries by bytes so the tail survives", () => { + // 1000 CJK chars fit the old char count but exceed 1200 bytes. + // Uncapped, ten such summaries pass DUMP_MAX_BYTES and clip the tail. + const dir = tempDir(); + try { + const path = join(dir, "cjk.sqlite"); + const store = new LcmStore(path); + try { + ingestEntries( + store, + Array.from({ length: 10 }, (_, i) => ({ + entryId: `k${i}`, + role: "user" as const, + text: `note ${i}`, + timestamp: i, + })), + ); + const ids = Array.from( + { length: 10 }, + (_, i) => store.messageByEntryId(`k${i}`)?.id, + ).filter((v): v is number => v !== undefined); + for (let s = 0; s < 10; s++) { + store.insertSummary({ + kind: "leaf", + text: `summary ${s}`, + tokens: 5, + depth: 0, + firstEntryId: `k${s}`, + lastEntryId: `k${s}`, + messageIds: [ids[s]!], + thoroughText: "字".repeat(1000), + }); + } + } finally { + store.close(); + } + const marker = "cjk-cap-marker"; + const result = dumpSidecar({ + path, + sessionId: SESSION, + redact: redactSecrets, + metricRows: [{ session: SESSION, kind: "usage", note: marker }], + }); + expect(result.status).toBe("dumped"); + if (result.status !== "dumped") return; + expect(result.text).toContain("[lcm:more"); + expect(result.text).toContain("bytes]"); + expect(result.text).toContain(marker); + expect(result.text).toContain("metrics:"); + expect(result.text).toContain("omitted:"); + expect(result.text).toContain("bytes each"); + expect(result.text).not.toContain("truncated at the"); + expect(Buffer.byteLength(result.text, "utf8")).toBeLessThanOrEqual(DUMP_MAX_BYTES); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("masks linked file previews and the echoed query, keeps paths", () => { + const dir = tempDir(); + try { + const path = join(dir, "file.sqlite"); + const store = new LcmStore(path); + try { + ingestEntries(store, [ + { entryId: "m1", role: "user", text: "note about harbors", timestamp: 1 }, + ]); + store.insertFile( + { + fileId: "file-secret-1", + path: "notes/keys.txt", + bytes: 64, + sha256: "0".repeat(64), + kind: "text", + preview: `${SECRET}\nsecond line of the file`, + }, + "m1", + ); + const first = store.messageByEntryId("m1"); + expect(first).toBeDefined(); + store.insertSummary({ + kind: "leaf", + text: "harbor summary", + tokens: 5, + depth: 0, + firstEntryId: "m1", + lastEntryId: "m1", + messageIds: [first!.id], + fileIds: ["file-secret-1"], + }); + } finally { + store.close(); + } + const result = dumpSidecar({ + path, + sessionId: SESSION, + redact: redactSecrets, + sampleQuery: `harbors ${SECRET}`, + }); + expect(result.status).toBe("dumped"); + if (result.status !== "dumped") return; + expect(result.text).not.toContain(SECRET); + expect(result.text).toContain("[REDACTED]"); + expect(result.text).toContain("notes/keys.txt"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("skips removed rows by default and labels them with --include-removed", () => { + const dir = tempDir(); + try { + const path = join(dir, "removed.sqlite"); + seedStore(path); + const hidden = dumpSidecar({ path, sessionId: SESSION, redact: redactSecrets }); + expect(hidden.status).toBe("dumped"); + if (hidden.status !== "dumped") return; + expect(hidden.text).not.toContain("second note about harbors"); + expect(hidden.text).not.toContain("[m2]"); + expect(hidden.text).toContain("first note about harbors"); + expect(hidden.text).toContain("[REDACTED]"); + + const shown = dumpSidecar({ + path, + sessionId: SESSION, + redact: redactSecrets, + includeRemoved: true, + }); + expect(shown.status).toBe("dumped"); + if (shown.status !== "dumped") return; + expect(shown.text).toContain("second note about harbors"); + expect(shown.text).toContain("[m2] (assistant) [removed]"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("saves through a private temp at mode 0600", async () => { + const dir = tempDir(); + const sidecars = new Sidecars(dir); + const texts: string[] = []; + const deps: LifecycleDeps = { + sidecars, + options: { + assembly: true, + swapAtRatio: 0.7, + recutAtRatio: 0.85, + keepRecentTokens: 20_000, + }, + version: () => "opencode-lossless test", + host: () => "opencode v2", + prompt: (_sessionID, text) => { + texts.push(text); + return Promise.resolve(undefined); + }, + record: () => {}, + heal: async () => { + throw new Error("heal is not scripted in this harness"); + }, + }; + try { + const store = sidecars.for(SESSION); + ingestEntries(store, [ + { entryId: `${SESSION}:prompt:msg_1`, role: "user", text: "saved dump note", timestamp: 1 }, + ]); + const target = join(dir, "lossless-dump.txt"); + writeFileSync(target, "stale bytes", { mode: 0o644 }); + chmodSync(target, 0o644); + await dumpCommand(deps).execute({ + sessionID: SESSION, + prompt: { text: `/lossless-dump --save ${target}` }, + delivery: {}, + } as CommandInvocation); + expect(statSync(target).mode & 0o777).toBe(0o600); + const written = readFileSync(target, "utf8"); + expect(written).toContain("LCM sidecar dump"); + expect(written).not.toContain("stale bytes"); + expect(existsSync(`${target}.tmp`)).toBe(false); + expect(texts.at(-1)).toContain("(mode 0600)"); + } finally { + sidecars.close(); + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("leaves a planted predictable temp untouched", async () => { + const dir = tempDir(); + const sidecars = new Sidecars(dir); + const texts: string[] = []; + const deps: LifecycleDeps = { + sidecars, + options: { + assembly: true, + swapAtRatio: 0.7, + recutAtRatio: 0.85, + keepRecentTokens: 20_000, + }, + version: () => "opencode-lossless test", + host: () => "opencode v2", + prompt: (_sessionID, text) => { + texts.push(text); + return Promise.resolve(undefined); + }, + record: () => {}, + heal: async () => { + throw new Error("heal is not scripted in this harness"); + }, + }; + try { + const store = sidecars.for(SESSION); + ingestEntries(store, [ + { entryId: `${SESSION}:prompt:msg_1`, role: "user", text: "saved dump note", timestamp: 1 }, + ]); + const target = join(dir, "lossless-dump-plant.txt"); + const decoy = `${target}.tmp`; + writeFileSync(decoy, "decoy bytes", { mode: 0o644 }); + await dumpCommand(deps).execute({ + sessionID: SESSION, + prompt: { text: `/lossless-dump --save ${target}` }, + delivery: {}, + } as CommandInvocation); + expect(readFileSync(decoy, "utf8")).toBe("decoy bytes"); + const written = readFileSync(target, "utf8"); + expect(written).toContain("LCM sidecar dump"); + expect(statSync(target).mode & 0o777).toBe(0o600); + expect(readdirSync(dir).filter((f) => f.endsWith(".tmp"))).toEqual([ + "lossless-dump-plant.txt.tmp", + ]); + } finally { + sidecars.close(); + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("removes its temp on every post-creation failure path", async () => { + // node:fs module namespaces are not configurable under ESM, so a write + // that creates the file then fails (disk full) resists spy fault + // injection. Pin the cleanup structure instead: the write catch unlinks + // the candidate before rethrowing, and EEXIST keeps its continue above + // without unlinking. + const source = readFileSync(new URL("../src/lifecycle.ts", import.meta.url), "utf8"); + const helper = source.slice( + source.indexOf("function writeDumpTarget"), + source.indexOf("cannot create temp file"), + ); + const eexist = helper.indexOf('code === "EEXIST"'); + const unlink = helper.indexOf("unlinkSync(candidate)"); + const rethrow = helper.indexOf("throw error"); + expect(eexist).toBeGreaterThanOrEqual(0); + expect(unlink).toBeGreaterThanOrEqual(0); + expect(rethrow).toBeGreaterThanOrEqual(0); + expect(eexist).toBeLessThan(unlink); + expect(unlink).toBeLessThan(rethrow); + // Success path leaves no temp behind beside the destination. + const dir = tempDir(); + const sidecars = new Sidecars(dir); + const texts: string[] = []; + const deps: LifecycleDeps = { + sidecars, + options: { + assembly: true, + swapAtRatio: 0.7, + recutAtRatio: 0.85, + keepRecentTokens: 20_000, + }, + version: () => "opencode-lossless test", + host: () => "opencode v2", + prompt: (_sessionID, text) => { + texts.push(text); + return Promise.resolve(undefined); + }, + record: () => {}, + heal: async () => { + throw new Error("heal is not scripted in this harness"); + }, + }; + try { + const store = sidecars.for(SESSION); + ingestEntries(store, [ + { entryId: `${SESSION}:prompt:msg_1`, role: "user", text: "saved dump note", timestamp: 1 }, + ]); + const target = join(dir, "lossless-dump-clean.txt"); + await dumpCommand(deps).execute({ + sessionID: SESSION, + prompt: { text: `/lossless-dump --save ${target}` }, + delivery: {}, + } as CommandInvocation); + expect(existsSync(target)).toBe(true); + expect(readdirSync(dir).filter((f) => f.endsWith(".tmp"))).toEqual([]); + expect(texts.at(-1)).toContain("(mode 0600)"); + } finally { + sidecars.close(); + rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/packages/pi/README.md b/packages/pi/README.md index 893dffd..741ff72 100644 --- a/packages/pi/README.md +++ b/packages/pi/README.md @@ -32,6 +32,8 @@ once context crosses the soft threshold (default: 70% of the model's window). /lcm backup # copy the live store to a timestamped file /lcm export # write the full store as JSONL (verbatim; see Security notes) /lcm report # metrics summary for the current session +/lcm dump # read-only dump of the session sidecar; add --include-removed + # to keep removed-branch rows, --save to write mode 0600 /lcm doctor # diagnostics: store checks, thresholds, summarizer chain, # footprint; add --repair to rebuild the index and drop the rows # nothing can reach @@ -61,7 +63,9 @@ a call, or a span needs, because an index cannot hold punctuation. A pattern is case-insensitive unless `case_sensitive` is set. Both paths page: hits come 20 at a time, `offset` reaches the next page, and the query path reports its total while the pattern path says whether more matches exist rather than claiming a -count it did not compute. +count it did not compute. Four filters narrow the search: `role` keeps +only rows stored with that role, `date_start` and `date_end` bound rows by +timestamp, and `tool_name` keeps only tool-result rows for that tool. All three take an address when you have one, so a known id does not have to be searched for: `lcm_grep` accepts `summary_id` to search inside one node, and diff --git a/packages/pi/skills/pi-lossless/SKILL.md b/packages/pi/skills/pi-lossless/SKILL.md index 387a88b..86c69e2 100644 --- a/packages/pi/skills/pi-lossless/SKILL.md +++ b/packages/pi/skills/pi-lossless/SKILL.md @@ -1,6 +1,6 @@ --- name: pi-lossless -description: "Use when a Pi session has pi-lossless installed: search compacted history with lcm_grep, lcm_describe, or lcm_expand_query; run /lcm status, settings, backup, export, report, doctor, gc, migrate, heal, or diagnose; or tune compaction thresholds." +description: "Use when a Pi session has pi-lossless installed: search compacted history with lcm_grep, lcm_describe, or lcm_expand_query; run /lcm status, settings, backup, export, report, doctor, gc, migrate, heal, dump, or diagnose; or tune compaction thresholds." license: MIT metadata: type: composition @@ -35,6 +35,7 @@ soft threshold (default 70% of the model window). - `/lcm backup` — copy the live store to a timestamped file. - `/lcm export` — write the full store as JSONL (verbatim bytes, secrets included). - `/lcm report` — metrics summary for the current session. +- `/lcm dump` prints a read-only dump of the session sidecar. `--include-removed` keeps removed-branch rows and `--save ` writes the file at mode 0600. - `/lcm doctor` — diagnostics: store checks, thresholds, summarizer chain, footprint; `--repair` rebuilds the index and drops unreachable rows. - `/lcm gc` — dry-run list of retired stores; `--apply` exports then DELETEs them (destructive; export lands beside the stores). - `/lcm migrate` — dry-run list of older-schema stores; `--apply` upgrades them in place for cross-session search. @@ -43,7 +44,7 @@ soft threshold (default 70% of the model window). ## Recall tools -- `lcm_grep` — search ALL history, including compacted-out content. `query` is full-text over the token index (words, quoted phrases, `AND`/`OR`, prefixes; first 100,000 chars per message). `pattern` is a case-insensitive regex over full text (up to 200,000 chars per message, 1s cap per call) — use it for pointers, calls, punctuation, or spans the index cannot hold. Hits page 20 at a time via `offset`; `query` reports a total, `pattern` reports whether more exist. `summary_id` scopes to one node; `scope: "sessions"` / `"all_sessions"` reaches past sessions. +- `lcm_grep` searches ALL history, including compacted-out content. `query` is full-text over the token index (words, quoted phrases, `AND`/`OR`, prefixes; first 100,000 chars per message). `pattern` is a case-insensitive regex over full text (up to 200,000 chars per message, 1s cap per call). Use it for pointers, calls, punctuation, or spans the index cannot hold. Hits page 20 at a time via `offset`; `query` reports a total, `pattern` reports whether more exist. `summary_id` scopes to one node; `scope: "sessions"` / `"all_sessions"` reaches past sessions. `role` keeps only rows stored with that role, `date_start` and `date_end` bound rows by timestamp, and `tool_name` keeps only tool-result rows for that tool. - `lcm_describe` — inspect a summary: what reading its originals would cost and which files it covers. - `lcm_expand_query` — bounded reader over stored originals; returns findings without that text entering the main conversation. Default budget 10,000 tokens over at most 6 reader turns; `max_tokens` changes the per-call budget. `summary_id` reads one node, `entry_id` one message. Cross-session: `lcm_expand_query({ session: "", entry_id: "" })` on a grep hit; findings are tagged as past-session history. Unknown ids are refused before any model call. diff --git a/packages/pi/src/index.ts b/packages/pi/src/index.ts index 4f4d488..0c7371f 100644 --- a/packages/pi/src/index.ts +++ b/packages/pi/src/index.ts @@ -429,7 +429,23 @@ export default function (pi: ExtensionAPI) { const probe = computeRawIngestDelta(raw, ingestWatermark); const delta = collectMessageEntries(probe.reset ? raw : raw.slice(probe.startIndex)); const ingestible = delta.map(toIngestible); - const opened = ingestible.some(isIngestible) ? openStore() : undefined; + const hasIngestible = ingestible.some(isIngestible); + if (ingestWatermark !== null && probe.reset && hasIngestible) { + appendMetric({ + event: "lcm", + kind: "ingest-rescan", + session: sessionTag, + watermark: ingestWatermark.id, + entries: raw.length, + }); + if (ctx.hasUI) { + ctx.ui.notify( + `LCM: watermark ${ingestWatermark.id} not found, rescanned ${raw.length} entries`, + "warning", + ); + } + } + const opened = hasIngestible ? openStore() : undefined; if (opened) { const stats = ingestEntries(opened, ingestible, { externalize: externalizePolicy(), @@ -617,6 +633,9 @@ export default function (pi: ExtensionAPI) { redact, session: sessionTag, stage: "blocking", + ...(config.mechanicalTools === undefined + ? {} + : { mechanicalTools: config.mechanicalTools }), }, ); const closed = store.commitRun(token); @@ -765,6 +784,9 @@ export default function (pi: ExtensionAPI) { redact, session: sessionTag, stage: "async", + ...(config.mechanicalTools === undefined + ? {} + : { mechanicalTools: config.mechanicalTools }), }, ) .then((outcome) => { diff --git a/packages/pi/src/tools/recall.ts b/packages/pi/src/tools/recall.ts index bb570b6..a9dd23e 100644 --- a/packages/pi/src/tools/recall.ts +++ b/packages/pi/src/tools/recall.ts @@ -58,6 +58,27 @@ const GrepParams = Type.Object({ description: "Include matches from a branch that left the active path (excluded by default)", }), ), + // Unsafe rather than a union of literals, for the same reason as `scope` + // above: an anyOf schema does not survive Google's API. + role: Type.Optional( + Type.Unsafe<"user" | "assistant" | "toolResult" | "custom">({ + type: "string", + enum: ["user", "assistant", "toolResult", "custom"], + description: "Keep only rows stored with this role", + }), + ), + date_start: Type.Optional( + Type.Number({ description: "Keep only rows at or after this timestamp" }), + ), + date_end: Type.Optional( + Type.Number({ description: "Keep only rows at or before this timestamp" }), + ), + tool_name: Type.Optional( + Type.String({ + description: + "Keep only tool-result rows for this tool. It matches [tool:Name] markers. It applies to query searches only.", + }), + ), // Unsafe rather than a union of literals: an anyOf schema does not survive // Google's API, and this parameter decides which store the call reads. Built // with this package's typebox so the emitted declaration names a portable @@ -215,6 +236,10 @@ export function createGrepTool(deps: RecallDeps) { limit: params.limit, summaryId: params.summary_id, includeRemoved: params.include_removed, + ...(params.role === undefined ? {} : { role: params.role }), + ...(params.date_start === undefined ? {} : { dateStart: params.date_start }), + ...(params.date_end === undefined ? {} : { dateEnd: params.date_end }), + ...(params.tool_name === undefined ? {} : { toolName: params.tool_name }), }); const hits = view.ok && typeof view.details.hits === "number" ? view.details.hits : undefined; recordRecall( @@ -230,7 +255,15 @@ export function createGrepTool(deps: RecallDeps) { function searchPastSessions( deps: RecallDeps, - params: { query?: string; pattern?: string; limit?: number }, + params: { + query?: string; + pattern?: string; + limit?: number; + role?: "user" | "assistant" | "toolResult" | "custom"; + date_start?: number; + date_end?: number; + tool_name?: string; + }, scope: "sessions" | "all_sessions", ): ToolResultShape { const roots = deps.sessions?.(); @@ -254,6 +287,10 @@ function searchPastSessions( }); const result = searchSessions(refs, params.query, { limit: Math.min(params.limit ?? SESSION_HITS_LIMIT, SESSION_HITS_LIMIT), + ...(params.role === undefined ? {} : { role: params.role }), + ...(params.date_start === undefined ? {} : { dateStart: params.date_start }), + ...(params.date_end === undefined ? {} : { dateEnd: params.date_end }), + ...(params.tool_name === undefined ? {} : { toolName: params.tool_name }), }); const skipped = result.skipped.missing + result.skipped.unreadable + result.skipped["older-generation"]; diff --git a/packages/pi/src/ui/command.ts b/packages/pi/src/ui/command.ts index 423a8f0..53efe77 100644 --- a/packages/pi/src/ui/command.ts +++ b/packages/pi/src/ui/command.ts @@ -4,11 +4,12 @@ import { existsSync, readFileSync, readdirSync, + renameSync, unlinkSync, writeFileSync, } from "node:fs"; import { homedir } from "node:os"; -import { basename, join, resolve } from "node:path"; +import { basename, dirname, join, resolve } from "node:path"; import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; import { lcmHomePath } from "lossless-core"; @@ -42,6 +43,7 @@ import { type LcmConfig, lcmConfigPath, } from "./config-io.ts"; +import { dumpSidecar } from "./sidecar-dump.ts"; export interface LcmCommandDeps { getStore(): LcmStore | undefined; @@ -65,6 +67,7 @@ export const LCM_SUBCOMMANDS = [ "export", "import", "report", + "dump", "doctor", "diagnose", "gc", @@ -111,6 +114,46 @@ function maxNodesValue(raw: string): MaxNodesFlag { return Number.isInteger(value) && value > 0 ? { kind: "value", value } : { kind: "invalid" }; } +function writeDumpTarget(target: string, text: string): void { + // Unique temp with exclusive creation, then an atomic rename: a planted + // predictable path is never touched and the destination swaps at once. + const dir = dirname(target); + const base = basename(target); + let tmp = ""; + for (let attempt = 0; attempt < 5; attempt++) { + const candidate = join( + dir, + `.${base}.${process.pid}.${Date.now()}.${Math.floor(Math.random() * 1e9)}.tmp`, + ); + try { + writeFileSync(candidate, text, { encoding: "utf8", mode: 0o600, flag: "wx" }); + tmp = candidate; + break; + } catch (error) { + if ((error as { code?: string })?.code === "EEXIST") continue; + // A write can create the private file and then fail (disk full), so + // remove the candidate before surfacing the error. EEXIST keeps its + // continue above and never unlinks a pre-existing collision. + try { + unlinkSync(candidate); + } catch {} + throw error; + } + } + if (tmp === "") throw new Error(`LCM dump: cannot create temp file beside ${target}.`); + try { + try { + chmodSync(tmp, 0o600); + } catch {} + renameSync(tmp, target); + } catch (error) { + try { + unlinkSync(tmp); + } catch {} + throw error; + } +} + /** A dry run reads the plan and makes no provider call. The type still carries * the sink, so `runHeal` records its start and complete rows. */ const healDryRunLlm: LlmComplete = Object.assign( @@ -596,6 +639,61 @@ export function registerLcmCommand( return; } + if (sub === "dump") { + const flags = (args ?? "").trim().split(/\s+/).slice(1); + const save = parseSaveFlag(flags); + if (save.kind === "missing-path") { + ctx.ui.notify( + "LCM dump --save: name the file to write, e.g. `/lcm dump --save lcm-dump.txt`.", + "warning", + ); + return; + } + const includeRemoved = flags.includes("--include-removed"); + const savePath = save.kind === "path" ? save.path : undefined; + const query = flags + .filter((flag, index) => { + if (flag.startsWith("--")) return false; + if (save.kind === "path" && flags[index - 1] === "--save") return false; + if (save.kind === "path" && flag === save.path && flags.includes("--save")) { + return false; + } + return true; + }) + .join(" "); + const tag = basename(db); + const result = dumpSidecar({ + path: db, + sessionId: tag, + redact: makeRedact(deps.getConfig), + ...(includeRemoved ? { includeRemoved: true as const } : {}), + metricRows: collectMetricsEvents().filter((e) => e.session === tag), + ...(query.length > 0 ? { sampleQuery: query } : {}), + }); + if (result.status === "skipped") { + ctx.ui.notify(`LCM dump skipped: ${result.reason} (${result.detail}).`, "warning"); + return; + } + if (savePath !== undefined) { + try { + writeDumpTarget(savePath, `${result.text}\n`); + } catch (error) { + ctx.ui.notify( + `LCM dump: cannot write ${savePath} (${error instanceof Error ? error.message : String(error)}).`, + "warning", + ); + return; + } + ctx.ui.notify( + `LCM dump: wrote ${Buffer.byteLength(result.text, "utf8")} bytes to ${savePath} (mode 0600).`, + "info", + ); + return; + } + ctx.ui.notify(result.text, "info"); + return; + } + if (sub === "doctor") { const usage = ctx.getContextUsage(); const window = usage?.contextWindow ?? 0; @@ -604,7 +702,13 @@ export function registerLcmCommand( const pin = deps.getCommitState(); const flags = (args ?? "").trim().split(/\s+/).slice(1); const repairs = flags.includes("--repair") ? repairIntegrity(store) : []; - const integrity = checkIntegrity(store); + // Pi sidecars are `.db` files under the lcm dir, which the default + // `.sqlite` scan would miss; the live store itself is never an orphan. + const integrity = checkIntegrity(store, { + sidecarDir: lcmHomePath("lcm"), + sidecarExtension: ".db", + liveSidecarPaths: [db], + }); const violations = integrity.findings.filter((f) => f.severity === "violation"); const chain = describeSummarizerChain(resolveModels(piModelHost(ctx), config)); const lines = [ @@ -677,7 +781,12 @@ export function registerLcmCommand( const window = usage?.contextWindow ?? 0; const mk = piModelKey(ctx); const z = config.zones?.[mk]; - const integrity = checkIntegrity(store); + // Same `.db` sidecar scope as doctor, so diagnose reports what doctor sees. + const integrity = checkIntegrity(store, { + sidecarDir: lcmHomePath("lcm"), + sidecarExtension: ".db", + liveSidecarPaths: [db], + }); const chain = describeSummarizerChain(resolveModels(piModelHost(ctx), config)); const effective = { ...DEFAULT_CONFIG, ...config }; const tag = basename(db); diff --git a/packages/pi/src/ui/config-io.ts b/packages/pi/src/ui/config-io.ts index 2613697..92c9616 100644 --- a/packages/pi/src/ui/config-io.ts +++ b/packages/pi/src/ui/config-io.ts @@ -14,6 +14,9 @@ import type { ZoneThresholds } from "lossless-core"; export interface LcmConfig { summarizer?: string | string[]; + /** Tool names whose spans compact without a model call. Empty keeps the + * LLM path; ingest still stores every message verbatim. */ + mechanicalTools?: readonly string[]; smartZone?: number; swapAtTokens?: number; recutAtTokens?: number; @@ -166,6 +169,7 @@ const KNOWN_KEYS: ReadonlySet = new Set([ ...NUMBER_KEYS, ...BOOLEAN_KEYS, "summarizer", + "mechanicalTools", "zones", ]); @@ -205,6 +209,8 @@ export function parseLcmConfig(raw: unknown): ParsedLcmConfig { } const summarizer = parseSummarizer(raw.summarizer, dropped, notes); if (summarizer !== undefined) config.summarizer = summarizer; + const mechanicalTools = parseMechanicalTools(raw.mechanicalTools, dropped); + if (mechanicalTools !== undefined) config.mechanicalTools = mechanicalTools; if (raw.zones !== undefined) { if (isPlainObject(raw.zones)) { const zones: Record = {}; @@ -284,6 +290,32 @@ function parseSummarizer( return entries.length === 1 ? entries[0] : entries; } +/** Tool names that skip the model. Accepts one name or a list; blanks and + * repeats drop quietly and the engine normalizes the rest. */ +function parseMechanicalTools(raw: unknown, dropped: string[]): string[] | undefined { + if (raw === undefined) return undefined; + const list = typeof raw === "string" ? [raw] : Array.isArray(raw) ? raw : undefined; + if (list === undefined) { + dropped.push("mechanicalTools (wrong type)"); + return undefined; + } + const entries: string[] = []; + const seen = new Set(); + for (const [i, entry] of list.entries()) { + if (typeof entry !== "string") { + dropped.push(`mechanicalTools[${i}] (wrong type)`); + continue; + } + const name = entry.trim(); + if (name.length === 0) continue; + const key = name.toLowerCase(); + if (seen.has(key)) continue; + seen.add(key); + entries.push(name); + } + return entries; +} + function copyAside(path: string, backup: string): string { try { copyFileSync(path, backup); diff --git a/packages/pi/src/ui/sidecar-dump.ts b/packages/pi/src/ui/sidecar-dump.ts new file mode 100644 index 0000000..b374582 --- /dev/null +++ b/packages/pi/src/ui/sidecar-dump.ts @@ -0,0 +1,279 @@ +import { existsSync } from "node:fs"; +import { basename } from "node:path"; + +import { + describeView, + expandView, + grepView, + historyManifest, + LcmStore, + StaleStoreError, +} from "lossless-core"; + +/** Max bytes one dump prints; callers copy the file when they need more. */ +export const DUMP_MAX_BYTES = 16_384; +/** Summaries described in full; the header still counts the rest. */ +export const DUMP_MAX_SUMMARIES = 10; +/** Metric rows kept from the tail of the session's rows. */ +export const DUMP_MAX_METRIC_ROWS = 20; +/** Top-level summaries the manifest names. */ +export const DUMP_MAX_HISTORY = 20; +/** Messages read as windows from the first summary. */ +export const DUMP_MAX_SAMPLE_MESSAGES = 3; +/** Chars one sampled message window holds. */ +export const DUMP_SAMPLE_CHARS = 400; +/** Bytes kept from each described summary; the tail names the cut. */ +export const DUMP_MAX_SUMMARY_BYTES = 1_200; + +/** Why a sidecar could not be dumped. Each reason is a count the caller + * reports, never an error it raises. */ +export type DumpSkipReason = "missing" | "unreadable" | "older-generation"; + +export type SidecarDumpResult = + | { status: "dumped"; text: string } + | { status: "skipped"; reason: DumpSkipReason; detail: string }; + +export interface SidecarDumpOptions { + /** Sidecar file to open read-only. Never created when absent. */ + readonly path: string; + /** Session id naming the dump and filtering metric rows. */ + readonly sessionId: string; + /** Masks secret-shaped text on every displayed seam. */ + readonly redact: (text: string) => string; + /** Include matches from a branch that left the active path. */ + readonly includeRemoved?: boolean; + /** Metric rows for this sidecar directory, oldest first. */ + readonly metricRows?: readonly Record[]; + /** Optional FTS5 query sampled through grepView. */ + readonly sampleQuery?: string; +} + +/** Read-only dump of one session sidecar plus its metrics. Opens the file + * with `queryOnly`, so SQLite refuses writes, and closes it before returning. + * An older generation reports a named skip instead of migrating. */ +export function dumpSidecar(options: SidecarDumpOptions): SidecarDumpResult { + if (!existsSync(options.path)) { + return { + status: "skipped", + reason: "missing", + detail: `no sidecar at ${basename(options.path)}`, + }; + } + let store: LcmStore | undefined; + try { + store = new LcmStore(options.path, { queryOnly: true }); + } catch (error) { + if (error instanceof StaleStoreError) { + return { + status: "skipped", + reason: "older-generation", + detail: + `generation ${error.generation} is older than this build. ` + + `Run migrate first. No rows were changed.`, + }; + } + return { + status: "skipped", + reason: "unreadable", + detail: error instanceof Error ? error.message.slice(0, 200) : String(error), + }; + } + try { + return { status: "dumped", text: buildDump(store, options) }; + } catch (error) { + return { + status: "skipped", + reason: "unreadable", + detail: error instanceof Error ? error.message.slice(0, 200) : String(error), + }; + } finally { + try { + store.close(); + } catch {} + } +} + +function buildDump(store: LcmStore, options: SidecarDumpOptions): string { + const includeRemoved = options.includeRemoved === true; + const stats = store.stats(); + const removed = stats.removedMessages > 0 ? ` (${stats.removedMessages} removed, retained)` : ""; + const head = [ + `LCM sidecar dump: ${options.sessionId}`, + `store: ${basename(options.path)}, generation ${store.generation}, ` + + `${stats.messages} messages${removed}, ${stats.summaries} summaries, ` + + `${(stats.dbBytes / 1024).toFixed(0)} KB`, + `history:`, + historyManifest(store, DUMP_MAX_HISTORY), + ]; + const summaries = store.allSummaries().slice(0, DUMP_MAX_SUMMARIES); + const totalSummaries = store.allSummaries().length; + const sections: string[] = [...head]; + if (summaries.length > 0) { + sections.push(`summaries: ${summaries.length} shown of ${totalSummaries}`); + for (const summary of summaries) { + const view = describeView(store, options.redact, summary.id); + sections.push(view.ok ? capSummaryText(view.text) : `Summary #${summary.id}: unreadable.`); + } + const first = summaries[0]!; + const coveredIds = store.coveredMessageIds(first.id); + const coveredMsgs = store.messagesByIds(coveredIds); + const coveredIdSet = new Set(coveredMsgs.map((m) => m.entryId)); + const removedIds = new Set( + coveredMsgs.filter((m) => m.removedAt !== undefined).map((m) => m.entryId), + ); + const visibleMsgs = includeRemoved + ? coveredMsgs + : coveredMsgs.filter((m) => m.removedAt === undefined); + const list = expandView(store, options.redact, { + mode: "list", + id: first.id, + offset: 0, + limit: DUMP_MAX_SAMPLE_MESSAGES, + maxChars: DUMP_SAMPLE_CHARS, + }); + sections.push( + `messages (summary #${first.id}):`, + list.ok + ? shapeSampleList( + list.text, + first.id, + coveredIdSet, + removedIds, + includeRemoved, + visibleMsgs.length, + ) + : "No messages.", + ); + for (const message of visibleMsgs.slice(0, DUMP_MAX_SAMPLE_MESSAGES)) { + const window = expandView(store, options.redact, { + mode: "message", + entryId: message.entryId, + charOffset: 0, + maxChars: DUMP_SAMPLE_CHARS, + }); + sections.push(window.ok ? window.text : `No stored message ${message.entryId}.`); + } + } else { + sections.push("summaries: none stored yet."); + } + if (options.sampleQuery !== undefined && options.sampleQuery.length > 0) { + const found = grepView(store, options.redact, { + query: options.sampleQuery, + limit: DUMP_MAX_SAMPLE_MESSAGES, + includeRemoved, + }); + sections.push( + `search ${JSON.stringify(options.sampleQuery)}${includeRemoved ? " (including removed)" : ""}:`, + found.text, + ); + } + const rows = (options.metricRows ?? []).filter((row) => row.session === options.sessionId); + const shown = rows.slice(Math.max(0, rows.length - DUMP_MAX_METRIC_ROWS)); + sections.push(`metrics: ${rows.length} row(s) for this session, ${shown.length} shown`); + for (const row of shown) sections.push(` ${JSON.stringify(sanitizeRow(row, options.redact))}`); + sections.push( + `omitted: summary text past ${DUMP_MAX_SUMMARY_BYTES} bytes each, message bodies past ${DUMP_SAMPLE_CHARS} chars per sample, ` + + `summaries past ${DUMP_MAX_SUMMARIES}, metric rows past ${DUMP_MAX_METRIC_ROWS}. ` + + `Removed rows carry [removed]. Read them with include_removed.`, + ); + const text = options.redact(sections.join("\n")); + if (Buffer.byteLength(text, "utf8") <= DUMP_MAX_BYTES) return text; + return clip(text); +} + +/** Reshape the sampled list for the dump: removed rows stay out by default + * and carry a [removed] label when --include-removed keeps them. Untouched + * when no covered row left the active path. */ +function shapeSampleList( + text: string, + summaryId: number, + coveredIds: ReadonlySet, + removedIds: ReadonlySet, + includeRemoved: boolean, + visibleTotal: number, +): string { + if (removedIds.size === 0) return text; + const groups: { id: string | undefined; chunks: string[] }[] = []; + for (const chunk of text.split("\n\n").slice(1)) { + const at = chunk.indexOf("\n"); + const first = at === -1 ? chunk : chunk.slice(0, at); + const id = /^\[([^\]]+)\] \((?:user|assistant|toolResult|custom)\)$/.exec(first)?.[1]; + if (id !== undefined && coveredIds.has(id)) groups.push({ id, chunks: [chunk] }); + else if (groups.length > 0) groups[groups.length - 1]!.chunks.push(chunk); + else groups.push({ id: undefined, chunks: [chunk] }); + } + const kept: string[] = []; + let shown = 0; + for (const group of groups) { + const removed = group.id !== undefined && removedIds.has(group.id); + if (removed && !includeRemoved) continue; + if (group.id !== undefined) shown++; + kept.push( + group.chunks + .map((chunk, index) => (index === 0 && removed ? labelRemoved(chunk) : chunk)) + .join("\n\n"), + ); + } + const head = + `Summary #${summaryId} → ${visibleTotal} original messages ` + + `(showing ${shown} from 0)${visibleTotal > shown ? ", use offset to page" : ""}`; + return [head, ...kept].join("\n\n"); +} + +/** Mark one kept list row as off the active path, grepView style. */ +function labelRemoved(chunk: string): string { + const at = chunk.indexOf("\n"); + return at === -1 ? `${chunk} [removed]` : `${chunk.slice(0, at)} [removed]${chunk.slice(at)}`; +} + +/** Keep one described summary bounded so large summaries cannot clip the tail. */ +function capSummaryText(text: string): string { + if (Buffer.byteLength(text, "utf8") <= DUMP_MAX_SUMMARY_BYTES) return text; + let used = 0; + let end = 0; + for (const chunk of text) { + const cost = Buffer.byteLength(chunk, "utf8"); + if (used + cost > DUMP_MAX_SUMMARY_BYTES) break; + used += cost; + end += chunk.length; + } + const more = Buffer.byteLength(text, "utf8") - used; + return `${text.slice(0, end)}\n[lcm:more ${more} bytes]`; +} + +/** Redact string seams in one metric row; keys and stub pointers stay intact. */ +function sanitizeRow( + row: Record, + redact: (text: string) => string, +): Record { + const out: Record = {}; + for (const [key, value] of Object.entries(row)) out[key] = sanitizeValue(value, redact); + return out; +} + +function sanitizeValue(value: unknown, redact: (text: string) => string): unknown { + if (typeof value === "string") return redact(value); + if (Array.isArray(value)) return value.map((item) => sanitizeValue(item, redact)); + if (typeof value === "object" && value !== null) { + const out: Record = {}; + for (const [key, item] of Object.entries(value as Record)) { + out[key] = sanitizeValue(item, redact); + } + return out; + } + return value; +} + +function clip(text: string): string { + const marker = `... truncated at the ${DUMP_MAX_BYTES} byte limit`; + const budget = DUMP_MAX_BYTES - Buffer.byteLength(marker, "utf8") - 1; + let kept = ""; + let used = 0; + for (const line of text.split("\n")) { + const cost = Buffer.byteLength(line, "utf8") + 1; + if (used + cost > budget) break; + kept += (kept.length === 0 ? "" : "\n") + line; + used += cost; + } + return `${kept}\n${marker}`; +} diff --git a/packages/pi/test/command.test.ts b/packages/pi/test/command.test.ts index 1475833..297938e 100644 --- a/packages/pi/test/command.test.ts +++ b/packages/pi/test/command.test.ts @@ -439,7 +439,7 @@ test("command: doctor with no summaries prints no depth rows", async () => { assert.ok(doctor); assert.match( doctor, - /^messages: 0, summaries: 0, size: \d+ KB\nintegrity: 9 checks, no findings\npinned projection: none$/m, + /^messages: 0, summaries: 0, size: \d+ KB\nintegrity: 10 checks, no findings\npinned projection: none$/m, ); liveStore.close(); } finally { @@ -578,7 +578,7 @@ test("command: doctor reports integrity, and --repair fixes what it can", async }); await handler("doctor"); const before = notifications.find((n) => n.startsWith("LCM doctor"))!; - assert.ok(before.includes("integrity: 9 checks, 2 finding(s)"), before); + assert.ok(before.includes("integrity: 10 checks, 2 finding(s)"), before); assert.ok(before.includes("summary-span: 1 x these summaries name an entry"), before); assert.ok(before.includes(" #1"), before); assert.ok(before.includes("summary-shape: 1 x 1 with a depth or provenance"), before); @@ -592,7 +592,7 @@ test("command: doctor reports integrity, and --repair fixes what it can", async after, ); assert.ok(after.includes("repair: dropped 1 summar(y|ies) with an unresolvable span"), after); - assert.ok(after.includes("integrity: 9 checks, no findings"), after); + assert.ok(after.includes("integrity: 10 checks, no findings"), after); assert.equal(levels.at(-1), "info"); liveStore.close(); } finally { @@ -600,6 +600,58 @@ test("command: doctor reports integrity, and --repair fixes what it can", async } }); +test("command: doctor reports an orphan Pi sidecar but never the live store", async () => { + const { env, liveStore, livePath, handler, notifications } = bootstrap(); + try { + const liveName = `${hashPath(livePath)}.db`; + const orphanDb = join(env.lcmDir, "deadbeefdeadbeef.db"); + const orphan = makeStoreWithMessage(orphanDb, "o1", "history from a gone session"); + orphan.close(); + await handler("doctor"); + const doctor = notifications.find((n) => n.startsWith("LCM doctor"))!; + assert.ok(doctor.includes("orphan-sidecar (info)"), doctor); + assert.ok(doctor.includes("sidecar(s) not open in this process"), doctor); + const sample = doctor + .split("\n") + .find((line) => line.startsWith(" ") && line.includes(orphanDb)); + assert.ok(sample, `the finding names the orphan path: ${doctor}`); + assert.ok(!sample.includes(liveName), `the live store never reports: ${sample}`); + liveStore.close(); + } finally { + env.restore(); + } +}); + +test("command: diagnose carries the orphan Pi sidecar finding", async () => { + const { env, liveStore, livePath, handler, notifications } = bootstrap({ + usage: { tokens: 60_000, contextWindow: 200_000 }, + }); + try { + const orphanStem = "deadbeefdeadbeef"; + const orphan = makeStoreWithMessage( + join(env.lcmDir, `${orphanStem}.db`), + "o1", + "history from a gone session", + ); + orphan.close(); + await handler("diagnose"); + const bundle = notifications.find((n) => n.startsWith("LCM diagnose bundle"))!; + assert.ok(bundle, notifications.join(" | ")); + assert.ok(bundle.includes("orphan-sidecar (info)"), bundle.slice(0, 2000)); + const sample = bundle + .split("\n") + .find((line) => line.includes(`${orphanStem}.db`) && line.includes("~/.pi/agent/lcm")); + assert.ok(sample, `the finding names the orphan path: ${bundle.slice(0, 2000)}`); + assert.ok( + !sample.includes(`${hashPath(livePath)}.db`), + `the live store never reports: ${sample}`, + ); + liveStore.close(); + } finally { + env.restore(); + } +}); + test("command: diagnose pastes one bundle, with no stored text and no home path", async () => { const { env, liveStore, livePath, handler, notifications, levels } = bootstrap({ usage: { tokens: 60_000, contextWindow: 200_000 }, diff --git a/packages/pi/test/config-io.test.ts b/packages/pi/test/config-io.test.ts index 140c54c..4776158 100644 --- a/packages/pi/test/config-io.test.ts +++ b/packages/pi/test/config-io.test.ts @@ -38,6 +38,23 @@ test("config-io: the values the cockpit writes all survive the parse", () => { assert.deepEqual(config, COCKPIT_CONFIG); }); +test("config-io: mechanicalTools survive the parse and stay opt-in", () => { + const { config, dropped, notes } = parseLcmConfig({ mechanicalTools: ["noisy-read", " grep "] }); + assert.deepEqual(dropped, []); + assert.deepEqual(notes, []); + assert.deepEqual(config, { mechanicalTools: ["noisy-read", "grep"] }); + assert.deepEqual(parseLcmConfig({}).config, {}); + assert.deepEqual(parseLcmConfig({ mechanicalTools: "noisy-read" }).config, { + mechanicalTools: ["noisy-read"], + }); + const bad = parseLcmConfig({ mechanicalTools: [42, ""] }); + assert.deepEqual(bad.dropped, ["mechanicalTools[0] (wrong type)"]); + assert.deepEqual(bad.config, { mechanicalTools: [] }); + assert.deepEqual(parseLcmConfig({ mechanicalTools: 42 }).dropped, [ + "mechanicalTools (wrong type)", + ]); +}); + test("config-io: the heal budget defaults to 120k", () => { assert.equal(DEFAULT_CONFIG.summarizerTokensPerHeal, 120_000); }); diff --git a/packages/pi/test/diagnose.test.ts b/packages/pi/test/diagnose.test.ts index c1f5b29..adb40e7 100644 --- a/packages/pi/test/diagnose.test.ts +++ b/packages/pi/test/diagnose.test.ts @@ -298,6 +298,32 @@ test("diagnose: the bundle never names the home directory", () => { assert.ok(text.includes("~/.pi/agent/lcm/x.db"), text); }); +test("diagnose: the bundle carries an orphan sidecar finding within the caps", () => { + const text = bundle({ + facts: facts({ + integrity: { + checks: 10, + repairs: [], + findings: [ + { + kind: "orphan-sidecar", + severity: "info", + count: 2, + detail: "2 sidecar(s) not open in this process. Deletion stays manual.", + sample: ["orphan.sqlite", "orphan.sqlite-wal"], + }, + ], + }, + }), + }); + assert.ok( + text.includes("orphan-sidecar (info): 2 x 2 sidecar(s) not open in this process"), + text, + ); + assert.ok(text.includes("orphan.sqlite"), text); + assert.ok(Buffer.byteLength(text, "utf8") <= MAX_BUNDLE_BYTES, String(text.length)); +}); + test("diagnose: the plugin version is this package's version, from src or dist", () => { const pkg = JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8")) as { name: string; diff --git a/packages/pi/test/hook-adapter.test.ts b/packages/pi/test/hook-adapter.test.ts index 005d2d5..2be8810 100644 --- a/packages/pi/test/hook-adapter.test.ts +++ b/packages/pi/test/hook-adapter.test.ts @@ -282,6 +282,7 @@ test("adapter: turn_end falls back to a full rescan when the watermark id is gon msgEntry("e5", "user", "five"), msgEntry("e6", "assistant", "six"), ]; + const mark = metricsMark(); await fake.handlers.get("turn_end")!( {} as never, { hasUI: false, sessionManager: sessionManagerOf(branch) } as never, @@ -290,6 +291,57 @@ test("adapter: turn_end falls back to a full rescan when the watermark id is gon assert.equal(reader.stats().messages, 4, "e1 kept (idempotent), e5 + e6 added, e2 retained"); assert.equal(reader.messagesInSpan("e5", "e6").length, 2); reader.close(); + const rescans = metricsSince(basename(lcmDbPath(sessionFile)), mark).filter( + (m) => m.kind === "ingest-rescan", + ); + assert.equal(rescans.length, 1, "the fallback rescan writes one metric row"); + assert.equal(rescans[0]!.watermark, "e2"); + assert.equal(rescans[0]!.entries, 3); +}); + +test("adapter: watermark rescan warns the UI once and stays silent on empty turns", async () => { + const fake = makeFakePi(); + mod(fake.pi as never); + const sessionFile = uniqueSessionFile(); + const tag = basename(lcmDbPath(sessionFile)); + const entries = [msgEntry("e1", "user", "one"), msgEntry("e2", "assistant", "two")]; + await startSession(fake.handlers, entries, sessionFile); + + const notified: Array<[string, string]> = []; + const uiCtx = (list: FakeEntry[]) => + ({ + hasUI: true, + ui: { + notify: (message: string, level: string) => void notified.push([message, level]), + setWidget: () => {}, + }, + sessionManager: sessionManagerOf(list), + }) as never; + + const branch = [msgEntry("e1", "user", "one"), msgEntry("e5", "user", "five")]; + const mark = metricsMark(); + await fake.handlers.get("turn_end")!({} as never, uiCtx(branch)); + assert.equal(notified.length, 1, "one warning for the rescan"); + assert.match(notified[0]![0], /watermark e2 not found/); + assert.equal(notified[0]![1], "warning"); + const rescans = metricsSince(tag, mark).filter((m) => m.kind === "ingest-rescan"); + assert.equal(rescans.length, 1, "one metric row for the rescan"); + assert.equal(rescans[0]!.watermark, "e2"); + assert.equal(rescans[0]!.entries, 2); + + notified.length = 0; + const quiet = [msgEntry("w1", "assistant", " ")]; + const mark2 = metricsMark(); + await fake.handlers.get("turn_end")!({} as never, uiCtx(quiet)); + assert.equal(notified.length, 0, "whitespace alone warns nothing"); + assert.deepEqual( + metricsSince(tag, mark2).filter((m) => m.kind === "ingest-rescan"), + [], + "whitespace alone writes no rescan row", + ); + const reader = new LcmStore(lcmDbPath(sessionFile)); + assert.equal(reader.stats().messages, 3, "e1, e2, e5 stored; whitespace added nothing"); + reader.close(); }); test("adapter: context hook returns undefined below the swap threshold", async () => { diff --git a/packages/pi/test/recall-tools.test.ts b/packages/pi/test/recall-tools.test.ts index 74001de..80c5a8c 100644 --- a/packages/pi/test/recall-tools.test.ts +++ b/packages/pi/test/recall-tools.test.ts @@ -328,7 +328,7 @@ test("grep tool: natural-language queries never throw raw sqlite errors", async ["AND", 'No matches for "AND" in session history.'], ["(", 'No matches for "(" in session history.'], ["a OR", 'No matches for "a OR" in session history.'], - ['unbalanced"', 'Invalid search query "unbalanced\\""; try simpler keywords.'], + ['unbalanced"', 'No matches for "unbalanced\\"" in session history.'], ]; for (const [q, message] of expected) { const r = await tool.execute("t", { query: q }); diff --git a/packages/pi/test/sidecar-dump.test.ts b/packages/pi/test/sidecar-dump.test.ts new file mode 100644 index 0000000..257ea87 --- /dev/null +++ b/packages/pi/test/sidecar-dump.test.ts @@ -0,0 +1,565 @@ +import assert from "node:assert/strict"; +import { + chmodSync, + existsSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { DatabaseSync } from "node:sqlite"; + +import { ingestEntries, LcmStore, redactSecrets } from "lossless-core"; +import { test } from "vite-plus/test"; + +import { registerLcmCommand, type LcmCommandDeps } from "../src/ui/command.ts"; +import { DUMP_MAX_BYTES, dumpSidecar } from "../src/ui/sidecar-dump.ts"; + +const SESSION = "ses_pi_dump"; +const SECRET = "sk-ant-abcdefghijklmnopqrstuvwxyz0123456789abcd"; + +function tempDir(): string { + return mkdtempSync(join(tmpdir(), "pi-lossless-dump-")); +} + +function seedStore(path: string): void { + const store = new LcmStore(path); + try { + ingestEntries(store, [ + { entryId: "m1", role: "user", text: `first note about anchors ${SECRET}`, timestamp: 1 }, + { entryId: "m2", role: "assistant", text: "second note about anchors", timestamp: 2 }, + ]); + const first = store.messageByEntryId("m1"); + const second = store.messageByEntryId("m2"); + if (first && second) { + store.insertSummary({ + kind: "leaf", + text: "anchor summary", + tokens: 5, + depth: 0, + firstEntryId: "m1", + lastEntryId: "m2", + messageIds: [first.id, second.id], + }); + store.setRemoved(["m2"], Date.now()); + } + } finally { + store.close(); + } +} + +function countMessages(path: string): number { + const store = new LcmStore(path, { queryOnly: true }); + try { + return store.messageCount(); + } finally { + store.close(); + } +} + +test("dump: live session is read-only, redacted, and labels removed rows", () => { + const dir = tempDir(); + try { + const path = join(dir, "live.sqlite"); + seedStore(path); + const before = countMessages(path); + const filesBefore = readdirSync(dir).sort(); + const result = dumpSidecar({ + path, + sessionId: SESSION, + redact: redactSecrets, + includeRemoved: true, + metricRows: [{ session: SESSION, kind: "context-decision", note: SECRET }], + sampleQuery: "anchors", + }); + assert.equal(result.status, "dumped"); + if (result.status !== "dumped") return; + assert.ok(result.text.includes("LCM sidecar dump")); + assert.ok(!result.text.includes(SECRET)); + assert.ok(result.text.includes("[REDACTED]")); + assert.ok(result.text.includes("[removed]")); + assert.equal(countMessages(path), before); + assert.deepEqual(readdirSync(dir).sort(), filesBefore); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("dump: old generation reports a named skip", () => { + const dir = tempDir(); + try { + const path = join(dir, "old.sqlite"); + seedStore(path); + const raw = new DatabaseSync(path); + try { + raw.exec("PRAGMA user_version = 1"); + } finally { + raw.close(); + } + const result = dumpSidecar({ path, sessionId: SESSION, redact: redactSecrets }); + assert.equal(result.status, "skipped"); + if (result.status !== "skipped") return; + assert.equal(result.reason, "older-generation"); + assert.ok(result.detail.length > 0); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("dump: missing sidecar reports a named skip", () => { + const dir = tempDir(); + try { + const result = dumpSidecar({ + path: join(dir, "absent.sqlite"), + sessionId: SESSION, + redact: redactSecrets, + }); + assert.equal(result.status, "skipped"); + if (result.status !== "skipped") return; + assert.equal(result.reason, "missing"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("dump: output stays within its byte cap", () => { + const dir = tempDir(); + try { + const path = join(dir, "big.sqlite"); + const store = new LcmStore(path); + try { + ingestEntries( + store, + Array.from({ length: 30 }, (_, i) => ({ + entryId: `e${i}`, + role: "user" as const, + text: `anchor note ${i} ${"filler ".repeat(200)}`, + timestamp: i, + })), + ); + } finally { + store.close(); + } + const result = dumpSidecar({ + path, + sessionId: SESSION, + redact: redactSecrets, + metricRows: Array.from({ length: 60 }, (_, i) => ({ + session: SESSION, + kind: "usage", + input: i, + note: `row ${i} ${"padding ".repeat(200)}`, + })), + }); + assert.equal(result.status, "dumped"); + if (result.status !== "dumped") return; + assert.ok(Buffer.byteLength(result.text, "utf8") <= DUMP_MAX_BYTES); + assert.ok(result.text.includes("truncated at the")); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("dump: caps per-summary text so metrics and footer survive", () => { + const dir = tempDir(); + try { + const path = join(dir, "capped.sqlite"); + const store = new LcmStore(path); + try { + ingestEntries( + store, + Array.from({ length: 10 }, (_, i) => ({ + entryId: `c${i}`, + role: "user" as const, + text: `note ${i}`, + timestamp: i, + })), + ); + const ids = Array.from({ length: 10 }, (_, i) => store.messageByEntryId(`c${i}`)?.id).filter( + (v): v is number => v !== undefined, + ); + for (let s = 0; s < 5; s++) { + store.insertSummary({ + kind: "leaf", + text: `summary ${s}`, + tokens: 1500, + depth: 0, + firstEntryId: `c${s * 2}`, + lastEntryId: `c${s * 2 + 1}`, + messageIds: [ids[s * 2]!, ids[s * 2 + 1]!], + thoroughText: `thorough ${s} ${"x".repeat(6000)}`, + }); + } + } finally { + store.close(); + } + const marker = "cap-marker-note"; + const result = dumpSidecar({ + path, + sessionId: SESSION, + redact: redactSecrets, + metricRows: [{ session: SESSION, kind: "usage", note: marker }], + sampleQuery: "note", + }); + assert.equal(result.status, "dumped"); + if (result.status !== "dumped") return; + assert.ok(result.text.includes("[lcm:more")); + assert.ok(result.text.includes(marker)); + assert.ok(result.text.includes("metrics:")); + assert.ok(result.text.includes("omitted:")); + assert.ok(!result.text.includes("no absolute paths")); + assert.ok(!result.text.includes("truncated at the")); + assert.ok(Buffer.byteLength(result.text, "utf8") <= DUMP_MAX_BYTES); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("dump: caps CJK summaries by bytes so the tail survives", () => { + // 1000 CJK chars fit the old char count but exceed 1200 bytes. + // Uncapped, ten such summaries pass DUMP_MAX_BYTES and clip the tail. + const dir = tempDir(); + try { + const path = join(dir, "cjk.sqlite"); + const store = new LcmStore(path); + try { + ingestEntries( + store, + Array.from({ length: 10 }, (_, i) => ({ + entryId: `k${i}`, + role: "user" as const, + text: `note ${i}`, + timestamp: i, + })), + ); + const ids = Array.from({ length: 10 }, (_, i) => store.messageByEntryId(`k${i}`)?.id).filter( + (v): v is number => v !== undefined, + ); + for (let s = 0; s < 10; s++) { + store.insertSummary({ + kind: "leaf", + text: `summary ${s}`, + tokens: 5, + depth: 0, + firstEntryId: `k${s}`, + lastEntryId: `k${s}`, + messageIds: [ids[s]!], + thoroughText: "字".repeat(1000), + }); + } + } finally { + store.close(); + } + const marker = "cjk-cap-marker"; + const result = dumpSidecar({ + path, + sessionId: SESSION, + redact: redactSecrets, + metricRows: [{ session: SESSION, kind: "usage", note: marker }], + }); + assert.equal(result.status, "dumped"); + if (result.status !== "dumped") return; + assert.ok(result.text.includes("[lcm:more")); + assert.ok(result.text.includes("bytes]")); + assert.ok(result.text.includes(marker)); + assert.ok(result.text.includes("metrics:")); + assert.ok(result.text.includes("omitted:")); + assert.ok(result.text.includes("bytes each")); + assert.ok(!result.text.includes("truncated at the")); + assert.ok(Buffer.byteLength(result.text, "utf8") <= DUMP_MAX_BYTES); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("dump: masks linked file previews and the echoed query, keeps paths", () => { + const dir = tempDir(); + try { + const path = join(dir, "file.sqlite"); + const store = new LcmStore(path); + try { + ingestEntries(store, [ + { entryId: "m1", role: "user", text: "note about harbors", timestamp: 1 }, + ]); + store.insertFile( + { + fileId: "file-secret-1", + path: "notes/keys.txt", + bytes: 64, + sha256: "0".repeat(64), + kind: "text", + preview: `${SECRET}\nsecond line of the file`, + }, + "m1", + ); + const first = store.messageByEntryId("m1"); + assert.ok(first); + store.insertSummary({ + kind: "leaf", + text: "harbor summary", + tokens: 5, + depth: 0, + firstEntryId: "m1", + lastEntryId: "m1", + messageIds: [first.id], + fileIds: ["file-secret-1"], + }); + } finally { + store.close(); + } + const result = dumpSidecar({ + path, + sessionId: SESSION, + redact: redactSecrets, + sampleQuery: `harbors ${SECRET}`, + }); + assert.equal(result.status, "dumped"); + if (result.status !== "dumped") return; + assert.ok(!result.text.includes(SECRET), result.text); + assert.ok(result.text.includes("[REDACTED]"), result.text); + assert.ok(result.text.includes("notes/keys.txt"), result.text); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("dump: default sampling skips removed rows, --include-removed labels them", () => { + const dir = tempDir(); + try { + const path = join(dir, "removed.sqlite"); + seedStore(path); + const hidden = dumpSidecar({ path, sessionId: SESSION, redact: redactSecrets }); + assert.equal(hidden.status, "dumped"); + if (hidden.status !== "dumped") return; + assert.ok(!hidden.text.includes("second note about anchors"), hidden.text); + assert.ok(!hidden.text.includes("[m2]"), hidden.text); + assert.ok(hidden.text.includes("first note about anchors"), hidden.text); + assert.ok(hidden.text.includes("[REDACTED]"), hidden.text); + + const shown = dumpSidecar({ + path, + sessionId: SESSION, + redact: redactSecrets, + includeRemoved: true, + }); + assert.equal(shown.status, "dumped"); + if (shown.status !== "dumped") return; + assert.ok(shown.text.includes("second note about anchors"), shown.text); + assert.ok(shown.text.includes("[m2] (assistant) [removed]"), shown.text); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("dump --save: writes through a private temp at mode 0600", async () => { + const dir = tempDir(); + const origHome = process.env.HOME; + process.env.HOME = dir; + try { + const dbPath = join(dir, "save.sqlite"); + const liveStore = new LcmStore(dbPath); + try { + ingestEntries(liveStore, [ + { entryId: "m1", role: "user", text: "saved dump note", timestamp: 1 }, + ]); + const notifications: string[] = []; + const commands = new Map< + string, + { handler: (args: string | undefined, ctx: unknown) => Promise } + >(); + const fakePi = { + registerCommand: (name: string, def: never) => { + commands.set(name, def); + }, + }; + const deps: LcmCommandDeps = { + getStore: () => liveStore, + openStore: () => liveStore, + getConfig: () => ({}), + patchConfig: () => {}, + resetConfig: () => {}, + clearCommitState: () => {}, + getCommitState: () => null, + getDbPath: () => dbPath, + keepRecentTokens: () => 20_000, + }; + registerLcmCommand(fakePi as never, deps); + const ctx = { + mode: "tui", + sessionManager: { + getSessionFile: () => join(dir, "sess.jsonl"), + getSessionId: () => "pi-dump-save", + }, + getContextUsage: () => undefined, + ui: { + notify: (msg: string) => { + notifications.push(msg); + }, + }, + }; + const target = join(dir, "lcm-dump.txt"); + writeFileSync(target, "stale bytes", { mode: 0o644 }); + chmodSync(target, 0o644); + await commands.get("lcm")!.handler(`dump --save ${target}`, ctx); + assert.equal(statSync(target).mode & 0o777, 0o600); + const written = readFileSync(target, "utf8"); + assert.ok(written.includes("LCM sidecar dump"), written); + assert.ok(!written.includes("stale bytes"), written); + assert.ok(!existsSync(`${target}.tmp`), "no temp file is left behind"); + assert.ok(notifications.at(-1)!.includes("(mode 0600)"), notifications.join(" | ")); + } finally { + liveStore.close(); + } + } finally { + process.env.HOME = origHome; + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("dump --save: leaves a planted predictable temp untouched", async () => { + const dir = tempDir(); + const origHome = process.env.HOME; + process.env.HOME = dir; + try { + const dbPath = join(dir, "save-plant.sqlite"); + const liveStore = new LcmStore(dbPath); + try { + ingestEntries(liveStore, [ + { entryId: "m1", role: "user", text: "saved dump note", timestamp: 1 }, + ]); + const notifications: string[] = []; + const commands = new Map< + string, + { handler: (args: string | undefined, ctx: unknown) => Promise } + >(); + const fakePi = { + registerCommand: (name: string, def: never) => { + commands.set(name, def); + }, + }; + const deps: LcmCommandDeps = { + getStore: () => liveStore, + openStore: () => liveStore, + getConfig: () => ({}), + patchConfig: () => {}, + resetConfig: () => {}, + clearCommitState: () => {}, + getCommitState: () => null, + getDbPath: () => dbPath, + keepRecentTokens: () => 20_000, + }; + registerLcmCommand(fakePi as never, deps); + const ctx = { + mode: "tui", + sessionManager: { + getSessionFile: () => join(dir, "sess.jsonl"), + getSessionId: () => "pi-dump-save", + }, + getContextUsage: () => undefined, + ui: { + notify: (msg: string) => { + notifications.push(msg); + }, + }, + }; + const target = join(dir, "lcm-dump-plant.txt"); + const decoy = `${target}.tmp`; + writeFileSync(decoy, "decoy bytes", { mode: 0o644 }); + await commands.get("lcm")!.handler(`dump --save ${target}`, ctx); + assert.equal(readFileSync(decoy, "utf8"), "decoy bytes"); + const written = readFileSync(target, "utf8"); + assert.ok(written.includes("LCM sidecar dump"), written); + assert.equal(statSync(target).mode & 0o777, 0o600); + assert.deepEqual( + readdirSync(dir).filter((f) => f.endsWith(".tmp")), + ["lcm-dump-plant.txt.tmp"], + ); + } finally { + liveStore.close(); + } + } finally { + process.env.HOME = origHome; + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("dump --save: write failure path removes its temp", async () => { + // node:fs module namespaces are not configurable under ESM, so a write that + // creates the file then fails (disk full) resists spy fault injection. Pin + // the cleanup structure instead: the write catch unlinks the candidate + // before rethrowing, and EEXIST keeps its continue above without unlinking. + const source = readFileSync(new URL("../src/ui/command.ts", import.meta.url), "utf8"); + const helper = source.slice( + source.indexOf("function writeDumpTarget"), + source.indexOf("cannot create temp file"), + ); + const eexist = helper.indexOf('code === "EEXIST"'); + const unlink = helper.indexOf("unlinkSync(candidate)"); + const rethrow = helper.indexOf("throw error"); + assert.ok(eexist >= 0 && unlink >= 0 && rethrow >= 0); + assert.ok(eexist < unlink && unlink < rethrow); + // Success path leaves no temp behind beside the destination. + const dir = tempDir(); + const origHome = process.env.HOME; + process.env.HOME = dir; + try { + const dbPath = join(dir, "save-clean.sqlite"); + const liveStore = new LcmStore(dbPath); + try { + ingestEntries(liveStore, [ + { entryId: "m1", role: "user", text: "saved dump note", timestamp: 1 }, + ]); + const commands = new Map< + string, + { handler: (args: string | undefined, ctx: unknown) => Promise } + >(); + const fakePi = { + registerCommand: (name: string, def: never) => { + commands.set(name, def); + }, + }; + const deps: LcmCommandDeps = { + getStore: () => liveStore, + openStore: () => liveStore, + getConfig: () => ({}), + patchConfig: () => {}, + resetConfig: () => {}, + clearCommitState: () => {}, + getCommitState: () => null, + getDbPath: () => dbPath, + keepRecentTokens: () => 20_000, + }; + registerLcmCommand(fakePi as never, deps); + const notifications: string[] = []; + const ctx = { + mode: "tui", + sessionManager: { + getSessionFile: () => join(dir, "sess.jsonl"), + getSessionId: () => "pi-dump-save", + }, + getContextUsage: () => undefined, + ui: { + notify: (msg: string) => { + notifications.push(msg); + }, + }, + }; + const target = join(dir, "lcm-dump-clean.txt"); + await commands.get("lcm")!.handler(`dump --save ${target}`, ctx); + assert.ok(existsSync(target)); + assert.deepEqual( + readdirSync(dir).filter((f) => f.endsWith(".tmp")), + [], + ); + } finally { + liveStore.close(); + } + } finally { + process.env.HOME = origHome; + rmSync(dir, { recursive: true, force: true }); + } +});