diff --git a/src/node/services/memoryLegacyAdoption.ts b/src/node/services/memoryLegacyAdoption.ts index e1342e646d..eb0d53803b 100644 --- a/src/node/services/memoryLegacyAdoption.ts +++ b/src/node/services/memoryLegacyAdoption.ts @@ -26,16 +26,16 @@ export function legacyAdoptionManifestPath(childSessionDir: string): string { /** * One adopted legacy file: content hash, child sidecar fingerprint, owner-store - * relPath, and whether the adoption CREATED that owner file (provenance: a + * relPath, and whether the adoption CREATED that owner file (provenance: only + * such a copy may be removed again when the legacy source disappears; a * pre-existing identical owner note is the owner's own). `pending`: written * BEFORE the copy lands (provenance must not depend on the copy's existence: a * retry finding the bytes already at the target could not tell an interrupted * adoption from an owner note); cleared once the sidecar fold completed. * * Every field beyond the three strings is optional and unknown fields are - * ignored on read, so later builds can extend the record (downgrade-time - * reconciliation of edited/deleted sources) without invalidating manifests - * written by this one. + * ignored on read, so a build that knows fewer of them still reads (and + * rewrites) a manifest written by this one; its records keep working here. */ export interface LegacyAdoptionRecord { content: string; @@ -45,20 +45,67 @@ export interface LegacyAdoptionRecord { pending?: boolean; /** * Identity of the owner file this adoption wrote (`ino:size:mtimeNs` right - * after the write): the copy is THIS generation of the file, not merely a - * file holding the adopted bytes — an owner who deleted and recreated (or - * edited and restored) the note to identical bytes owns the new file. - * Absent (write before stamping, or the stamp could not be taken): the copy - * is never treated as this adoption's. + * after the write). Deletion reconciliation requires the copy to be THIS + * generation of the file, not merely to hold the adopted bytes: an owner + * who deleted and recreated (or edited and restored) the note to identical + * bytes owns the new file, and a byte match alone would let a downgraded + * child's source deletion remove it. Absent (write before stamping, or the + * stamp could not be taken): never unchanged — the copy is preserved. */ targetStamp?: string; + /** + * The copy this adoption created was since replaced outside it (rewritten, + * or deleted and recreated to identical bytes: `targetStamp` no longer + * matches), so the file is the owner's own. Kept apart from a note the + * owner already had when it was first adopted (`created` never set): that + * one still folds the child's pin toggles, a replaced copy never does — + * `created` alone cannot tell the two apart once provenance is lost. + */ + replaced?: boolean; + /** + * Hash of the bytes an in-place replacement is about to write (set on the + * pending prior record, cleared once the pass completes). With `content` + * (the pre-write bytes) this lets a retry recognize the copy as this + * adoption's on either side of an interrupted write. + */ + replacementContent?: string; + /** + * Identity (`ino:size:mtimeNs`) of the staged bytes an in-place replacement + * is about to install, taken on the staging entry before the install (a + * rename keeps it) and set together with `replacementContent`. A retry + * finds the installed copy by this stamp; a byte match alone never counts. + */ + replacementStamp?: string; + /** + * Reconciliation of a deleted source is under way: the copy is about to be + * (or was just) removed. Set before the removal so a crash between the + * removal and the tombstone write is recovered as "removed by us" rather + * than "changed by the owner". + */ + pendingDeletion?: boolean; + /** + * The legacy source was deleted (or renamed away) on a downgraded build and + * the copy reconciled. Kept rather than dropped: the child's pre-sharing + * refinement rows for this note (a delete's restore inverse, a rename's + * mirrored rename) still address the legacy path and need the mapping to + * be rolled back into the shared store; a reappearing source is adopted + * afresh (the record's other fields are stale then). Written together + * with `pending: true`: a build that knows neither this flag nor the + * reconciliation ignores it and would otherwise read the settled hash as + * "folded in earlier" — reporting a complete handover for a reappearing + * source while no copy exists. Pending, it re-adopts instead. Here, + * `deleted` takes precedence: a tombstone is not an interrupted adoption. + */ + deleted?: boolean; } /** * Parse one manifest record. Lifecycle flags are raw JSON: a value that is - * neither absent nor boolean fails CLOSED — `pending` reads as set (the pass - * is redone), `created` as unset (no destructive provenance) — so a corrupted - * flag can never make an interrupted pass look settled. + * neither absent nor boolean fails CLOSED — `pending`/`pendingDeletion` read + * as set (the pass is redone), `created`/`deleted` as unset (no destructive + * provenance; the source is reconciled as a plain unlisted note), `replaced` + * as set (the child's pin no longer reaches the file) — so a corrupted flag + * can never make an interrupted pass look settled. */ function parseLegacyAdoptionRecord(value: unknown): LegacyAdoptionRecord | null { if (typeof value !== "object" || value === null) return null; @@ -70,7 +117,19 @@ function parseLegacyAdoptionRecord(value: unknown): LegacyAdoptionRecord | null ) { return null; } + // A present but non-string replacement hash is a malformed RECORD (not a + // flag to fail closed on): without it, a replacement pass that crashed + // after writing the new owner bytes leaves a copy reconciliation cannot + // recognize as this adoption's — a later source deletion would tombstone + // it as owner-owned and removal would report a complete handover while the + // adoption-created note stays visible without provenance. + if (record.replacementContent !== undefined && typeof record.replacementContent !== "string") { + return null; + } if (record.targetStamp !== undefined && typeof record.targetStamp !== "string") return null; + if (record.replacementStamp !== undefined && typeof record.replacementStamp !== "string") { + return null; + } const flag = (raw: unknown, malformed: boolean): boolean | undefined => raw === undefined ? undefined : typeof raw === "boolean" ? raw : malformed; return { @@ -79,6 +138,11 @@ function parseLegacyAdoptionRecord(value: unknown): LegacyAdoptionRecord | null target: record.target, created: flag(record.created, false), pending: flag(record.pending, true), + pendingDeletion: flag(record.pendingDeletion, true), + deleted: flag(record.deleted, false), + replaced: flag(record.replaced, true), + replacementContent: record.replacementContent, + replacementStamp: record.replacementStamp, targetStamp: record.targetStamp, }; } diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 2d56da8d05..bada783ae2 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -27,7 +27,12 @@ import { } from "@/common/types/refinement"; import { applyRefinementInverse, readRefinementEvents } from "./refinement/refinementTestHelpers"; import { rollbackRefinement } from "./refinement/refinementRollback"; -import { legacyAdoptionManifestPath } from "./memoryLegacyAdoption"; +import { + adoptionTargetStamp, + legacyAdoptionManifestPath, + readLegacyAdoptionManifest, +} from "./memoryLegacyAdoption"; +import { sha256Hex } from "./refinement/refinementJournal"; import { workspaceRemovalTombstonePath } from "./workspaceRemoval"; import { TestTempDir } from "./tools/testHelpers"; @@ -2283,6 +2288,1312 @@ describe("MemoryService", () => { .map((e) => e.relPath) ).toEqual(["real.md"]); }); + + it("preserves a leading BOM through adoption and never matches it against a BOM-less owner note", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.mkdir(ownerRoot, { recursive: true }); + const bom = Buffer.from([0xef, 0xbb, 0xbf]); + await fsPromises.writeFile( + path.join(legacyRoot, "fresh.md"), + Buffer.concat([bom, Buffer.from("fresh")]) + ); + await fsPromises.writeFile( + path.join(legacyRoot, "clash.md"), + Buffer.concat([bom, Buffer.from("same text")]) + ); + await fsPromises.writeFile(path.join(ownerRoot, "clash.md"), "same text"); + await fixture.service.adoptLegacyPrivateStoreForRemoval("ws-child", "ws-owner"); + // Byte-exact copy: the BOM is part of the note, not decoder noise. + expect( + (await fsPromises.readFile(path.join(ownerRoot, "fresh.md"))).equals( + Buffer.concat([bom, Buffer.from("fresh")]) + ) + ).toBe(true); + // A BOM-less owner note is different content: the legacy note lands + // beside it instead of being settled as already present. + expect( + ( + await fsPromises.readFile(path.join(ownerRoot, "imported", "ws-child", "clash.md")) + ).equals(Buffer.concat([bom, Buffer.from("same text")])) + ).toBe(true); + expect(await fsPromises.readFile(path.join(ownerRoot, "clash.md"), "utf-8")).toBe( + "same text" + ); + }); + + it("classifies untyped dirents by lstat in the strict legacy walk instead of dropping them", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(path.join(legacyRoot, "nested"), { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "top.md"), "top"); + await fsPromises.writeFile(path.join(legacyRoot, "nested", "deep.md"), "deep"); + // A filesystem reporting DT_UNKNOWN: every type predicate of the dirent + // is false, for regular files and directories alike. + const realReaddir = fsPromises.readdir.bind(fsPromises); + const untyped = spyOn(fsPromises, "readdir").mockImplementation((async ( + target: string, + options: unknown + ) => { + const entries = (await realReaddir( + target, + options as { withFileTypes: true } + )) as unknown as Array>; + if (!String(target).startsWith(legacyRoot)) return entries; + const no = () => false; + return entries.map((entry) => ({ + ...entry, + name: entry.name, + isFile: no, + isDirectory: no, + isSymbolicLink: no, + isFIFO: no, + isSocket: no, + isBlockDevice: no, + isCharacterDevice: no, + })); + }) as unknown as typeof fsPromises.readdir); + try { + await fixture.service.adoptLegacyPrivateStoreForRemoval("ws-child", "ws-owner"); + } finally { + untyped.mockRestore(); + } + expect(await fsPromises.readFile(path.join(ownerRoot, "top.md"), "utf-8")).toBe("top"); + expect(await fsPromises.readFile(path.join(ownerRoot, "nested", "deep.md"), "utf-8")).toBe( + "deep" + ); + }); + + it("imports conflicting notes of a child whose id the path grammar rejects under an escaped segment", async () => { + using fixture = await createFixture("proj~1-child"); + await fixture.config.editConfig((cfg) => { + cfg.projects.set(FIXTURE_PROJECT_PATH, { + workspaces: [ + { id: "ws-owner", name: "owner", path: "/checkouts/owner" }, + { + id: "proj~1-child", + name: "child", + path: "/checkouts/child", + parentWorkspaceId: "ws-owner", + }, + ], + }); + return cfg; + }); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "proj~1-child", "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.mkdir(ownerRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "clash.md"), "child"); + await fsPromises.writeFile(path.join(ownerRoot, "clash.md"), "owner"); + await fixture.service.adoptLegacyPrivateStoreForRemoval("proj~1-child", "ws-owner"); + // `~` is not a memory path character: verbatim, the copy would be + // written but invisible to the index and unaddressable. + const target = "imported/proj=7E1-child/clash.md"; + expect(await fsPromises.readFile(path.join(ownerRoot, target), "utf-8")).toBe("child"); + expect(await pathExists(path.join(ownerRoot, "imported", "proj~1-child"))).toBe(false); + const ownerCtx = { ...fixture.ctx, workspaceId: "ws-owner" }; + expect( + (await fixture.service.listIndexEntries(ownerCtx)) + .filter((e) => e.scope === "workspace") + .map((e) => e.relPath) + .sort() + ).toEqual(["clash.md", target]); + const manifest = JSON.parse( + await fsPromises.readFile(legacyAdoptionManifestPath(path.dirname(legacyRoot)), "utf-8") + ) as Record; + expect(manifest["clash.md"].target).toBe(target); + }); + + it("re-adopts a legacy note renamed onto its own conflict-copy path as a fresh copy", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(ownerRoot, { recursive: true }); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + // Conflict: the owner has a different a.md, so the child's is adopted + // under imported//a.md. + await fsPromises.writeFile(path.join(ownerRoot, "a.md"), "owner's a"); + await fsPromises.writeFile(path.join(legacyRoot, "a.md"), "child's a"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + const importedCopy = path.join(ownerRoot, "imported", "ws-child", "a.md"); + expect(await fsPromises.readFile(importedCopy, "utf-8")).toBe("child's a"); + // The downgraded build renames the source to exactly that imported + // path: the old record's reconciliation removes its copy first, then + // the new name is adopted at the now-free path — the note stays + // represented, with provenance on the new record. + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.mkdir(path.join(legacyRoot, "imported", "ws-child"), { recursive: true }); + await fsPromises.rename( + path.join(legacyRoot, "a.md"), + path.join(legacyRoot, "imported", "ws-child", "a.md") + ); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await fsPromises.readFile(importedCopy, "utf-8")).toBe("child's a"); + const manifest = JSON.parse( + await fsPromises.readFile(legacyAdoptionManifestPath(path.dirname(legacyRoot)), "utf-8") + ) as Record; + // The old record stays as a tombstone (rollbacks of the child's + // pre-sharing rows for a.md still need its mapping). + expect(Object.keys(manifest).sort()).toEqual(["a.md", "imported/ws-child/a.md"]); + expect(manifest["a.md"]).toMatchObject({ deleted: true }); + expect(manifest["imported/ws-child/a.md"]).toMatchObject({ + target: "imported/ws-child/a.md", + created: true, + }); + // With provenance transferred, deleting the renamed source removes the copy. + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.rm(path.join(legacyRoot, "imported", "ws-child", "a.md")); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await pathExists(importedCopy)).toBe(false); + }); + + it("re-adopts a note renamed onto its conflict-copy path across an interrupted replacement", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(ownerRoot, { recursive: true }); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(ownerRoot, "a.md"), "owner's a"); + await fsPromises.writeFile(path.join(legacyRoot, "a.md"), "child's a"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + const importedCopy = path.join(ownerRoot, "imported", "ws-child", "a.md"); + const manifestPath = legacyAdoptionManifestPath(path.dirname(legacyRoot)); + const settled = (await readLegacyAdoptionManifest(manifestPath)).get("a.md")!; + // The downgraded build edits a.md; the replacement pass installed the + // new bytes (a new generation) but crashed before settling: the record + // is pending with the OVERWRITTEN generation's targetStamp and the + // installed one's replacementStamp. + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.writeFile(path.join(legacyRoot, "a.md"), "child's a2"); + await fsPromises.rm(importedCopy); + await fsPromises.writeFile(importedCopy, "child's a2"); + const installed = (await adoptionTargetStamp(importedCopy))!; + expect(installed).not.toBe(settled.targetStamp); + await fsPromises.writeFile( + manifestPath, + JSON.stringify({ + "a.md": { + ...settled, + pending: true, + replacementContent: sha256Hex("child's a2"), + replacementStamp: installed, + }, + }) + ); + // Before the retry, the source is renamed onto the conflict-copy path. + // The pending record still recognizes the installed file by its + // replacement receipt, so the old record's reconciliation removes it + // (rather than leaving it as the owner's), and the new name is adopted + // as a fresh copy with its own generation. + await fsPromises.mkdir(path.join(legacyRoot, "imported", "ws-child"), { recursive: true }); + await fsPromises.rename( + path.join(legacyRoot, "a.md"), + path.join(legacyRoot, "imported", "ws-child", "a.md") + ); + await new MemoryService( + fixture.config, + new MemoryMetaService(fixture.xumHome) + ).listIndexEntries({ ...fixture.ctx }); + const manifest = await readLegacyAdoptionManifest(manifestPath); + expect(manifest.get("a.md")).toMatchObject({ deleted: true, created: true }); + const successor = manifest.get("imported/ws-child/a.md")!; + expect(successor).toMatchObject({ target: "imported/ws-child/a.md", created: true }); + expect(successor.pending).toBeUndefined(); + expect(successor.targetStamp).toBe((await adoptionTargetStamp(importedCopy)) ?? undefined); + expect(await fsPromises.readFile(importedCopy, "utf-8")).toBe("child's a2"); + // With its own generation, deleting the renamed source removes the copy. + await fsPromises.rm(path.join(legacyRoot, "imported", "ws-child", "a.md")); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await pathExists(importedCopy)).toBe(false); + }); + + it("keeps an owner-edited conflict copy the owner's when a renamed legacy note lands on it", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerCtx = { ...fixture.ctx, workspaceId: "ws-owner" }; + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(ownerRoot, { recursive: true }); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(ownerRoot, "a.md"), "owner's a"); + await fsPromises.writeFile(path.join(legacyRoot, "a.md"), "child's a"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + // The owner edits the conflict copy: it is the owner's now. + await fixture.service.strReplace( + ownerCtx, + "/memories/workspace/imported/ws-child/a.md", + "child's a", + "owner's edit", + "agent" + ); + // The downgraded build renames the source onto that path with the + // owner's bytes: the new record reuses the file, but no provenance + // transfers — the old copy no longer holds the adopted bytes. + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.mkdir(path.join(legacyRoot, "imported", "ws-child"), { recursive: true }); + await fsPromises.rm(path.join(legacyRoot, "a.md")); + await fsPromises.writeFile( + path.join(legacyRoot, "imported", "ws-child", "a.md"), + "owner's edit" + ); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + const manifest = JSON.parse( + await fsPromises.readFile(legacyAdoptionManifestPath(path.dirname(legacyRoot)), "utf-8") + ) as Record; + expect(manifest["imported/ws-child/a.md"].created).not.toBe(true); + // ...and the obsolete record's tombstone drops its destructive + // provenance: the child's old rows may not map onto the owner's note. + expect(manifest["a.md"]).toMatchObject({ deleted: true }); + expect(manifest["a.md"].created).not.toBe(true); + // Deleting the renamed source leaves the owner's note in place. + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.rm(path.join(legacyRoot, "imported", "ws-child", "a.md")); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect( + await fsPromises.readFile(path.join(ownerRoot, "imported", "ws-child", "a.md"), "utf-8") + ).toBe("owner's edit"); + }); + + it("re-adopts when a downgraded build edits a nested legacy note in place or only its pin", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(path.join(legacyRoot, "sub"), { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "sub", "note.md"), "v1"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await fsPromises.readFile(path.join(ownerRoot, "sub", "note.md"), "utf-8")).toBe("v1"); + // In-place edit of an existing nested file on the old build: neither the + // legacy root's mtime nor the (unknown to it) store clock moves. + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.writeFile(path.join(legacyRoot, "sub", "note.md"), "v2 (downgrade)"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + // The copy this adoption created was untouched by the owner: the new + // bytes replace it in place (an imported/ duplicate would strand the + // old copy, provenance lost, in the shared notebook). + expect(await fsPromises.readFile(path.join(ownerRoot, "sub", "note.md"), "utf-8")).toBe( + "v2 (downgrade)" + ); + expect(await pathExists(path.join(ownerRoot, "imported", "ws-child", "sub", "note.md"))).toBe( + false + ); + // Sidecar-only change (a pin toggled on the old build under the child + // key): no file stat changes at all, yet the owner key must follow. + const childKey = memoryLogicalKey("workspace", "sub/note.md", { + projectPath: "", + workspaceId: "ws-child", + }); + await fixture.metaService.setPinned(childKey, true); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect( + (await fixture.metaService.getPinnedKeys()).has( + memoryLogicalKey("workspace", "sub/note.md", { + projectPath: "", + workspaceId: "ws-owner", + }) + ) + ).toBe(true); + // Once the OWNER edited the copy it is the owner's: a further legacy + // edit is placed anew under imported/. + await fixture.service.strReplace( + { ...fixture.ctx, workspaceId: "ws-owner" }, + "/memories/workspace/sub/note.md", + "v2", + "owner's v3", + "agent" + ); + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.writeFile(path.join(legacyRoot, "sub", "note.md"), "v4 (downgrade)"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect( + await fsPromises.readFile( + path.join(ownerRoot, "imported", "ws-child", "sub", "note.md"), + "utf-8" + ) + ).toBe("v4 (downgrade)"); + expect(await fsPromises.readFile(path.join(ownerRoot, "sub", "note.md"), "utf-8")).toBe( + "owner's v3 (downgrade)" + ); + }); + + it("follows legacy deletions and renames for copies the adoption created, never owner notes", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(ownerRoot, { recursive: true }); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + // `same.md` pre-exists identically on the owner side (reused, not created); + // `mine.md` and `moved.md` are created by the adoption; `edited.md` too, + // but the owner edits it afterwards. + await fsPromises.writeFile(path.join(ownerRoot, "same.md"), "identical"); + for (const [name, body] of [ + ["same.md", "identical"], + ["mine.md", "child note"], + ["moved.md", "to be renamed"], + ["edited.md", "child draft"], + ]) { + await fsPromises.writeFile(path.join(legacyRoot, name), body); + } + await fixture.service.listIndexEntries({ ...fixture.ctx }); + for (const name of ["same.md", "mine.md", "moved.md", "edited.md"]) { + expect( + await fsPromises.stat(path.join(ownerRoot, name)).then( + () => true, + () => false + ) + ).toBe(true); + } + await fixture.service.setPinned({ ...fixture.ctx }, "/memories/workspace/mine.md", true); + await fixture.service.strReplace( + { ...fixture.ctx }, + "/memories/workspace/edited.md", + "draft", + "final", + "agent" + ); + // The downgraded build deletes same.md and mine.md, renames moved.md, and + // deletes edited.md. + await new Promise((resolve) => setTimeout(resolve, 5)); + for (const name of ["same.md", "mine.md", "edited.md"]) { + await fsPromises.rm(path.join(legacyRoot, name)); + } + await fsPromises.rename( + path.join(legacyRoot, "moved.md"), + path.join(legacyRoot, "renamed.md") + ); + const relisted = (await fixture.service.listIndexEntries({ ...fixture.ctx })) + .filter((entry) => entry.scope === "workspace") + .map((entry) => entry.relPath) + .sort(); + // Created + unchanged copies are gone (mine.md, moved.md); the reused + // owner note and the owner-edited copy stay; the rename's new name is + // adopted. + expect(relisted).toEqual(["edited.md", "renamed.md", "same.md"]); + expect(await fsPromises.readFile(path.join(ownerRoot, "edited.md"), "utf-8")).toBe( + "child final" + ); + // The removed copy's owner-side pin went with it. + expect( + (await fixture.metaService.getPinnedKeys()).has( + memoryLogicalKey("workspace", "mine.md", { projectPath: "", workspaceId: "ws-owner" }) + ) + ).toBe(false); + // Idempotent: a further pass changes nothing. + const again = (await fixture.service.listIndexEntries({ ...fixture.ctx })) + .filter((entry) => entry.scope === "workspace") + .map((entry) => entry.relPath) + .sort(); + expect(again).toEqual(relisted); + // A lossy legacy listing (readdir failure tolerated by listFiles) is not + // proof of deletion: the copies stay while the sources provably exist. + await fsPromises.writeFile(path.join(legacyRoot, "renamed.md"), "to be renamed (v2)"); + const lossy = spyOn(fsPromises, "readdir").mockImplementationOnce((() => + Promise.reject(Object.assign(new Error("EIO"), { code: "EIO" }))) as never); + try { + await fixture.service.listIndexEntries({ ...fixture.ctx }); + } finally { + lossy.mockRestore(); + } + // ...and the edited source replaces its untouched adopted copy in place. + expect( + (await fixture.service.listIndexEntries({ ...fixture.ctx })) + .filter((entry) => entry.scope === "workspace") + .map((entry) => entry.relPath) + .sort() + ).toEqual(["edited.md", "renamed.md", "same.md"]); + expect(await fsPromises.readFile(path.join(ownerRoot, "renamed.md"), "utf-8")).toBe( + "to be renamed (v2)" + ); + }); + + it("imports a legacy edit beside an adopted copy the owner recreated with the same bytes", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "v1"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + const ownerCopy = path.join(ownerRoot, "note.md"); + expect(await fsPromises.readFile(ownerCopy, "utf-8")).toBe("v1"); + // The owner deletes the copy and recreates it with the adopted bytes — + // the owner's generation now — and, before any pass re-inspects it, + // the downgraded build edits the legacy source. The bytes still hash + // to the record's, but the stamp no longer matches: the edit must not + // replace the owner's note in place; it lands in the import directory. + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.rm(ownerCopy); + await fsPromises.writeFile(ownerCopy, "v1"); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "v2"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await fsPromises.readFile(ownerCopy, "utf-8")).toBe("v1"); + expect( + await fsPromises.readFile(path.join(ownerRoot, "imported", "ws-child", "note.md"), "utf-8") + ).toBe("v2"); + const record = ( + await readLegacyAdoptionManifest( + legacyAdoptionManifestPath(path.join(fixture.config.sessionsDir, "ws-child")) + ) + ).get("note.md")!; + expect(record.target).toBe("imported/ws-child/note.md"); + expect(record.created).toBe(true); + expect(record.targetStamp).toBe( + (await adoptionTargetStamp(path.join(ownerRoot, "imported", "ws-child", "note.md"))) ?? + undefined + ); + // The staged bytes were installed by rename; nothing lingers (the + // staging dir sits beside the memory root, outside the namespace). + expect(await pathExists(path.join(path.dirname(ownerRoot), "memory-adoption-staging"))).toBe( + false + ); + }); + + it("never claims a copy by byte match alone: a pending record without its receipt's generation", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "same bytes"); + // A fresh adoption crashed after recording its pending manifest but + // BEFORE installing the copy — the receipt names staged bytes that + // never reached the target. Another backend (or the owner) then created + // an owner note with the very same bytes at the planned target. + const manifestPath = legacyAdoptionManifestPath(path.dirname(legacyRoot)); + await fsPromises.mkdir(path.dirname(manifestPath), { recursive: true }); + await fsPromises.writeFile( + manifestPath, + JSON.stringify({ + "note.md": { + content: sha256Hex("same bytes"), + sidecar: "", + target: "note.md", + created: true, + pending: true, + targetStamp: "1:10:1", + }, + }) + ); + await fsPromises.mkdir(ownerRoot, { recursive: true }); + await fsPromises.writeFile(path.join(ownerRoot, "note.md"), "same bytes"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + const record = (await readLegacyAdoptionManifest(manifestPath)).get("note.md")!; + expect(record.pending).toBeUndefined(); + expect(record.created).toBe(false); + expect(record.targetStamp).toBeUndefined(); + // The same for an older build's stamp-less pending record: ambiguous, + // so it claims nothing. + await fsPromises.writeFile( + manifestPath, + JSON.stringify({ + "note.md": { + content: sha256Hex("same bytes"), + sidecar: "", + target: "note.md", + created: true, + pending: true, + }, + }) + ); + await new MemoryService( + fixture.config, + new MemoryMetaService(fixture.xumHome) + ).listIndexEntries({ ...fixture.ctx }); + expect((await readLegacyAdoptionManifest(manifestPath)).get("note.md")!.created).toBe(false); + // Deleting the legacy source therefore leaves the owner's note alone. + await fsPromises.rm(path.join(legacyRoot, "note.md")); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await fsPromises.readFile(path.join(ownerRoot, "note.md"), "utf-8")).toBe( + "same bytes" + ); + // No staged bytes linger beside the owner store. + expect(await pathExists(path.join(path.dirname(ownerRoot), "memory-adoption-staging"))).toBe( + false + ); + }); + + it("retains adoption provenance while the copy of a deleted legacy note cannot be inspected", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "child notes"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + const target = path.join(ownerRoot, "note.md"); + expect(await pathExists(target)).toBe(true); + // The downgraded build deletes the source while the copy's stat fails + // transiently: neither the copy nor its provenance may go. + await fsPromises.rm(path.join(legacyRoot, "note.md")); + const realStat = fsPromises.lstat.bind(fsPromises); + const unreadable = spyOn(fsPromises, "lstat").mockImplementation((( + p: Parameters[0], + ...rest: unknown[] + ) => + String(p) === target + ? Promise.reject(Object.assign(new Error("EIO"), { code: "EIO" })) + : (realStat as (...args: unknown[]) => unknown)(p, ...rest)) as never); + try { + await fixture.service.listIndexEntries({ ...fixture.ctx }); + } finally { + unreadable.mockRestore(); + } + expect(await pathExists(target)).toBe(true); + const manifest = JSON.parse( + await fsPromises.readFile(legacyAdoptionManifestPath(path.dirname(legacyRoot)), "utf-8") + ) as Record; + expect(Object.keys(manifest)).toEqual(["note.md"]); + // Recovered: the retained provenance lets the copy follow its source out. + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await pathExists(target)).toBe(false); + }); + + it("treats a legacy directory replaced by a note as deleting its adopted descendants", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(path.join(legacyRoot, "dir"), { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "dir", "note.md"), "nested"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await pathExists(path.join(ownerRoot, "dir", "note.md"))).toBe(true); + // The downgraded build replaces dir/ with a regular note: the old + // descendant's probe fails ENOTDIR — proof of deletion, like ENOENT. + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.rm(path.join(legacyRoot, "dir"), { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "dir"), "now a note"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await pathExists(path.join(ownerRoot, "dir", "note.md"))).toBe(false); + // The new note itself lands under imported/ (the owner still has a + // directory at that path). + expect( + await fsPromises.readFile(path.join(ownerRoot, "imported", "ws-child", "dir"), "utf-8") + ).toBe("now a note"); + }); + + it("retains adoption provenance while the copy of a deleted legacy note cannot be read", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "child notes"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + const target = path.join(ownerRoot, "note.md"); + await fsPromises.rm(path.join(legacyRoot, "note.md")); + // stat succeeds, the content read fails: not "changed" — keep the entry. + const realOpen = fsPromises.open.bind(fsPromises); + const unreadable = spyOn(fsPromises, "open").mockImplementation((( + p: Parameters[0], + ...rest: unknown[] + ) => + String(p) === target + ? Promise.reject(Object.assign(new Error("EIO"), { code: "EIO" })) + : (realOpen as (...args: unknown[]) => unknown)(p, ...rest)) as never); + try { + await fixture.service.listIndexEntries({ ...fixture.ctx }); + } finally { + unreadable.mockRestore(); + } + expect(await pathExists(target)).toBe(true); + expect( + Object.keys( + JSON.parse( + await fsPromises.readFile(legacyAdoptionManifestPath(path.dirname(legacyRoot)), "utf-8") + ) as Record + ) + ).toEqual(["note.md"]); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await pathExists(target)).toBe(false); + }); + + it("keeps a tombstoned record for a deleted legacy source and re-adopts a reappearing one", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "v1"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.rm(path.join(legacyRoot, "note.md")); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await pathExists(path.join(ownerRoot, "note.md"))).toBe(false); + const manifestPath = legacyAdoptionManifestPath(path.dirname(legacyRoot)); + const tombstoned = JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record< + string, + { deleted?: boolean; target: string } + >; + expect(tombstoned["note.md"]).toMatchObject({ target: "note.md", deleted: true }); + // A copy restored into the shared store (a rollback of the deletion) + // is not reconciled away again: the tombstone is final. + await fsPromises.writeFile(path.join(ownerRoot, "note.md"), "v1"); + await fixture.service.create(fixture.ctx, "/memories/workspace/other.md", "o", "agent"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await pathExists(path.join(ownerRoot, "note.md"))).toBe(true); + // The source reappears on the old build: adopted as a fresh note. + await fsPromises.rm(path.join(ownerRoot, "note.md")); + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "v2"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await fsPromises.readFile(path.join(ownerRoot, "note.md"), "utf-8")).toBe("v2"); + const readopted = JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record< + string, + { deleted?: boolean; created?: boolean } + >; + expect(readopted["note.md"]).toMatchObject({ created: true }); + expect(readopted["note.md"].deleted).toBeUndefined(); + }); + + it("preserves an owner note recreated with the adopted bytes when the legacy source is deleted", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + const ownerCtx = { ...fixture.ctx, workspaceId: "ws-owner" }; + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "v1"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await fsPromises.readFile(path.join(ownerRoot, "note.md"), "utf-8")).toBe("v1"); + // ABA on the owner side: the owner deletes the adopted copy and later + // writes a note of its own at the same path with the same bytes (or + // edits and restores it). The bytes match the record; the file is not + // this adoption's copy any more. + await fixture.service.deletePath(ownerCtx, "/memories/workspace/note.md", "agent"); + await fixture.service.create(ownerCtx, "/memories/workspace/note.md", "v1", "agent"); + // The downgraded child then deletes its source. + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.rm(path.join(legacyRoot, "note.md")); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await fsPromises.readFile(path.join(ownerRoot, "note.md"), "utf-8")).toBe("v1"); + const manifest = JSON.parse( + await fsPromises.readFile(legacyAdoptionManifestPath(path.dirname(legacyRoot)), "utf-8") + ) as Record; + // Tombstoned as owner-owned: no destructive provenance survives. + expect(manifest["note.md"]).toMatchObject({ deleted: true, created: false }); + }); + + it("recovers an interrupted in-place replacement without duplicating the note", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "v1"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + // The downgraded build edits the note; the replacement pass crashed + // after recording its pending state but before writing the bytes — + // the on-disk state that leaves: the PRIOR record marked pending, the + // owner copy still holding the old bytes. + const manifestPath = legacyAdoptionManifestPath(path.dirname(legacyRoot)); + const prior = ( + JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record< + string, + { content: string; sidecar: string; target: string; created?: boolean } + > + )["note.md"]; + expect(prior).toMatchObject({ target: "note.md", created: true }); + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "v2"); + await fsPromises.writeFile( + manifestPath, + JSON.stringify({ "note.md": { ...prior, pending: true } }) + ); + expect(await fsPromises.readFile(path.join(ownerRoot, "note.md"), "utf-8")).toBe("v1"); + // The retry recognizes the surviving old bytes as this adoption's copy + // and replaces them in place — no imported/ duplicate, provenance kept. + const restarted = new MemoryService(fixture.config, new MemoryMetaService(fixture.xumHome)); + await restarted.listIndexEntries({ ...fixture.ctx }); + expect(await fsPromises.readFile(path.join(ownerRoot, "note.md"), "utf-8")).toBe("v2"); + expect(await pathExists(path.join(ownerRoot, "imported", "ws-child", "note.md"))).toBe(false); + const manifest = JSON.parse( + await fsPromises.readFile(legacyAdoptionManifestPath(path.dirname(legacyRoot)), "utf-8") + ) as Record; + expect(Object.keys(manifest)).toEqual(["note.md"]); + expect(manifest["note.md"]).toMatchObject({ target: "note.md", created: true }); + expect(manifest["note.md"].pending).toBeUndefined(); + // A pin the child toggled together with an edit survives an interrupted + // replacement: the pending record keeps the PRIOR sidecar state, so the + // retry still sees the transition and applies it over the owner's pin. + const childKey = memoryLogicalKey("workspace", "note.md", { + projectPath: "", + workspaceId: "ws-child", + }); + const ownerKey = memoryLogicalKey("workspace", "note.md", { + projectPath: "", + workspaceId: "ws-owner", + }); + await fixture.metaService.setPinned(ownerKey, false); + const settled = ( + JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record< + string, + { content: string; sidecar: string; target: string; created?: boolean } + > + )["note.md"]; + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "v3"); + await fixture.metaService.setPinned(childKey, true); + await fsPromises.writeFile( + manifestPath, + JSON.stringify({ "note.md": { ...settled, pending: true } }) + ); + await new MemoryService( + fixture.config, + new MemoryMetaService(fixture.xumHome) + ).listIndexEntries({ + ...fixture.ctx, + }); + expect(await fsPromises.readFile(path.join(ownerRoot, "note.md"), "utf-8")).toBe("v3"); + // Read through a fresh instance: the fixture's sidecar service caches + // its last read and the fold above was written by another instance. + expect((await new MemoryMetaService(fixture.xumHome).getPinnedKeys()).has(ownerKey)).toBe( + true + ); + // A crash after the replacement write but before the settled manifest: + // the pending record still carries the OVERWRITTEN generation's stamp. + // The retry binds the replacement on disk (r75) — settling the stale + // stamp would refuse the child's rollbacks as "replaced" and leave the + // copy behind when the source is deleted. + const settled2 = ( + JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record< + string, + { content: string; sidecar: string; target: string; targetStamp?: string } + > + )["note.md"]; + expect(settled2.targetStamp).toBeDefined(); + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "v3-replaced"); + await fsPromises.writeFile(path.join(ownerRoot, "note.md"), "v3-replaced"); + // The receipt the pass took on the staged bytes (a rename keeps it). + const receipt = async () => (await adoptionTargetStamp(path.join(ownerRoot, "note.md")))!; + await fsPromises.writeFile( + manifestPath, + JSON.stringify({ + "note.md": { + ...settled2, + pending: true, + replacementContent: sha256Hex("v3-replaced"), + replacementStamp: await receipt(), + }, + }) + ); + await new MemoryService( + fixture.config, + new MemoryMetaService(fixture.xumHome) + ).listIndexEntries({ + ...fixture.ctx, + }); + const rebound = ( + JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record< + string, + { pending?: boolean; targetStamp?: string } + > + )["note.md"]; + expect(rebound.pending).toBeUndefined(); + expect(rebound.targetStamp).not.toBe(settled2.targetStamp); + expect(rebound.targetStamp).toBe( + (await adoptionTargetStamp(path.join(ownerRoot, "note.md"))) ?? undefined + ); + // The opposite crash window: the replacement bytes landed but the final + // manifest write did not, and the downgraded build deletes the source + // before the retry. The pending record names both hashes, so the copy + // is still recognized as this adoption's and follows the source out. + const settled3 = ( + JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record< + string, + { content: string; sidecar: string; target: string; created?: boolean } + > + )["note.md"]; + await fsPromises.writeFile(path.join(ownerRoot, "note.md"), "v4"); + await fsPromises.writeFile( + manifestPath, + JSON.stringify({ + "note.md": { + ...settled3, + pending: true, + replacementContent: sha256Hex("v4"), + replacementStamp: await receipt(), + }, + }) + ); + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.rm(path.join(legacyRoot, "note.md")); + await new MemoryService( + fixture.config, + new MemoryMetaService(fixture.xumHome) + ).listIndexEntries({ + ...fixture.ctx, + }); + expect(await pathExists(path.join(ownerRoot, "note.md"))).toBe(false); + }); + + it("recovers a deletion interrupted between the copy's removal and the tombstone", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "v1"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + const manifestPath = legacyAdoptionManifestPath(path.dirname(legacyRoot)); + const prior = ( + JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record< + string, + { content: string; sidecar: string; target: string; created?: boolean } + > + )["note.md"]; + // The crash state: source deleted, deletion recorded as pending, copy + // already removed, tombstone never written. + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.rm(path.join(legacyRoot, "note.md")); + await fsPromises.rm(path.join(ownerRoot, "note.md")); + await fsPromises.writeFile( + manifestPath, + JSON.stringify({ "note.md": { ...prior, pendingDeletion: true } }) + ); + await new MemoryService( + fixture.config, + new MemoryMetaService(fixture.xumHome) + ).listIndexEntries({ + ...fixture.ctx, + }); + const tombstone = ( + JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record< + string, + { deleted?: boolean; created?: boolean; pendingDeletion?: boolean } + > + )["note.md"]; + // Removed by us, not changed by the owner: destructive provenance kept, + // so the child's delete row still maps onto the shared store. + expect(tombstone).toMatchObject({ deleted: true, created: true }); + expect(tombstone.pendingDeletion).toBeUndefined(); + }); + + it("does not read owner state at a pending-deletion target as removed by us", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "v1"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + const manifestPath = legacyAdoptionManifestPath(path.dirname(legacyRoot)); + const prior = ( + JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record< + string, + { content: string; sidecar: string; target: string; created?: boolean } + > + )["note.md"]; + // The deletion was recorded pending but failed before the removal; the + // owner replaced the copy with a directory of its own in the meantime. + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.rm(path.join(legacyRoot, "note.md")); + await fsPromises.rm(path.join(ownerRoot, "note.md")); + await fsPromises.mkdir(path.join(ownerRoot, "note.md")); + await fsPromises.writeFile(path.join(ownerRoot, "note.md", "inner.md"), "owner's"); + await fsPromises.writeFile( + manifestPath, + JSON.stringify({ "note.md": { ...prior, pendingDeletion: true } }) + ); + await new MemoryService( + fixture.config, + new MemoryMetaService(fixture.xumHome) + ).listIndexEntries({ + ...fixture.ctx, + }); + const tombstone = ( + JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record< + string, + { deleted?: boolean; created?: boolean } + > + )["note.md"]; + expect(tombstone.deleted).toBe(true); + expect(tombstone.created).not.toBe(true); + expect(await fsPromises.readFile(path.join(ownerRoot, "note.md", "inner.md"), "utf-8")).toBe( + "owner's" + ); + // Same for a containment failure: an escaping symlink at the target is + // owner state, not proof of absence. + await fsPromises.writeFile(path.join(legacyRoot, "link.md"), "v1"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + const linkPrior = ( + JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record + )["link.md"] as Record; + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.rm(path.join(legacyRoot, "link.md")); + await fsPromises.rm(path.join(ownerRoot, "link.md")); + await fsPromises.symlink( + path.join(fixture.xumHome, "outside.md"), + path.join(ownerRoot, "link.md") + ); + await fsPromises.writeFile(path.join(fixture.xumHome, "outside.md"), "outside"); + const current = JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record< + string, + unknown + >; + await fsPromises.writeFile( + manifestPath, + JSON.stringify({ ...current, "link.md": { ...linkPrior, pendingDeletion: true } }) + ); + await new MemoryService( + fixture.config, + new MemoryMetaService(fixture.xumHome) + ).listIndexEntries({ + ...fixture.ctx, + }); + const linkTombstone = ( + JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record< + string, + { deleted?: boolean; created?: boolean } + > + )["link.md"]; + expect(linkTombstone.deleted).toBe(true); + expect(linkTombstone.created).not.toBe(true); + expect((await fsPromises.lstat(path.join(ownerRoot, "link.md"))).isSymbolicLink()).toBe(true); + }); + + it("reads malformed manifest lifecycle flags fail-closed", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "v1"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + const manifestPath = legacyAdoptionManifestPath(path.dirname(legacyRoot)); + const prior = ( + JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record + )["note.md"] as Record; + // An interrupted adoption's `pending` corrupted to a string: the copy + // was never written. The record must not read as settled — removal's + // handover reconstructs the copy instead of reporting completion. + await fsPromises.rm(path.join(ownerRoot, "note.md")); + await fsPromises.writeFile( + manifestPath, + JSON.stringify({ "note.md": { ...prior, pending: "true" } }) + ); + await new MemoryService( + fixture.config, + new MemoryMetaService(fixture.xumHome) + ).adoptLegacyPrivateStoreForRemoval("ws-child", "ws-owner"); + expect(await fsPromises.readFile(path.join(ownerRoot, "note.md"), "utf-8")).toBe("v1"); + const settled = ( + JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record< + string, + { created?: boolean; pending?: boolean } + > + )["note.md"]; + expect(settled).toMatchObject({ created: true }); + expect(settled.pending).toBeUndefined(); + }); + + it("re-adopts a source that reappeared identically while its deletion was pending", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "v1"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + const manifestPath = legacyAdoptionManifestPath(path.dirname(legacyRoot)); + const prior = ( + JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record + )["note.md"] as Record; + // Crash after the copy's removal, before the tombstone; the downgraded + // build then recreates the source with the same bytes. + await fsPromises.rm(path.join(ownerRoot, "note.md")); + await fsPromises.writeFile( + manifestPath, + JSON.stringify({ "note.md": { ...prior, pendingDeletion: true } }) + ); + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.utimes(path.join(legacyRoot, "note.md"), new Date(), new Date()); + // Removal's handover must restore the copy, not report "nothing to do" + // and delete the only remaining note with the child session. + await new MemoryService( + fixture.config, + new MemoryMetaService(fixture.xumHome) + ).adoptLegacyPrivateStoreForRemoval("ws-child", "ws-owner"); + expect(await fsPromises.readFile(path.join(ownerRoot, "note.md"), "utf-8")).toBe("v1"); + const settled = ( + JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record< + string, + { created?: boolean; pendingDeletion?: boolean } + > + )["note.md"]; + expect(settled).toMatchObject({ created: true }); + expect(settled.pendingDeletion).toBeUndefined(); + }); + + it("keeps the owner's pin when a downgraded build only viewed the adopted note", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "v1"); + const childKey = memoryLogicalKey("workspace", "note.md", { + projectPath: "", + workspaceId: "ws-child", + }); + const ownerKey = memoryLogicalKey("workspace", "note.md", { + projectPath: "", + workspaceId: "ws-owner", + }); + // Adopted before the child ever had a sidecar entry (no view, no pin). + await fixture.service.listIndexEntries({ ...fixture.ctx }); + // The owner pins the shared copy... + await fixture.metaService.setPinned(ownerKey, true); + // ...then the downgraded build merely views the legacy note: the child + // sidecar gains a usage-only entry — the default unpinned state, not a + // pin transition — so the owner's pin stands. + await fixture.metaService.recordAccess(childKey, { write: false }); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect((await fixture.metaService.getPinnedKeys()).has(ownerKey)).toBe(true); + // Another view once an entry exists: usage changes, the pin bit does not. + await fixture.metaService.recordAccess(childKey, { write: false }); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect((await fixture.metaService.getPinnedKeys()).has(ownerKey)).toBe(true); + // A pin the child actually toggles on the old build is the newer intent. + await fixture.metaService.setPinned(ownerKey, false); + await fixture.metaService.setPinned(childKey, true); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect((await fixture.metaService.getPinnedKeys()).has(ownerKey)).toBe(true); + // ...but not once the copy is the OWNER's generation (deleted and + // recreated with identical bytes): a later child toggle no longer folds + // in (r79), and the record stops claiming the copy — the same rule + // deletion reconciliation and the rollback remapper apply. + const ownerCopy = path.join(fixture.config.sessionsDir, "ws-owner", "memory", "note.md"); + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.rm(ownerCopy); + await fsPromises.writeFile(ownerCopy, "v1"); + await fixture.metaService.setPinned(childKey, false); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect((await fixture.metaService.getPinnedKeys()).has(ownerKey)).toBe(true); + const record = ( + await readLegacyAdoptionManifest( + legacyAdoptionManifestPath(path.join(fixture.config.sessionsDir, "ws-child")) + ) + ).get("note.md")!; + expect(record.created).toBe(false); + expect(record.targetStamp).toBeUndefined(); + expect(record.replaced).toBe(true); + expect(await fsPromises.readFile(ownerCopy, "utf-8")).toBe("v1"); + // The record now persists without `created`, like a note the owner had + // all along — but that one folds child toggles, this one must not: the + // next toggle (a fresh process, so nothing is remembered in memory) + // leaves the owner-owned replacement alone too (r80). + await fixture.metaService.setPinned(childKey, true); + await fixture.metaService.setPinned(ownerKey, false); + await new MemoryService( + fixture.config, + new MemoryMetaService(fixture.xumHome) + ).listIndexEntries({ ...fixture.ctx }); + // (A fresh sidecar instance: the fold ran in another one.) + expect((await new MemoryMetaService(fixture.xumHome).getPinnedKeys()).has(ownerKey)).toBe( + false + ); + }); + + it("gives a descendant its own copy when the identical owner file is a sibling's adoption", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const childCtx = { ...fixture.ctx }; + const grandchildCtx = { ...fixture.ctx, workspaceId: "ws-grandchild" }; + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const childRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + const grandchildRoot = path.join(fixture.config.sessionsDir, "ws-grandchild", "memory"); + const key = (rel: string, workspaceId: string) => + memoryLogicalKey("workspace", rel, { projectPath: "", workspaceId }); + await fsPromises.mkdir(ownerRoot, { recursive: true }); + await fsPromises.mkdir(childRoot, { recursive: true }); + await fsPromises.mkdir(grandchildRoot, { recursive: true }); + // shared.md: the child's adoption creates the owner copy (child viewed + // it, unpinned). own.md: a note the owner wrote itself, pinned. + await fsPromises.writeFile(path.join(childRoot, "shared.md"), "same note"); + await fixture.metaService.recordAccess(key("shared.md", "ws-child"), { write: false }); + await fixture.service.listIndexEntries(childCtx); + await fsPromises.writeFile(path.join(ownerRoot, "own.md"), "owner's own"); + await fixture.metaService.setPinned(key("own.md", "ws-owner"), true); + // The grandchild holds both notes byte-identical, pinned. + await fsPromises.writeFile(path.join(grandchildRoot, "shared.md"), "same note"); + await fsPromises.writeFile(path.join(grandchildRoot, "own.md"), "owner's own"); + await fixture.metaService.setPinned(key("shared.md", "ws-grandchild"), true); + await fixture.metaService.setPinned(key("own.md", "ws-grandchild"), true); + await fixture.service.listIndexEntries(grandchildCtx); + // The sibling's copy is not reused: the grandchild gets its own, with + // its own pin; the child's copy and pin state are untouched. + const ownCopy = path.join(ownerRoot, "imported", "ws-grandchild", "shared.md"); + expect(await fsPromises.readFile(ownCopy, "utf-8")).toBe("same note"); + const pinned = await fixture.metaService.getPinnedKeys(); + expect(pinned.has(key("imported/ws-grandchild/shared.md", "ws-owner"))).toBe(true); + expect(pinned.has(key("shared.md", "ws-owner"))).toBe(false); + // The owner's own identical note IS reused (no slot, the owner's pin + // stands), as before. + expect(await pathExists(path.join(ownerRoot, "imported", "ws-grandchild", "own.md"))).toBe( + false + ); + expect(pinned.has(key("own.md", "ws-owner"))).toBe(true); + const manifest = JSON.parse( + await fsPromises.readFile(legacyAdoptionManifestPath(path.dirname(grandchildRoot)), "utf-8") + ) as Record; + expect(manifest["shared.md"]).toMatchObject({ + target: "imported/ws-grandchild/shared.md", + created: true, + }); + expect(manifest["own.md"]).toMatchObject({ target: "own.md", created: false }); + // The creator edits, then deletes, its source: only ITS copy follows; + // the grandchild's copy is a separate file and stays. + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.writeFile(path.join(childRoot, "shared.md"), "child's v2"); + await fixture.service.listIndexEntries(childCtx); + expect(await fsPromises.readFile(path.join(ownerRoot, "shared.md"), "utf-8")).toBe( + "child's v2" + ); + expect(await fsPromises.readFile(ownCopy, "utf-8")).toBe("same note"); + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.rm(path.join(childRoot, "shared.md")); + await fixture.service.listIndexEntries(childCtx); + expect(await pathExists(path.join(ownerRoot, "shared.md"))).toBe(false); + expect(await fsPromises.readFile(ownCopy, "utf-8")).toBe("same note"); + expect( + (await fixture.metaService.getPinnedKeys()).has( + key("imported/ws-grandchild/shared.md", "ws-owner") + ) + ).toBe(true); + }); + + it("waits instead of reusing an identical owner file while a sibling's manifest cannot be read", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const childCtx = { ...fixture.ctx }; + const grandchildCtx = { ...fixture.ctx, workspaceId: "ws-grandchild" }; + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const childRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + const grandchildRoot = path.join(fixture.config.sessionsDir, "ws-grandchild", "memory"); + const key = (workspaceId: string) => + memoryLogicalKey("workspace", "shared.md", { projectPath: "", workspaceId }); + await fsPromises.mkdir(childRoot, { recursive: true }); + await fsPromises.mkdir(grandchildRoot, { recursive: true }); + await fsPromises.writeFile(path.join(childRoot, "shared.md"), "same note"); + await fixture.service.listIndexEntries(childCtx); + await fixture.metaService.setPinned(key("ws-owner"), false); + await fsPromises.writeFile(path.join(grandchildRoot, "shared.md"), "same note"); + await fixture.metaService.setPinned(key("ws-grandchild"), true); + // The child's manifest is malformed: whether the identical owner file + // is the child's copy cannot be told, so the note is neither reused + // (its pin would land on a possibly foreign copy) nor duplicated yet. + const childManifest = legacyAdoptionManifestPath(path.dirname(childRoot)); + const intact = await fsPromises.readFile(childManifest, "utf-8"); + await fsPromises.writeFile(childManifest, "{not json"); + const passes = spyOn( + fixture.service as unknown as { readOrQuarantineAdoptionManifest: () => Promise }, + "readOrQuarantineAdoptionManifest" + ); + await fixture.service.listIndexEntries(grandchildCtx); + expect(await pathExists(path.join(ownerRoot, "imported"))).toBe(false); + expect((await fixture.metaService.getPinnedKeys()).has(key("ws-owner"))).toBe(false); + expect(await pathExists(legacyAdoptionManifestPath(path.dirname(grandchildRoot)))).toBe( + false + ); + // Transient: not memoized. Once readable again, the note lands as the + // grandchild's own copy. + await fsPromises.writeFile(childManifest, intact); + await fixture.service.listIndexEntries(grandchildCtx); + expect(passes).toHaveBeenCalledTimes(2); + expect( + await fsPromises.readFile( + path.join(ownerRoot, "imported", "ws-grandchild", "shared.md"), + "utf-8" + ) + ).toBe("same note"); + expect((await fixture.metaService.getPinnedKeys()).has(key("ws-owner"))).toBe(false); + }); + + it("lands a downgraded rename in one pass when the owner store is at capacity", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(ownerRoot, { recursive: true }); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + // The adopted note takes the owner store's last slot. + await Promise.all( + Array.from({ length: MEMORY_MAX_FILES_PER_SCOPE - 1 }, (_, i) => + fsPromises.writeFile(path.join(ownerRoot, `o${String(i).padStart(4, "0")}.md`), "o") + ) + ); + await fsPromises.writeFile(path.join(legacyRoot, "old.md"), "note"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await fsPromises.readFile(path.join(ownerRoot, "old.md"), "utf-8")).toBe("note"); + const passes = spyOn( + fixture.service as unknown as { readOrQuarantineAdoptionManifest: () => Promise }, + "readOrQuarantineAdoptionManifest" + ); + // The downgraded build renames it: the slot its copy frees is credited + // to the same pass, so the new name lands at once — not skipped as + // "full" (and then memoized) while the old copy still held the slot. + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.rename(path.join(legacyRoot, "old.md"), path.join(legacyRoot, "new.md")); + const files = (await fixture.service.listIndexEntries({ ...fixture.ctx })) + .filter((e) => e.scope === "workspace") + .map((e) => e.relPath); + expect(passes).toHaveBeenCalledTimes(1); + expect(files).toHaveLength(MEMORY_MAX_FILES_PER_SCOPE); + expect(files).toContain("new.md"); + expect(files).not.toContain("old.md"); + expect(await fsPromises.readFile(path.join(ownerRoot, "new.md"), "utf-8")).toBe("note"); + // Strict removal agrees the handover is complete. + await fixture.service.adoptLegacyPrivateStoreForRemoval("ws-child", "ws-owner"); + }); + + it("writes tombstones the previous build reads as unsettled, so a reappearing source is re-adopted there too", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "v1"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.rm(path.join(legacyRoot, "note.md")); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await pathExists(path.join(ownerRoot, "note.md"))).toBe(false); + const manifestPath = legacyAdoptionManifestPath(path.dirname(legacyRoot)); + const tombstone = ( + JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record< + string, + Record + > + )["note.md"]; + expect(tombstone).toMatchObject({ deleted: true, pending: true }); + // The previous build knows only content/sidecar/target/created/pending/ + // targetStamp: this is the record as it parses (and rewrites) it. With + // the source recreated identically, its forced handover must NOT read + // the settled hash as "folded in earlier" and delete the child session + // while no copy exists — pending, it re-adopts. + const asPreviousBuild = Object.fromEntries( + Object.entries(tombstone).filter(([field]) => + ["content", "sidecar", "target", "created", "pending", "targetStamp"].includes(field) + ) + ); + expect(asPreviousBuild.pending).toBe(true); + await fsPromises.writeFile(manifestPath, JSON.stringify({ "note.md": asPreviousBuild })); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "v1"); + await new MemoryService( + fixture.config, + new MemoryMetaService(fixture.xumHome) + ).adoptLegacyPrivateStoreForRemoval("ws-child", "ws-owner"); + expect(await fsPromises.readFile(path.join(ownerRoot, "note.md"), "utf-8")).toBe("v1"); + }); }); describe("memory index entries", () => { diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 8ae65ce29d..8dd1c1db91 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -18,6 +18,7 @@ */ import { EventEmitter } from "events"; import { createHash, randomUUID } from "node:crypto"; +import type { Dirent } from "node:fs"; import * as fsPromises from "node:fs/promises"; import * as path from "node:path"; import writeFileAtomic from "write-file-atomic"; @@ -426,6 +427,34 @@ interface MemoryStore { } const LEGACY_IMPORT_DIR = "imported"; +/** + * The per-child directory a conflicting legacy note is imported under. A + * workspace id is not a memory path segment by construction — a legacy id + * keeps its project basename's `~`, and an id may carry `..`, `%2e`, control + * or XML characters the grammar rejects (parseMemoryPath) — and a copy placed + * under such a segment would be written and settled yet filtered out of the + * index and unaddressable by every command, while removal then deletes the + * legacy source. Ids the grammar admits are used verbatim (every manifest + * written so far names them that way); the rest are escaped per UTF-8 byte + * as `=XX` (a `.` cannot be percent-encoded: `%2e` is itself rejected). A + * verbatim segment never contains `=` (such ids are escaped too), so the two + * forms cannot collide and an escaped segment decodes unambiguously. + */ +function legacyImportSegment(childId: string): string { + if (!childId.includes("=")) { + try { + parseMemoryPath(toVirtualPath("workspace", `${LEGACY_IMPORT_DIR}/${childId}/x`)); + return childId; + } catch { + // escaped below + } + } + return Array.from(Buffer.from(childId, "utf-8"), (byte) => + /[A-Za-z0-9_-]/.test(String.fromCharCode(byte)) + ? String.fromCharCode(byte) + : `=${byte.toString(16).toUpperCase().padStart(2, "0")}` + ).join(""); +} /** * Directory beside the owner's memory root (in its session dir, OUTSIDE the * model-writable memory namespace — a legacy note may legitimately live under @@ -494,6 +523,19 @@ function isMissingPathError(error: unknown): boolean { return code === "ENOENT" || code === "ENOTDIR"; } +/** DT_UNKNOWN: readdir could not type the entry (no predicate holds). */ +function isDirentTypeUnknown(entry: Dirent): boolean { + return !( + entry.isFile() || + entry.isDirectory() || + entry.isSymbolicLink() || + entry.isFIFO() || + entry.isSocket() || + entry.isBlockDevice() || + entry.isCharacterDevice() + ); +} + /** * Link-aware kind of a path: symlinks are reported as such, never followed. * "missing" only when proven (ENOENT/ENOTDIR); any other failure (EACCES, @@ -588,9 +630,25 @@ class LocalMemoryStore implements MemoryStore { if (options?.strict !== true && results.length > MEMORY_MAX_FILES_PER_SCOPE) return; if (options?.includeDotfiles !== true && entry.name.startsWith(".")) continue; const childRel = dirRel === "" ? entry.name : `${dirRel}/${entry.name}`; + // A filesystem may report DT_UNKNOWN: every type predicate is false + // and the entry would drop out of the walk. Strict callers (removal's + // legacy handover) would then see a complete listing that omits a + // regular note or a whole subtree, so they classify by lstat instead; + // an unclassifiable entry fails the listing like an unreadable dir. + let kind: "dir" | "file" | "other"; if (entry.isDirectory()) { - await walk(childRel); + kind = "dir"; } else if (entry.isFile()) { + kind = "file"; + } else if (options?.strict === true && isDirentTypeUnknown(entry)) { + const stat = await fsPromises.lstat(this.abs(childRel)); + kind = stat.isDirectory() ? "dir" : stat.isFile() ? "file" : "other"; + } else { + kind = "other"; + } + if (kind === "dir") { + await walk(childRel); + } else if (kind === "file") { results.push(childRel); } } @@ -1388,6 +1446,33 @@ export class MemoryService extends EventEmitter { const manifestPath = legacyAdoptionManifestPath(childSessionDir); const adopted = await this.readOrQuarantineAdoptionManifest(manifestPath, childId); const sidecarEntries = await this.metaService.getEntriesOrThrow(); + // An adoption-created copy belongs to exactly ONE descendant: a second + // descendant whose note is byte-identical never reuses a sibling's + // copy (one child's in-place replacement would rewrite bytes the + // other still represents, one child's source deletion would remove a + // copy the other still needs, and their pins would collide on one + // file) — it gets its own under imported//. Ownership is by + // LIVE generation: a sibling's settled `created` record naming the + // path whose receipt (targetStamp, or replacementStamp on the far side + // of an interrupted replacement) equals the stamp of the file on disk. + // A path plus flags alone would read an owner-edited or recreated copy + // as the sibling's. The sibling manifests are read strictly, once per + // pass and only when a candidate is identical: an unreadable or + // malformed one cannot answer, and the note waits (transient skip) + // rather than reuse — or clear the pins of — a copy that may be a + // sibling's. + let siblingRecords: LegacyAdoptionRecord[] | null = null; + const siblingOwns = async (targetRelPath: string, liveStamp: string | null) => { + if (liveStamp === null) return false; + siblingRecords ??= await this.descendantAdoptionRecords(owner, childId); + return siblingRecords.some( + (record) => + record.target === targetRelPath && + record.created === true && + record.deleted !== true && + (record.targetStamp === liveStamp || record.replacementStamp === liveStamp) + ); + }; // The per-scope file cap is a store invariant (create/rename enforce // it): the copy stops at the owner store's remaining capacity so a // combined notebook cannot exceed it — an over-full scope is silently @@ -1408,6 +1493,128 @@ export class MemoryService extends EventEmitter { writeFileAtomic(manifestPath, JSON.stringify(Object.fromEntries(adopted)), { encoding: "utf-8", }); + // Legacy notes deleted or renamed on the downgraded build: a copy THIS + // adoption created, still holding the adopted bytes, follows the source + // out of the shared notebook (a rename's new name is adopted below like + // a fresh note). Reconciled BEFORE the listed notes are placed: the + // slot a removed copy frees is credited to this pass, so a rename in + // an owner store at capacity lands in the same pass instead of being + // skipped as "full" while its old copy still holds the slot; and a + // rename onto the path of its own conflict copy finds that path free + // rather than a file to reuse. Provenance and unchanged content are + // both required — an owner note that merely happened to be identical, + // or an adopted copy the owner has since edited, is the owner's and + // stays. Unlisted sources are only ever judged against the listing + // that succeeded above; a failed listing never reaches this point. + const listed = new Set(files); + for (const [relPath, previous] of adopted) { + if (listed.has(relPath) || previous.deleted === true) continue; + // Absence from the listing is not proof enough on its own: only a + // provable ENOENT on the source itself counts; any other outcome + // keeps the entry (and the copy) for a later pass. ENOTDIR is proof + // too: the downgraded build replaced `dir/` with a regular note, + // deleting every descendant. + const sourceGone = await fsPromises.lstat(path.join(legacyRoot, relPath)).then( + () => false, + (error: unknown) => isMissingPathError(error) + ); + if (!sourceGone) continue; + let unchangedForTombstone = false; + if (previous.created === true) { + // Strict probe: a target that merely could not be inspected is not + // "changed" — dropping the entry on that basis would lose the + // provenance for good and leave the obsolete copy visible forever + // once the filesystem recovers. Keep the entry (and the pass + // incomplete) so the next access reconciles it. A directory, + // symlink, non-regular entry, escaping component or over-cap / + // non-UTF-8 file there is owner state (content null). + const targetContained = await store.assertContained(previous.target).then( + () => true, + () => false + ); + let destination: "free" | { content: string | null } = "free"; + try { + if (targetContained) { + destination = await this.inspectAdoptionDestination(store, previous.target); + } + } catch (error) { + log.warn( + "[MemoryService] cannot inspect an adopted legacy note's copy; retrying later", + { childId, owner, relPath, target: previous.target, error } + ); + skipped++; + transientSkips++; + continue; + } + const current = destination === "free" ? null : destination.content; + // Ours only while it is a generation this adoption installed + // (targetStamp; replacementStamp on the far side of an interrupted + // in-place replacement — both receipts taken on the staged bytes, + // so a crash cannot have kept them from being recorded): identical + // bytes in a file the owner deleted and recreated, or edited and + // restored, are the owner's, and a record without a stamp preserves. + const currentHash = current === null ? null : sha256Hex(current); + const stamp = await adoptionTargetStamp(store.physicalPath(previous.target)); + const unchanged = + currentHash !== null && + stamp !== null && + ((currentHash === previous.content && stamp === previous.targetStamp) || + (previous.pending === true && + currentHash === previous.replacementContent && + stamp === previous.replacementStamp)); + // A target PROVEN absent (contained path, strict probe ENOENT) while + // a deletion was pending was removed by the interrupted pass, not + // changed by the owner. + const removedByUs = + previous.pendingDeletion === true && targetContained && destination === "free"; + unchangedForTombstone = unchanged || removedByUs; + if (unchanged) { + // Deletion provenance first: a crash after the removal but before + // the tombstone write must not make the retry read the missing + // copy as owner-changed (and drop the child's rollback mapping). + adopted.set(relPath, { ...previous, pendingDeletion: true }); + await writeManifest(); + // Metadata next: a sidecar failure then aborts the pass with the + // file and manifest entry intact, so the retry repeats both; + // the reverse order would strand the owner-key pin/usage once + // the file was gone and the entry dropped. + await this.metaService.removeKeys( + memoryLogicalKey("workspace", previous.target, { + projectPath: ctx.projectPath, + workspaceId: owner, + }) + ); + await store.remove(previous.target); + remainingCapacity++; + adoptedCount++; + log.info("[MemoryService] removed an adopted legacy note deleted on the old build", { + childId, + owner, + relPath, + target: previous.target, + }); + } + } + // Kept as a tombstone, not dropped: the child's pre-sharing rows for + // this note still need relPath → target to be rolled back into the + // shared store (a delete's restore lands at the reconciled target; + // the reconciliation above never runs again for it). + // Destructive provenance survives only while the target was still + // this adoption's copy: a copy the owner edited is not the old + // path's to delete or restore any more. + // `pending` too (see LegacyAdoptionRecord.deleted): a build that + // predates the tombstone reads it as an unsettled adoption and + // re-adopts a reappearing source, instead of taking the settled hash + // for "folded in earlier" while no copy exists. + adopted.set(relPath, { + ...previous, + pendingDeletion: undefined, + deleted: true, + pending: true, + created: previous.created === true && unchangedForTombstone, + }); + manifestDirty = true; + } for (const relPath of files) { // Same gates as a memory command. Name first: a legacy file whose // name the path grammar rejects (traversal-looking segments, control @@ -1442,10 +1649,13 @@ export class MemoryService extends EventEmitter { } // Strict decode: invalid UTF-8 cannot be carried by a text write, but a // note that legitimately contains U+FFFD must not be mistaken for one - // (a lossy decode would make the two indistinguishable). + // (a lossy decode would make the two indistinguishable). BOM kept: a + // memory write admits a leading U+FEFF, and the default decoder would + // swallow it — the copy and its hash would then differ from the + // byte-exact source (and a BOM-less owner note would read as equal). let content: string; try { - content = new TextDecoder("utf-8", { fatal: true }).decode(bytes); + content = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(bytes); } catch { skipped++; continue; @@ -1461,29 +1671,46 @@ export class MemoryService extends EventEmitter { target: "", created: false, }; - const previous = adopted.get(relPath); + // A record whose source was reconciled as deleted is kept only for + // the path mapping (rollbacks); a reappearing source is a fresh note. + const priorRecord = adopted.get(relPath); + const previous = priorRecord?.deleted === true ? undefined : priorRecord; if ( previous?.content === record.content && previous.sidecar === record.sidecar && - previous.pending !== true + previous.pending !== true && + // A deletion under way removed (or is about to remove) the copy: a + // source reappearing with the same bytes must be adopted anew. + previous.pendingDeletion !== true ) { continue; // folded in earlier, nothing changed since } - let target: { relPath: string; write: boolean } | null = null; + // `generation`: the stamp of the copy an in-place replacement installs + // over, re-checked right before the install. + let target: { + relPath: string; + write: boolean; + replaces?: boolean; + generation?: string; + } | null = null; // A child's pin toggle folds into the copy only while the copy is // this adoption's generation (see below) or the owner's identical // note it was folded into at first adoption; a copy the owner - // replaced since keeps the owner's pin. + // replaced since keeps the owner's pin — recorded as `replaced`, so + // the next pass still knows (its `created` is gone either way). let foldChildPin = true; if (previous !== undefined) { // The recorded target is reused only while it still holds bytes // this adoption put there — the owner may have edited, replaced or // deleted it since, and the child's note must not land on unrelated // content or a missing file. Unchanged legacy bytes (only the - // sidecar moved): reuse without a write. Otherwise the note is - // placed anew (legacyImportTarget: a legacy note edited on the - // downgraded build lands under imported// beside the copy of - // its earlier bytes). Inspected strictly: a prior target that + // sidecar moved): reuse without a write. Legacy bytes edited on the + // downgraded build while the copy THIS adoption created is still + // untouched: the copy is replaced in place — placing the new bytes + // elsewhere would strand the old copy, provenance lost, in the + // model-visible notebook (and, both destinations taken, skip the + // edit for good). Otherwise the note is placed anew + // (legacyImportTarget). Inspected strictly: a prior target that // merely cannot be stat'ed or read right now is not "replaced" — // retry on the next access instead (the pass stays incomplete). let priorContent: string | null; @@ -1498,27 +1725,62 @@ export class MemoryService extends EventEmitter { transientSkips++; continue; } - if (priorContent === content) { - // Ours only while the copy is the generation this adoption - // installed (LegacyAdoptionRecord.targetStamp — a receipt taken - // on the staged bytes BEFORE they appear at the target, so even a - // pass interrupted between manifest and install left one). - // Identical bytes in another generation are the owner's: the - // owner may have deleted and recreated the note with the very - // same bytes, or — a pass interrupted before its install — - // another backend may have created it at the planned target. - // `pending` alone is never provenance: a stamp-less pending - // record (an older build's) is ambiguous and claims nothing. - const currentStamp = - (await adoptionTargetStamp(store.physicalPath(previous.target))) ?? undefined; - const ours = - previous.created === true && - currentStamp !== undefined && - currentStamp === previous.targetStamp; + // Ours only while the copy is a generation this adoption installed + // (LegacyAdoptionRecord.targetStamp / replacementStamp — receipts + // taken on the staged bytes BEFORE they appear at the target, so + // even a pass interrupted between manifest and install left one). + // Identical bytes in another generation are the owner's (the same + // rule deletion reconciliation applies): the owner may have deleted + // and recreated the note with the very same bytes, or — a pass + // interrupted before its install — another backend may have created + // it at the planned target. `pending` alone is never provenance: a + // stamp-less pending record (an older build's) is ambiguous and + // claims nothing. + const currentStamp = + (await adoptionTargetStamp(store.physicalPath(previous.target))) ?? undefined; + const ours = + previous.created === true && + currentStamp !== undefined && + (currentStamp === previous.targetStamp || currentStamp === previous.replacementStamp); + // A recorded target that is not ours may be a sibling's copy (an + // earlier build reused identical bytes across descendants, or the + // owner's replacement was itself a sibling's fresh adoption): then + // it is not this note's to reuse, and the note is placed anew. + let siblings = false; + if (!ours && priorContent === content) { + try { + siblings = await siblingOwns(previous.target, currentStamp ?? null); + } catch (error) { + log.warn( + "[MemoryService] cannot read a sibling's adoption manifest; retrying later", + { childId, owner, relPath, target: previous.target, error } + ); + skipped++; + transientSkips++; + continue; + } + } + if (priorContent === content && !siblings) { target = { relPath: previous.target, write: false }; record.created = ours; record.targetStamp = ours ? currentStamp : undefined; - foldChildPin = !(previous.created === true && !ours); + const replaced = previous.replaced === true || (previous.created === true && !ours); + if (replaced) record.replaced = true; + foldChildPin = !replaced; + } else if ( + ours && + priorContent !== null && + [previous.content, previous.replacementContent].includes(sha256Hex(priorContent)) + ) { + // Legacy bytes edited on the downgraded build while the copy is + // still this adoption's (either side of an interrupted + // replacement): replaced in place. + target = { + relPath: previous.target, + write: true, + replaces: true, + generation: currentStamp, + }; } } if (target === null) { @@ -1526,7 +1788,7 @@ export class MemoryService extends EventEmitter { // its lstat or read) is neither free nor different: the note waits // for the next pass with no copy made and no record written. try { - target = await this.legacyImportTarget(store, childId, relPath, content); + target = await this.legacyImportTarget(store, childId, relPath, content, siblingOwns); } catch (error) { log.warn("[MemoryService] cannot inspect a legacy note's destination; retrying later", { childId, @@ -1544,7 +1806,7 @@ export class MemoryService extends EventEmitter { } } if (target.write) { - if (remainingCapacity <= 0) { + if (target.replaces !== true && remainingCapacity <= 0) { capacityExhausted = true; skipped++; continue; @@ -1573,7 +1835,10 @@ export class MemoryService extends EventEmitter { // a file not yet there (nothing claimed) or names the installed // generation by stamp; a plain byte match never has to stand in // for provenance. Without the record, an installed copy would read - // as the owner's own note. + // as the owner's own note, and a legacy deletion could then never + // follow it out of the shared store. A replacement keeps the PRIOR + // record (old hash and stamp, same target) while pending: on either + // side of the install the retry recognizes the file by its stamp. const stagingPath = path.join(stagingDir, randomUUID()); try { await fsPromises.mkdir(stagingDir, { recursive: true }); @@ -1595,20 +1860,35 @@ export class MemoryService extends EventEmitter { transientSkips++; continue; } - adopted.set(relPath, { - ...record, - target: target.relPath, - created: true, - pending: true, - targetStamp: stagedStamp, - }); + adopted.set( + relPath, + target.replaces === true && previous !== undefined + ? { + ...previous, + pending: true, + replacementContent: record.content, + replacementStamp: stagedStamp, + } + : { + ...record, + target: target.relPath, + created: true, + pending: true, + targetStamp: stagedStamp, + } + ); await writeManifest(); // The destination as decided above, re-checked under the lock right - // before the install: a fresh placement must still be free. + // before the install: a fresh placement must still be free, a + // replacement must still be the generation it was decided against. // Anything else is owner state the rename must not clobber — the // staged bytes are dropped, the record restored, and the note is // placed on the next pass. - const installable = (await store.kind(target.relPath, { strict: true })) === null; + const installable = + target.replaces === true + ? (await adoptionTargetStamp(store.physicalPath(target.relPath))) === + target.generation + : (await store.kind(target.relPath, { strict: true })) === null; const restoreRecord = async () => { await fsPromises.rm(stagingPath, { force: true }); if (previous === undefined) adopted.delete(relPath); @@ -1648,7 +1928,7 @@ export class MemoryService extends EventEmitter { transientSkips++; continue; } - remainingCapacity--; + if (target.replaces !== true) remainingCapacity--; imported++; record.created = true; // The generation of the file just installed (see targetStamp): the @@ -1671,7 +1951,9 @@ export class MemoryService extends EventEmitter { if (childEntry !== undefined) { // A pending record still carries the sidecar state it was recorded // with: a fresh adoption's is the child's current state (no - // transition → first-adoption semantics). + // transition → first-adoption semantics), a pending replacement's + // is the prior record's — the child's toggle since must not be + // lost to the interrupted pass. const priorPinned = previous === undefined ? null : legacySidecarPinned(previous.sidecar); // Only an actual boolean transition of the child's pin overrides // the owner's; an unknown prior state never does. @@ -1782,17 +2064,24 @@ export class MemoryService extends EventEmitter { /** * Where a legacy file lands in the owner store: its own relPath when free - * (write) or already identical (no write); the per-child import directory - * when the owner has different content there; null when even that slot is - * taken by different content (the file stays only in the legacy directory). + * (write) or already identical and the owner's own (no write); the + * per-child import directory when the owner has different content there + * or the identical file is another descendant's adoption-created copy; + * null when even that slot is taken by different content (the file stays + * only in the legacy directory). Throws when a sibling manifest the + * decision needs cannot be read (callers skip the note transiently). */ private async legacyImportTarget( store: MemoryStore, childId: string, relPath: string, - content: string + content: string, + siblingOwns: (targetRelPath: string, liveStamp: string | null) => Promise ): Promise<{ relPath: string; write: boolean } | null> { - for (const candidate of [relPath, `${LEGACY_IMPORT_DIR}/${childId}/${relPath}`]) { + for (const candidate of [ + relPath, + `${LEGACY_IMPORT_DIR}/${legacyImportSegment(childId)}/${relPath}`, + ]) { // Never even compare through an escaping path (the write site re-checks). const contained = await store.assertContained(candidate).then( () => true, @@ -1801,11 +2090,47 @@ export class MemoryService extends EventEmitter { if (!contained) continue; const destination = await this.inspectAdoptionDestination(store, candidate); if (destination === "free") return { relPath: candidate, write: true }; - if (destination.content === content) return { relPath: candidate, write: false }; + // Identical: the owner's own note is reused (no slot, the owner's pin + // stands); another descendant's adoption-created copy is not — this + // note gets its own copy at the next candidate. + if ( + destination.content === content && + !(await siblingOwns(candidate, await adoptionTargetStamp(store.physicalPath(candidate)))) + ) { + return { relPath: candidate, write: false }; + } } return null; } + /** + * The settled adoption records of the owner's OTHER descendants, read + * strictly: the pass decides on their authority whether an identical owner + * file may be reused, so an unreadable or malformed sibling manifest fails + * the question (callers skip the note transiently) instead of answering + * "not a sibling's". + */ + private async descendantAdoptionRecords( + owner: string, + childId: string + ): Promise { + const cfg = this.config.loadConfigOrDefault(); + const resolve = workspaceMemoryOwnerResolver(cfg); + const records: LegacyAdoptionRecord[] = []; + for (const project of cfg.projects.values()) { + for (const workspace of project.workspaces) { + const id = workspace.id; + if (id === undefined || id === childId || id === owner || resolve(id) !== owner) continue; + const manifest = await readLegacyAdoptionManifest( + legacyAdoptionManifestPath(path.join(this.config.sessionsDir, id)), + { strict: true } + ); + records.push(...manifest.values()); + } + } + return records; + } + /** * Content of an adopted note's copy in the owner store, or null when no * regular listed file is there (absent, a directory, a symlink, or grown @@ -1853,7 +2178,7 @@ export class MemoryService extends EventEmitter { const bytes = await store.readFilePrefixBytes(relPath, MEMORY_MAX_FILE_BYTES + 1); if (bytes.length > MEMORY_MAX_FILE_BYTES) return { content: null }; try { - return { content: new TextDecoder("utf-8", { fatal: true }).decode(bytes) }; + return { content: new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(bytes) }; } catch { return { content: null }; }