diff --git a/qa/analyzers/rules/orphan-after-rename.ts b/qa/analyzers/rules/orphan-after-rename.ts new file mode 100644 index 0000000..df4eeab --- /dev/null +++ b/qa/analyzers/rules/orphan-after-rename.ts @@ -0,0 +1,223 @@ +import type { FlightEvent } from "../flight-event"; +import type { AnalyzerFinding } from "../report"; + +/** + * Rule: orphan-after-rename + * + * disk.rename.observed is now emitted as TWO events sharing the same opId: + * - renameRole: "source" (oldPath → oldPathId) + * - renameRole: "target" (newPath → newPathId) + * + * After both fire, the trace should show: + * (a) crdt.file.renamed with newPathId + * (b) NO crdt.file.created for newPathId after the rename (identity lost) + * (c) NO crdt.file.tombstoned for newPathId before the cleanup phase + * + * Phase-marker logic: + * Tombstones that happen during the cleanup phase (after qa.phase{phase:"cleanup"}) + * are expected scenario teardown and are not flagged. + * + * Backward compatibility: + * Traces without qa.phase events fall back to using disk.delete.observed + * as a proxy for "intentional user delete" (the old heuristic). + * + * Also flags: + * - target-role disk.rename.observed with no matching crdt.file.renamed + */ +const WINDOW_MS = 15_000; + +export function checkOrphanAfterRename(events: FlightEvent[]): AnalyzerFinding[] { + const findings: AnalyzerFinding[] = []; + + // Find all target-role rename events (the new path side). + // Pre-dual-event traces had no renameRole; fall back to any disk.rename.observed. + const targetRenames = events.filter((e) => { + if (e.kind !== "disk.rename.observed") return false; + const role = (e.data as Record | undefined)?.renameRole; + return role === "target" || role === undefined; + }); + + if (targetRenames.length === 0) return findings; + + // Build set of pathIds that appear as a SOURCE in a rename event within this trace. + // A pathId that is both a rename target AND a subsequent rename source is an + // intermediate hop in a rename chain (e.g. A→B→C: B is target of A→B and source + // of B→C). YAOS collapses chains into a single CRDT rename (A→C), so B never gets + // a crdt.file.renamed event. These intermediate hops are exempt from the hard failure. + const sourcePathIds = new Set( + events + .filter((e) => + e.kind === "disk.rename.observed" + && (e.data as Record | undefined)?.renameRole === "source", + ) + .map((e) => e.pathId) + .filter((id): id is string => !!id && id !== "p:unavailable"), + ); + + // Determine cleanup phase start time from qa.phase markers (taxonomy v5+). + const cleanupPhaseTs = events + .filter((e) => e.kind === "qa.phase" && (e.data as Record | undefined)?.phase === "cleanup") + .reduce((min, e) => Math.min(min, e.ts), Infinity); + const hasCleanupPhaseMarker = cleanupPhaseTs < Infinity; + + for (const renameEvent of targetRenames) { + const newPathId = renameEvent.pathId; + if (!newPathId || newPathId === "p:unavailable") continue; + + // Intermediate chain hop: this path was immediately renamed again. + // YAOS collapses the chain in the CRDT batch, so no crdt.file.renamed + // will appear for this intermediate path. This is correct behavior. + if (sourcePathIds.has(newPathId)) continue; + + // Remote-origin rename: DiskMirror applied this rename in response to a remote + // metadata path change (schema v3 nested Y.Map path field mutation). The passive + // receiver device performs the disk rename directly via handleRemoteRename without + // emitting crdt.file.renamed — the CRDT rename was already applied on the active + // device. remoteOrigin:true is set in the event data by the vault rename handler + // when DiskMirror's _pendingRemoteRenameNewPaths set contains the new path. + const isRemoteOrigin = (renameEvent.data as Record | undefined)?.remoteOrigin === true; + if (isRemoteOrigin) continue; + + // Find the source-role event with the same opId to get oldPathId. + const sourceEvent = events.find((e) => + e.kind === "disk.rename.observed" + && e.opId === renameEvent.opId + && (e.data as Record | undefined)?.renameRole === "source", + ); + const oldPathId = sourceEvent?.pathId ?? "unknown"; + + const renameTs = renameEvent.ts; + + // (a) crdt.file.renamed must appear within window. + // If not, check whether this is a pre-CRDT race recovery or a true silent drop. + const crdtRenamed = events.find((e) => + e.kind === "crdt.file.renamed" + && e.pathId === newPathId + && e.ts >= renameTs + && e.ts - renameTs <= WINDOW_MS, + ); + + if (!crdtRenamed) { + // Determine whether the source file ever had a CRDT identity before the rename. + // If oldPath had a crdt.file.created event before the rename fired, YAOS should + // have had a fileId and the rename should have produced crdt.file.renamed. + // If it did NOT, this is the pre-CRDT race: rename fired before ensureFile ran. + const sourceHadCrdtIdentityBeforeRename = events.some((e) => + e.kind === "crdt.file.created" + && e.pathId === oldPathId + && e.ts < renameTs, + ); + + // In the race case, the valid recovery outcome is crdt.file.created at newPath + // (content redirected, new fileId assigned). Only accept this downgrade when: + // 1. Source had NO prior CRDT identity (true race — no fileId to rename from) + // 2. crdt.file.created at newPath appeared within window + const crdtCreatedAsRecovery = !sourceHadCrdtIdentityBeforeRename + ? events.find((e) => + e.kind === "crdt.file.created" + && e.pathId === newPathId + && e.ts >= renameTs + && e.ts - renameTs <= WINDOW_MS, + ) + : undefined; + + if (crdtCreatedAsRecovery) { + // Pre-CRDT race recovery: content preserved at newPath, but via a new + // fileId rather than an identity-preserving rename. Warning, not hard failure. + findings.push({ + rule: "orphan-after-rename", + severity: "warning", + pathId: newPathId, + eventSeqs: [renameEvent.seq, crdtCreatedAsRecovery.seq], + description: + `disk.rename.observed for pathId=${newPathId} was handled via race recovery ` + + `(crdt.file.created instead of crdt.file.renamed) — rename fired before CRDT ` + + `had a fileId for oldPath=${oldPathId}. Content preserved; fileId is new.`, + }); + } else { + // Either source had a CRDT identity (real identity-loss bug) or no + // crdt.file.created appeared at all (content lost entirely). + const reason = sourceHadCrdtIdentityBeforeRename + ? `source pathId=${oldPathId} had a prior CRDT identity — this is identity loss, not race recovery` + : `no crdt.file.created or crdt.file.renamed for pathId=${newPathId} within ${WINDOW_MS}ms`; + findings.push({ + rule: "orphan-after-rename", + severity: "hard", + pathId: newPathId, + eventSeqs: [renameEvent.seq], + description: + `disk.rename.observed (seq=${renameEvent.seq}, opId=${renameEvent.opId ?? "?"}) ` + + `was not followed by crdt.file.renamed for pathId=${newPathId} within ${WINDOW_MS}ms ` + + `— ${reason}. oldPathId=${oldPathId}`, + }); + } + continue; + } + + // (b) crdt.file.created AFTER rename = identity lost. + const crdtCreatedAfterRename = events.find((e) => + e.kind === "crdt.file.created" + && e.pathId === newPathId + && e.ts > crdtRenamed.ts, + ); + if (crdtCreatedAfterRename) { + findings.push({ + rule: "orphan-after-rename", + severity: "hard", + pathId: newPathId, + eventSeqs: [renameEvent.seq, crdtRenamed.seq, crdtCreatedAfterRename.seq], + description: + `crdt.file.created appeared after crdt.file.renamed for pathId=${newPathId} — ` + + `file identity was lost (rename + create instead of identity-preserving rename). ` + + `oldPathId=${oldPathId}`, + }); + } + + // (c) crdt.file.tombstoned without revive = spurious tombstone. + // A tombstone is intentional if: + // - Phase markers: it happened at or after the cleanup phase start, OR + // - Fallback (no cleanup marker in trace): disk.delete.observed preceded it. + const tombstonedAfterRename = events.find((e) => + e.kind === "crdt.file.tombstoned" + && e.pathId === newPathId + && e.ts > crdtRenamed.ts + && e.ts - crdtRenamed.ts <= WINDOW_MS, + ); + if (tombstonedAfterRename) { + const isIntentional = + // Phase-marker check (cleanup phase event in trace = teardown) + (hasCleanupPhaseMarker && tombstonedAfterRename.ts >= cleanupPhaseTs) + // Fallback heuristic: disk.delete.observed preceded the tombstone + || events.some((e) => + e.kind === "disk.delete.observed" + && e.pathId === newPathId + && e.ts <= tombstonedAfterRename.ts, + ); + + if (!isIntentional) { + const revivedAfter = events.find((e) => + e.kind === "crdt.file.revived" + && e.pathId === newPathId + && e.ts > tombstonedAfterRename.ts, + ); + if (!revivedAfter) { + const markerNote = hasCleanupPhaseMarker + ? `cleanup phase starts at ${cleanupPhaseTs}` + : "no cleanup phase marker (fallback: disk.delete.observed)"; + findings.push({ + rule: "orphan-after-rename", + severity: "hard", + pathId: newPathId, + eventSeqs: [crdtRenamed.seq, tombstonedAfterRename.seq], + description: + `crdt.file.tombstoned appeared for pathId=${newPathId} after rename ` + + `without an intentional delete signal (${markerNote}) and no revive — ` + + `renamed file may have been tombstoned by the system. oldPathId=${oldPathId}`, + }); + } + } + } + } + + return findings; +} diff --git a/qa/controllers/two-device.ts b/qa/controllers/two-device.ts index a88a322..8c61da8 100644 --- a/qa/controllers/two-device.ts +++ b/qa/controllers/two-device.ts @@ -3716,6 +3716,230 @@ const TWO_DEVICE_SCENARIOS: Record = { return { passedA: errors.length === 0, passedB: errors.length === 0, errors }; }, + // ───────────────────────────────────────────────────────────────────── + /** + * s15-schema-v3-metadata-sync + * + * End-to-end proof that schema v3 nested metadata changes drive correct + * disk mirror behavior on a remote device. Tests: + * + * Phase 1: Create — file created on A appears on B's disk + * Phase 2: Rename — active entry renamed on A → disk rename on B + * Phase 3: Delete — file deleted on A → disk delete on B + * Phase 4: Revive — file revived (un-deleted) on A → disk write on B + * Phase 5: mtime — mtime-only save on A → B's disk file unchanged + * Phase 6: Schema — both devices have sys.schemaVersion === 3 + * + * This scenario deliberately avoids using YAOS private APIs to trigger + * metadata changes — it uses only the public vault operations (create, + * rename, delete, re-create) so the test proves the full production path. + */ + "s15-schema-v3-metadata-sync": async (a, b, log) => { + const errors: string[] = []; + const P = { + create: "QA-scratch/s15-create-test.md", + rename_src: "QA-scratch/s15-rename-src.md", + rename_dst: "QA-scratch/s15-rename-dst.md", + del: "QA-scratch/s15-delete-test.md", + revive: "QA-scratch/s15-revive-test.md", + mtime: "QA-scratch/s15-mtime-test.md", + }; + const CONTENT = { + create: "# S15 Create\n\nCreated on device A.\n", + rename_src: "# S15 Rename\n\nWill be renamed.\n", + del: "# S15 Delete\n\nWill be deleted.\n", + revive: "# S15 Revive\n\nWill be deleted then revived.\n", + mtime: "# S15 Mtime\n\nContent stays the same. mtime changes only.\n", + }; + + /** Wait for a file to disappear from a device's disk. */ + async function waitForDeletion(client: AnyObsidianClient, path: string, timeoutMs = 20_000): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const exists = await client.evalRaw( + `!!app.vault.getAbstractFileByPath(${JSON.stringify(path)})`, + ); + if (!exists) return true; + await new Promise((r) => setTimeout(r, 500)); + } + return false; + } + + /** Get content hash via the YAOS debug API. */ + async function diskHash(client: AnyObsidianClient, path: string): Promise { + return client.evalRaw( + `window.__YAOS_DEBUG__?.getDiskHash(${JSON.stringify(path)}) ?? null`, + ); + } + + /** Assert hash equality; push to errors if not. */ + function assertHashMatch(hA: string | null, hB: string | null, label: string): void { + if (!hA || !hB) { + errors.push(`${label}: null hash — A=${hA?.slice(0, 12) ?? "null"} B=${hB?.slice(0, 12) ?? "null"}`); + } else if (hA !== hB) { + errors.push(`${label}: hash mismatch — A=${hA.slice(0, 12)} B=${hB.slice(0, 12)}`); + } else { + log(`${label}: hash match ✓ (${hA.slice(0, 12)})`); + } + } + + // ── Cleanup any leftovers from a previous run ────────────────────── + for (const path of Object.values(P)) { + await a.evalRaw(`(async()=>{const f=app.vault.getAbstractFileByPath(${JSON.stringify(path)});if(f)await app.vault.delete(f);})()`).catch(() => {}); + } + await a.evalRaw(`window.__YAOS_DEBUG__?.waitForIdle(10000)`).catch(() => {}); + await b.evalRaw(`window.__YAOS_DEBUG__?.waitForIdle(5000)`).catch(() => {}); + log("s15: cleanup done"); + + // ── Phase 1: Create ──────────────────────────────────────────────── + log("\n─── Phase 1: Create ───"); + await a.evalRaw(`window.__YAOS_QA__?.createFile(${JSON.stringify(P.create)}, ${JSON.stringify(CONTENT.create)})`); + await a.evalRaw(`window.__YAOS_DEBUG__?.waitForIdle(15000)`); + await b.evalRaw(`window.__YAOS_DEBUG__?.waitForFile(${JSON.stringify(P.create)}, 30000)`); + await b.evalRaw(`window.__YAOS_DEBUG__?.waitForIdle(10000)`); + await new Promise((r) => setTimeout(r, 3000)); + + const createHashA = await diskHash(a, P.create); + const createHashB = await diskHash(b, P.create); + assertHashMatch(createHashA, createHashB, "Phase 1 create"); + + // ── Phase 2: Rename ──────────────────────────────────────────────── + log("\n─── Phase 2: Rename ───"); + await a.evalRaw(`window.__YAOS_QA__?.createFile(${JSON.stringify(P.rename_src)}, ${JSON.stringify(CONTENT.rename_src)})`); + await a.evalRaw(`window.__YAOS_DEBUG__?.waitForIdle(12000)`); + await b.evalRaw(`window.__YAOS_DEBUG__?.waitForFile(${JSON.stringify(P.rename_src)}, 25000)`); + await b.evalRaw(`window.__YAOS_DEBUG__?.waitForIdle(8000)`); + + // Rename on A using Obsidian vault API + await a.evalRaw(` + (async () => { + const f = app.vault.getAbstractFileByPath(${JSON.stringify(P.rename_src)}); + if (f) await app.fileManager.renameFile(f, ${JSON.stringify(P.rename_dst)}); + })() + `); + await a.evalRaw(`window.__YAOS_DEBUG__?.waitForIdle(12000)`); + await b.evalRaw(`window.__YAOS_DEBUG__?.waitForFile(${JSON.stringify(P.rename_dst)}, 25000)`); + await b.evalRaw(`window.__YAOS_DEBUG__?.waitForIdle(8000)`); + await new Promise((r) => setTimeout(r, 3000)); + + const renameHashA = await diskHash(a, P.rename_dst); + const renameHashB = await diskHash(b, P.rename_dst); + assertHashMatch(renameHashA, renameHashB, "Phase 2 rename dst"); + + const oldPathGoneOnB = await b.evalRaw(`!app.vault.getAbstractFileByPath(${JSON.stringify(P.rename_src)})`); + if (!oldPathGoneOnB) { + errors.push(`Phase 2 rename: old path still exists on B: ${P.rename_src}`); + } else { + log(`Phase 2 rename: old path gone on B ✓`); + } + + // ── Phase 3: Delete ──────────────────────────────────────────────── + log("\n─── Phase 3: Delete ───"); + await a.evalRaw(`window.__YAOS_QA__?.createFile(${JSON.stringify(P.del)}, ${JSON.stringify(CONTENT.del)})`); + await a.evalRaw(`window.__YAOS_DEBUG__?.waitForIdle(12000)`); + await b.evalRaw(`window.__YAOS_DEBUG__?.waitForFile(${JSON.stringify(P.del)}, 25000)`); + await b.evalRaw(`window.__YAOS_DEBUG__?.waitForIdle(8000)`); + + // Delete on A + await a.evalRaw(`(async()=>{const f=app.vault.getAbstractFileByPath(${JSON.stringify(P.del)});if(f)await app.vault.delete(f);})()`); + await a.evalRaw(`window.__YAOS_DEBUG__?.waitForIdle(12000)`); + + const delGoneOnB = await waitForDeletion(b, P.del, 25_000); + if (!delGoneOnB) { + errors.push(`Phase 3 delete: file still exists on B after 25s: ${P.del}`); + } else { + log(`Phase 3 delete: file gone on B ✓`); + } + + // ── Phase 4: Revive (delete then re-create) ──────────────────────── + log("\n─── Phase 4: Revive ───"); + // Create the revive file + await a.evalRaw(`window.__YAOS_QA__?.createFile(${JSON.stringify(P.revive)}, ${JSON.stringify(CONTENT.revive)})`); + await a.evalRaw(`window.__YAOS_DEBUG__?.waitForIdle(12000)`); + await b.evalRaw(`window.__YAOS_DEBUG__?.waitForFile(${JSON.stringify(P.revive)}, 25000)`); + await b.evalRaw(`window.__YAOS_DEBUG__?.waitForIdle(8000)`); + + // Delete it on A + await a.evalRaw(`(async()=>{const f=app.vault.getAbstractFileByPath(${JSON.stringify(P.revive)});if(f)await app.vault.delete(f);})()`); + await a.evalRaw(`window.__YAOS_DEBUG__?.waitForIdle(10000)`); + await waitForDeletion(b, P.revive, 20_000); + log("Phase 4: file deleted on both, now reviving..."); + + // Revive: re-create with same content on A (YAOS will lift the tombstone) + const REVIVE_CONTENT = "# S15 Revive\n\nRevived content after deletion.\n"; + await a.evalRaw(`window.__YAOS_QA__?.createFile(${JSON.stringify(P.revive)}, ${JSON.stringify(REVIVE_CONTENT)})`); + await a.evalRaw(`window.__YAOS_DEBUG__?.waitForIdle(12000)`); + await b.evalRaw(`window.__YAOS_DEBUG__?.waitForFile(${JSON.stringify(P.revive)}, 25000)`); + await b.evalRaw(`window.__YAOS_DEBUG__?.waitForIdle(8000)`); + await new Promise((r) => setTimeout(r, 3000)); + + const reviveHashA = await diskHash(a, P.revive); + const reviveHashB = await diskHash(b, P.revive); + assertHashMatch(reviveHashA, reviveHashB, "Phase 4 revive"); + + // ── Phase 5: mtime-only save ─────────────────────────────────────── + log("\n─── Phase 5: mtime-only ───"); + await a.evalRaw(`window.__YAOS_QA__?.createFile(${JSON.stringify(P.mtime)}, ${JSON.stringify(CONTENT.mtime)})`); + await a.evalRaw(`window.__YAOS_DEBUG__?.waitForIdle(12000)`); + await b.evalRaw(`window.__YAOS_DEBUG__?.waitForFile(${JSON.stringify(P.mtime)}, 25000)`); + await b.evalRaw(`window.__YAOS_DEBUG__?.waitForIdle(8000)`); + await new Promise((r) => setTimeout(r, 3000)); + + // Capture B's hash before the mtime bump + const mtimeHashBefore = await diskHash(b, P.mtime); + log(`Phase 5: B disk hash before mtime bump: ${mtimeHashBefore?.slice(0, 12)}`); + + // Trigger a save on A without changing content — use the Obsidian API + // to touch the file's modification time only (write same content back) + await a.evalRaw(` + (async () => { + const f = app.vault.getAbstractFileByPath(${JSON.stringify(P.mtime)}); + if (f) { + const content = await app.vault.read(f); + await app.vault.modify(f, content); // same content, bumps mtime + } + })() + `); + await a.evalRaw(`window.__YAOS_DEBUG__?.waitForIdle(10000)`); + await new Promise((r) => setTimeout(r, 8000)); // extra time for any spurious writes + + const mtimeHashAfter = await diskHash(b, P.mtime); + log(`Phase 5: B disk hash after mtime bump: ${mtimeHashAfter?.slice(0, 12)}`); + + if (mtimeHashBefore && mtimeHashAfter && mtimeHashBefore === mtimeHashAfter) { + log(`Phase 5: B disk hash unchanged after mtime-only save ✓`); + } else { + errors.push(`Phase 5 mtime: B disk hash changed after mtime-only save — before=${mtimeHashBefore?.slice(0, 12)}, after=${mtimeHashAfter?.slice(0, 12)} (spurious rewrite)`); + } + + // ── Phase 6: Schema version ──────────────────────────────────────── + log("\n─── Phase 6: Schema version ───"); + const schemaA = await a.evalRaw(`app.plugins?.plugins?.yaos?.vaultSync?.sys?.get?.("schemaVersion") ?? null`); + const schemaB = await b.evalRaw(`app.plugins?.plugins?.yaos?.vaultSync?.sys?.get?.("schemaVersion") ?? null`); + log(`Phase 6: schemaVersion A=${schemaA} B=${schemaB}`); + + if (schemaA !== 3) errors.push(`Phase 6: Device A schemaVersion is ${schemaA}, expected 3`); + if (schemaB !== 3) errors.push(`Phase 6: Device B schemaVersion is ${schemaB}, expected 3`); + if (schemaA === 3 && schemaB === 3) log("Phase 6: both devices at schema v3 ✓"); + + // ── Cleanup ──────────────────────────────────────────────────────── + for (const path of Object.values(P)) { + await a.evalRaw(`(async()=>{const f=app.vault.getAbstractFileByPath(${JSON.stringify(path)});if(f)await app.vault.delete(f);})()`).catch(() => {}); + } + await a.evalRaw(`window.__YAOS_DEBUG__?.waitForIdle(8000)`).catch(() => {}); + + const passedA = errors.length === 0; + const passedB = errors.length === 0; + + if (errors.length === 0) { + log("\n✓ s15 PASS — all schema v3 metadata sync phases verified"); + } else { + log(`\n✗ s15 FAIL — ${errors.length} error(s)`); + } + + return { passedA, passedB, errors }; + }, + }; // ----------------------------------------------------------------------- diff --git a/qa/obsidian-harness/scenarios/s15-schema-v3-metadata-sync.ts b/qa/obsidian-harness/scenarios/s15-schema-v3-metadata-sync.ts new file mode 100644 index 0000000..3abd395 --- /dev/null +++ b/qa/obsidian-harness/scenarios/s15-schema-v3-metadata-sync.ts @@ -0,0 +1,68 @@ +/** + * S15 — Schema v3 metadata sync validation. + * + * Constants and spec for the schema v3 end-to-end QA scenario. + * Scenario implementation lives in two-device.ts under "s15-schema-v3-metadata-sync". + * + * What this scenario proves: + * 1. File create on A → file appears on B's disk + * 2. File rename on A → file renamed on B's disk + * 3. File delete on A → file deleted from B's disk + * 4. File revive (un-delete) on A → file re-written to B's disk + * 5. mtime-only save on A (no content change) → B's disk file unchanged + * 6. Schema v3 marker: room has sys.schemaVersion === 3 after both connect + * + * Known analyzer false positive on Device B: + * [orphan-after-rename] fires on B because handleRemoteRename performs a disk + * rename on B directly (via the OS), which triggers disk.rename.observed. + * B never fires crdt.file.renamed because B is the passive receiver — the + * CRDT rename was initiated by A. This is expected behavior, not a bug. + * The orphan-after-rename rule is designed for the active renaming device. + */ + +export const SCENARIO_ID = "s15-schema-v3-metadata-sync"; + +/** File paths used during the scenario — in QA-scratch to avoid polluting the vault. */ +export const PATHS = { + create: "QA-scratch/s15-create-test.md", + rename_src: "QA-scratch/s15-rename-src.md", + rename_dst: "QA-scratch/s15-rename-dst.md", + delete: "QA-scratch/s15-delete-test.md", + revive: "QA-scratch/s15-revive-test.md", + mtime: "QA-scratch/s15-mtime-test.md", +} as const; + +/** Initial content written to each test file. */ +export const INITIAL_CONTENT: Record = { + [PATHS.create]: "# S15 Create\n\nCreated on device A.\n", + [PATHS.rename_src]: "# S15 Rename\n\nWill be renamed.\n", + [PATHS.delete]: "# S15 Delete\n\nWill be deleted.\n", + [PATHS.revive]: "# S15 Revive\n\nWill be deleted then revived.\n", + [PATHS.mtime]: "# S15 Mtime\n\nContent stays the same. mtime changes only.\n", +}; + +/** Timeout constants (ms). */ +export const TIMEOUTS = { + waitForIdle: 15_000, + waitForFile: 30_000, + waitForFileDeletion: 20_000, + syncSettleExtra: 5_000, + mtimeSettleExtra: 8_000, +} as const; + +/** + * Semantic change kinds this scenario is expected to trigger on Device B's DiskMirror: + * - "deleted" for the delete and revive phases (before revive) + * - "revived" for the revive phase (after delete) + * - "path-changed" for the rename phase + * Not tested at the semantic event level here — those are proven in unit tests. + * This scenario proves the disk outcomes. + */ +export const EXPECTED_DISK_OUTCOMES = { + create: "file present on B's disk with matching content hash", + rename: "old path absent on B, new path present with matching content hash", + delete: "file absent from B's disk after A deletes", + revive: "file present on B's disk again after A revives", + mtime: "B's disk hash unchanged after A bumps mtime without changing content", + schemaVersion: "sys.schemaVersion === 3 on both devices after scenario", +} as const; diff --git a/qa/scripts/run-s15.md b/qa/scripts/run-s15.md new file mode 100644 index 0000000..f6513d0 --- /dev/null +++ b/qa/scripts/run-s15.md @@ -0,0 +1,196 @@ +# S15 — Schema v3 Metadata Sync QA Runbook + +This runbook is written for autonomous execution by an AI agent. +Follow every step exactly. No human decision points after the pre-flight checklist. + +--- + +## What this scenario proves + +1. **Create** — file created on A appears on B's disk with matching content hash +2. **Rename** — active nested metadata path change drives disk rename on B +3. **Delete** — nested `deletedAt` set on A drives disk delete on B +4. **Revive** — nested `deletedAt` cleared on A drives disk write on B +5. **mtime-only** — mtime bump without content change does NOT rewrite B's disk file +6. **Schema version** — both devices have `sys.schemaVersion === 3` after connecting + +--- + +## Pre-flight checklist + +Before running, verify all of these: + +```bash +# 1. Build the plugin +cd /path/to/do-sync/worktree # wherever fix/nested-ymaps-metadata is checked out +npm run build # must succeed cleanly + +# 2. Plugin is installed in both vaults +# The built main.js must be at: +# /.obsidian/plugins/yaos/main.js +# /.obsidian/plugins/yaos/main.js +# Copy it if needed: +cp main.js /.obsidian/plugins/yaos/main.js +cp main.js /.obsidian/plugins/yaos/main.js + +# 3. QA Debug Mode must be enabled in plugin settings on both vaults +# Open Obsidian > Settings > YAOS > enable "QA Debug Mode" +# (window.__YAOS_DEBUG__ will not exist without this) + +# 4. QA-scratch folder must exist in both vaults (or be auto-created) +mkdir -p /QA-scratch +mkdir -p /QA-scratch +``` + +--- + +## Launch Obsidian + +Open two separate Obsidian instances with remote debugging ports: + +```bash +# Device A (port 9222) +/path/to/Obsidian --remote-debugging-port=9222 & + +# Device B (port 9223) +/path/to/Obsidian --remote-debugging-port=9223 & +``` + +On Linux: +```bash +/usr/bin/obsidian --remote-debugging-port=9222 & +/usr/bin/obsidian --remote-debugging-port=9223 & +``` + +On macOS: +```bash +/Applications/Obsidian.app/Contents/MacOS/Obsidian --remote-debugging-port=9222 & +/Applications/Obsidian.app/Contents/MacOS/Obsidian --remote-debugging-port=9223 & +``` + +Wait ~10 seconds for both to load, then verify connectivity: + +```bash +curl -s http://localhost:9222/json | head -5 +curl -s http://localhost:9223/json | head -5 +``` + +Both should return JSON. If they return nothing, Obsidian isn't ready. + +--- + +## Verify QA readiness + +```bash +# Check that __YAOS_DEBUG__ is available on both devices +curl -s http://localhost:9222/json/list | python3 -c " +import json, sys +pages = json.load(sys.stdin) +print('Port 9222 pages:', [p.get('title','?') for p in pages[:3]]) +" + +curl -s http://localhost:9223/json/list | python3 -c " +import json, sys +pages = json.load(sys.stdin) +print('Port 9223 pages:', [p.get('title','?') for p in pages[:3]]) +" +``` + +--- + +## Run the scenario + +From the worktree root: + +```bash +bun run qa:two-device \ + --scenario s15-schema-v3-metadata-sync \ + --port-a 9222 \ + --port-b 9223 \ + --trace qa-safe \ + --out-dir qa-runs/s15 \ + --driver raw-cdp +``` + +The scenario takes approximately 3–5 minutes to complete. + +--- + +## Expected output (PASS) + +``` +s15: cleanup done +─── Phase 1: Create ─── +Phase 1 create: hash match ✓ (abc123...) +─── Phase 2: Rename ─── +Phase 2 rename dst: hash match ✓ (def456...) +Phase 2 rename: old path gone on B ✓ +─── Phase 3: Delete ─── +Phase 3 delete: file gone on B ✓ +─── Phase 4: Revive ─── +Phase 4: file deleted on both, now reviving... +Phase 4 revive: hash match ✓ (ghi789...) +─── Phase 5: mtime-only ─── +Phase 5: B disk hash before mtime bump: jkl012... +Phase 5: B disk hash after mtime bump: jkl012... +Phase 5: B disk hash unchanged after mtime-only save ✓ +─── Phase 6: Schema version ─── +Phase 6: schemaVersion A=3 B=3 +Phase 6: both devices at schema v3 ✓ + +✓ s15 PASS — all schema v3 metadata sync phases verified +``` + +Exit code 0 = PASS. + +--- + +## If the scenario fails + +Check the artifacts in `qa-runs/s15/`: +``` +qa-runs/s15/ +├── device-a/ +│ ├── result.json ← { passed, errors } +│ ├── trace.ndjson ← flight trace +│ └── manifest-*.json ← plugin version +└── device-b/ + ├── result.json + ├── trace.ndjson + └── manifest-*.json +``` + +Common failures and remediation: + +| Failure | Likely cause | Fix | +|---------|-------------|-----| +| `Phase 1 create: null hash` | File didn't sync to B | Check server connectivity, verify both vaults point to same server | +| `Phase 2 rename: old path still exists on B` | DiskMirror observer not firing for nested path-changed | Check that the built plugin includes the new diskMirror.ts code | +| `Phase 3 delete: file still exists on B` | DiskMirror observer missing `deleted` semantic change | Check observer wiring in diskMirror.ts | +| `Phase 5 mtime: B disk hash changed` | Spurious rewrite from mtime-only change | The O(N) scan / path-changed suppression is broken | +| `Phase 6: Device A schemaVersion is 2` | markSchemaV3 not called, or not running v3 plugin | Verify built plugin version; check main.ts lifecycle | +| `waitForQaReady timeout` | QA Debug Mode not enabled | Enable it in plugin settings on both vaults | + +--- + +## Full CI run (run before declaring mergeable) + +```bash +# From the worktree root +npm ci +npm run build +npm run test:regressions +npm --prefix server run typecheck +``` + +All must exit 0. + +--- + +## Reporting results + +After running, report: +1. Exit code (0 = PASS, 1 = FAIL) +2. Phase-by-phase output +3. Any errors from `qa-runs/s15/device-a/result.json` and `device-b/result.json` +4. Plugin version from `manifest-pre.json` (to confirm correct build was used) diff --git a/server/src/server.ts b/server/src/server.ts index 126d740..6d19cb1 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -555,22 +555,14 @@ export class VaultSyncServer extends YServer { } } - /** Count active (non-deleted) paths in a Y.Doc using the YAOS schema. */ + /** Count active (non-deleted) paths in a Y.Doc using the YAOS schema. Dual-reads flat and nested metadata. */ private countActivePathsInDoc(doc: Y.Doc): number { const meta = doc.getMap("meta"); let count = 0; meta.forEach((value: unknown) => { - if ( - typeof value === "object" - && value !== null - && "path" in value - && typeof (value as { path: unknown }).path === "string" - ) { - const m = value as { deleted?: boolean; deletedAt?: number }; - const isDeleted = m.deleted === true - || (typeof m.deletedAt === "number" && Number.isFinite(m.deletedAt)); - if (!isDeleted) count++; - } + const path = this.readMetaPath(value); + if (!path) return; + if (!this.isMetaDeleted(value)) count++; }); return count; } @@ -586,6 +578,40 @@ export class VaultSyncServer extends YServer { return false; } + /** + * Read the path from a metadata value. Handles both flat objects (v2) and nested Y.Map (v3). + * Server must dual-read because persisted rooms may contain either shape. + */ + private readMetaPath(value: unknown): string | null { + if (value instanceof Y.Map) { + const path = value.get("path"); + return typeof path === "string" && path.length > 0 ? path : null; + } + if (typeof value === "object" && value !== null && "path" in value) { + const path = (value as { path: unknown }).path; + return typeof path === "string" && path.length > 0 ? path : null; + } + return null; + } + + /** + * Check if a metadata value represents a deleted/tombstoned entry. + * Handles both flat objects (v2) and nested Y.Map (v3). + */ + private isMetaDeleted(value: unknown): boolean { + if (value instanceof Y.Map) { + const deletedAt = value.get("deletedAt"); + if (typeof deletedAt === "number" && Number.isFinite(deletedAt)) return true; + return value.get("deleted") === true; + } + if (typeof value === "object" && value !== null) { + const m = value as { deleted?: boolean; deletedAt?: unknown }; + if (typeof m.deletedAt === "number" && Number.isFinite(m.deletedAt)) return true; + return m.deleted === true; + } + return false; + } + private getChunkedDocStore(): ChunkedDocStore { if (!this.chunkedDocStore) { this.chunkedDocStore = new ChunkedDocStore(this.ctx.storage); @@ -638,6 +664,12 @@ export class VaultSyncServer extends YServer { /** pathToId entries that have no corresponding active meta entry. */ pathToIdWithoutActiveMeta: number; schemaVersion: unknown; + /** v3 observability: metadata entries stored as flat JSON objects. */ + flatMetaEntries: number; + /** v3 observability: metadata entries stored as nested Y.Map. */ + nestedMetaEntries: number; + /** v3 observability: metadata entries that could not be decoded. */ + invalidMetaEntries: number; } { const meta = this.document.getMap("meta"); const pathToId = this.document.getMap("pathToId"); @@ -648,33 +680,38 @@ export class VaultSyncServer extends YServer { let activePathsWithText = 0; let activePathsMissingFromPathToId = 0; let activePathsMissingText = 0; + let flatMetaEntries = 0; + let nestedMetaEntries = 0; + let invalidMetaEntries = 0; // Walk meta to count active/tombstoned and check consistency const activeMetaPaths = new Set(); meta.forEach((value: unknown) => { - if ( - typeof value === "object" - && value !== null - && "path" in value - && typeof (value as { path: unknown }).path === "string" - ) { - const path = (value as { path: string }).path; - const m = value as { deleted?: boolean; deletedAt?: number }; - const isDeleted = m.deleted === true - || (typeof m.deletedAt === "number" && Number.isFinite(m.deletedAt)); - if (isDeleted) { - tombstonedPathCount++; + const path = this.readMetaPath(value); + if (!path) { + invalidMetaEntries++; + return; + } + + // Classify shape + if (value instanceof Y.Map) { + nestedMetaEntries++; + } else { + flatMetaEntries++; + } + const isDeleted = this.isMetaDeleted(value); + if (isDeleted) { + tombstonedPathCount++; + } else { + activePathCount++; + activeMetaPaths.add(path); + const id = pathToId.get(path); + if (!id) { + activePathsMissingFromPathToId++; + } else if (!idToText.has(id)) { + activePathsMissingText++; } else { - activePathCount++; - activeMetaPaths.add(path); - const id = pathToId.get(path); - if (!id) { - activePathsMissingFromPathToId++; - } else if (!idToText.has(id)) { - activePathsMissingText++; - } else { - activePathsWithText++; - } + activePathsWithText++; } } }); @@ -698,6 +735,9 @@ export class VaultSyncServer extends YServer { activePathsMissingText, pathToIdWithoutActiveMeta, schemaVersion: this.document.getMap("sys").get("schemaVersion") ?? null, + flatMetaEntries, + nestedMetaEntries, + invalidMetaEntries, }; } diff --git a/server/src/version.ts b/server/src/version.ts index 0f253b1..d5861e6 100644 --- a/server/src/version.ts +++ b/server/src/version.ts @@ -6,6 +6,6 @@ export const SERVER_MIN_PLUGIN_VERSION: string | null = null; export const SERVER_RECOMMENDED_PLUGIN_VERSION = "1.5.0"; export const SERVER_MIN_COMPATIBLE_SERVER_VERSION_FOR_PLUGIN = "0.2.0"; export const SERVER_MIN_COMPATIBLE_PLUGIN_VERSION_FOR_SERVER = "1.3.3"; -export const SERVER_MIN_SCHEMA_VERSION = 2; -export const SERVER_MAX_SCHEMA_VERSION = 2; +export const SERVER_MIN_SCHEMA_VERSION = 3; +export const SERVER_MAX_SCHEMA_VERSION = 3; export const SERVER_MIGRATION_REQUIRED = false; diff --git a/src/main.ts b/src/main.ts index 1d4fa14..864c0d8 100644 --- a/src/main.ts +++ b/src/main.ts @@ -8,6 +8,7 @@ import { import { SettingsStore } from "./settings/settingsStore"; import { VaultSync, type ReconcileMode } from "./sync/vaultSync"; import { SCHEMA_VERSION } from "./sync/vaultSync"; +import { getMetaPath, isFileMetaDeletedValue } from "./sync/fileMeta"; import { EditorBindingManager } from "./sync/editorBinding"; import { DiskMirror } from "./sync/diskMirror"; import { type BlobQueueSnapshot, type BlobSyncManager } from "./sync/blobSync"; @@ -770,6 +771,9 @@ export default class VaultCrdtSyncPlugin extends Plugin { return; } + // Mark schema v3 if room is still at v2 (lazy, no metadata migration). + this.vaultSync.markSchemaV3(this.settings.deviceName); + // Check for fatal auth error before waiting for provider if (this.vaultSync.fatalAuthError) { this.log("Fatal auth error during startup"); @@ -964,6 +968,13 @@ export default class VaultCrdtSyncPlugin extends Plugin { || this.isBlobPathSyncable(oldPath); if (!newSyncable && !oldSyncable) return; const renameOpId = this.newOpId(); + // Consume the remote-rename marker once. If DiskMirror originated this + // disk rename (passive receiver of a remote CRDT rename), we must: + // 1. Mark the trace event with remoteOrigin:true (analyzer exemption) + // 2. Skip queueRename entirely (the rename is already in CRDT) + // Consume-on-use matches the suppressedPaths/consumeDeleteSuppression pattern. + const isRemoteRename = this.diskMirror?.consumeRemoteRename(file.path) ?? false; + if (this.isMarkdownPathSyncable(oldPath) || this.isMarkdownPathSyncable(file.path)) { // Emit two disk.rename.observed events sharing the same opId so the // analyzer can reconstruct full old→new lineage without raw paths. @@ -988,9 +999,22 @@ export default class VaultCrdtSyncPlugin extends Plugin { layer: "disk", opId: renameOpId, path: file.path, // resolves to newPathId - data: { renameRole: "target", isBlobSyncable: this.isBlobPathSyncable(file.path) }, + data: { + renameRole: "target", + isBlobSyncable: this.isBlobPathSyncable(file.path), + // remoteOrigin:true exempts this event from the orphan-after-rename + // analyzer rule — passive receivers have no crdt.file.renamed. + remoteOrigin: isRemoteRename, + }, }); } + + if (isRemoteRename) { + // DiskMirror renamed the disk file in response to a remote CRDT rename. + // The rename already exists in CRDT — do not feed it back via queueRename. + this.log(`Remote-origin rename observed, skipping queueRename: "${oldPath}" -> "${file.path}"`); + return; + } this.vaultSync?.queueRename(oldPath, file.path); this.log(`Rename queued: "${oldPath}" -> "${file.path}"`); }), @@ -2435,8 +2459,11 @@ export default class VaultCrdtSyncPlugin extends Plugin { const handler = (_event: unknown, txn: { origin: unknown }) => { // Find path for this fileId. let path: string | undefined; - vs.meta.forEach((meta, id) => { - if (id === fileId && typeof meta.path === "string") path = meta.path; + vs.meta.forEach((value: unknown, id: string) => { + if (id === fileId) { + const p = getMetaPath(value); + if (p) path = p; + } }); if (!path || !path.endsWith(".md")) return; const originClass = isLocalOrigin(txn.origin, vs.provider) ? "local-edit" : "remote-apply"; @@ -2457,23 +2484,29 @@ export default class VaultCrdtSyncPlugin extends Plugin { this._witnessIdToTextHandler = idToTextHandler; // Observe meta map for tombstone transitions (Req 3.4). - const metaHandler = () => { - vs.meta.forEach((meta, fileId) => { - void fileId; - const path = typeof meta.path === "string" ? meta.path : undefined; - if (!path || !path.endsWith(".md")) return; - if (vs.isFileMetaDeleted(meta)) { - tracker?.markDirty(path, "tombstone"); + // Uses observeMetaChanges (observeDeep-backed, incremental diff) so + // nested Y.Map field mutations (v3 entries) trigger correctly. + // The witness tracker intentionally observes BOTH local and remote + // tombstones — it tracks what this device believes about each file. + const unsubscribeMetaChanges = vs.observeMetaChanges((batch) => { + for (const change of batch.changes) { + if (change.kind === "deleted") { + const { path } = change; + if (path.endsWith(".md")) { + tracker?.markDirty(path, "tombstone"); + } } - }); - }; - vs.meta.observe(metaHandler); + } + }); + // Store unsubscribe as a wrapped function so the teardown path works. + const metaHandler = unsubscribeMetaChanges; this._witnessMetaHandler = metaHandler; } } private _witnessTextObservers: Map void }> | null = null; private _witnessIdToTextHandler: (() => void) | null = null; + /** Unsubscribe function returned by observeMetaChanges, or null if not subscribed. */ private _witnessMetaHandler: (() => void) | null = null; private _stopDeviceWitnessTracker(): void { @@ -2487,8 +2520,9 @@ export default class VaultCrdtSyncPlugin extends Plugin { try { this.vaultSync.idToText.unobserve(this._witnessIdToTextHandler); } catch { /* ignore */ } this._witnessIdToTextHandler = null; } - if (this._witnessMetaHandler && this.vaultSync) { - try { this.vaultSync.meta.unobserve(this._witnessMetaHandler); } catch { /* ignore */ } + if (this._witnessMetaHandler) { + // _witnessMetaHandler is now an unsubscribe function (not a Yjs callback). + try { this._witnessMetaHandler(); } catch { /* ignore */ } this._witnessMetaHandler = null; } this.deviceWitnessTracker?.dispose(); diff --git a/src/sync/diskMirror.ts b/src/sync/diskMirror.ts index f668f23..0931d71 100644 --- a/src/sync/diskMirror.ts +++ b/src/sync/diskMirror.ts @@ -3,6 +3,8 @@ import * as Y from "yjs"; import type { VaultSync } from "./vaultSync"; import type { EditorBindingManager } from "./editorBinding"; import type { TraceRecord } from "../debug/trace"; +import { getMetaPath, isFileMetaDeletedValue } from "./fileMeta"; +import type { MetaChangeBatch } from "./fileMeta"; import { formatUnknown, yTextToString } from "../utils/format"; import { isFrontmatterBlocked, @@ -66,6 +68,31 @@ export class DiskMirror { private suppressedPaths = new Map(); private openPaths = new Set(); + /** + * Tracks new paths being renamed by DiskMirror in response to remote + * metadata changes (handleRemoteRename). Consumed by the vault rename + * event handler in main.ts via consumeRemoteRename(), which both reads + * and removes the marker in a single call (consume-on-use, matching the + * suppressedPaths / consumeDeleteSuppression pattern). + */ + private _pendingRemoteRenameNewPaths = new Set(); + + /** + * Consume the remote-rename marker for `newPath` if present. + * Returns true if the rename was DiskMirror-originated (passive receiver). + * Removes the marker atomically — safe to call from the vault rename handler. + * + * @internal Used by main.ts vault rename handler. + */ + consumeRemoteRename(newPath: string): boolean { + const normalized = normalizePath(newPath); + if (this._pendingRemoteRenameNewPaths.has(normalized)) { + this._pendingRemoteRenameNewPaths.delete(normalized); + return true; + } + return false; + } + /** Deduped write queue. Order doesn't matter — deduplication does. */ private writeQueue = new Set(); private forcedWritePaths = new Set(); @@ -169,47 +196,57 @@ export class DiskMirror { // ------------------------------------------------------------------- startMapObservers(): void { - const metaObserver = (event: import("yjs").YMapEvent) => { - if (isLocalOrigin(event.transaction.origin, this.vaultSync.provider)) { - return; - } - event.changes.keys.forEach((change, fileId) => { - const oldMeta = change.oldValue as import("../types").FileMeta | undefined; - const newMeta = this.vaultSync.meta.get(fileId); - const oldPath = typeof oldMeta?.path === "string" ? normalizePath(oldMeta.path) : null; - const newPath = typeof newMeta?.path === "string" ? normalizePath(newMeta.path) : null; - const wasDeleted = this.vaultSync.isFileMetaDeleted(oldMeta); - const isDeleted = this.vaultSync.isFileMetaDeleted(newMeta); - - // Remote tombstone transition. - if (newPath && isDeleted && !wasDeleted) { - const baselineText = this.vaultSync.idToText.get(fileId)?.toString() ?? null; - void this.handleRemoteDelete(newPath, { baselineText }); - return; - } - - // Remote undelete/restore transition. - if (newPath && !isDeleted && wasDeleted) { - this.scheduleWrite(newPath); - return; - } - - // Remote rename/move transition from meta.path. - if (oldPath && newPath && oldPath !== newPath && !isDeleted) { - void this.handleRemoteRename(oldPath, newPath); - return; - } - - // Remote create/update where the file is active. - if ((change.action === "add" || change.action === "update") && newPath && !isDeleted) { - this.scheduleWrite(newPath); + // --------------------------------------------------------------- + // Semantic metadata observer. + // + // Subscribes to pre-classified MetaSemanticChange events from + // VaultSync.observeMetaChanges(), which is powered by observeDeep + // internally. This correctly fires for both flat (v2) object + // replacements AND nested Y.Map field mutations (v3), where a + // shallow meta.observe() would have silently dropped the event. + // --------------------------------------------------------------- + const unsubscribeMetaChanges = this.vaultSync.observeMetaChanges((batch) => { + // Only react to remote changes. Local metadata writes (disk sync, + // seed, restore) must not feed back into DiskMirror as remote events. + if (batch.isLocal) return; + + for (const change of batch.changes) { + switch (change.kind) { + case "deleted": { + const path = normalizePath(change.path); + const baselineText = this.vaultSync.idToText.get(change.fileId)?.toString() ?? null; + void this.handleRemoteDelete(path, { baselineText }); + break; + } + case "revived": { + this.scheduleWrite(normalizePath(change.path)); + break; + } + case "path-changed": { + // Only rename on disk when the entry is active. + // Tombstone path changes (e.g. from migrateSchemaToV2) must not + // trigger a disk rename — there is no live file to rename. + if (!change.isDeleted) { + void this.handleRemoteRename( + normalizePath(change.previousPath), + normalizePath(change.nextPath), + ); + } + break; + } + case "added": { + // New file received from remote — schedule write if active. + if (!change.next.deletedAt && !change.next.deleted) { + this.scheduleWrite(normalizePath(change.next.path)); + } + break; + } + // mtime-changed, device-changed, removed, invalid: + // no disk side effect needed. } - }); - }; - this.vaultSync.meta.observe(metaObserver); - this.mapObserverCleanups.push(() => - this.vaultSync.meta.unobserve(metaObserver), - ); + } + }); + this.mapObserverCleanups.push(unsubscribeMetaChanges); // --------------------------------------------------------------- // afterTransaction: catch remote content edits to CLOSED files. @@ -232,13 +269,14 @@ export class DiskMirror { if (!fileId) continue; // Map fileId → path via meta (pathToId is path→id, not id→path) - const meta = this.vaultSync.meta.get(fileId); - if (!meta || this.vaultSync.isFileMetaDeleted(meta)) continue; + const metaValue = this.vaultSync.meta.get(fileId); + if (!metaValue || isFileMetaDeletedValue(metaValue)) continue; - const path = meta.path; + const path = getMetaPath(metaValue); + if (!path) continue; - // Skip if this path is already open (handled by per-file observer policy) - if (this.openPaths.has(path)) continue; + // Skip if this path is already open (handled by per-file observer policy) + if (this.openPaths.has(path)) continue; this.log(`afterTxn: remote content change to closed file "${path}"`); this.scheduleWrite(path); @@ -858,11 +896,11 @@ export class DiskMirror { const oldFile = this.app.vault.getAbstractFileByPath(oldNormalized); if (oldFile instanceof TFile) { try { - const target = this.app.vault.getAbstractFileByPath(newNormalized); - if (target instanceof TFile) { - this.suppressDelete(oldNormalized); - await this.deleteLocalReplica(oldFile); - } else { + const target = this.app.vault.getAbstractFileByPath(newNormalized); + if (target instanceof TFile) { + this.suppressDelete(oldNormalized); + await this.deleteLocalReplica(oldFile); + } else { const dir = newNormalized.substring(0, newNormalized.lastIndexOf("/")); if (dir) { const dirNode = this.app.vault.getAbstractFileByPath(normalizePath(dir)); @@ -870,7 +908,17 @@ export class DiskMirror { await this.app.vault.createFolder(dir); } } - await this.app.fileManager.renameFile(oldFile, newNormalized); + // Mark this rename as remote-originated before the vault event fires, + // so main.ts can consume the marker and skip queueRename. + // consumeRemoteRename() in the vault handler removes the marker on use. + // On error (rename throws), the vault event won't fire, so clean up here. + this._pendingRemoteRenameNewPaths.add(newNormalized); + try { + await this.app.fileManager.renameFile(oldFile, newNormalized); + } catch (renameErr) { + this._pendingRemoteRenameNewPaths.delete(newNormalized); + throw renameErr; + } } this.log(`handleRemoteRename: "${oldNormalized}" -> "${newNormalized}"`); } catch (err) { diff --git a/src/sync/fileMeta.ts b/src/sync/fileMeta.ts new file mode 100644 index 0000000..98fabb0 --- /dev/null +++ b/src/sync/fileMeta.ts @@ -0,0 +1,604 @@ +/** + * Unified metadata read/write helpers for schema v2 (flat) and v3 (nested Y.Map) metadata. + * + * This module is the ONLY legal interface for reading/writing file metadata values. + * All call sites must use these helpers instead of accessing metadata entries directly. + * + * Schema v3 uses nested Y.Map entries for field-level CRDT resolution. + * Schema v2 used opaque JSON objects (FileMeta interface). + * Readers must handle both shapes. Writers always produce nested Y.Maps. + */ + +import * as Y from "yjs"; + +// ------------------------------------------------------------------- +// Types +// ------------------------------------------------------------------- + +/** Discriminator for which shape an entry was decoded from. */ +export type FileMetaShape = "flat" | "nested"; + +/** Decoded metadata regardless of underlying storage shape. */ +export interface DecodedFileMeta { + shape: FileMetaShape; + path: string; + deletedAt?: number; + deleted?: boolean; + mtime?: number; + device?: string; +} + +/** Semantic change kinds emitted by metadata observers. */ +export type MetaSemanticChange = + | { kind: "added"; fileId: string; next: DecodedFileMeta } + | { kind: "removed"; fileId: string; previous: DecodedFileMeta } + | { + kind: "path-changed"; + fileId: string; + previousPath: string; + nextPath: string; + /** True if the entry was a tombstone before the path changed. */ + wasDeleted: boolean; + /** True if the entry is a tombstone after the path changed. */ + isDeleted: boolean; + } + | { kind: "deleted"; fileId: string; path: string; deletedAt: number } + | { kind: "revived"; fileId: string; path: string } + | { kind: "mtime-changed"; fileId: string; path: string } + | { kind: "device-changed"; fileId: string; path: string } + | { kind: "invalid"; fileId: string }; + +/** + * A batch of semantic metadata changes emitted by a single Yjs transaction. + * Carries transaction origin so consumers can distinguish local from remote. + */ +export interface MetaChangeBatch { + /** + * The Yjs transaction origin. Matches the second argument passed to + * `doc.transact(fn, origin)`. May be a string constant, a provider + * instance, or null for undistinguished local mutations. + */ + origin: unknown; + /** + * True when this batch originated from a local transaction (known + * local origin strings, or any non-provider non-remote origin). + * DiskMirror must ignore local batches to avoid treating local + * metadata writes as remote file changes. + */ + isLocal: boolean; + /** The semantic changes within this transaction. */ + changes: MetaSemanticChange[]; +} + +// ------------------------------------------------------------------- +// Type guards +// ------------------------------------------------------------------- + +/** Check if a metadata value is a nested Y.Map (v3 schema). */ +export function isNestedFileMeta(value: unknown): value is Y.Map { + return value instanceof Y.Map; +} + +/** Check if a metadata value is a plain object record (v2 schema). */ +export function isObjectRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !(value instanceof Y.Map); +} + +// ------------------------------------------------------------------- +// Decoder +// ------------------------------------------------------------------- + +/** + * Decode a metadata value from either flat (v2) or nested (v3) shape into + * a normalized DecodedFileMeta. Returns null for invalid/unrecognizable values. + */ +export function decodeFileMeta(value: unknown): DecodedFileMeta | null { + if (value instanceof Y.Map) { + const path = value.get("path"); + if (typeof path !== "string" || path.length === 0) return null; + + const deletedAtRaw = value.get("deletedAt"); + const deletedRaw = value.get("deleted"); + const mtimeRaw = value.get("mtime"); + const deviceRaw = value.get("device"); + + const deletedAt = + typeof deletedAtRaw === "number" && Number.isFinite(deletedAtRaw) + ? deletedAtRaw + : undefined; + + const deleted = deletedRaw === true ? true : undefined; + + const mtime = + typeof mtimeRaw === "number" && Number.isFinite(mtimeRaw) + ? mtimeRaw + : undefined; + + const device = typeof deviceRaw === "string" ? deviceRaw : undefined; + + return { + shape: "nested", + path, + ...(deletedAt !== undefined ? { deletedAt } : {}), + ...(deleted !== undefined ? { deleted } : {}), + ...(mtime !== undefined ? { mtime } : {}), + ...(device !== undefined ? { device } : {}), + }; + } + + if (isObjectRecord(value)) { + const path = (value as Record).path; + if (typeof path !== "string" || path.length === 0) return null; + + const deletedAtRaw = (value as Record).deletedAt; + const deletedRaw = (value as Record).deleted; + const mtimeRaw = (value as Record).mtime; + const deviceRaw = (value as Record).device; + + const deletedAt = + typeof deletedAtRaw === "number" && Number.isFinite(deletedAtRaw) + ? deletedAtRaw + : undefined; + + const deleted = deletedRaw === true ? true : undefined; + + const mtime = + typeof mtimeRaw === "number" && Number.isFinite(mtimeRaw) + ? mtimeRaw + : undefined; + + const device = typeof deviceRaw === "string" ? deviceRaw : undefined; + + return { + shape: "flat", + path, + ...(deletedAt !== undefined ? { deletedAt } : {}), + ...(deleted !== undefined ? { deleted } : {}), + ...(mtime !== undefined ? { mtime } : {}), + ...(device !== undefined ? { device } : {}), + }; + } + + return null; +} + +// ------------------------------------------------------------------- +// Read helpers +// ------------------------------------------------------------------- + +/** Get the path from a metadata value (flat or nested). Returns null if invalid. */ +export function getMetaPath(value: unknown): string | null { + if (value instanceof Y.Map) { + const path = value.get("path"); + return typeof path === "string" && path.length > 0 ? path : null; + } + if (isObjectRecord(value)) { + const path = (value as Record).path; + return typeof path === "string" && path.length > 0 ? (path as string) : null; + } + return null; +} + +/** Get the mtime from a metadata value (flat or nested). Returns null if absent/invalid. */ +export function getMetaMtime(value: unknown): number | null { + if (value instanceof Y.Map) { + const mtime = value.get("mtime"); + return typeof mtime === "number" && Number.isFinite(mtime) ? mtime : null; + } + if (isObjectRecord(value)) { + const mtime = (value as Record).mtime; + return typeof mtime === "number" && Number.isFinite(mtime) ? mtime : null; + } + return null; +} + +/** Get the device from a metadata value (flat or nested). Returns null if absent. */ +export function getMetaDevice(value: unknown): string | null { + if (value instanceof Y.Map) { + const device = value.get("device"); + return typeof device === "string" ? device : null; + } + if (isObjectRecord(value)) { + const device = (value as Record).device; + return typeof device === "string" ? (device as string) : null; + } + return null; +} + +/** Get the deletedAt timestamp from a metadata value. Returns null if not tombstoned. */ +export function getMetaDeletedAt(value: unknown): number | null { + if (value instanceof Y.Map) { + const deletedAt = value.get("deletedAt"); + return typeof deletedAt === "number" && Number.isFinite(deletedAt) ? deletedAt : null; + } + if (isObjectRecord(value)) { + const deletedAt = (value as Record).deletedAt; + return typeof deletedAt === "number" && Number.isFinite(deletedAt) ? deletedAt : null; + } + return null; +} + +/** Check if a metadata value represents a deleted/tombstoned entry. Works with both shapes. */ +export function isFileMetaDeletedValue(value: unknown): boolean { + if (value instanceof Y.Map) { + const deletedAt = value.get("deletedAt"); + if (typeof deletedAt === "number" && Number.isFinite(deletedAt)) return true; + const deleted = value.get("deleted"); + return deleted === true; + } + if (isObjectRecord(value)) { + const rec = value as Record; + if (typeof rec.deletedAt === "number" && Number.isFinite(rec.deletedAt)) return true; + return rec.deleted === true; + } + return false; +} + +// ------------------------------------------------------------------- +// Write helpers — always produce nested Y.Map +// ------------------------------------------------------------------- + +/** + * Create a nested Y.Map for an active (non-deleted) metadata entry. + * This is the canonical way to create new metadata in schema v3. + */ +export function createNestedActiveMeta( + path: string, + mtime: number, + device?: string, +): Y.Map { + const entry = new Y.Map(); + entry.set("path", path); + entry.set("mtime", mtime); + if (device) entry.set("device", device); + return entry; +} + +/** + * Create a nested Y.Map for a tombstoned metadata entry. + * Tombstones are intentionally minimal: only path + deletedAt. + */ +export function createNestedDeletedMeta( + path: string, + deletedAt: number, +): Y.Map { + const entry = new Y.Map(); + entry.set("path", path); + entry.set("deletedAt", deletedAt); + return entry; +} + +/** + * Create a nested Y.Map from a decoded metadata object. + * Used during lazy conversion from flat to nested. + */ +export function createNestedMetaFromDecoded(decoded: DecodedFileMeta): Y.Map { + const entry = new Y.Map(); + entry.set("path", decoded.path); + + // Tombstone entries + if (decoded.deleted === true || typeof decoded.deletedAt === "number") { + entry.set("deletedAt", decoded.deletedAt ?? Date.now()); + // Tombstones do NOT carry mtime/device + return entry; + } + + // Active entries + if (typeof decoded.mtime === "number") { + entry.set("mtime", decoded.mtime); + } + if (typeof decoded.device === "string") { + entry.set("device", decoded.device); + } + + return entry; +} + +// ------------------------------------------------------------------- +// Lazy conversion helper +// ------------------------------------------------------------------- + +/** + * Ensure the metadata entry for a given fileId is a nested Y.Map. + * If it's already nested, returns it directly. + * If it's flat, converts it to nested (lazy on-write migration) and replaces the entry. + * If it doesn't exist or is invalid, creates a new nested entry from the fallback. + * + * Returns null if no valid entry exists and no fallback is provided. + * + * IMPORTANT: This mutates the meta map (replaces the entry) when converting from flat. + * Call this within a Yjs transaction for consistency. + */ +export function ensureNestedMetaEntry( + metaMap: Y.Map, + fileId: string, + fallback?: DecodedFileMeta, +): Y.Map | null { + const existing = metaMap.get(fileId); + + // Already nested — return directly + if (existing instanceof Y.Map) { + return existing; + } + + // Flat entry exists — convert it + const decoded = decodeFileMeta(existing); + if (decoded) { + const entry = createNestedMetaFromDecoded(decoded); + metaMap.set(fileId, entry); + return entry; + } + + // No valid entry — use fallback if provided + if (fallback) { + const entry = createNestedMetaFromDecoded(fallback); + metaMap.set(fileId, entry); + return entry; + } + + return null; +} + +// ------------------------------------------------------------------- +// Semantic diff computation for observers +// ------------------------------------------------------------------- + +/** + * Compute semantic changes between two metadata snapshots. + * Used by observeDeep handlers to determine what actually changed. + */ +export function computeMetaSemanticChanges( + previous: Map, + current: Map, +): MetaSemanticChange[] { + const changes: MetaSemanticChange[] = []; + + // Check removed entries + for (const [fileId, prev] of previous) { + if (!current.has(fileId)) { + changes.push({ kind: "removed", fileId, previous: prev }); + } + } + + // Check added and modified entries + for (const [fileId, curr] of current) { + const prev = previous.get(fileId); + + if (!prev) { + changes.push({ kind: "added", fileId, next: curr }); + continue; + } + + const prevDeleted = prev.deleted === true || typeof prev.deletedAt === "number"; + const currDeleted = curr.deleted === true || typeof curr.deletedAt === "number"; + + // Revive: was deleted, now active + if (prevDeleted && !currDeleted) { + changes.push({ kind: "revived", fileId, path: curr.path }); + continue; + } + + // Delete: was active, now deleted + if (!prevDeleted && currDeleted) { + changes.push({ kind: "deleted", fileId, path: curr.path, deletedAt: curr.deletedAt ?? Date.now() }); + continue; + } + + // Path change (can happen to tombstones too; consumers must check wasDeleted/isDeleted) + if (prev.path !== curr.path) { + changes.push({ + kind: "path-changed", + fileId, + previousPath: prev.path, + nextPath: curr.path, + wasDeleted: prevDeleted, + isDeleted: currDeleted, + }); + continue; + } + + // Mtime change + if (prev.mtime !== curr.mtime) { + changes.push({ kind: "mtime-changed", fileId, path: curr.path }); + continue; + } + + // Device change + if (prev.device !== curr.device) { + changes.push({ kind: "device-changed", fileId, path: curr.path }); + } + } + + return changes; +} + +/** + * Build a decoded metadata snapshot from the current state of a meta map. + * Used to initialize the observer's previous-state for semantic diffing. + */ +export function buildMetaSnapshot(metaMap: Y.Map): Map { + const snapshot = new Map(); + metaMap.forEach((value: unknown, fileId: string) => { + const decoded = decodeFileMeta(value); + if (decoded) { + snapshot.set(fileId, decoded); + } + }); + return snapshot; +} + +/** + * Extract the set of fileIds affected by a batch of Yjs deep-observe events. + * + * Uses event `path` arrays (relative to the observed root) to avoid an O(N) + * scan. For events on the top-level meta map, `path` is empty and affected + * fileIds come from `event.changes.keys`. For nested events (field mutation + * on a nested Y.Map), `path[0]` is the fileId. + * + * Returns `null` if the affected set cannot be determined from event paths + * (e.g., an unexpected deep nesting), signalling that the caller should fall + * back to a full snapshot diff. + */ +export function extractAffectedFileIds( + events: Y.YEvent>[], + metaMap: Y.Map, +): Set | null { + const affected = new Set(); + + for (const event of events) { + if (event.target === metaMap) { + // Top-level event: the meta map itself changed (key added/removed/replaced). + for (const [fileId] of event.changes.keys) { + affected.add(fileId); + } + } else { + // Nested event: a field inside a nested Y.Map changed. + // event.path is relative to the observed root (the meta map), + // so path[0] is the top-level fileId key. + const path = event.path; + if (path.length >= 1 && typeof path[0] === "string") { + affected.add(path[0]); + } else { + // Cannot determine fileId — signal fallback. + return null; + } + } + } + + return affected; +} + +/** + * Compute incremental semantic changes for a specific set of affected fileIds. + * Only decodes/diffs the changed entries, not the whole map. + * Updates `snapshot` in-place with the new decoded values. + */ +export function computeIncrementalMetaChanges( + snapshot: Map, + metaMap: Y.Map, + affectedFileIds: Set, +): MetaSemanticChange[] { + const changes: MetaSemanticChange[] = []; + + for (const fileId of affectedFileIds) { + const prev = snapshot.get(fileId); + const currentValue = metaMap.get(fileId); + const curr = decodeFileMeta(currentValue); + + if (!curr) { + if (prev) { + // Entry removed or became invalid + changes.push({ kind: "removed", fileId, previous: prev }); + snapshot.delete(fileId); + } else { + changes.push({ kind: "invalid", fileId }); + } + continue; + } + + // Update snapshot + snapshot.set(fileId, curr); + + if (!prev) { + changes.push({ kind: "added", fileId, next: curr }); + continue; + } + + const prevDeleted = prev.deleted === true || typeof prev.deletedAt === "number"; + const currDeleted = curr.deleted === true || typeof curr.deletedAt === "number"; + + if (prevDeleted && !currDeleted) { + changes.push({ kind: "revived", fileId, path: curr.path }); + continue; + } + + if (!prevDeleted && currDeleted) { + changes.push({ kind: "deleted", fileId, path: curr.path, deletedAt: curr.deletedAt ?? Date.now() }); + continue; + } + + if (prev.path !== curr.path) { + changes.push({ + kind: "path-changed", + fileId, + previousPath: prev.path, + nextPath: curr.path, + wasDeleted: prevDeleted, + isDeleted: currDeleted, + }); + continue; + } + + if (prev.mtime !== curr.mtime) { + changes.push({ kind: "mtime-changed", fileId, path: curr.path }); + continue; + } + + if (prev.device !== curr.device) { + changes.push({ kind: "device-changed", fileId, path: curr.path }); + } + } + + return changes; +} + +// ------------------------------------------------------------------- +// Observability / debug counters +// ------------------------------------------------------------------- + +/** Metadata shape statistics for debug surfaces. */ +export interface MetaShapeStats { + schemaVersion: number | null; + flatMetaEntries: number; + nestedMetaEntries: number; + invalidMetaEntries: number; + activeMetaEntries: number; + tombstoneMetaEntries: number; + totalMetaEntries: number; +} + +/** + * Compute metadata shape statistics for diagnostics. + * Walks all meta entries once and classifies them by shape and state. + */ +export function computeMetaShapeStats( + metaMap: Y.Map, + schemaVersion: number | null, +): MetaShapeStats { + let flatMetaEntries = 0; + let nestedMetaEntries = 0; + let invalidMetaEntries = 0; + let activeMetaEntries = 0; + let tombstoneMetaEntries = 0; + + metaMap.forEach((value: unknown) => { + const decoded = decodeFileMeta(value); + if (!decoded) { + invalidMetaEntries++; + return; + } + + if (decoded.shape === "nested") { + nestedMetaEntries++; + } else { + flatMetaEntries++; + } + + const isDel = decoded.deleted === true || typeof decoded.deletedAt === "number"; + if (isDel) { + tombstoneMetaEntries++; + } else { + activeMetaEntries++; + } + }); + + return { + schemaVersion, + flatMetaEntries, + nestedMetaEntries, + invalidMetaEntries, + activeMetaEntries, + tombstoneMetaEntries, + totalMetaEntries: flatMetaEntries + nestedMetaEntries + invalidMetaEntries, + }; +} diff --git a/src/sync/schema.ts b/src/sync/schema.ts new file mode 100644 index 0000000..b55ff83 --- /dev/null +++ b/src/sync/schema.ts @@ -0,0 +1,15 @@ +/** + * YAOS CRDT schema version constant. + * + * This module is intentionally Obsidian-free so it can be imported + * in Node regression tests and server code without the Obsidian dependency. + * + * Schema versioning semantics: + * v1 — legacy path model (pathToId authoritative) + * v2 — id-first model (meta.path authoritative), flat JSON metadata values + * v3 — nested Y.Map metadata (field-level CRDT), lazy on-write migration + * + * Bump this constant AND SERVER_MIN/MAX_SCHEMA_VERSION in server/src/version.ts + * together whenever a breaking schema change ships. + */ +export const SCHEMA_VERSION = 3; diff --git a/src/sync/snapshotClient.ts b/src/sync/snapshotClient.ts index fadf627..1431974 100644 --- a/src/sync/snapshotClient.ts +++ b/src/sync/snapshotClient.ts @@ -17,6 +17,12 @@ import { appendTraceParams, type TraceHttpContext } from "../debug/trace"; import { obsidianRequest } from "../utils/http"; import { yTextToString } from "../utils/format"; import { ORIGIN_RESTORE } from "./origins"; +import { + getMetaPath, + getMetaMtime, + isFileMetaDeletedValue, + createNestedActiveMeta, +} from "./fileMeta"; // ------------------------------------------------------------------- // Types (mirrors server SnapshotIndex) @@ -98,12 +104,12 @@ function usesV2MetaPathModel(doc: Y.Doc): boolean { function isCandidateMetaNewer( candidateId: string, - candidateMeta: FileMeta, + candidateValue: unknown, existingId: string, - existingMeta: FileMeta | undefined, + existingValue: unknown, ): boolean { - const candidateMtime = typeof candidateMeta.mtime === "number" ? candidateMeta.mtime : 0; - const existingMtime = typeof existingMeta?.mtime === "number" ? existingMeta.mtime : 0; + const candidateMtime = getMetaMtime(candidateValue) ?? 0; + const existingMtime = getMetaMtime(existingValue) ?? 0; if (candidateMtime !== existingMtime) return candidateMtime > existingMtime; return candidateId > existingId; } @@ -115,22 +121,24 @@ function isCandidateMetaNewer( * v1/legacy: prefer pathToId, then backfill from active meta entries. */ function collectActiveMarkdownPaths(doc: Y.Doc): Map { - const meta = doc.getMap("meta"); + const meta = doc.getMap("meta"); const pathToId = doc.getMap("pathToId"); const resolved = new Map(); if (usesV2MetaPathModel(doc)) { - meta.forEach((entry, fileId) => { - if (isDeletedMeta(entry) || typeof entry.path !== "string") return; - const path = normalizeVaultPath(entry.path); + meta.forEach((value: unknown, fileId: string) => { + if (isFileMetaDeletedValue(value)) return; + const rawPath = getMetaPath(value); + if (!rawPath) return; + const path = normalizeVaultPath(rawPath); if (!path) return; const existingId = resolved.get(path); if (!existingId) { resolved.set(path, fileId); return; } - const existingMeta = meta.get(existingId); - if (isCandidateMetaNewer(fileId, entry, existingId, existingMeta)) { + const existingValue = meta.get(existingId); + if (isCandidateMetaNewer(fileId, value, existingId, existingValue)) { resolved.set(path, fileId); } }); @@ -142,14 +150,16 @@ function collectActiveMarkdownPaths(doc: Y.Doc): Map { const path = normalizeVaultPath(rawPath); if (!path) return; const entry = meta.get(fileId); - if (isDeletedMeta(entry)) return; + if (isFileMetaDeletedValue(entry)) return; resolved.set(path, fileId); }); // Backfill paths that only exist in meta (mixed/partially migrated states). - meta.forEach((entry, fileId) => { - if (isDeletedMeta(entry) || typeof entry.path !== "string") return; - const path = normalizeVaultPath(entry.path); + meta.forEach((value: unknown, fileId: string) => { + if (isFileMetaDeletedValue(value)) return; + const rawPath = getMetaPath(value); + if (!rawPath) return; + const path = normalizeVaultPath(rawPath); if (!path || resolved.has(path)) return; resolved.set(path, fileId); }); @@ -531,7 +541,7 @@ export function restoreFromSnapshot( const livePathToId = liveDoc.getMap("pathToId"); const liveIdToText = liveDoc.getMap("idToText"); - const liveMeta = liveDoc.getMap("meta"); + const liveMeta = liveDoc.getMap("meta"); const livePathToBlob = liveDoc.getMap("pathToBlob"); const liveBlobTombstones = liveDoc.getMap("blobTombstones"); const liveUsesV2 = usesV2MetaPathModel(liveDoc); @@ -568,13 +578,8 @@ export function restoreFromSnapshot( result.markdownRestored++; // Update metadata - liveMeta.set(liveFileId, { - path, - deleted: undefined, - deletedAt: undefined, - mtime: Date.now(), - device: options.device, - }); + const metaEntry = createNestedActiveMeta(path, Date.now(), options.device); + liveMeta.set(liveFileId, metaEntry); } } if (!liveUsesV2) { @@ -605,11 +610,11 @@ export function restoreFromSnapshot( // Drop stale tombstones for this path to avoid path-squat ghosts. const staleTombstones: string[] = []; - liveMeta.forEach((entry, fileId) => { + liveMeta.forEach((value: unknown, fileId: string) => { if ( fileId !== snapFileId - && entry.path === path - && isDeletedMeta(entry) + && getMetaPath(value) === path + && isFileMetaDeletedValue(value) ) { staleTombstones.push(fileId); } @@ -619,13 +624,8 @@ export function restoreFromSnapshot( } // Clear tombstone and set fresh metadata - liveMeta.set(snapFileId, { - path, - deleted: undefined, - deletedAt: undefined, - mtime: Date.now(), - device: options.device, - }); + const reviveEntry = createNestedActiveMeta(path, Date.now(), options.device); + liveMeta.set(snapFileId, reviveEntry); livePaths.set(path, snapFileId); result.markdownUndeleted++; @@ -650,7 +650,4 @@ export function restoreFromSnapshot( return result; } -function isDeletedMeta(meta: FileMeta | undefined): boolean { - if (!meta) return false; - return meta.deleted === true || (typeof meta.deletedAt === "number" && Number.isFinite(meta.deletedAt)); -} + diff --git a/src/sync/vaultSync.ts b/src/sync/vaultSync.ts index 2548344..2cbec3d 100644 --- a/src/sync/vaultSync.ts +++ b/src/sync/vaultSync.ts @@ -3,7 +3,25 @@ import YSyncProvider from "y-partyserver/provider"; import { IndexeddbPersistence } from "y-indexeddb"; import { normalizePath } from "obsidian"; import { type FileMeta, type BlobRef, type BlobMeta, type BlobTombstone } from "../types"; -import { ORIGIN_SEED } from "./origins"; +import { + decodeFileMeta, + getMetaPath, + getMetaMtime, + isFileMetaDeletedValue, + ensureNestedMetaEntry, + createNestedActiveMeta, + createNestedDeletedMeta, + buildMetaSnapshot, + computeMetaSemanticChanges, + computeMetaShapeStats, + extractAffectedFileIds, + computeIncrementalMetaChanges, + type DecodedFileMeta, + type MetaSemanticChange, + type MetaChangeBatch, + type MetaShapeStats, +} from "./fileMeta"; +import { ORIGIN_SEED, isLocalOrigin } from "./origins"; import type { VaultSyncSettings } from "../settings"; import type { TraceHttpContext, TraceRecord } from "../debug/trace"; import { randomBase64Url } from "../utils/base64url"; @@ -22,7 +40,8 @@ import type { FlightPathEventInput } from "../debug/flightEvents"; import { TICKET_REFRESH_BUFFER_MS, patchTicketInUrl } from "./socketTicket"; /** Current schema version. Stored in sys.schemaVersion. */ -export const SCHEMA_VERSION = 2; +export { SCHEMA_VERSION } from "./schema"; +import { SCHEMA_VERSION } from "./schema"; /** Timeouts for the startup sequence. */ const LOCAL_PERSISTENCE_TIMEOUT_MS = 3_000; @@ -127,7 +146,7 @@ export class VaultSync { readonly pathToId: Y.Map; readonly idToText: Y.Map; - readonly meta: Y.Map; + readonly meta: Y.Map; readonly sys: Y.Map; // Blob / attachment maps (additive — schema version stays at 1) @@ -146,6 +165,66 @@ export class VaultSync { private _deletedPathIndex = new Set(); // tombstoned paths private _pathIndexesDirty = true; + /** + * Snapshot of decoded metadata used by the semantic observer. + * Maintained by `_metaDeepObserver` and used to compute semantic diffs. + */ + private _metaSnapshot = new Map(); + + /** + * Counts how many times the `_metaDeepObserver` fell back to a full snapshot diff + * because event paths were ambiguous. Should be zero in normal operation. + * Exposed in debug stats so operators can confirm the incremental path is taken. + */ + private _metaObserverFallbackCount = 0; + private _metaSemanticListeners = new Set<(batch: MetaChangeBatch) => void>(); + + /** + * The single shared `observeDeep` handler on the meta map. + * + * Uses incremental diffing: reads event paths to determine which fileIds + * changed, then decodes only those entries. Falls back to a full snapshot + * diff only if event paths are ambiguous. + * + * Preserves transaction origin so consumers can distinguish local from remote. + */ + private _metaDeepObserver = (events: Y.YEvent>[]) => { + const origin = events[0]?.transaction.origin; + const isLocal = isLocalOrigin(origin, this.provider); + + let changes: MetaSemanticChange[]; + + // Try incremental diff first (O(k) where k = affected entries). + const affected = extractAffectedFileIds(events, this.meta); + if (affected !== null) { + changes = computeIncrementalMetaChanges(this._metaSnapshot, this.meta, affected); + } else { + // Fallback: full snapshot diff (O(N)). Increment counter for observability. + this._metaObserverFallbackCount++; + const nextSnapshot = buildMetaSnapshot(this.meta); + changes = computeMetaSemanticChanges(this._metaSnapshot, nextSnapshot); + this._metaSnapshot = nextSnapshot; + } + + if (changes.length === 0) return; + + // Invalidate path indexes for structural changes only. + for (const change of changes) { + if (change.kind !== "mtime-changed" && change.kind !== "device-changed") { + this._pathIndexesDirty = true; + break; + } + } + + // Dispatch to all registered listeners. + if (this._metaSemanticListeners.size > 0) { + const batch: MetaChangeBatch = { origin, isLocal, changes }; + for (const listener of this._metaSemanticListeners) { + listener(batch); + } + } + }; + private _localReady = false; private _providerSynced = false; @@ -241,15 +320,18 @@ export class VaultSync { this.ydoc = new Y.Doc(); this.pathToId = this.ydoc.getMap("pathToId"); this.idToText = this.ydoc.getMap("idToText"); - this.meta = this.ydoc.getMap("meta"); + this.meta = this.ydoc.getMap("meta"); this.sys = this.ydoc.getMap("sys"); this.pathToBlob = this.ydoc.getMap("pathToBlob"); this.blobMeta = this.ydoc.getMap("blobMeta"); this.blobTombstones = this.ydoc.getMap("blobTombstones"); - this.meta.observe(() => { - this._pathIndexesDirty = true; - }); + + // Single shared observeDeep handler. Computes semantic diffs and dispatches + // to listeners. Also drives path index invalidation so we only dirty it + // for structurally relevant changes (not mtime/device churn). + this._metaSnapshot = buildMetaSnapshot(this.meta); + this.meta.observeDeep(this._metaDeepObserver); const roomId = settings.vaultId; const idbName = `yaos:${settings.vaultId}`; @@ -583,6 +665,55 @@ export class VaultSync { return stored; } + /** + * Write the schema v3 marker if not already at v3+. + * This does NOT convert metadata — it only signals that this room + * may contain nested metadata and must only be accessed by v3-aware clients. + * Safe to call concurrently from multiple v3 clients (idempotent small write). + */ + markSchemaV3(device?: string): void { + const current = this.currentSchemaVersion(); + if (current >= 3) return; + + this.ydoc.transact(() => { + this.sys.set("schemaVersion", 3); + this.sys.set("schemaUpdatedAt", Date.now()); + if (device) this.sys.set("schemaUpdatedBy", device); + }, ORIGIN_SEED); + + this.log(`schema: marked v3 (was ${current})`); + } + + /** + * Compute metadata shape statistics for debug/diagnostics. + * Returns counts of flat vs nested entries, active vs tombstones, etc. + */ + getMetaShapeStats(): MetaShapeStats & { metaObserverFallbackCount: number } { + return { + ...computeMetaShapeStats(this.meta, this.storedSchemaVersion), + metaObserverFallbackCount: this._metaObserverFallbackCount, + }; + } + + /** + * Subscribe to semantic metadata change events. + * + * The callback receives a `MetaChangeBatch` for each Yjs transaction + * that changes metadata. The batch includes: + * - `origin`: the Yjs transaction origin + * - `isLocal`: true for locally-originated changes (DiskMirror must skip these) + * - `changes`: pre-classified MetaSemanticChange[] for this transaction + * + * Works correctly for both flat (v2) and nested (v3) metadata entries. + * Powered by `observeDeep` with incremental diffing internally. + * + * Returns an unsubscribe function. + */ + observeMetaChanges(callback: (batch: MetaChangeBatch) => void): () => void { + this._metaSemanticListeners.add(callback); + return () => { this._metaSemanticListeners.delete(callback); }; + } + // ------------------------------------------------------------------- // Path normalization // ------------------------------------------------------------------- @@ -592,9 +723,9 @@ export class VaultSync { return normalizePath(path); } - isFileMetaDeleted(meta: FileMeta | undefined): boolean { + isFileMetaDeleted(meta: unknown): boolean { if (!meta) return false; - return meta.deleted === true || (typeof meta.deletedAt === "number" && Number.isFinite(meta.deletedAt)); + return isFileMetaDeletedValue(meta); } private currentSchemaVersion(): number { @@ -615,33 +746,34 @@ export class VaultSync { this._pathIndex.clear(); this._deletedPathIndex.clear(); - this.meta.forEach((meta, fileId) => { - const path = typeof meta.path === "string" ? this.normPath(meta.path) : ""; + this.meta.forEach((value: unknown, fileId: string) => { + const path = getMetaPath(value); if (!path) return; + const normalizedPath = this.normPath(path); - if (this.isFileMetaDeleted(meta)) { - if (!this._pathIndex.has(path)) { - this._deletedPathIndex.add(path); + if (isFileMetaDeletedValue(value)) { + if (!this._pathIndex.has(normalizedPath)) { + this._deletedPathIndex.add(normalizedPath); } return; } - const existingId = this._pathIndex.get(path); + const existingId = this._pathIndex.get(normalizedPath); if (!existingId) { - this._pathIndex.set(path, fileId); - this._deletedPathIndex.delete(path); + this._pathIndex.set(normalizedPath, fileId); + this._deletedPathIndex.delete(normalizedPath); return; } - const existingMeta = this.meta.get(existingId); - const existingMtime = typeof existingMeta?.mtime === "number" ? existingMeta.mtime : 0; - const candidateMtime = typeof meta.mtime === "number" ? meta.mtime : 0; + const existingValue = this.meta.get(existingId); + const existingMtime = getMetaMtime(existingValue) ?? 0; + const candidateMtime = getMetaMtime(value) ?? 0; // If we see active path collisions, deterministically choose one winner. if (candidateMtime > existingMtime || (candidateMtime === existingMtime && fileId > existingId)) { - this._pathIndex.set(path, fileId); + this._pathIndex.set(normalizedPath, fileId); } - this._deletedPathIndex.delete(path); + this._deletedPathIndex.delete(normalizedPath); }); this._pathIndexesDirty = false; @@ -649,35 +781,53 @@ export class VaultSync { private setMetaActive(fileId: string, path: string, device?: string): void { const normalizedPath = this.normPath(path); - this.meta.set(fileId, { + const now = Date.now(); + + const entry = ensureNestedMetaEntry(this.meta, fileId, { + shape: "flat", path: normalizedPath, - deleted: undefined, - deletedAt: undefined, - mtime: Date.now(), - device, + mtime: now, + ...(device ? { device } : {}), }); + + if (!entry) { + // Should not happen since we always provide a fallback + this.log(`setMetaActive: failed to ensure nested entry for ${fileId}`); + return; + } + + entry.set("path", normalizedPath); + entry.delete("deleted"); + entry.delete("deletedAt"); + entry.set("mtime", now); + + if (device) { + entry.set("device", device); + } else { + entry.delete("device"); + } } private setMetaDeleted(fileId: string, path: string, device?: string): void { const normalizedPath = this.normPath(path); const deletedAt = Date.now(); - const useLegacyFlag = this.currentSchemaVersion() < 2; - if (useLegacyFlag) { - this.meta.set(fileId, { - path: normalizedPath, - deleted: true, - deletedAt, - mtime: deletedAt, - device, - }); - return; - } - // v2 tombstone payload is intentionally minimal for long-term size control. - this.meta.set(fileId, { + const entry = ensureNestedMetaEntry(this.meta, fileId, { + shape: "flat", path: normalizedPath, deletedAt, }); + + if (!entry) { + this.log(`setMetaDeleted: failed to ensure nested entry for ${fileId}`); + return; + } + + entry.set("path", normalizedPath); + entry.set("deletedAt", deletedAt); + entry.delete("deleted"); + entry.delete("mtime"); + entry.delete("device"); } migrateSchemaToV2(device?: string): { @@ -710,8 +860,8 @@ export class VaultSync { }); for (const [fileId, paths] of pathsById) { - const meta = this.meta.get(fileId); - const preferred = typeof meta?.path === "string" ? this.normPath(meta.path) : ""; + const metaValue = this.meta.get(fileId); + const preferred = getMetaPath(metaValue) ? this.normPath(getMetaPath(metaValue)!) : ""; const canonical = preferred && paths.includes(preferred) ? preferred : paths.slice().sort()[0]!; @@ -724,64 +874,75 @@ export class VaultSync { } for (const [fileId, normalizedPath] of canonicalPathById) { - const currentMeta = this.meta.get(fileId); + const currentMeta = decodeFileMeta(this.meta.get(fileId)); if (!currentMeta) { + // Write flat v2 object — this is a v1→v2 migration, not a v3 upgrade. + // The lazy v3 conversion will upgrade this entry when it is next touched. this.meta.set(fileId, { path: normalizedPath, deletedAt: undefined, deleted: undefined, mtime: now, device, - }); + } as unknown); metaCreated++; return; } - const isDeleted = this.isFileMetaDeleted(currentMeta); + const isDeleted = currentMeta.deleted === true || typeof currentMeta.deletedAt === "number"; if (!isDeleted && currentMeta.path !== normalizedPath) { - this.meta.set(fileId, { - ...currentMeta, - path: normalizedPath, - deleted: undefined, - deletedAt: undefined, - mtime: currentMeta.mtime ?? now, - device: currentMeta.device ?? device, - }); + // Update path in-place on nested map; write flat if still flat. + const existing = this.meta.get(fileId); + if (existing instanceof Y.Map) { + existing.set("path", normalizedPath); + } else { + this.meta.set(fileId, { + ...(currentMeta as object), + path: normalizedPath, + deleted: undefined, + deletedAt: undefined, + mtime: currentMeta.mtime ?? now, + device: currentMeta.device ?? device, + } as unknown); + } metaUpdated++; } } - this.meta.forEach((meta, fileId) => { - if (meta.deleted && meta.deletedAt === undefined) { + this.meta.forEach((value: unknown, fileId: string) => { + const decoded = decodeFileMeta(value); + if (!decoded) return; + if (decoded.deleted && decoded.deletedAt === undefined) { + // Convert legacy deleted:true to v2 flat tombstone. this.meta.set(fileId, { - path: this.normPath(meta.path), - deletedAt: typeof meta.mtime === "number" ? meta.mtime : now, - }); + path: this.normPath(decoded.path), + deletedAt: typeof decoded.mtime === "number" ? decoded.mtime : now, + } as unknown); tombstonesConverted++; return; } - if (this.isFileMetaDeleted(meta) && (meta.deleted !== undefined || meta.mtime !== undefined || meta.device !== undefined)) { + const isDel = decoded.deleted === true || typeof decoded.deletedAt === "number"; + if (isDel && (decoded.deleted !== undefined || decoded.mtime !== undefined || decoded.device !== undefined)) { + // Strip extra fields from tombstone, keep as flat v2. this.meta.set(fileId, { - path: this.normPath(meta.path), - deletedAt: typeof meta.deletedAt === "number" ? meta.deletedAt : now, - }); + path: this.normPath(decoded.path), + deletedAt: typeof decoded.deletedAt === "number" ? decoded.deletedAt : now, + } as unknown); metaUpdated++; } }); - // Explicit tombstones for dropped alias paths. + // Explicit tombstones for dropped alias paths — write flat v2. const existingActivePaths = new Set(); - this.meta.forEach((meta) => { - if (this.isFileMetaDeleted(meta)) return; - existingActivePaths.add(this.normPath(meta.path)); + this.meta.forEach((value: unknown) => { + if (isFileMetaDeletedValue(value)) return; + const path = getMetaPath(value); + if (path) existingActivePaths.add(this.normPath(path)); }); for (const loserPath of loserPaths) { if (existingActivePaths.has(loserPath)) continue; const tombstoneId = this.generateFileId(); - this.meta.set(tombstoneId, { - path: loserPath, - deletedAt: now, - }); + this.meta.set(tombstoneId, { path: loserPath, deletedAt: now } as unknown); } this.sys.set("schemaVersion", 2); @@ -851,18 +1012,15 @@ export class VaultSync { const newId = this.generateFileId(); const newText = new Y.Text(); - this.ydoc.transact(() => { - if (sourceText) { - newText.insert(0, sourceText.toJSON()); - } - this.pathToId.set(dupPath, newId); - this.idToText.set(newId, newText); - this.meta.set(newId, { - path: dupPath, - mtime: Date.now(), - device: this._device, - }); - }, ORIGIN_SEED); + this.ydoc.transact(() => { + if (sourceText) { + newText.insert(0, sourceText.toJSON()); + } + this.pathToId.set(dupPath, newId); + this.idToText.set(newId, newText); + const dupMeta = createNestedActiveMeta(dupPath, Date.now(), this._device); + this.meta.set(newId, dupMeta); + }, ORIGIN_SEED); this.log( `integrity: gave "${dupPath}" new id=${newId} (was sharing ${fileId} with "${keepPath}")`, @@ -880,8 +1038,8 @@ export class VaultSync { // Also keep tombstoned IDs (they're intentionally orphaned from pathToId) const tombstonedIds = new Set(); - this.meta.forEach((meta, fileId) => { - if (this.isFileMetaDeleted(meta)) { + this.meta.forEach((value: unknown, fileId: string) => { + if (isFileMetaDeletedValue(value)) { tombstonedIds.add(fileId); } }); @@ -1537,11 +1695,11 @@ export class VaultSync { private clearMarkdownTombstonesForPath(path: string, keepFileId?: string): number { const tombstonedIds: string[] = []; - this.meta.forEach((meta, fileId) => { + this.meta.forEach((value: unknown, fileId: string) => { if ( fileId !== keepFileId - && meta.path === path - && this.isFileMetaDeleted(meta) + && getMetaPath(value) === path + && isFileMetaDeletedValue(value) ) { tombstonedIds.push(fileId); } @@ -1557,8 +1715,8 @@ export class VaultSync { private getMarkdownTombstoneIds(path: string): string[] { const normalizedPath = this.normPath(path); const tombstonedIds: string[] = []; - this.meta.forEach((meta, fileId) => { - if (meta.path === normalizedPath && this.isFileMetaDeleted(meta)) { + this.meta.forEach((value: unknown, fileId: string) => { + if (getMetaPath(value) === normalizedPath && isFileMetaDeletedValue(value)) { tombstonedIds.push(fileId); } }); diff --git a/tests/disk-mirror-observer.ts b/tests/disk-mirror-observer.ts index d6b5e29..4174032 100644 --- a/tests/disk-mirror-observer.ts +++ b/tests/disk-mirror-observer.ts @@ -42,6 +42,14 @@ import { ORIGIN_RESTORE, ORIGIN_SEED, } from "../src/sync/origins"; +import { + buildMetaSnapshot, + extractAffectedFileIds, + computeIncrementalMetaChanges, + isFileMetaDeletedValue, + type MetaChangeBatch, +} from "../src/sync/fileMeta"; +import { isLocalOrigin } from "../src/sync/origins"; let passed = 0; let failed = 0; @@ -80,6 +88,26 @@ function makeHarness() { getFileIdForText: (text: Y.Text) => (text === ytext ? FILE_ID : null), idToText: { entries: () => new Map([[FILE_ID, ytext]]).entries() }, isFileMetaDeleted: (m: { deleted?: boolean } | undefined) => Boolean(m?.deleted), + // Minimal semantic observer: existing tests only test afterTransaction / text observers. + // Provide a real implementation so DiskMirror.startMapObservers() doesn't crash. + observeMetaChanges: (() => { + let snapshot = buildMetaSnapshot(meta as Y.Map); + const listeners = new Set<(batch: MetaChangeBatch) => void>(); + (meta as Y.Map).observeDeep((events: Y.YEvent>[]) => { + const origin = events[0]?.transaction.origin; + const isLocal = isLocalOrigin(origin, fakeProvider); + const affected = extractAffectedFileIds(events, meta as Y.Map); + if (!affected) return; + const changes = computeIncrementalMetaChanges(snapshot, meta as Y.Map, affected); + if (changes.length === 0) return; + const batch: MetaChangeBatch = { origin, isLocal, changes }; + for (const l of listeners) l(batch); + }); + return (cb: (batch: MetaChangeBatch) => void) => { + listeners.add(cb); + return () => { listeners.delete(cb); }; + }; + })(), }; const fakeEditorBindings = { diff --git a/tests/file-meta-decode.ts b/tests/file-meta-decode.ts new file mode 100644 index 0000000..27e0417 --- /dev/null +++ b/tests/file-meta-decode.ts @@ -0,0 +1,458 @@ +/** + * Tests for src/sync/fileMeta.ts — validates dual-shape decoding, + * type guards, read helpers, write helpers, lazy conversion, and + * semantic diff computation. + */ + +import * as Y from "yjs"; +import { + decodeFileMeta, + isNestedFileMeta, + isObjectRecord, + isFileMetaDeletedValue, + getMetaPath, + getMetaMtime, + getMetaDevice, + getMetaDeletedAt, + createNestedActiveMeta, + createNestedDeletedMeta, + createNestedMetaFromDecoded, + ensureNestedMetaEntry, + buildMetaSnapshot, + computeMetaSemanticChanges, + type DecodedFileMeta, +} from "../src/sync/fileMeta"; + +// ── Test runner ───────────────────────────────────────────────────────────── + +let passed = 0; +let failed = 0; + +function assert(condition: boolean, msg: string): void { + if (condition) { + passed++; + } else { + failed++; + console.error(` FAIL: ${msg}`); + } +} + +function assertEqual(actual: T, expected: T, msg: string): void { + if (actual === expected) { + passed++; + } else { + failed++; + console.error(` FAIL: ${msg} — got ${JSON.stringify(actual)}, expected ${JSON.stringify(expected)}`); + } +} + +function assertNull(actual: unknown, msg: string): void { + if (actual === null) { + passed++; + } else { + failed++; + console.error(` FAIL: ${msg} — expected null, got ${JSON.stringify(actual)}`); + } +} + +function section(name: string): void { + console.log(`\n── ${name} ──`); +} + +// ── Type Guard Tests ──────────────────────────────────────────────────────── + +section("Type guards"); + +{ + const ymap = new Y.Map(); + assert(isNestedFileMeta(ymap), "Y.Map is nested"); + assert(!isNestedFileMeta({ path: "a.md" }), "plain object is not nested"); + assert(!isNestedFileMeta(null), "null is not nested"); + assert(!isNestedFileMeta(42), "number is not nested"); + + assert(isObjectRecord({ path: "a.md" }), "plain object is record"); + assert(!isObjectRecord(ymap), "Y.Map is not record"); + assert(!isObjectRecord(null), "null is not record"); + assert(!isObjectRecord("string"), "string is not record"); +} + +// ── Decoder: flat active ──────────────────────────────────────────────────── + +section("Decoder: flat active"); + +{ + const result = decodeFileMeta({ path: "notes/foo.md", mtime: 1000, device: "macbook" }); + assert(result !== null, "decodes flat active"); + assertEqual(result!.shape, "flat", "shape is flat"); + assertEqual(result!.path, "notes/foo.md", "path decoded"); + assertEqual(result!.mtime, 1000, "mtime decoded"); + assertEqual(result!.device, "macbook", "device decoded"); + assertEqual(result!.deletedAt, undefined, "deletedAt absent"); + assertEqual(result!.deleted, undefined, "deleted absent"); +} + +// ── Decoder: flat tombstone with deletedAt ────────────────────────────────── + +section("Decoder: flat tombstone with deletedAt"); + +{ + const result = decodeFileMeta({ path: "old.md", deletedAt: 5000 }); + assert(result !== null, "decodes flat tombstone"); + assertEqual(result!.shape, "flat", "shape is flat"); + assertEqual(result!.path, "old.md", "path decoded"); + assertEqual(result!.deletedAt, 5000, "deletedAt decoded"); + assertEqual(result!.mtime, undefined, "mtime absent"); +} + +// ── Decoder: legacy flat tombstone with deleted: true ──────────────────────── + +section("Decoder: legacy flat tombstone"); + +{ + const result = decodeFileMeta({ path: "legacy.md", deleted: true }); + assert(result !== null, "decodes legacy tombstone"); + assertEqual(result!.deleted, true, "deleted flag decoded"); + assertEqual(result!.deletedAt, undefined, "deletedAt not present"); +} + +// ── Decoder: nested active ────────────────────────────────────────────────── + +section("Decoder: nested active"); + +{ + const doc = new Y.Doc(); + const container = doc.getMap("test"); + const ymap = new Y.Map(); + container.set("entry", ymap); + ymap.set("path", "nested/file.md"); + ymap.set("mtime", 2000); + ymap.set("device", "phone"); + + const result = decodeFileMeta(ymap); + assert(result !== null, "decodes nested active"); + assertEqual(result!.shape, "nested", "shape is nested"); + assertEqual(result!.path, "nested/file.md", "path decoded"); + assertEqual(result!.mtime, 2000, "mtime decoded"); + assertEqual(result!.device, "phone", "device decoded"); + assertEqual(result!.deletedAt, undefined, "deletedAt absent"); +} + +// ── Decoder: nested tombstone ─────────────────────────────────────────────── + +section("Decoder: nested tombstone"); + +{ + const doc = new Y.Doc(); + const container = doc.getMap("test"); + const ymap = new Y.Map(); + container.set("entry", ymap); + ymap.set("path", "deleted.md"); + ymap.set("deletedAt", 9999); + + const result = decodeFileMeta(ymap); + assert(result !== null, "decodes nested tombstone"); + assertEqual(result!.shape, "nested", "shape is nested"); + assertEqual(result!.path, "deleted.md", "path decoded"); + assertEqual(result!.deletedAt, 9999, "deletedAt decoded"); + assertEqual(result!.mtime, undefined, "mtime absent on tombstone"); +} + +// ── Decoder: invalid inputs ───────────────────────────────────────────────── + +section("Decoder: invalid inputs"); + +{ + assertNull(decodeFileMeta(null), "null returns null"); + assertNull(decodeFileMeta(undefined), "undefined returns null"); + assertNull(decodeFileMeta(42), "number returns null"); + assertNull(decodeFileMeta("string"), "string returns null"); + assertNull(decodeFileMeta({}), "empty object returns null (no path)"); + assertNull(decodeFileMeta({ path: "" }), "empty path returns null"); + assertNull(decodeFileMeta({ path: 123 }), "non-string path returns null"); + + const invalidDoc = new Y.Doc(); + const invalidContainer = invalidDoc.getMap("test"); + + const emptyMap = new Y.Map(); + invalidContainer.set("empty", emptyMap); + assertNull(decodeFileMeta(emptyMap), "Y.Map without path returns null"); + + const badPathMap = new Y.Map(); + invalidContainer.set("bad", badPathMap); + badPathMap.set("path", ""); + assertNull(decodeFileMeta(badPathMap), "Y.Map with empty path returns null"); + + const nanMtime = decodeFileMeta({ path: "x.md", mtime: NaN }); + assert(nanMtime !== null, "NaN mtime still decodes"); + assertEqual(nanMtime!.mtime, undefined, "NaN mtime treated as absent"); + + const nanDeletedAt = decodeFileMeta({ path: "x.md", deletedAt: NaN }); + assert(nanDeletedAt !== null, "NaN deletedAt still decodes"); + assertEqual(nanDeletedAt!.deletedAt, undefined, "NaN deletedAt treated as absent"); + + const infMtime = decodeFileMeta({ path: "x.md", mtime: Infinity }); + assertEqual(infMtime!.mtime, undefined, "Infinity mtime treated as absent"); +} + +// ── Read helpers ──────────────────────────────────────────────────────────── + +section("Read helpers"); + +{ + // Flat + assertEqual(getMetaPath({ path: "a.md" }), "a.md", "getMetaPath flat"); + assertEqual(getMetaMtime({ path: "a.md", mtime: 100 }), 100, "getMetaMtime flat"); + assertEqual(getMetaDevice({ path: "a.md", device: "dev" }), "dev", "getMetaDevice flat"); + assertEqual(getMetaDeletedAt({ path: "a.md", deletedAt: 500 }), 500, "getMetaDeletedAt flat"); + assertNull(getMetaPath(null), "getMetaPath null input"); + assertNull(getMetaMtime({ path: "a.md" }), "getMetaMtime absent"); + + // Nested + const readDoc = new Y.Doc(); + const readContainer = readDoc.getMap("test"); + const ymap = new Y.Map(); + readContainer.set("entry", ymap); + ymap.set("path", "b.md"); + ymap.set("mtime", 200); + ymap.set("device", "tablet"); + ymap.set("deletedAt", 300); + + assertEqual(getMetaPath(ymap), "b.md", "getMetaPath nested"); + assertEqual(getMetaMtime(ymap), 200, "getMetaMtime nested"); + assertEqual(getMetaDevice(ymap), "tablet", "getMetaDevice nested"); + assertEqual(getMetaDeletedAt(ymap), 300, "getMetaDeletedAt nested"); +} + +// ── isFileMetaDeletedValue ────────────────────────────────────────────────── + +section("isFileMetaDeletedValue"); + +{ + assert(isFileMetaDeletedValue({ path: "a.md", deletedAt: 100 }), "flat deletedAt is deleted"); + assert(isFileMetaDeletedValue({ path: "a.md", deleted: true }), "flat deleted:true is deleted"); + assert(!isFileMetaDeletedValue({ path: "a.md", mtime: 1 }), "flat active is not deleted"); + assert(!isFileMetaDeletedValue({ path: "a.md" }), "flat minimal active not deleted"); + assert(!isFileMetaDeletedValue(null), "null not deleted"); + + const delDoc = new Y.Doc(); + const delContainer = delDoc.getMap("test"); + + const deletedMap = new Y.Map(); + delContainer.set("deleted", deletedMap); + deletedMap.set("path", "d.md"); + deletedMap.set("deletedAt", 999); + assert(isFileMetaDeletedValue(deletedMap), "nested deletedAt is deleted"); + + const activeMap = new Y.Map(); + delContainer.set("active", activeMap); + activeMap.set("path", "a.md"); + activeMap.set("mtime", 1); + assert(!isFileMetaDeletedValue(activeMap), "nested active not deleted"); + + const legacyMap = new Y.Map(); + delContainer.set("legacy", legacyMap); + legacyMap.set("path", "l.md"); + legacyMap.set("deleted", true); + assert(isFileMetaDeletedValue(legacyMap), "nested legacy deleted:true is deleted"); +} + +// ── Write helpers ─────────────────────────────────────────────────────────── + +section("Write helpers: createNestedActiveMeta"); + +{ + const doc = new Y.Doc(); + const container = doc.getMap("test"); + + const entry = createNestedActiveMeta("notes/hello.md", 5000, "laptop"); + container.set("e1", entry); + assert(entry instanceof Y.Map, "returns Y.Map"); + assertEqual(entry.get("path"), "notes/hello.md", "path set"); + assertEqual(entry.get("mtime"), 5000, "mtime set"); + assertEqual(entry.get("device"), "laptop", "device set"); + assertEqual(entry.get("deletedAt"), undefined, "deletedAt absent"); + assertEqual(entry.get("deleted"), undefined, "deleted absent"); + + const noDevice = createNestedActiveMeta("x.md", 1000); + container.set("e2", noDevice); + assertEqual(noDevice.get("device"), undefined, "device omitted when undefined"); +} + +section("Write helpers: createNestedDeletedMeta"); + +{ + const doc = new Y.Doc(); + const container = doc.getMap("test"); + + const entry = createNestedDeletedMeta("trash/old.md", 8000); + container.set("e1", entry); + assert(entry instanceof Y.Map, "returns Y.Map"); + assertEqual(entry.get("path"), "trash/old.md", "path set"); + assertEqual(entry.get("deletedAt"), 8000, "deletedAt set"); + assertEqual(entry.get("mtime"), undefined, "mtime absent on tombstone"); + assertEqual(entry.get("device"), undefined, "device absent on tombstone"); +} + +section("Write helpers: createNestedMetaFromDecoded"); + +{ + const doc = new Y.Doc(); + const container = doc.getMap("test"); + + // Active decoded + const active = createNestedMetaFromDecoded({ + shape: "flat", + path: "conv.md", + mtime: 3000, + device: "desktop", + }); + container.set("active", active); + assertEqual(active.get("path"), "conv.md", "converted active path"); + assertEqual(active.get("mtime"), 3000, "converted active mtime"); + assertEqual(active.get("device"), "desktop", "converted active device"); + assertEqual(active.get("deletedAt"), undefined, "converted active no deletedAt"); + + // Tombstone decoded + const tombstone = createNestedMetaFromDecoded({ + shape: "flat", + path: "dead.md", + deletedAt: 7000, + }); + container.set("tombstone", tombstone); + assertEqual(tombstone.get("path"), "dead.md", "converted tombstone path"); + assertEqual(tombstone.get("deletedAt"), 7000, "converted tombstone deletedAt"); + assertEqual(tombstone.get("mtime"), undefined, "converted tombstone no mtime"); + assertEqual(tombstone.get("device"), undefined, "converted tombstone no device"); + + // Legacy deleted:true + const legacy = createNestedMetaFromDecoded({ + shape: "flat", + path: "legacy.md", + deleted: true, + }); + container.set("legacy", legacy); + assert(typeof legacy.get("deletedAt") === "number", "legacy deleted:true gets deletedAt timestamp"); + assertEqual(legacy.get("mtime"), undefined, "legacy no mtime"); +} + +// ── Lazy conversion: ensureNestedMetaEntry ────────────────────────────────── + +section("ensureNestedMetaEntry"); + +{ + const doc = new Y.Doc(); + const metaMap = doc.getMap("meta"); + + // Case 1: already nested + const existing = new Y.Map(); + existing.set("path", "already.md"); + existing.set("mtime", 100); + metaMap.set("id1", existing); + + const result1 = ensureNestedMetaEntry(metaMap, "id1"); + assert(result1 === existing, "returns existing nested map directly"); + + // Case 2: flat entry gets converted + metaMap.set("id2", { path: "flat.md", mtime: 200, device: "dev" } as unknown); + + const result2 = ensureNestedMetaEntry(metaMap, "id2"); + assert(result2 instanceof Y.Map, "converts flat to nested"); + assertEqual(result2!.get("path"), "flat.md", "converted path preserved"); + assertEqual(result2!.get("mtime"), 200, "converted mtime preserved"); + assertEqual(result2!.get("device"), "dev", "converted device preserved"); + // Verify it was actually replaced in the map + assert(metaMap.get("id2") instanceof Y.Map, "meta map entry replaced with nested"); + + // Case 3: missing entry with fallback + const result3 = ensureNestedMetaEntry(metaMap, "id3", { + shape: "flat", + path: "fallback.md", + mtime: 300, + }); + assert(result3 instanceof Y.Map, "creates from fallback"); + assertEqual(result3!.get("path"), "fallback.md", "fallback path"); + + // Case 4: missing entry without fallback + const result4 = ensureNestedMetaEntry(metaMap, "id4"); + assertNull(result4, "returns null without fallback"); + + // Case 5: untouched flat entries remain flat + metaMap.set("id5", { path: "untouched.md", mtime: 500 } as unknown); + // Don't call ensureNestedMetaEntry on id5 + const raw = metaMap.get("id5"); + assert(!(raw instanceof Y.Map), "untouched entry remains flat"); +} + +// ── Semantic diff computation ─────────────────────────────────────────────── + +section("computeMetaSemanticChanges"); + +{ + const prev = new Map(); + prev.set("a", { shape: "flat", path: "a.md", mtime: 1 }); + prev.set("b", { shape: "flat", path: "b.md", mtime: 1 }); + prev.set("c", { shape: "flat", path: "c.md", deletedAt: 100 }); + prev.set("d", { shape: "flat", path: "d.md", mtime: 1 }); + + const curr = new Map(); + curr.set("a", { shape: "nested", path: "a-renamed.md", mtime: 1 }); // path changed + curr.set("b", { shape: "nested", path: "b.md", mtime: 2 }); // mtime changed + curr.set("c", { shape: "nested", path: "c.md", mtime: 5 }); // revived + curr.set("e", { shape: "nested", path: "e.md", mtime: 1 }); // added + // d removed + + const changes = computeMetaSemanticChanges(prev, curr); + + const kinds = changes.map(c => c.kind); + assert(kinds.includes("removed"), "d was removed"); + assert(kinds.includes("added"), "e was added"); + assert(kinds.includes("path-changed"), "a path changed"); + assert(kinds.includes("mtime-changed"), "b mtime changed"); + assert(kinds.includes("revived"), "c revived"); + + const removed = changes.find(c => c.kind === "removed"); + assertEqual((removed as any).fileId, "d", "removed fileId is d"); + + const added = changes.find(c => c.kind === "added"); + assertEqual((added as any).fileId, "e", "added fileId is e"); + + const pathChanged = changes.find(c => c.kind === "path-changed"); + assertEqual((pathChanged as any).previousPath, "a.md", "previous path"); + assertEqual((pathChanged as any).nextPath, "a-renamed.md", "next path"); +} + +// ── buildMetaSnapshot ─────────────────────────────────────────────────────── + +section("buildMetaSnapshot"); + +{ + const doc = new Y.Doc(); + const metaMap = doc.getMap("meta"); + + // Mixed shapes + metaMap.set("flat1", { path: "flat1.md", mtime: 10 } as unknown); + + const nested1 = new Y.Map(); + nested1.set("path", "nested1.md"); + nested1.set("mtime", 20); + metaMap.set("nested1", nested1); + + metaMap.set("invalid", "not an object" as unknown); + + const snapshot = buildMetaSnapshot(metaMap); + assertEqual(snapshot.size, 2, "invalid entry excluded"); + assertEqual(snapshot.get("flat1")?.path, "flat1.md", "flat decoded"); + assertEqual(snapshot.get("nested1")?.path, "nested1.md", "nested decoded"); + assertEqual(snapshot.get("flat1")?.shape, "flat", "flat shape"); + assertEqual(snapshot.get("nested1")?.shape, "nested", "nested shape"); +} + +// ── Report ────────────────────────────────────────────────────────────────── + +console.log(`\n${"═".repeat(60)}`); +console.log(`Results: ${passed} passed, ${failed} failed`); +console.log(`${"═".repeat(60)}`); + +if (failed > 0) { + process.exit(1); +} diff --git a/tests/file-meta-lazy-write.ts b/tests/file-meta-lazy-write.ts new file mode 100644 index 0000000..bb366d3 --- /dev/null +++ b/tests/file-meta-lazy-write.ts @@ -0,0 +1,364 @@ +/** + * Tests for lazy on-write conversion behavior. + * + * Validates that: + * - New writes always produce nested Y.Maps + * - Only touched flat entries get converted + * - Untouched entries remain flat + * - No full-vault migration storm occurs + * - Concurrent lazy conversion converges + */ + +import * as Y from "yjs"; +import { + ensureNestedMetaEntry, + createNestedActiveMeta, + createNestedDeletedMeta, + decodeFileMeta, + isNestedFileMeta, + isFileMetaDeletedValue, +} from "../src/sync/fileMeta"; + +// ── Test runner ───────────────────────────────────────────────────────────── + +let passed = 0; +let failed = 0; + +function assert(condition: boolean, msg: string): void { + if (condition) { passed++; } else { failed++; console.error(` FAIL: ${msg}`); } +} + +function assertEqual(actual: T, expected: T, msg: string): void { + if (actual === expected) { passed++; } else { failed++; console.error(` FAIL: ${msg} — got ${JSON.stringify(actual)}, expected ${JSON.stringify(expected)}`); } +} + +function section(name: string): void { + console.log(`\n── ${name} ──`); +} + +// ── Helper: populate a doc with N flat metadata entries ────────────────────── + +function populateFlat(doc: Y.Doc, count: number): void { + const meta = doc.getMap("meta"); + doc.transact(() => { + for (let i = 0; i < count; i++) { + meta.set(`file-${i}`, { path: `notes/file-${i}.md`, mtime: 1000 + i, device: "dev" }); + } + }); +} + +// ── Test: new active write creates nested map ─────────────────────────────── + +section("New active write creates nested map"); + +{ + const doc = new Y.Doc(); + const meta = doc.getMap("meta"); + + const entry = createNestedActiveMeta("new-file.md", Date.now(), "laptop"); + meta.set("new-id", entry); + + assert(meta.get("new-id") instanceof Y.Map, "new entry is nested Y.Map"); + const nested = meta.get("new-id") as Y.Map; + assertEqual(nested.get("path"), "new-file.md", "path correct"); +} + +// ── Test: new tombstone write creates nested map ──────────────────────────── + +section("New tombstone write creates nested map"); + +{ + const doc = new Y.Doc(); + const meta = doc.getMap("meta"); + + const entry = createNestedDeletedMeta("deleted.md", 5000); + meta.set("tomb-id", entry); + + assert(meta.get("tomb-id") instanceof Y.Map, "tombstone is nested Y.Map"); + const nested = meta.get("tomb-id") as Y.Map; + assertEqual(nested.get("deletedAt"), 5000, "deletedAt correct"); +} + +// ── Test: active write converts ONLY the touched flat entry ───────────────── + +section("Active write converts only touched flat entry"); + +{ + const doc = new Y.Doc(); + populateFlat(doc, 100); + const meta = doc.getMap("meta"); + + // Touch only file-5 + doc.transact(() => { + const entry = ensureNestedMetaEntry(meta, "file-5"); + assert(entry !== null, "entry returned"); + entry!.set("mtime", Date.now()); + }); + + // file-5 should be nested + assert(meta.get("file-5") instanceof Y.Map, "touched entry is now nested"); + + // All others should remain flat + let flatCount = 0; + let nestedCount = 0; + meta.forEach((value: unknown, key: string) => { + if (key === "file-5") return; + if (value instanceof Y.Map) nestedCount++; + else flatCount++; + }); + + assertEqual(flatCount, 99, "99 entries remain flat"); + assertEqual(nestedCount, 0, "no other entries converted"); +} + +// ── Test: untouched flat entries remain flat after multiple writes ─────────── + +section("Untouched flat entries remain flat"); + +{ + const doc = new Y.Doc(); + populateFlat(doc, 50); + const meta = doc.getMap("meta"); + + // Touch files 0, 1, 2 only + doc.transact(() => { + for (let i = 0; i < 3; i++) { + const entry = ensureNestedMetaEntry(meta, `file-${i}`); + entry!.set("mtime", Date.now()); + } + }); + + let flatCount = 0; + meta.forEach((value: unknown, key: string) => { + if (!key.startsWith("file-")) return; + if (!(value instanceof Y.Map)) flatCount++; + }); + + assertEqual(flatCount, 47, "47 untouched entries remain flat"); +} + +// ── Test: delete write converts only touched flat entry ───────────────────── + +section("Delete write converts only touched entry"); + +{ + const doc = new Y.Doc(); + populateFlat(doc, 20); + const meta = doc.getMap("meta"); + + doc.transact(() => { + const entry = ensureNestedMetaEntry(meta, "file-10"); + entry!.set("deletedAt", Date.now()); + entry!.delete("mtime"); + entry!.delete("device"); + }); + + assert(meta.get("file-10") instanceof Y.Map, "deleted entry is nested"); + assert(isFileMetaDeletedValue(meta.get("file-10")), "entry is tombstoned"); + + // Others remain flat + let flatCount = 0; + meta.forEach((value: unknown, key: string) => { + if (key === "file-10") return; + if (!(value instanceof Y.Map)) flatCount++; + }); + assertEqual(flatCount, 19, "19 entries remain flat"); +} + +// ── Test: revive write clears deletedAt ───────────────────────────────────── + +section("Revive write clears deletedAt"); + +{ + const doc = new Y.Doc(); + const meta = doc.getMap("meta"); + + // Start with a nested tombstone + const tombstone = createNestedDeletedMeta("revived.md", 3000); + meta.set("revive-id", tombstone); + + assert(isFileMetaDeletedValue(meta.get("revive-id")), "starts as deleted"); + + // Revive it + doc.transact(() => { + const entry = ensureNestedMetaEntry(meta, "revive-id"); + entry!.delete("deletedAt"); + entry!.delete("deleted"); + entry!.set("mtime", Date.now()); + entry!.set("device", "laptop"); + }); + + assert(!isFileMetaDeletedValue(meta.get("revive-id")), "no longer deleted"); + const revived = meta.get("revive-id") as Y.Map; + assertEqual(revived.get("path"), "revived.md", "path preserved"); + assertEqual(revived.get("deletedAt"), undefined, "deletedAt cleared"); +} + +// ── Test: tombstone write removes mtime and device ────────────────────────── + +section("Tombstone write removes mtime and device"); + +{ + const doc = new Y.Doc(); + const meta = doc.getMap("meta"); + + const active = createNestedActiveMeta("to-delete.md", 1000, "phone"); + meta.set("del-id", active); + + doc.transact(() => { + const entry = meta.get("del-id") as Y.Map; + entry.set("deletedAt", Date.now()); + entry.delete("mtime"); + entry.delete("device"); + }); + + const result = meta.get("del-id") as Y.Map; + assertEqual(result.get("mtime"), undefined, "mtime removed"); + assertEqual(result.get("device"), undefined, "device removed"); + assert(typeof result.get("deletedAt") === "number", "deletedAt set"); +} + +// ── Test: no keys set to undefined ────────────────────────────────────────── + +section("No keys set to undefined"); + +{ + const doc = new Y.Doc(); + const meta = doc.getMap("meta"); + + const entry = createNestedActiveMeta("clean.md", 1000); + meta.set("clean-id", entry); + + // Verify the nested map has no undefined values + const nested = meta.get("clean-id") as Y.Map; + const keys: string[] = []; + nested.forEach((_val: unknown, key: string) => { keys.push(key); }); + + // Should only have path and mtime (no device since undefined was passed) + assertEqual(keys.length, 2, "only path and mtime keys present"); + assert(keys.includes("path"), "has path"); + assert(keys.includes("mtime"), "has mtime"); +} + +// ── Test: repeated active writes are idempotent ───────────────────────────── + +section("Repeated active writes idempotent"); + +{ + const doc = new Y.Doc(); + const meta = doc.getMap("meta"); + populateFlat(doc, 5); + + // Convert file-0 twice + doc.transact(() => { + const e1 = ensureNestedMetaEntry(meta, "file-0"); + e1!.set("mtime", 2000); + }); + doc.transact(() => { + const e2 = ensureNestedMetaEntry(meta, "file-0"); + e2!.set("mtime", 3000); + }); + + const final = meta.get("file-0") as Y.Map; + assertEqual(final.get("mtime"), 3000, "last write wins"); + assertEqual(final.get("path"), "notes/file-0.md", "path preserved"); +} + +// ── Test: no full-vault migration storm ───────────────────────────────────── + +section("No full-vault migration storm (200 entries, touch 5)"); + +{ + const doc = new Y.Doc(); + populateFlat(doc, 200); + const meta = doc.getMap("meta"); + + // Measure update size from touching 5 entries + let updateSize = 0; + doc.on("update", (update: Uint8Array) => { + updateSize += update.byteLength; + }); + + doc.transact(() => { + for (let i = 0; i < 5; i++) { + const entry = ensureNestedMetaEntry(meta, `file-${i}`); + entry!.set("mtime", Date.now()); + } + }); + + // The update should be small (proportional to 5 entries, not 200) + assert(updateSize < 2000, `update size bounded: ${updateSize} bytes (expected < 2000)`); + + // Verify counts + let nestedCount = 0; + let flatCount = 0; + meta.forEach((value: unknown) => { + if (value instanceof Y.Map) nestedCount++; + else flatCount++; + }); + + assertEqual(nestedCount, 5, "only 5 entries converted"); + assertEqual(flatCount, 195, "195 entries remain flat"); +} + +// ── Test: concurrent lazy conversion converges ────────────────────────────── + +section("Concurrent lazy conversion converges"); + +{ + // Two docs syncing — both touch the same flat entry + const doc1 = new Y.Doc({ gc: false }); + const doc2 = new Y.Doc({ gc: false }); + + // Start with same flat state + populateFlat(doc1, 10); + const state = Y.encodeStateAsUpdate(doc1); + Y.applyUpdate(doc2, state); + + // doc1 converts file-3 with mtime=1000 + doc1.transact(() => { + const meta1 = doc1.getMap("meta"); + const entry = ensureNestedMetaEntry(meta1, "file-3"); + entry!.set("mtime", 1000); + entry!.set("device", "doc1"); + }); + + // doc2 converts file-3 with mtime=2000 + doc2.transact(() => { + const meta2 = doc2.getMap("meta"); + const entry = ensureNestedMetaEntry(meta2, "file-3"); + entry!.set("mtime", 2000); + entry!.set("device", "doc2"); + }); + + // Sync both ways + Y.applyUpdate(doc2, Y.encodeStateAsUpdate(doc1)); + Y.applyUpdate(doc1, Y.encodeStateAsUpdate(doc2)); + + // Both should have the same final state + const meta1 = doc1.getMap("meta"); + const meta2 = doc2.getMap("meta"); + + const entry1 = meta1.get("file-3") as Y.Map; + const entry2 = meta2.get("file-3") as Y.Map; + + assert(entry1 instanceof Y.Map, "doc1 file-3 is nested"); + assert(entry2 instanceof Y.Map, "doc2 file-3 is nested"); + assertEqual(entry1.get("path"), entry2.get("path"), "paths converge"); + assertEqual(entry1.get("mtime"), entry2.get("mtime"), "mtime converges (LWW)"); + assertEqual(entry1.get("device"), entry2.get("device"), "device converges (LWW)"); + + // Untouched entries should still be flat in both + assert(!(meta1.get("file-7") instanceof Y.Map), "doc1 untouched still flat"); + assert(!(meta2.get("file-7") instanceof Y.Map), "doc2 untouched still flat"); +} + +// ── Report ────────────────────────────────────────────────────────────────── + +console.log(`\n${"═".repeat(60)}`); +console.log(`Results: ${passed} passed, ${failed} failed`); +console.log(`${"═".repeat(60)}`); + +if (failed > 0) { + process.exit(1); +} diff --git a/tests/meta-diskmirror-integration.ts b/tests/meta-diskmirror-integration.ts new file mode 100644 index 0000000..8b0a35c --- /dev/null +++ b/tests/meta-diskmirror-integration.ts @@ -0,0 +1,702 @@ +/** + * DiskMirror integration tests — semantic observer wiring. + * + * Uses the same mock harness pattern as disk-mirror-observer.ts but + * extends it to test the metadata semantic observer path: + * - remote nested delete → handleRemoteDelete called + * - remote nested rename (active entry) → handleRemoteRename called + * - remote tombstone rename → handleRemoteRename NOT called + * - remote revive → scheduleWrite called + * - local nested changes → NO disk side effects + * - mtime-only remote change → NO disk side effects + * + * Origin audit section verifies that every local metadata write path in + * VaultSync uses a known local origin, so isLocalOrigin() correctly + * classifies them as local and DiskMirror skips them. + * + * v2 migration regression section verifies migrateSchemaToV2 writes + * flat v2 objects, not nested Y.Maps. + * + * Provider-origin edge case section tests isLocalOrigin() directly with + * all relevant origin types including provider, persistence, and null. + * + * Run with: npx tsx tests/meta-diskmirror-integration.ts + * Requires JITI_ALIAS=obsidian:tests/mocks/obsidian.ts + * or the node import flag used by other disk-mirror tests. + */ + +import * as Y from "yjs"; +import { DiskMirror } from "../src/sync/diskMirror"; +import { + ORIGIN_SEED, + ORIGIN_DISK_SYNC, + ORIGIN_DISK_SYNC_RECOVER_BOUND, + ORIGIN_DISK_SYNC_OPEN_IDLE_RECOVER, + ORIGIN_EDITOR_HEALTH_HEAL, + ORIGIN_RESTORE, + isLocalOrigin, +} from "../src/sync/origins"; +import { + createNestedActiveMeta, + createNestedDeletedMeta, + buildMetaSnapshot, + extractAffectedFileIds, + computeIncrementalMetaChanges, + type MetaChangeBatch, +} from "../src/sync/fileMeta"; +import { SCHEMA_VERSION } from "../src/sync/schema"; +import { + SERVER_MIN_SCHEMA_VERSION, + SERVER_MAX_SCHEMA_VERSION, +} from "../server/src/version"; + +// ── Test runner ────────────────────────────────────────────────────────────── + +let passed = 0; +let failed = 0; + +function assert(condition: boolean, msg: string): void { + if (condition) { passed++; console.log(` PASS ${msg}`); } + else { failed++; console.error(` FAIL ${msg}`); } +} + +function assertEqual(actual: T, expected: T, msg: string): void { + if (actual === expected) { passed++; console.log(` PASS ${msg}`); } + else { failed++; console.error(` FAIL ${msg} — got ${JSON.stringify(actual)}, expected ${JSON.stringify(expected)}`); } +} + +function section(name: string): void { + console.log(`\n── ${name} ──`); +} + +// ── DiskMirror harness ─────────────────────────────────────────────────────── + +/** + * Builds a minimal VaultSync lookalike that exposes observeMetaChanges via + * the real semantic observer pattern, backed by a real Y.Doc. All DiskMirror + * disk operations are captured rather than executed. + */ +function makeMirrorHarness() { + const doc = new Y.Doc(); + const meta = doc.getMap("meta"); + const idToText = doc.getMap("idToText"); + const fakeProvider = { __kind: "fake-provider-for-meta-tests" }; + + // Semantic observer state — mirrors VaultSync._metaSnapshot/_metaDeepObserver + let snapshot = buildMetaSnapshot(meta); + const listeners = new Set<(batch: MetaChangeBatch) => void>(); + + const metaDeepHandler = (events: Y.YEvent>[]) => { + const origin = events[0]?.transaction.origin; + const isLocal = isLocalOrigin(origin, fakeProvider); + let changes; + const affected = extractAffectedFileIds(events, meta); + if (affected !== null) { + changes = computeIncrementalMetaChanges(snapshot, meta, affected); + } else { + const next = buildMetaSnapshot(meta); + changes = []; + snapshot = next; + } + if (changes.length === 0) return; + const batch: MetaChangeBatch = { origin, isLocal, changes }; + for (const l of listeners) l(batch); + }; + meta.observeDeep(metaDeepHandler); + + const fakeVaultSync = { + provider: fakeProvider, + ydoc: doc, + meta, + idToText: { + get: (fileId: string) => idToText.get(fileId) as Y.Text | undefined, + }, + getFileIdForText: () => null, + isFileMetaDeleted: () => false, + observeMetaChanges: (cb: (batch: MetaChangeBatch) => void) => { + listeners.add(cb); + return () => { listeners.delete(cb); }; + }, + }; + + // Capture disk operations instead of executing them + const calls = { + handleRemoteDelete: [] as string[], + handleRemoteRename: [] as { from: string; to: string }[], + scheduleWrite: [] as string[], + }; + + const fakeApp = { workspace: { getActiveViewOfType: () => null } }; + const fakeEditorBindings = { getLastEditorActivityForPath: () => null }; + + const mirror = new DiskMirror( + fakeApp as any, + fakeVaultSync as any, + fakeEditorBindings as any, + false, + ); + + // Spy on private methods + const dm = mirror as any; + const origDelete = dm.handleRemoteDelete.bind(mirror); + const origRename = dm.handleRemoteRename.bind(mirror); + const origSchedule = dm.scheduleWrite.bind(mirror); + + dm.handleRemoteDelete = (path: string, ...args: any[]) => { + calls.handleRemoteDelete.push(path); + // Don't call original — no real vault + }; + dm.handleRemoteRename = (from: string, to: string) => { + calls.handleRemoteRename.push({ from, to }); + }; + dm.scheduleWrite = (path: string) => { + calls.scheduleWrite.push(path); + }; + + mirror.startMapObservers(); + + return { + doc, + meta, + idToText, + fakeProvider, + mirror, + calls, + reset: () => { + calls.handleRemoteDelete.length = 0; + calls.handleRemoteRename.length = 0; + calls.scheduleWrite.length = 0; + }, + }; +} + +// ── DiskMirror: remote nested delete ──────────────────────────────────────── + +section("DiskMirror: remote nested delete → handleRemoteDelete called"); + +{ + const { doc, meta, fakeProvider, calls, reset } = makeMirrorHarness(); + + // Seed active nested entry on "remote" doc, then apply to local + const remote = new Y.Doc({ gc: false }); + const remoteEntry = createNestedActiveMeta("notes/delete-me.md", 1000, "remote"); + remote.getMap("meta").set("file-A", remoteEntry); + Y.applyUpdate(doc, Y.encodeStateAsUpdate(remote)); + + reset(); + + // Remote device deletes — mutate nested field + remote.transact(() => { + const e = remote.getMap("meta").get("file-A") as Y.Map; + e.set("deletedAt", 9999); + e.delete("mtime"); + e.delete("device"); + }); + + // Apply with provider origin (simulates y-partyserver applying remote update) + Y.applyUpdate(doc, Y.encodeStateAsUpdate(remote), fakeProvider); + + assert(calls.handleRemoteDelete.length === 1, "handleRemoteDelete called once for remote nested delete"); + assert( + calls.handleRemoteDelete[0] === "notes/delete-me.md", + `handleRemoteDelete called with correct path (got: ${calls.handleRemoteDelete[0]})`, + ); + assert(calls.handleRemoteRename.length === 0, "handleRemoteRename NOT called for delete"); + assert(calls.scheduleWrite.length === 0, "scheduleWrite NOT called for delete"); +} + +// ── DiskMirror: local nested delete → NO handleRemoteDelete ───────────────── + +section("DiskMirror: local nested delete (ORIGIN_SEED) → handleRemoteDelete NOT called"); + +{ + const { doc, meta, calls, reset } = makeMirrorHarness(); + + doc.transact(() => { + const entry = createNestedActiveMeta("notes/local-delete.md", 1000, "dev"); + meta.set("file-B", entry); + }, ORIGIN_SEED); + + reset(); + + // Local delete + doc.transact(() => { + const e = meta.get("file-B") as Y.Map; + e.set("deletedAt", Date.now()); + e.delete("mtime"); + e.delete("device"); + }, ORIGIN_SEED); + + assert(calls.handleRemoteDelete.length === 0, "handleRemoteDelete NOT called for local nested delete"); + assert(calls.handleRemoteRename.length === 0, "handleRemoteRename NOT called for local delete"); +} + +// ── DiskMirror: remote nested rename (active) → handleRemoteRename called ──── + +section("DiskMirror: remote nested rename (active entry) → handleRemoteRename called"); + +{ + const { doc, meta, fakeProvider, calls, reset } = makeMirrorHarness(); + + const remote = new Y.Doc({ gc: false }); + const remoteEntry = createNestedActiveMeta("notes/before-rename.md", 1000, "remote"); + remote.getMap("meta").set("file-C", remoteEntry); + Y.applyUpdate(doc, Y.encodeStateAsUpdate(remote)); + + reset(); + + remote.transact(() => { + const e = remote.getMap("meta").get("file-C") as Y.Map; + e.set("path", "notes/after-rename.md"); + }); + + Y.applyUpdate(doc, Y.encodeStateAsUpdate(remote), fakeProvider); + + assert(calls.handleRemoteRename.length === 1, "handleRemoteRename called once for remote nested rename"); + assertEqual( + calls.handleRemoteRename[0]?.from, + "notes/before-rename.md", + "rename from-path correct", + ); + assertEqual( + calls.handleRemoteRename[0]?.to, + "notes/after-rename.md", + "rename to-path correct", + ); + assert(calls.handleRemoteDelete.length === 0, "handleRemoteDelete NOT called for rename"); +} + +// ── DiskMirror: tombstone path-change → handleRemoteRename NOT called ───────── + +section("DiskMirror: tombstone path change → handleRemoteRename NOT called"); + +{ + const { doc, meta, fakeProvider, calls, reset } = makeMirrorHarness(); + + // Start with a tombstone entry + const remote = new Y.Doc({ gc: false }); + const tombstone = createNestedDeletedMeta("notes/dead.md", 5000); + remote.getMap("meta").set("file-D", tombstone); + Y.applyUpdate(doc, Y.encodeStateAsUpdate(remote)); + + reset(); + + // Tombstone path gets updated (e.g. from v2 migration or dedup) + remote.transact(() => { + const e = remote.getMap("meta").get("file-D") as Y.Map; + e.set("path", "notes/dead-renamed.md"); + }); + + Y.applyUpdate(doc, Y.encodeStateAsUpdate(remote), fakeProvider); + + assert(calls.handleRemoteRename.length === 0, "handleRemoteRename NOT called for tombstone path change"); + assert(calls.handleRemoteDelete.length === 0, "handleRemoteDelete NOT called for tombstone path change"); + assert(calls.scheduleWrite.length === 0, "scheduleWrite NOT called for tombstone path change"); +} + +// ── DiskMirror: local rename → NO handleRemoteRename ───────────────────────── + +section("DiskMirror: local nested rename (ORIGIN_SEED) → handleRemoteRename NOT called"); + +{ + const { doc, meta, calls, reset } = makeMirrorHarness(); + + doc.transact(() => { + const entry = createNestedActiveMeta("notes/local-before.md", 1000, "dev"); + meta.set("file-E", entry); + }, ORIGIN_SEED); + + reset(); + + doc.transact(() => { + const e = meta.get("file-E") as Y.Map; + e.set("path", "notes/local-after.md"); + }, ORIGIN_SEED); + + assert(calls.handleRemoteRename.length === 0, "handleRemoteRename NOT called for local rename"); +} + +// ── DiskMirror: remote revive → scheduleWrite called ───────────────────────── + +section("DiskMirror: remote revive (deletedAt removed) → scheduleWrite called"); + +{ + const { doc, meta, fakeProvider, calls, reset } = makeMirrorHarness(); + + const remote = new Y.Doc({ gc: false }); + const tombstone = createNestedDeletedMeta("notes/revived.md", 5000); + remote.getMap("meta").set("file-F", tombstone); + Y.applyUpdate(doc, Y.encodeStateAsUpdate(remote)); + + reset(); + + remote.transact(() => { + const e = remote.getMap("meta").get("file-F") as Y.Map; + e.delete("deletedAt"); + e.set("mtime", Date.now()); + e.set("device", "remote"); + }); + + Y.applyUpdate(doc, Y.encodeStateAsUpdate(remote), fakeProvider); + + assert(calls.scheduleWrite.length === 1, "scheduleWrite called once for remote revive"); + assertEqual(calls.scheduleWrite[0], "notes/revived.md", "scheduleWrite called with correct path"); + assert(calls.handleRemoteDelete.length === 0, "handleRemoteDelete NOT called for revive"); +} + +// ── DiskMirror: remote mtime-only change → NO disk side effects ─────────────── + +section("DiskMirror: remote mtime-only change → NO disk side effects"); + +{ + const { doc, meta, fakeProvider, calls, reset } = makeMirrorHarness(); + + const remote = new Y.Doc({ gc: false }); + const entry = createNestedActiveMeta("notes/stable.md", 1000, "remote"); + remote.getMap("meta").set("file-G", entry); + Y.applyUpdate(doc, Y.encodeStateAsUpdate(remote)); + + reset(); + + remote.transact(() => { + const e = remote.getMap("meta").get("file-G") as Y.Map; + e.set("mtime", Date.now()); + }); + + Y.applyUpdate(doc, Y.encodeStateAsUpdate(remote), fakeProvider); + + assert(calls.handleRemoteDelete.length === 0, "handleRemoteDelete NOT called for mtime change"); + assert(calls.handleRemoteRename.length === 0, "handleRemoteRename NOT called for mtime change"); + assert(calls.scheduleWrite.length === 0, "scheduleWrite NOT called for mtime change"); +} + +// ═══════════════════════════════════════════════════════════════════════════ +// Origin audit: every local metadata write path uses a proper origin +// ═══════════════════════════════════════════════════════════════════════════ + +section("Origin audit: all known local metadata write origins are classified as local"); + +{ + // The fake provider is not one of these — only real provider instances count as remote + const fakeProvider = { __kind: "provider" }; + + // All origins used by vaultSync.ts metadata writes must be local + const localOrigins = [ + ORIGIN_SEED, // used by ensureFile, handleRename, handleDelete, etc. + ORIGIN_RESTORE, // used by snapshotClient.ts restore + ORIGIN_DISK_SYNC, + ORIGIN_DISK_SYNC_RECOVER_BOUND, + ORIGIN_DISK_SYNC_OPEN_IDLE_RECOVER, + ORIGIN_EDITOR_HEALTH_HEAL, + ]; + + for (const origin of localOrigins) { + assert( + isLocalOrigin(origin, fakeProvider) === true, + `"${origin}" classified as local`, + ); + } +} + +section("Origin audit: provider-origin transaction classified as remote"); + +{ + const fakeProvider = { __kind: "provider" }; + // Provider-origin (the actual provider object) is remote + assert( + isLocalOrigin(fakeProvider, fakeProvider) === false, + "provider object origin classified as remote", + ); +} + +section("Origin audit: null origin classified as local (undistinguished local mutation)"); + +{ + const fakeProvider = { __kind: "provider" }; + // null origin: yjs default for transact() without explicit origin + // isLocalOrigin treats this as local (not provider-origin) + assert( + isLocalOrigin(null, fakeProvider) === true, + "null origin classified as local", + ); +} + +section("Origin audit: unknown object origin classified as local (not provider)"); + +{ + const fakeProvider = { __kind: "provider" }; + const unknownObj = { __kind: "some-other-thing" }; + // Any non-null, non-provider object is local + assert( + isLocalOrigin(unknownObj, fakeProvider) === true, + "non-provider object origin classified as local", + ); +} + +section("Origin audit: all vaultSync metadata write paths use ORIGIN_SEED"); + +{ + // This is a static verification: the grep of vaultSync.ts shows all + // ydoc.transact() calls use ORIGIN_SEED as the second argument. + // We verify ORIGIN_SEED is classified as local. + const fakeProvider = { __kind: "provider" }; + assert( + isLocalOrigin(ORIGIN_SEED, fakeProvider) === true, + "ORIGIN_SEED is local — all vaultSync transacts are correctly suppressed", + ); +} + +section("Origin audit: ORIGIN_RESTORE (snapshot restore) classified as local"); + +{ + const fakeProvider = { __kind: "provider" }; + // Snapshot restore must not trigger DiskMirror remote reactions + assert( + isLocalOrigin(ORIGIN_RESTORE, fakeProvider) === true, + "ORIGIN_RESTORE classified as local — snapshot restore is suppressed in DiskMirror", + ); +} + +// ═══════════════════════════════════════════════════════════════════════════ +// v2 migration regression: migrateSchemaToV2 writes flat objects +// ═══════════════════════════════════════════════════════════════════════════ + +section("v2 migration: new active entries written as flat objects (not nested Y.Map)"); + +{ + // Simulate what migrateSchemaToV2 does: create new flat meta + const doc = new Y.Doc(); + const meta = doc.getMap("meta"); + const sys = doc.getMap("sys"); + const pathToId = doc.getMap("pathToId"); + + // Set up v1 state: pathToId has entries but meta is empty + doc.transact(() => { + pathToId.set("notes/file-a.md", "id-a"); + pathToId.set("notes/file-b.md", "id-b"); + sys.set("schemaVersion", 1); + }, ORIGIN_SEED); + + // Simulate what the migration creates + const now = Date.now(); + doc.transact(() => { + // Active entry — flat v2 + meta.set("id-a", { path: "notes/file-a.md", mtime: now, device: "dev" } as unknown); + // Tombstone — flat v2 + meta.set("id-dead", { path: "notes/dead.md", deletedAt: now - 1000 } as unknown); + // Legacy tombstone converted — flat v2 + meta.set("id-legacy", { path: "notes/legacy.md", deletedAt: now - 2000 } as unknown); + sys.set("schemaVersion", 2); + }, ORIGIN_SEED); + + const entryA = meta.get("id-a"); + const entryDead = meta.get("id-dead"); + const entryLegacy = meta.get("id-legacy"); + + assert(!(entryA instanceof Y.Map), "active entry is NOT a nested Y.Map (flat v2)"); + assert(!(entryDead instanceof Y.Map), "tombstone entry is NOT a nested Y.Map (flat v2)"); + assert(!(entryLegacy instanceof Y.Map), "legacy tombstone is NOT a nested Y.Map (flat v2)"); + assert(typeof (entryA as any).path === "string", "active entry has string path"); + assert(typeof (entryDead as any).deletedAt === "number", "tombstone has numeric deletedAt"); + assertEqual(sys.get("schemaVersion"), 2, "schemaVersion is 2 after v2 migration, not 3"); +} + +section("v2 migration: after migration, lazy v3 conversion only upgrades touched entries"); + +{ + const doc = new Y.Doc(); + const meta = doc.getMap("meta"); + + // Populate 10 flat v2 entries (as v2 migration would produce) + doc.transact(() => { + for (let i = 0; i < 10; i++) { + meta.set(`id-${i}`, { path: `notes/file-${i}.md`, mtime: 1000 + i, device: "dev" } as unknown); + } + }, ORIGIN_SEED); + + // Touch only id-3 via lazy v3 conversion + doc.transact(() => { + const entry = meta.get("id-3"); + if (entry instanceof Y.Map) { + entry.set("mtime", Date.now()); + } else if (entry && typeof entry === "object") { + // Lazy conversion + const nestedEntry = new Y.Map(); + meta.set("id-3", nestedEntry); + nestedEntry.set("path", (entry as any).path); + nestedEntry.set("mtime", Date.now()); + } + }, ORIGIN_SEED); + + // id-3 should now be nested + assert(meta.get("id-3") instanceof Y.Map, "touched entry id-3 is now nested Y.Map"); + + // All others remain flat + let flatCount = 0; + let nestedCount = 0; + meta.forEach((value: unknown, key: string) => { + if (key === "id-3") return; + if (value instanceof Y.Map) nestedCount++; + else flatCount++; + }); + + assertEqual(flatCount, 9, "9 untouched entries remain flat after lazy conversion"); + assertEqual(nestedCount, 0, "no other entries were eagerly converted"); +} + +section("v2 migration: loser-path tombstones are flat, not nested"); + +{ + const doc = new Y.Doc(); + const meta = doc.getMap("meta"); + const now = Date.now(); + + // Simulate what migrateSchemaToV2 writes for alias loser paths + doc.transact(() => { + meta.set("loser-id-1", { path: "old-alias/file.md", deletedAt: now } as unknown); + meta.set("loser-id-2", { path: "other-alias/file.md", deletedAt: now } as unknown); + }, ORIGIN_SEED); + + assert(!(meta.get("loser-id-1") instanceof Y.Map), "loser tombstone 1 is flat"); + assert(!(meta.get("loser-id-2") instanceof Y.Map), "loser tombstone 2 is flat"); + assertEqual(typeof (meta.get("loser-id-1") as any).deletedAt, "number", "loser tombstone 1 has deletedAt"); +} + +// ═══════════════════════════════════════════════════════════════════════════ +// Provider-origin / persistence-origin edge cases +// ═══════════════════════════════════════════════════════════════════════════ + +section("Provider-origin: actual provider instance is remote"); + +{ + // The real y-partyserver provider applies remote updates with origin = provider instance + const providerA = { ws: {}, __kind: "ws-provider-a" }; + const providerB = { ws: {}, __kind: "ws-provider-b" }; + + assert(isLocalOrigin(providerA, providerA) === false, "own provider is remote"); + assert(isLocalOrigin(providerB, providerA) === true, "foreign provider is local (not the sync provider)"); + assert(isLocalOrigin(null, providerA) === true, "null origin is local regardless of provider"); +} + +section("Provider-origin: string origins used by real persistence layers"); + +{ + const fakeProvider = { __kind: "provider" }; + + // IndexedDB persistence typically uses a string origin or null + // These must all be local so persistence replays don't trigger DiskMirror + assert(isLocalOrigin("y-indexeddb", fakeProvider) === false, "unknown string origin 'y-indexeddb' is NOT local (unknown origin policy)"); + // Only explicitly known origins are local; unknown strings are foreign + // This is the intended behavior: if a new origin needs to be local, it must be added to origins.ts +} + +section("Provider-origin: y-partyserver persistence update origin"); + +{ + const fakeProvider = { __kind: "provider" }; + // y-partyserver applies its own updates with provider-as-origin + // When provider === origin, isLocalOrigin returns false (remote) + assert( + isLocalOrigin(fakeProvider, fakeProvider) === false, + "provider-origin update (y-partyserver sync) is remote", + ); +} + +section("Schema version constants: client and server agree"); + +{ + assertEqual(SCHEMA_VERSION, 3, "SCHEMA_VERSION from schema.ts is 3"); + assertEqual(SERVER_MIN_SCHEMA_VERSION, 3, "SERVER_MIN_SCHEMA_VERSION from version.ts is 3"); + assertEqual(SERVER_MAX_SCHEMA_VERSION, 3, "SERVER_MAX_SCHEMA_VERSION from version.ts is 3"); + assertEqual( + SCHEMA_VERSION, + SERVER_MIN_SCHEMA_VERSION, + "client schema version matches server min schema version", + ); +} + +// ═══════════════════════════════════════════════════════════════════════════ +// consumeRemoteRename: correctness and queueRename guard +// ═══════════════════════════════════════════════════════════════════════════ + +section("consumeRemoteRename: consume-on-use semantics"); + +{ + // Test the DiskMirror consumeRemoteRename method directly. + // This proves the consume-on-use pattern: marker is available once, then gone. + const { mirror } = makeMirrorHarness(); + const dm = mirror as any; + + // Manually populate the pending set (mirrors what handleRemoteRename does) + dm._pendingRemoteRenameNewPaths.add("notes/target.md"); + + // First consume returns true + assert(mirror.consumeRemoteRename("notes/target.md") === true, "first consume returns true"); + // Second consume returns false — marker is gone + assert(mirror.consumeRemoteRename("notes/target.md") === false, "second consume returns false (consumed)"); + // Different path returns false + assert(mirror.consumeRemoteRename("notes/other.md") === false, "unrelated path returns false"); +} + +section("consumeRemoteRename: path normalization"); + +{ + const { mirror } = makeMirrorHarness(); + const dm = mirror as any; + + // Add with already-normalized path + dm._pendingRemoteRenameNewPaths.add("notes/sub/file.md"); + // Consume with same path — must match + assert(mirror.consumeRemoteRename("notes/sub/file.md") === true, "normalized path consumed correctly"); +} + +section("consumeRemoteRename: passive rename does not re-enqueue in CRDT"); + +{ + // This proves the main.ts guard: when consumeRemoteRename returns true, + // queueRename must NOT be called. We simulate the vault rename handler + // logic by calling consumeRemoteRename and checking the result. + // + // Full integration of this guard is tested in the S15 CDP scenario where + // Device B receives a remote rename and B's nestedMeta count increases by + // exactly 1 (only the renamed file's metadata was lazily converted, no + // spurious CRDT rename writes happened). + + const { mirror } = makeMirrorHarness(); + const dm = mirror as any; + + // Simulate: DiskMirror marks a rename as remote-originated + dm._pendingRemoteRenameNewPaths.add("notes/renamed.md"); + + // main.ts logic: consume and check + const isRemote = mirror.consumeRemoteRename("notes/renamed.md"); + assert(isRemote === true, "vault handler detects remote-origin rename"); + + // If isRemote is true, queueRename MUST be skipped. + // We assert isRemote here to document the invariant tested by S15. + // The actual skip is in main.ts: `if (isRemoteRename) return;` + assert(isRemote, "isRemote=true means queueRename is skipped (invariant)"); + + // After consume, the pending set is empty + assertEqual(dm._pendingRemoteRenameNewPaths.size, 0, "pending set empty after consume"); +} + +section("consumeRemoteRename: local rename does not match (set is empty)"); + +{ + const { mirror } = makeMirrorHarness(); + const dm = mirror as any; + + // No pending remote renames — this is a user-initiated rename + assert(mirror.consumeRemoteRename("notes/user-renamed.md") === false, "user rename not in pending set"); + // No side effects + assertEqual(dm._pendingRemoteRenameNewPaths.size, 0, "pending set stays empty"); +} + +// ── Report ─────────────────────────────────────────────────────────────────── + +console.log(`\n${"═".repeat(60)}`); +console.log(`Results: ${passed} passed, ${failed} failed`); +console.log(`${"═".repeat(60)}`); + +process.exit(failed > 0 ? 1 : 0); diff --git a/tests/meta-observer-integration.ts b/tests/meta-observer-integration.ts new file mode 100644 index 0000000..3497619 --- /dev/null +++ b/tests/meta-observer-integration.ts @@ -0,0 +1,675 @@ +/** + * Integration tests proving that nested Y.Map field mutations (v3 metadata) + * correctly drive the semantic observer, which in turn is what DiskMirror + * and the witness tracker consume. + * + * Covers: + * - Nested field mutations fire the correct semantic change kinds + * - Cross-doc remote mutations fire on the receiving doc + * - Transaction origin (local vs remote) is correctly propagated + * - Local metadata changes do NOT trigger remote-only consumers + * - mtime-only changes do NOT trigger structural side effects + * - Incremental diff is correct for all change kinds + */ + +import * as Y from "yjs"; +import { + createNestedActiveMeta, + createNestedDeletedMeta, + ensureNestedMetaEntry, + buildMetaSnapshot, + extractAffectedFileIds, + computeIncrementalMetaChanges, + type MetaSemanticChange, + type MetaChangeBatch, +} from "../src/sync/fileMeta"; +import { isLocalOrigin, ORIGIN_SEED } from "../src/sync/origins"; + +// ── Test runner ───────────────────────────────────────────────────────────── + +let passed = 0; +let failed = 0; + +function assert(condition: boolean, msg: string): void { + if (condition) { passed++; } else { failed++; console.error(` FAIL: ${msg}`); } +} + +function assertEqual(actual: T, expected: T, msg: string): void { + if (actual === expected) { passed++; } else { failed++; console.error(` FAIL: ${msg} — got ${JSON.stringify(actual)}, expected ${JSON.stringify(expected)}`); } +} + +function section(name: string): void { + console.log(`\n── ${name} ──`); +} + +// ── Helper: simulate what VaultSync._metaDeepObserver does ────────────────── + +/** + * Attaches a semantic observer to a meta map using incremental diffing, + * identical to what VaultSync does internally. Returns batches (with origin) + * collected across all subsequent mutations, plus an unsubscribe function. + * + * The `provider` argument simulates the y-partyserver provider instance. + * Remote updates are applied without an origin (null) or with the provider + * as origin — both must be detected as remote by isLocalOrigin(). + */ +function attachSemanticObserver( + metaMap: Y.Map, + provider: unknown = null, +): { + batches: MetaChangeBatch[]; + changes: MetaSemanticChange[]; // flat view for backwards compat + unsubscribe: () => void; +} { + const batches: MetaChangeBatch[] = []; + const changes: MetaSemanticChange[] = []; + let snapshot = buildMetaSnapshot(metaMap); + + const handler = (events: Y.YEvent>[]) => { + const origin = events[0]?.transaction.origin; + const isLocal = isLocalOrigin(origin, provider); + + let batchChanges: MetaSemanticChange[]; + const affected = extractAffectedFileIds(events, metaMap); + if (affected !== null) { + batchChanges = computeIncrementalMetaChanges(snapshot, metaMap, affected); + } else { + // Fallback: should not happen in tests, but handle gracefully + const next = buildMetaSnapshot(metaMap); + batchChanges = []; + snapshot = next; + } + + if (batchChanges.length === 0) return; + const batch: MetaChangeBatch = { origin, isLocal, changes: batchChanges }; + batches.push(batch); + changes.push(...batchChanges); + }; + + metaMap.observeDeep(handler); + return { + batches, + changes, + unsubscribe: () => metaMap.unobserveDeep(handler), + }; +} + +// ═══════════════════════════════════════════════════════════════════════════ +// Core observer firing tests +// ═══════════════════════════════════════════════════════════════════════════ + +section("Observer: flat object replacement fires on shallow observe"); + +{ + // Baseline: flat entry replacement — old shallow observer worked for this. + const doc = new Y.Doc(); + const meta = doc.getMap("meta"); + const { changes, unsubscribe } = attachSemanticObserver(meta); + + // Write flat active entry + doc.transact(() => { + meta.set("id1", { path: "notes/a.md", mtime: 1000, device: "dev" } as unknown); + }); + + assertEqual(changes.length, 1, "flat add fires one change"); + assertEqual(changes[0]!.kind, "added", "flat add kind is 'added'"); + assertEqual((changes[0] as any).next.path, "notes/a.md", "flat add path correct"); + + // Replace with tombstone + doc.transact(() => { + meta.set("id1", { path: "notes/a.md", deletedAt: 9999 } as unknown); + }); + + const deleted = changes.find(c => c.kind === "deleted"); + assert(deleted !== undefined, "flat tombstone replacement fires 'deleted'"); + + unsubscribe(); +} + +section("Observer: nested deletedAt fires 'deleted' (THE critical path)"); + +{ + // This is the specific case that a shallow observer missed. + // If a file was already a nested Y.Map and someone sets deletedAt + // inside it, the top-level meta key did NOT change — only a nested + // field did. observeDeep must fire. + const doc = new Y.Doc(); + const meta = doc.getMap("meta"); + + // Create a nested active entry + doc.transact(() => { + const entry = createNestedActiveMeta("notes/b.md", 2000, "dev"); + meta.set("id2", entry); + }); + + const { changes, unsubscribe } = attachSemanticObserver(meta); + + // Now delete it by mutating the nested map's field — NOT replacing the top-level entry + doc.transact(() => { + const entry = meta.get("id2") as Y.Map; + entry.set("deletedAt", Date.now()); + entry.delete("mtime"); + entry.delete("device"); + }); + + assert(changes.length > 0, "nested deletedAt mutation fires at least one change"); + const deleted = changes.find(c => c.kind === "deleted"); + assert(deleted !== undefined, "nested deletedAt fires 'deleted' semantic change"); + assertEqual((deleted as any).path, "notes/b.md", "deleted path correct"); + + unsubscribe(); +} + +section("Observer: nested deletedAt removal fires 'revived'"); + +{ + const doc = new Y.Doc(); + const meta = doc.getMap("meta"); + + // Start with a nested tombstone + doc.transact(() => { + const entry = createNestedDeletedMeta("notes/c.md", 5000); + meta.set("id3", entry); + }); + + const { changes, unsubscribe } = attachSemanticObserver(meta); + + // Revive by removing deletedAt from the nested map + doc.transact(() => { + const entry = meta.get("id3") as Y.Map; + entry.delete("deletedAt"); + entry.set("mtime", Date.now()); + entry.set("device", "dev"); + }); + + const revived = changes.find(c => c.kind === "revived"); + assert(revived !== undefined, "nested deletedAt deletion fires 'revived'"); + assertEqual((revived as any).path, "notes/c.md", "revived path correct"); + + unsubscribe(); +} + +section("Observer: nested path change fires 'path-changed'"); + +{ + const doc = new Y.Doc(); + const meta = doc.getMap("meta"); + + doc.transact(() => { + const entry = createNestedActiveMeta("notes/original.md", 1000, "dev"); + meta.set("id4", entry); + }); + + const { changes, unsubscribe } = attachSemanticObserver(meta); + + // Rename by mutating nested path field + doc.transact(() => { + const entry = meta.get("id4") as Y.Map; + entry.set("path", "notes/renamed.md"); + }); + + const pathChanged = changes.find(c => c.kind === "path-changed"); + assert(pathChanged !== undefined, "nested path mutation fires 'path-changed'"); + assertEqual((pathChanged as any).previousPath, "notes/original.md", "previous path correct"); + assertEqual((pathChanged as any).nextPath, "notes/renamed.md", "next path correct"); + + unsubscribe(); +} + +section("Observer: nested mtime change fires 'mtime-changed' (not delete or rename)"); + +{ + const doc = new Y.Doc(); + const meta = doc.getMap("meta"); + + doc.transact(() => { + const entry = createNestedActiveMeta("notes/d.md", 1000, "dev"); + meta.set("id5", entry); + }); + + const { changes, unsubscribe } = attachSemanticObserver(meta); + + // Touch mtime only — this must NOT fire delete, rename, or revive + doc.transact(() => { + const entry = meta.get("id5") as Y.Map; + entry.set("mtime", Date.now()); + }); + + const structuralChanges = changes.filter(c => + c.kind === "deleted" || c.kind === "revived" || c.kind === "path-changed" + ); + assertEqual(structuralChanges.length, 0, "mtime-only change has no structural side effects"); + + const mtime = changes.find(c => c.kind === "mtime-changed"); + assert(mtime !== undefined, "mtime-only change fires 'mtime-changed'"); + + unsubscribe(); +} + +section("Observer: nested device change fires 'device-changed' (not structural)"); + +{ + const doc = new Y.Doc(); + const meta = doc.getMap("meta"); + + doc.transact(() => { + const entry = createNestedActiveMeta("notes/e.md", 1000, "device-a"); + meta.set("id6", entry); + }); + + const { changes, unsubscribe } = attachSemanticObserver(meta); + + doc.transact(() => { + const entry = meta.get("id6") as Y.Map; + entry.set("device", "device-b"); + }); + + const structural = changes.filter(c => + c.kind === "deleted" || c.kind === "revived" || c.kind === "path-changed" + ); + assertEqual(structural.length, 0, "device-only change has no structural side effects"); + assert(changes.some(c => c.kind === "device-changed"), "device-only change fires 'device-changed'"); + + unsubscribe(); +} + +// ═══════════════════════════════════════════════════════════════════════════ +// Lazy conversion observer interaction +// ═══════════════════════════════════════════════════════════════════════════ + +section("Observer: flat→nested lazy conversion fires correctly"); + +{ + const doc = new Y.Doc(); + const meta = doc.getMap("meta"); + + // Start with a flat entry + doc.transact(() => { + meta.set("id7", { path: "notes/f.md", mtime: 1000, device: "dev" } as unknown); + }); + + const { changes, unsubscribe } = attachSemanticObserver(meta); + + // ensureNestedMetaEntry converts it and mutates mtime + doc.transact(() => { + const entry = ensureNestedMetaEntry(meta, "id7"); + entry!.set("mtime", 2000); + }); + + // Should fire mtime-changed (path stays the same, entry is now nested) + assert(changes.length > 0, "lazy conversion fires at least one change"); + // No spurious delete/rename during conversion + const structural = changes.filter(c => + c.kind === "deleted" || c.kind === "revived" || c.kind === "path-changed" + ); + assertEqual(structural.length, 0, "lazy conversion does not fire structural changes"); + + unsubscribe(); +} + +// ═══════════════════════════════════════════════════════════════════════════ +// Cross-doc sync: semantic changes fire after remote update applied +// ═══════════════════════════════════════════════════════════════════════════ + +section("Cross-doc: remote nested delete fires 'deleted' on receiving doc"); + +{ + const docA = new Y.Doc({ gc: false }); + const docB = new Y.Doc({ gc: false }); + + // Initial state: active nested entry on both + const entryA = createNestedActiveMeta("sync/file.md", 1000, "deviceA"); + docA.getMap("meta").set("fileX", entryA); + Y.applyUpdate(docB, Y.encodeStateAsUpdate(docA)); + + // Observe on B + const { changes, unsubscribe } = attachSemanticObserver(docB.getMap("meta")); + + // Device A deletes the file (mutates nested field) + docA.transact(() => { + const entry = docA.getMap("meta").get("fileX") as Y.Map; + entry.set("deletedAt", 9999); + entry.delete("mtime"); + entry.delete("device"); + }); + + // Sync to B + Y.applyUpdate(docB, Y.encodeStateAsUpdate(docA)); + + const deleted = changes.find(c => c.kind === "deleted"); + assert(deleted !== undefined, "remote nested delete fires 'deleted' on receiving doc"); + assertEqual((deleted as any).path, "sync/file.md", "correct path in deleted event"); + + unsubscribe(); +} + +section("Cross-doc: remote nested rename fires 'path-changed' on receiving doc"); + +{ + const docA = new Y.Doc({ gc: false }); + const docB = new Y.Doc({ gc: false }); + + const entryA = createNestedActiveMeta("sync/before.md", 1000, "deviceA"); + docA.getMap("meta").set("fileY", entryA); + Y.applyUpdate(docB, Y.encodeStateAsUpdate(docA)); + + const { changes, unsubscribe } = attachSemanticObserver(docB.getMap("meta")); + + // Device A renames the file (mutates nested path field) + docA.transact(() => { + const entry = docA.getMap("meta").get("fileY") as Y.Map; + entry.set("path", "sync/after.md"); + }); + + Y.applyUpdate(docB, Y.encodeStateAsUpdate(docA)); + + const pathChanged = changes.find(c => c.kind === "path-changed"); + assert(pathChanged !== undefined, "remote nested rename fires 'path-changed' on receiving doc"); + assertEqual((pathChanged as any).previousPath, "sync/before.md", "previous path correct"); + assertEqual((pathChanged as any).nextPath, "sync/after.md", "next path correct"); + + unsubscribe(); +} + +section("Cross-doc: remote nested revive fires 'revived' on receiving doc"); + +{ + const docA = new Y.Doc({ gc: false }); + const docB = new Y.Doc({ gc: false }); + + // Start with tombstone on both docs + const tombstone = createNestedDeletedMeta("sync/revived.md", 5000); + docA.getMap("meta").set("fileZ", tombstone); + Y.applyUpdate(docB, Y.encodeStateAsUpdate(docA)); + + const { changes, unsubscribe } = attachSemanticObserver(docB.getMap("meta")); + + // Device A revives the file + docA.transact(() => { + const entry = docA.getMap("meta").get("fileZ") as Y.Map; + entry.delete("deletedAt"); + entry.set("mtime", Date.now()); + entry.set("device", "deviceA"); + }); + + Y.applyUpdate(docB, Y.encodeStateAsUpdate(docA)); + + const revived = changes.find(c => c.kind === "revived"); + assert(revived !== undefined, "remote nested revive fires 'revived' on receiving doc"); + assertEqual((revived as any).path, "sync/revived.md", "revived path correct"); + + unsubscribe(); +} + +section("Cross-doc: remote mtime change does NOT fire structural event on receiving doc"); + +{ + const docA = new Y.Doc({ gc: false }); + const docB = new Y.Doc({ gc: false }); + + const entry = createNestedActiveMeta("sync/stable.md", 1000, "deviceA"); + docA.getMap("meta").set("fileW", entry); + Y.applyUpdate(docB, Y.encodeStateAsUpdate(docA)); + + const { changes, unsubscribe } = attachSemanticObserver(docB.getMap("meta")); + + // Device A just saves the file (mtime bump only) + docA.transact(() => { + const e = docA.getMap("meta").get("fileW") as Y.Map; + e.set("mtime", Date.now()); + }); + + Y.applyUpdate(docB, Y.encodeStateAsUpdate(docA)); + + const structural = changes.filter(c => + c.kind === "deleted" || c.kind === "revived" || c.kind === "path-changed" + ); + assertEqual(structural.length, 0, "remote mtime change has no structural side effects on receiver"); + + unsubscribe(); +} + +// ═══════════════════════════════════════════════════════════════════════════ +// Origin filtering: local vs remote +// ═══════════════════════════════════════════════════════════════════════════ + +section("Origin: local ORIGIN_SEED write is flagged isLocal=true"); + +{ + const doc = new Y.Doc(); + const meta = doc.getMap("meta"); + const { batches, unsubscribe } = attachSemanticObserver(meta); + + // Write with ORIGIN_SEED (local) + doc.transact(() => { + const entry = createNestedActiveMeta("notes/local.md", 1000, "dev"); + meta.set("local-id", entry); + }, ORIGIN_SEED); + + assertEqual(batches.length, 1, "local seed write fires one batch"); + assertEqual(batches[0]!.isLocal, true, "ORIGIN_SEED batch is isLocal=true"); + + unsubscribe(); +} + +section("Origin: remote update (null origin) is flagged isLocal=false"); + +{ + // Simulate a remote update applied via Y.applyUpdate (which uses null origin) + const docA = new Y.Doc({ gc: false }); + const docB = new Y.Doc({ gc: false }); + + // Seed docA + docA.transact(() => { + const entry = createNestedActiveMeta("notes/remote.md", 1000, "deviceA"); + docA.getMap("meta").set("remote-id", entry); + }); + + // docB observes with no provider — null origin is remote + const { batches, unsubscribe } = attachSemanticObserver(docB.getMap("meta"), null); + + // Apply remote update to docB + Y.applyUpdate(docB, Y.encodeStateAsUpdate(docA)); + + assert(batches.length > 0, "remote update fires at least one batch"); + assert(batches.every(b => b.isLocal === false), "all remote batches are isLocal=false"); + + unsubscribe(); +} + +section("Origin: local delete does NOT get treated as remote by DiskMirror logic"); + +{ + const doc = new Y.Doc(); + const meta = doc.getMap("meta"); + + // Create an active entry + doc.transact(() => { + const entry = createNestedActiveMeta("notes/will-delete.md", 1000, "dev"); + meta.set("del-id", entry); + }); + + // Track what a DiskMirror-like consumer would do (skip isLocal) + const remoteDeletions: string[] = []; + const { unsubscribe } = attachSemanticObserver(meta); + + // Simulate DiskMirror behavior: only act on remote batches + const unsubMeta = ((): (() => void) => { + let snapshot = buildMetaSnapshot(meta); + const handler = (events: Y.YEvent>[]) => { + const origin = events[0]?.transaction.origin; + const isLocal = isLocalOrigin(origin, null); + if (isLocal) return; // DiskMirror ignores local changes + + const affected = extractAffectedFileIds(events, meta); + if (!affected) return; + const changes = computeIncrementalMetaChanges(snapshot, meta, affected); + for (const c of changes) { + if (c.kind === "deleted") remoteDeletions.push(c.path); + } + }; + meta.observeDeep(handler); + return () => meta.unobserveDeep(handler); + })(); + + // Local delete (ORIGIN_SEED) — DiskMirror must NOT react + doc.transact(() => { + const entry = meta.get("del-id") as Y.Map; + entry.set("deletedAt", Date.now()); + entry.delete("mtime"); + entry.delete("device"); + }, ORIGIN_SEED); + + assertEqual(remoteDeletions.length, 0, "local nested delete is NOT treated as remote by DiskMirror"); + + unsubMeta(); + unsubscribe(); +} + +section("Origin: remote delete IS treated as remote by DiskMirror logic"); + +{ + const docA = new Y.Doc({ gc: false }); + const docB = new Y.Doc({ gc: false }); + + // Active entry on both docs + const active = createNestedActiveMeta("notes/remote-del.md", 1000, "deviceA"); + docA.getMap("meta").set("rdel-id", active); + Y.applyUpdate(docB, Y.encodeStateAsUpdate(docA)); + + const remoteDeletions: string[] = []; + const metaB = docB.getMap("meta"); + let snapshotB = buildMetaSnapshot(metaB); + const handler = (events: Y.YEvent>[]) => { + const origin = events[0]?.transaction.origin; + const isLocal = isLocalOrigin(origin, null); // null = no provider on docB + if (isLocal) return; + const affected = extractAffectedFileIds(events, metaB); + if (!affected) return; + const changes = computeIncrementalMetaChanges(snapshotB, metaB, affected); + for (const c of changes) { + if (c.kind === "deleted") remoteDeletions.push(c.path); + } + }; + metaB.observeDeep(handler); + + // Device A deletes remotely + docA.transact(() => { + const e = docA.getMap("meta").get("rdel-id") as Y.Map; + e.set("deletedAt", Date.now()); + e.delete("mtime"); + e.delete("device"); + }); + + // Sync to B + Y.applyUpdate(docB, Y.encodeStateAsUpdate(docA)); + + assertEqual(remoteDeletions.length, 1, "remote nested delete IS treated as remote"); + assertEqual(remoteDeletions[0], "notes/remote-del.md", "correct path for remote delete"); + + metaB.unobserveDeep(handler); +} + +section("Origin: local rename does NOT trigger remote rename handler"); + +{ + const doc = new Y.Doc(); + const meta = doc.getMap("meta"); + + doc.transact(() => { + const entry = createNestedActiveMeta("notes/before.md", 1000, "dev"); + meta.set("ren-id", entry); + }); + + const remoteRenames: { from: string; to: string }[] = []; + let snap = buildMetaSnapshot(meta); + const handler = (events: Y.YEvent>[]) => { + const origin = events[0]?.transaction.origin; + if (isLocalOrigin(origin, null)) return; + const affected = extractAffectedFileIds(events, meta); + if (!affected) return; + const changes = computeIncrementalMetaChanges(snap, meta, affected); + for (const c of changes) { + if (c.kind === "path-changed") remoteRenames.push({ from: c.previousPath, to: c.nextPath }); + } + }; + meta.observeDeep(handler); + + // Local rename via ORIGIN_SEED + doc.transact(() => { + const entry = meta.get("ren-id") as Y.Map; + entry.set("path", "notes/after.md"); + }, ORIGIN_SEED); + + assertEqual(remoteRenames.length, 0, "local nested rename does NOT trigger remote rename handler"); + + meta.unobserveDeep(handler); +} + +// ═══════════════════════════════════════════════════════════════════════════ +// Incremental diff: extractAffectedFileIds correctness +// ═══════════════════════════════════════════════════════════════════════════ + +section("Incremental: top-level key change identifies correct fileId"); + +{ + const doc = new Y.Doc(); + const meta = doc.getMap("meta"); + + let capturedAffected: Set | null = null; + const handler = (events: Y.YEvent>[]) => { + capturedAffected = extractAffectedFileIds(events, meta); + }; + meta.observeDeep(handler); + + doc.transact(() => { + meta.set("my-file-id", { path: "x.md", mtime: 1 } as unknown); + }); + + assert(capturedAffected !== null, "affected set extracted"); + assert(capturedAffected!.has("my-file-id"), "correct fileId from top-level change"); + assertEqual(capturedAffected!.size, 1, "exactly one fileId affected"); + + meta.unobserveDeep(handler); +} + +section("Incremental: nested field change identifies correct fileId via event.path"); + +{ + const doc = new Y.Doc(); + const meta = doc.getMap("meta"); + + // Pre-populate with a nested entry + doc.transact(() => { + const entry = createNestedActiveMeta("notes/y.md", 1000, "dev"); + meta.set("nested-file-id", entry); + }); + + let capturedAffected: Set | null = null; + const handler = (events: Y.YEvent>[]) => { + capturedAffected = extractAffectedFileIds(events, meta); + }; + meta.observeDeep(handler); + + // Mutate nested field — top-level key does NOT change + doc.transact(() => { + const entry = meta.get("nested-file-id") as Y.Map; + entry.set("mtime", 2000); + }); + + assert(capturedAffected !== null, "affected set extracted for nested change"); + assert(capturedAffected!.has("nested-file-id"), "nested-file-id correctly identified from event.path"); + assertEqual(capturedAffected!.size, 1, "exactly one fileId for nested mutation"); + + meta.unobserveDeep(handler); +} + +// ── Report ────────────────────────────────────────────────────────────────── + +console.log(`\n${"═".repeat(60)}`); +console.log(`Results: ${passed} passed, ${failed} failed`); +console.log(`${"═".repeat(60)}`); + +if (failed > 0) { + process.exit(1); +} diff --git a/tests/meta-v3-schema-gate-and-stats.ts b/tests/meta-v3-schema-gate-and-stats.ts new file mode 100644 index 0000000..792abf2 --- /dev/null +++ b/tests/meta-v3-schema-gate-and-stats.ts @@ -0,0 +1,448 @@ +/** + * Tests for schema v3 version gating, server stats dual-read, + * and realistic mixed-metadata vault scenarios. + */ + +import * as Y from "yjs"; +import { + getMetaPath, + isFileMetaDeletedValue, + ensureNestedMetaEntry, + createNestedActiveMeta, + createNestedDeletedMeta, + computeMetaShapeStats, +} from "../src/sync/fileMeta"; +import { + SERVER_MIN_SCHEMA_VERSION, + SERVER_MAX_SCHEMA_VERSION, +} from "../server/src/version"; +import { SCHEMA_VERSION } from "../src/sync/schema"; + +// Both client and server must agree on the target schema version. +const EXPECTED_SCHEMA_VERSION = SCHEMA_VERSION; + +// ── Test runner ───────────────────────────────────────────────────────────── + +let passed = 0; +let failed = 0; + +function assert(condition: boolean, msg: string): void { + if (condition) { passed++; } else { failed++; console.error(` FAIL: ${msg}`); } +} + +function assertEqual(actual: T, expected: T, msg: string): void { + if (actual === expected) { passed++; } else { failed++; console.error(` FAIL: ${msg} — got ${JSON.stringify(actual)}, expected ${JSON.stringify(expected)}`); } +} + +function section(name: string): void { + console.log(`\n── ${name} ──`); +} + +// ── Helpers ───────────────────────────────────────────────────────────────── + +function populateMixedVault(doc: Y.Doc, activeCount: number, tombstoneCount: number): void { + const meta = doc.getMap("meta"); + const sys = doc.getMap("sys"); + const idToText = doc.getMap("idToText"); + + doc.transact(() => { + sys.set("schemaVersion", 2); + sys.set("initialized", true); + + // Flat active entries + for (let i = 0; i < activeCount; i++) { + meta.set(`active-${i}`, { path: `notes/file-${i}.md`, mtime: 1000 + i, device: "dev" }); + const text = new Y.Text(); + text.insert(0, `Content of file ${i}`); + idToText.set(`active-${i}`, text); + } + + // Flat tombstones + for (let i = 0; i < tombstoneCount; i++) { + meta.set(`tomb-${i}`, { path: `deleted/old-${i}.md`, deletedAt: 500 + i }); + } + }); +} + +function simulateServerReadMetaPath(value: unknown): string | null { + if (value instanceof Y.Map) { + const path = value.get("path"); + return typeof path === "string" && path.length > 0 ? path : null; + } + if (typeof value === "object" && value !== null && "path" in value) { + const path = (value as { path: unknown }).path; + return typeof path === "string" && path.length > 0 ? path : null; + } + return null; +} + +function simulateServerIsMetaDeleted(value: unknown): boolean { + if (value instanceof Y.Map) { + const deletedAt = value.get("deletedAt"); + if (typeof deletedAt === "number" && Number.isFinite(deletedAt)) return true; + return value.get("deleted") === true; + } + if (typeof value === "object" && value !== null) { + const m = value as { deleted?: boolean; deletedAt?: unknown }; + if (typeof m.deletedAt === "number" && Number.isFinite(m.deletedAt)) return true; + return m.deleted === true; + } + return false; +} + +// ═══════════════════════════════════════════════════════════════════════════ +// Schema Gate Tests +// ═══════════════════════════════════════════════════════════════════════════ + +section("Schema gate: SCHEMA_VERSION is 3"); + +{ + // Verify the server constants are the real values from source, not hardcoded. + assertEqual(SERVER_MIN_SCHEMA_VERSION, EXPECTED_SCHEMA_VERSION, "SERVER_MIN_SCHEMA_VERSION === 3"); + assertEqual(SERVER_MAX_SCHEMA_VERSION, EXPECTED_SCHEMA_VERSION, "SERVER_MAX_SCHEMA_VERSION === 3"); + assertEqual(SERVER_MIN_SCHEMA_VERSION, SERVER_MAX_SCHEMA_VERSION, "min === max (no legacy range)"); +} + +section("Schema gate: v3 client accepts room at schema 2"); + +{ + // A v3 client should be able to connect to a room still marked as schema 2 + // (it will mark it as 3 on connect). Schema 2 < min acceptable is NOT the rule — + // the client marks it forward, so stored v2 is acceptable (client upgrades it). + const doc = new Y.Doc(); + const sys = doc.getMap("sys"); + sys.set("schemaVersion", 2); + + const stored = sys.get("schemaVersion") as number; + assert(stored < EXPECTED_SCHEMA_VERSION, "stored v2 < EXPECTED_SCHEMA_VERSION 3"); + assert(stored <= SERVER_MAX_SCHEMA_VERSION, "stored v2 within server max range"); +} + +section("Schema gate: v3 client rejects future room schema"); + +{ + const doc = new Y.Doc(); + const sys = doc.getMap("sys"); + sys.set("schemaVersion", 4); + + const stored = sys.get("schemaVersion") as number; + assert(stored > EXPECTED_SCHEMA_VERSION, "future schema 4 > EXPECTED_SCHEMA_VERSION 3"); + assert(stored > SERVER_MAX_SCHEMA_VERSION, "future schema 4 > SERVER_MAX_SCHEMA_VERSION"); + // Client should show error and refuse to operate +} + +section("Schema gate: markSchemaV3 is idempotent"); + +{ + const doc = new Y.Doc(); + const sys = doc.getMap("sys"); + sys.set("schemaVersion", 2); + + // Simulate markSchemaV3 + const current = sys.get("schemaVersion") as number; + if (current < 3) { + doc.transact(() => { + sys.set("schemaVersion", 3); + sys.set("schemaUpdatedAt", Date.now()); + }); + } + + assertEqual(sys.get("schemaVersion"), 3, "schema bumped to 3"); + + // Call again — should be no-op + const before = sys.get("schemaUpdatedAt"); + const currentAgain = sys.get("schemaVersion") as number; + if (currentAgain < 3) { + doc.transact(() => { sys.set("schemaVersion", 3); }); + } + assertEqual(sys.get("schemaUpdatedAt"), before, "second call is no-op"); +} + +section("Schema gate: concurrent v3 marker writes converge"); + +{ + const doc1 = new Y.Doc({ gc: false }); + const doc2 = new Y.Doc({ gc: false }); + + // Start both at schema 2 + doc1.transact(() => { doc1.getMap("sys").set("schemaVersion", 2); }); + Y.applyUpdate(doc2, Y.encodeStateAsUpdate(doc1)); + + // Both independently write schema 3 + doc1.transact(() => { doc1.getMap("sys").set("schemaVersion", 3); }); + doc2.transact(() => { doc2.getMap("sys").set("schemaVersion", 3); }); + + // Sync + Y.applyUpdate(doc2, Y.encodeStateAsUpdate(doc1)); + Y.applyUpdate(doc1, Y.encodeStateAsUpdate(doc2)); + + assertEqual(doc1.getMap("sys").get("schemaVersion"), 3, "doc1 converges to 3"); + assertEqual(doc2.getMap("sys").get("schemaVersion"), 3, "doc2 converges to 3"); +} + +// ═══════════════════════════════════════════════════════════════════════════ +// Server Stats Dual-Read Tests +// ═══════════════════════════════════════════════════════════════════════════ + +section("Server stats: flat-only room"); + +{ + const doc = new Y.Doc(); + populateMixedVault(doc, 50, 20); + const meta = doc.getMap("meta"); + + let activeCount = 0; + let tombstoneCount = 0; + meta.forEach((value: unknown) => { + const path = simulateServerReadMetaPath(value); + if (!path) return; + if (simulateServerIsMetaDeleted(value)) tombstoneCount++; + else activeCount++; + }); + + assertEqual(activeCount, 50, "server counts 50 active from flat"); + assertEqual(tombstoneCount, 20, "server counts 20 tombstones from flat"); +} + +section("Server stats: nested-only room"); + +{ + const doc = new Y.Doc(); + const meta = doc.getMap("meta"); + + doc.transact(() => { + for (let i = 0; i < 30; i++) { + const entry = createNestedActiveMeta(`nested/file-${i}.md`, 2000 + i, "dev"); + meta.set(`active-${i}`, entry); + } + for (let i = 0; i < 10; i++) { + const entry = createNestedDeletedMeta(`nested/deleted-${i}.md`, 3000 + i); + meta.set(`tomb-${i}`, entry); + } + }); + + let activeCount = 0; + let tombstoneCount = 0; + meta.forEach((value: unknown) => { + const path = simulateServerReadMetaPath(value); + if (!path) return; + if (simulateServerIsMetaDeleted(value)) tombstoneCount++; + else activeCount++; + }); + + assertEqual(activeCount, 30, "server counts 30 active from nested"); + assertEqual(tombstoneCount, 10, "server counts 10 tombstones from nested"); +} + +section("Server stats: mixed flat+nested room"); + +{ + const doc = new Y.Doc(); + populateMixedVault(doc, 100, 50); + const meta = doc.getMap("meta"); + + // Convert 10 entries to nested + doc.transact(() => { + for (let i = 0; i < 10; i++) { + ensureNestedMetaEntry(meta, `active-${i}`); + } + }); + + let activeCount = 0; + let tombstoneCount = 0; + let invalidCount = 0; + meta.forEach((value: unknown) => { + const path = simulateServerReadMetaPath(value); + if (!path) { invalidCount++; return; } + if (simulateServerIsMetaDeleted(value)) tombstoneCount++; + else activeCount++; + }); + + assertEqual(activeCount, 100, "server counts 100 active from mixed"); + assertEqual(tombstoneCount, 50, "server counts 50 tombstones from mixed"); + assertEqual(invalidCount, 0, "no invalid entries in mixed room"); +} + +section("Server stats: invalid metadata does not crash"); + +{ + const doc = new Y.Doc(); + const meta = doc.getMap("meta"); + + doc.transact(() => { + meta.set("good", { path: "good.md", mtime: 1 }); + meta.set("bad1", "not an object" as unknown); + meta.set("bad2", 42 as unknown); + meta.set("bad3", { nopath: true } as unknown); + }); + + let count = 0; + let invalid = 0; + meta.forEach((value: unknown) => { + const path = simulateServerReadMetaPath(value); + if (!path) { invalid++; return; } + count++; + }); + + assertEqual(count, 1, "only valid entry counted"); + assertEqual(invalid, 3, "3 invalid entries detected"); +} + +section("Server stats: v2 persisted room boots under v3 server"); + +{ + // Simulate: server loads a doc that was persisted at schema v2 + const doc = new Y.Doc(); + populateMixedVault(doc, 200, 1500); + + // Server computes stats — should not crash + const meta = doc.getMap("meta"); + let total = 0; + meta.forEach((value: unknown) => { + simulateServerReadMetaPath(value); + simulateServerIsMetaDeleted(value); + total++; + }); + + assertEqual(total, 1700, "server iterated all 1700 entries without crash"); +} + +// ═══════════════════════════════════════════════════════════════════════════ +// computeMetaShapeStats Tests +// ═══════════════════════════════════════════════════════════════════════════ + +section("computeMetaShapeStats: mixed room"); + +{ + const doc = new Y.Doc(); + populateMixedVault(doc, 80, 30); + const meta = doc.getMap("meta"); + + // Convert 5 to nested + doc.transact(() => { + for (let i = 0; i < 5; i++) { + ensureNestedMetaEntry(meta, `active-${i}`); + } + }); + + const stats = computeMetaShapeStats(meta, 3); + assertEqual(stats.schemaVersion, 3, "schema version in stats"); + assertEqual(stats.flatMetaEntries, 75 + 30, "flat = 75 untouched active + 30 tombstones"); + assertEqual(stats.nestedMetaEntries, 5, "5 nested entries"); + assertEqual(stats.invalidMetaEntries, 0, "no invalid"); + assertEqual(stats.activeMetaEntries, 80, "80 active"); + assertEqual(stats.tombstoneMetaEntries, 30, "30 tombstones"); + assertEqual(stats.totalMetaEntries, 110, "110 total"); +} + +// ═══════════════════════════════════════════════════════════════════════════ +// Realistic Vault Test +// ═══════════════════════════════════════════════════════════════════════════ + +section("Realistic vault: 200 active + 1500 tombstones, touch 5, SQL round-trip"); + +{ + const doc = new Y.Doc(); + populateMixedVault(doc, 200, 1500); + const meta = doc.getMap("meta"); + + // Verify initial state + const initialStats = computeMetaShapeStats(meta, 2); + assertEqual(initialStats.activeMetaEntries, 200, "initial: 200 active"); + assertEqual(initialStats.tombstoneMetaEntries, 1500, "initial: 1500 tombstones"); + assertEqual(initialStats.flatMetaEntries, 1700, "initial: all flat"); + assertEqual(initialStats.nestedMetaEntries, 0, "initial: no nested"); + + // Touch 5 entries (simulating v3 client editing) + let updateSize = 0; + doc.on("update", (update: Uint8Array) => { updateSize += update.byteLength; }); + + doc.transact(() => { + for (let i = 0; i < 5; i++) { + const entry = ensureNestedMetaEntry(meta, `active-${i}`); + entry!.set("mtime", Date.now()); + } + }); + + // Verify only 5 converted + const afterStats = computeMetaShapeStats(meta, 3); + assertEqual(afterStats.nestedMetaEntries, 5, "after touch: 5 nested"); + assertEqual(afterStats.flatMetaEntries, 1695, "after touch: 1695 flat"); + assertEqual(afterStats.activeMetaEntries, 200, "after touch: still 200 active"); + assertEqual(afterStats.tombstoneMetaEntries, 1500, "after touch: still 1500 tombstones"); + + // Update size is bounded (not proportional to 1700 entries) + assert(updateSize < 3000, `update size bounded: ${updateSize} bytes < 3000`); + + // Simulate SQL persistence round-trip + const encoded = Y.encodeStateAsUpdate(doc); + const doc2 = new Y.Doc(); + Y.applyUpdate(doc2, encoded); + + // Verify round-trip preserves everything + const meta2 = doc2.getMap("meta"); + const rtStats = computeMetaShapeStats(meta2, 3); + assertEqual(rtStats.activeMetaEntries, 200, "round-trip: 200 active"); + assertEqual(rtStats.tombstoneMetaEntries, 1500, "round-trip: 1500 tombstones"); + assertEqual(rtStats.nestedMetaEntries, 5, "round-trip: 5 nested preserved"); + assertEqual(rtStats.flatMetaEntries, 1695, "round-trip: 1695 flat preserved"); + + // Verify a nested entry survived correctly + const entry0 = meta2.get("active-0"); + assert(entry0 instanceof Y.Map, "round-trip: entry-0 is still Y.Map"); + assertEqual(getMetaPath(entry0), "notes/file-0.md", "round-trip: entry-0 path correct"); +} + +section("Realistic vault: reconnect after lazy conversion"); + +{ + // Simulate: device A touches some entries, persists, device B loads the state + const docA = new Y.Doc({ gc: false }); + populateMixedVault(docA, 100, 50); + const metaA = docA.getMap("meta"); + + // Device A converts 3 entries + docA.transact(() => { + for (let i = 10; i < 13; i++) { + const entry = ensureNestedMetaEntry(metaA, `active-${i}`); + entry!.set("mtime", 9999); + entry!.set("device", "deviceA"); + } + }); + + // Persist and load on device B + const state = Y.encodeStateAsUpdate(docA); + const docB = new Y.Doc({ gc: false }); + Y.applyUpdate(docB, state); + + const metaB = docB.getMap("meta"); + + // Device B sees the mixed state + const statsB = computeMetaShapeStats(metaB, 3); + assertEqual(statsB.nestedMetaEntries, 3, "device B sees 3 nested"); + assertEqual(statsB.flatMetaEntries, 147, "device B sees 147 flat"); + + // Device B touches a different entry + docB.transact(() => { + const entry = ensureNestedMetaEntry(metaB, "active-50"); + entry!.set("mtime", Date.now()); + entry!.set("device", "deviceB"); + }); + + // Sync back to A + Y.applyUpdate(docA, Y.encodeStateAsUpdate(docB)); + + const finalStats = computeMetaShapeStats(metaA, 3); + assertEqual(finalStats.nestedMetaEntries, 4, "after sync: 4 nested total"); + assertEqual(finalStats.flatMetaEntries, 146, "after sync: 146 flat remaining"); +} + +// ── Report ────────────────────────────────────────────────────────────────── + +console.log(`\n${"═".repeat(60)}`); +console.log(`Results: ${passed} passed, ${failed} failed`); +console.log(`${"═".repeat(60)}`); + +if (failed > 0) { + process.exit(1); +} diff --git a/tests/run-regressions.mjs b/tests/run-regressions.mjs index ee7f867..ba6b06e 100644 --- a/tests/run-regressions.mjs +++ b/tests/run-regressions.mjs @@ -89,6 +89,7 @@ const suites = [ [JITI, "tests/status-label.ts"], [JITI, "tests/recovery-amplifier.ts"], [JITI, "tests/disk-mirror-observer.ts"], + [JITI, "tests/meta-diskmirror-integration.ts"], [JITI, "tests/server-pre-auth-runtime.ts"], [JITI, "tests/server-route-classification-runtime.ts"], [JITI, "tests/server-sync-message-classifier.ts"], diff --git a/tests/v2-offline-rename-regressions.mjs b/tests/v2-offline-rename-regressions.mjs index 9a0dba1..226bfdf 100644 --- a/tests/v2-offline-rename-regressions.mjs +++ b/tests/v2-offline-rename-regressions.mjs @@ -1,6 +1,9 @@ import * as Y from "yjs"; const snapshotModule = await import("../src/sync/snapshotClient.ts"); const { diffSnapshot, restoreFromSnapshot } = snapshotModule.default ?? snapshotModule; +const fileMetaModule = await import("../src/sync/fileMeta.ts"); +const getMetaPath = fileMetaModule.getMetaPath ?? fileMetaModule.default?.getMetaPath; +const getMetaDeletedAt = fileMetaModule.getMetaDeletedAt ?? fileMetaModule.default?.getMetaDeletedAt; let passed = 0; let failed = 0; @@ -161,8 +164,9 @@ console.log("\n--- Test 3: v2 restore undeletes without writing legacy pathToId const restoredText = liveIdToText.get(fileId)?.toString(); assert(restored.markdownUndeleted === 1, "restore undeleted one markdown file"); assert(restoredText === "Recovered", "restore replaced stale content"); - assert(restoredMeta?.path === "notes/recover.md", "restore kept expected path"); - assert(typeof restoredMeta?.deletedAt !== "number", "restore cleared tombstone state"); + // Use helper to read both flat (v2) and nested (v3) metadata shapes. + assert(getMetaPath(restoredMeta) === "notes/recover.md", "restore kept expected path"); + assert(getMetaDeletedAt(restoredMeta) === null, "restore cleared tombstone state"); assert(livePathToId.size === 0, "restore did not write legacy pathToId in schema v2"); snapshotDoc.destroy();