From 1198e1789a5d5954b9fc095b93bad1ad3e9f8d26 Mon Sep 17 00:00:00 2001 From: Tomasz Zajac Date: Mon, 14 Sep 2026 21:01:48 +0200 Subject: [PATCH] feat: lazy-load node descriptions for md-folder documents MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The md-folder backend (one .md file per node) read and parsed every node's full body into memory the instant a document was opened, and kept it there for the whole session — for a very large model, that's the dominant memory cost, paid up front regardless of how much of the model the user actually looks at. Second step (after #81) of scaling the model to very large sizes. deserializeFromMdFolder gains a `lazy` option: node metadata (frontmatter) still loads eagerly as before, but bodies are left unset and their file paths returned separately (bodyPaths) instead. diagramStore tracks which nodes are still pending (pendingBodyNodeIds) and hydrateNode fetches one node's body on demand (wired to the properties panel opening a node) via a new single-file read path for both Electron (reusing the existing file:read IPC) and web (a new readOneFileFromHandle in webFolder.ts). Nothing in the app tolerated a partially-loaded model before this: the Elements panel search and Quick Search (Cmd/Ctrl+P) both scan every node's description synchronously. Rather than build a permanent in-memory search index (which would just duplicate the same text and give up most of the memory win), the existing instant search keeps working as-is on label/type/ technology (already resident) and simply won't match text in an unopened node's body — degraded, not broken. A new explicit "Search descriptions" action (scanDescriptions) does an opt-in scan of only the still-pending nodes for a query; matches get merged into the live model so opening them next is instant, non-matches are read and discarded rather than cached. The one correctness hazard this had to avoid: saveDocument always writes every node's description back to disk, so serializing an unhydrated node's description (undefined) verbatim would silently wipe its saved content on the next save. Rather than a riskier byte-patching write path, saveDocument transiently re-reads any still-pending node's file just for the purposes of building the write payload — never merged into the live model, so a save can't defeat the laziness, and the serialize/write code path itself is completely unchanged. Scope is the `md` document source only — `fs` (single JSON file) and `ls` (localStorage) are inherently all-or-nothing blobs with nothing to split. The AI query layer (ai/queryLanguage.ts) still sees undefined descriptions for unhydrated nodes on a lazy doc; that's a known, deliberately deferred follow-up rather than something fixed here. Co-Authored-By: Claude Sonnet 5 --- src/renderer/src/components/QuickSearch.tsx | 36 +++++++ src/renderer/src/components/RightPanel.tsx | 53 +++++++++++ src/renderer/src/persist/mdFolder.ts | 48 +++++++++- src/renderer/src/persist/webFolder.ts | 13 +++ src/renderer/src/store/diagramStore.ts | 67 +++++++++++++ src/renderer/src/store/documentStore.ts | 94 ++++++++++++++++-- tests/lazyNodeHydration.test.ts | 100 ++++++++++++++++++++ tests/mdFolderDocument.test.ts | 34 ++++++- tests/mdFolderPersistence.test.ts | 6 +- 9 files changed, 434 insertions(+), 17 deletions(-) create mode 100644 tests/lazyNodeHydration.test.ts diff --git a/src/renderer/src/components/QuickSearch.tsx b/src/renderer/src/components/QuickSearch.tsx index 236c8c2..8df0ce9 100644 --- a/src/renderer/src/components/QuickSearch.tsx +++ b/src/renderer/src/components/QuickSearch.tsx @@ -32,6 +32,9 @@ export function QuickSearch(): React.ReactElement | null { const selectEdge = useDiagramStore((s) => s.selectEdge) const setSelectedNodeIds = useDiagramStore((s) => s.setSelectedNodeIds) const presentationActive = useDiagramStore((s) => s.presentationActive) + const pendingBodyNodeIds = useDiagramStore((s) => s.pendingBodyNodeIds) + const scanDescriptions = useDiagramStore((s) => s.scanDescriptions) + const [descScanning, setDescScanning] = useState(false) const [q, setQ] = useState('') const [hover, setHover] = useState(0) @@ -92,6 +95,17 @@ export function QuickSearch(): React.ReactElement | null { } }, [aiSettings.active]) + // Descriptions on unopened nodes of a lazily-loaded doc aren't in memory + // yet, so `results` below can't see them. This is an explicit opt-in scan + // over just those nodes; matches get merged into c4Nodes by the action, so + // they flow back into `results` automatically once it resolves. + const handleScanDescriptions = useCallback(() => { + const query = q.trim() + if (!query) return + setDescScanning(true) + scanDescriptions(query).finally(() => setDescScanning(false)) + }, [q, scanDescriptions]) + const runAI = useCallback(async (text: string): Promise => { const prompt = text.trim() if (!prompt || aiBusy) return @@ -703,6 +717,28 @@ export function QuickSearch(): React.ReactElement | null { ))} + {/* Explicit opt-in scan of not-yet-opened node descriptions (lazy md-folder docs) */} + {!aiMode && q.trim() !== '' && Object.keys(pendingBodyNodeIds).length > 0 && ( +
e.preventDefault()} + onClick={handleScanDescriptions} + role="button" + tabIndex={-1} + aria-disabled={descScanning} + title={`Also search the ${Object.keys(pendingBodyNodeIds).length} node description(s) not yet opened`} + > + … + + + {descScanning + ? 'Searching descriptions…' + : `Search ${Object.keys(pendingBodyNodeIds).length} unopened description(s)`} + + +
+ )} + {/* Ask AI action — visible while typing in search mode (only when configured) */} {!aiMode && aiConfigured && q.trim() !== '' && !aiBusy && (
s.selectNode) const selectEdge = useDiagramStore((s) => s.selectEdge) const hubTemplates = useDiagramStore((s) => s.hubTemplates) + const pendingBodyNodeIds = useDiagramStore((s) => s.pendingBodyNodeIds) + const hydrateNode = useDiagramStore((s) => s.hydrateNode) + + // Lazily-loaded md-folder docs don't have the description in memory until + // the node is opened — fetch it now. + useEffect(() => { + if (selectedNodeId) hydrateNode(selectedNodeId) + }, [selectedNodeId, hydrateNode]) // ── Node ────────────────────────────────────────────────────────────────── if (selectedNodeId && c4Nodes[selectedNodeId]) { const node = c4Nodes[selectedNodeId] + const descriptionPending = !!pendingBodyNodeIds[node.id] // Generic field renderer — handles built-in typed fields and arbitrary // extra properties stored on the node by the metamodel property definitions. @@ -1672,6 +1681,14 @@ function PropertiesContent({ readOnly = false }: { readOnly?: boolean }) { ) } if (type === 'textarea') { + if (key === 'description' && descriptionPending) { + return ( +
+ + {}} readOnly /> +
+ ) + } return (
@@ -1732,6 +1749,14 @@ function PropertiesContent({ readOnly = false }: { readOnly?: boolean }) { ) } if (p.type === 'textarea') { + if (p.key === 'description' && descriptionPending) { + return ( +
+ + {}} readOnly /> +
+ ) + } return (
@@ -1994,6 +2019,8 @@ export function RightPanel({ readOnly = false, collapsed = false, onToggleCollap const nodesCount = useDiagramStore((s) => Object.keys(s.c4Nodes).length) const relationsCount = useDiagramStore((s) => Object.keys(s.c4Relations).length) const sequencesCount = useDiagramStore((s) => Object.keys(s.sequences).length) + const pendingBodyNodeIds = useDiagramStore((s) => s.pendingBodyNodeIds) + const scanDescriptions = useDiagramStore((s) => s.scanDescriptions) const [openSection, setOpenSection] = useState<'nodes' | 'relations' | 'sequences'>('nodes') // ── Nodes tree filter: free-text search + type chips ────────────────────── @@ -2001,6 +2028,21 @@ export function RightPanel({ readOnly = false, collapsed = false, onToggleCollap const [nodeTypeFilter, setNodeTypeFilter] = useState>(new Set()) const nodeFilterActive = nodeSearch.trim() !== '' || nodeTypeFilter.size > 0 + // Descriptions on unopened nodes of a lazily-loaded doc aren't in memory + // yet, so the instant search below can't see them. "Search descriptions" + // is an explicit opt-in scan over just those nodes (see scanDescriptions). + // Matches get merged into c4Nodes by that action, so they flow back into + // the synchronous search below automatically once it resolves — no local + // result-merging needed here. + const pendingCount = Object.keys(pendingBodyNodeIds).length + const [descScanning, setDescScanning] = useState(false) + const handleScanDescriptions = () => { + const q = nodeSearch.trim() + if (!q) return + setDescScanning(true) + scanDescriptions(q).finally(() => setDescScanning(false)) + } + const { nodesMatchedSet, nodesVisibleSet, nodeTypeCounts } = useMemo(() => { const counts = new Map() for (const n of Object.values(allNodes)) counts.set(n.type, (counts.get(n.type) ?? 0) + 1) @@ -2088,6 +2130,17 @@ export function RightPanel({ readOnly = false, collapsed = false, onToggleCollap onClearFilter={handleClearNodeFilter} /> )} + {nodeFilterActive && pendingCount > 0 && ( + + )} {rootNodes.length === 0 ?
{nodeFilterActive ? 'No matches.' : 'No nodes.'} diff --git a/src/renderer/src/persist/mdFolder.ts b/src/renderer/src/persist/mdFolder.ts index 7fa7e0a..e5381a3 100644 --- a/src/renderer/src/persist/mdFolder.ts +++ b/src/renderer/src/persist/mdFolder.ts @@ -296,10 +296,43 @@ function writeJsonIfPresent( // ─── Deserialize: folder files → DiagramData ───────────────────────────────── -export function deserializeFromMdFolder(files: FolderFiles): DiagramData { +export interface DeserializeMdFolderOptions { + /** + * When true, node bodies (descriptions) are NOT loaded into memory — + * instead their relative file paths are returned via `bodyPaths` so a + * caller can fetch a single node's body on demand later. Everything else + * (frontmatter-derived fields, layout, sidecars) loads exactly as today. + * + * Note: `description` is only ever set on a node when its body is + * non-empty (see below) — so `description === undefined` already + * legitimately means "no description" on a normal (non-lazy) load. + * Callers must track "not yet hydrated" separately (via `bodyPaths`), + * never by checking whether `description` is falsy. + */ + lazy?: boolean +} + +export interface DeserializeMdFolderResult { + data: DiagramData + /** Present only when `lazy: true` — nodeId → relative .md file path, for + * nodes whose body was not loaded. */ + bodyPaths?: Record +} + +/** Extract just the body (description) text from one node's raw .md content. */ +export function extractNodeBody(content: string): string { + return parseMarkdown(content).body +} + +export function deserializeFromMdFolder( + files: FolderFiles, + opts?: DeserializeMdFolderOptions, +): DeserializeMdFolderResult { + const lazy = opts?.lazy ?? false const layout = readJson(files[LAYOUT_FILE]) ?? { nodes: {} } interface ParsedNode { + path: string front: Record body: string /** Directory key that identifies this node as a container (or null). */ @@ -317,13 +350,15 @@ export function deserializeFromMdFolder(files: FolderFiles): DiagramData { const isIndex = path.endsWith('/' + INDEX_BASENAME) if (isIndex) { const ownDirKey = dirname(path) // e.g. nodes/system-a - parsed.push({ front, body, ownDirKey, parentDirKey: dirname(ownDirKey) }) + parsed.push({ path, front, body, ownDirKey, parentDirKey: dirname(ownDirKey) }) parsedByDirKey.set(ownDirKey, String(front.id)) } else { - parsed.push({ front, body, ownDirKey: null, parentDirKey: dirname(path) }) + parsed.push({ path, front, body, ownDirKey: null, parentDirKey: dirname(path) }) } } + const bodyPaths: Record = {} + const nodes: C4Node[] = parsed.map((p) => { const id = String(p.front.id) const parentId = p.parentDirKey === NODES_DIR ? undefined : parsedByDirKey.get(p.parentDirKey) @@ -339,7 +374,10 @@ export function deserializeFromMdFolder(files: FolderFiles): DiagramData { collapsed: !!lay.collapsed, } if (parentId) node.parentId = parentId - if (p.body) node.description = p.body + if (p.body) { + if (lazy) bodyPaths[id] = p.path + else node.description = p.body + } for (const [key, value] of Object.entries(p.front)) { if (key === 'id' || key === 'type' || key === 'label') continue node[key] = value @@ -366,7 +404,7 @@ export function deserializeFromMdFolder(files: FolderFiles): DiagramData { if (layout.defaultPositions) data.defaultPositions = layout.defaultPositions if (layout.defaultViewport !== undefined) data.defaultViewport = layout.defaultViewport - return data + return lazy ? { data, bodyPaths } : { data } } /** True when the file map looks like a Radical md-folder (has the manifest). */ diff --git a/src/renderer/src/persist/webFolder.ts b/src/renderer/src/persist/webFolder.ts index a8c43e9..a9d020a 100644 --- a/src/renderer/src/persist/webFolder.ts +++ b/src/renderer/src/persist/webFolder.ts @@ -91,6 +91,19 @@ export async function readFolderFromHandle(handle: FsDirHandle): Promise { + const parts = relPath.split('/') + let dir = handle + for (let i = 0; i < parts.length - 1; i++) { + dir = await dir.getDirectoryHandle(parts[i]) + } + const fileHandle = await dir.getFileHandle(parts[parts.length - 1]) + const file = await fileHandle.getFile() + return file.text() +} + /** Write a file map into a directory handle, then prune our own stale managed * files (`.md` under `nodes/` and known sidecars) that are no longer present. */ export async function writeFolderToHandle(handle: FsDirHandle, files: FolderFiles): Promise { diff --git a/src/renderer/src/store/diagramStore.ts b/src/renderer/src/store/diagramStore.ts index 5c2fbd7..3f34ddc 100644 --- a/src/renderer/src/store/diagramStore.ts +++ b/src/renderer/src/store/diagramStore.ts @@ -904,6 +904,10 @@ interface DiagramStore { defaultPositions: Record /** Camera state (pan + zoom) for the "All" (default) view */ defaultViewport: { x: number; y: number; zoom: number } | null + /** Node ids on a lazily-loaded md-folder doc whose `description` hasn't + * been fetched from disk yet — see `hydrateNode`. Empty for other + * document sources. */ + pendingBodyNodeIds: Record // ── derived React Flow state ── rfNodes: Node[] @@ -933,6 +937,17 @@ interface DiagramStore { // ── actions: nodes ── addNode: (node: Omit) => string updateNode: (id: string, updates: Partial>) => void + /** Fetch a lazily-loaded node's description from disk and merge it in. + * No-op if the node isn't pending. Not a user edit: doesn't push undo or + * mark a milestone dirty. */ + hydrateNode: (id: string) => void + /** Explicit "search descriptions" action: scans every not-yet-hydrated + * node's body for `query` (case-insensitive substring). Matches are + * merged into the live model (so opening them next is instant); + * non-matches are read but discarded, not cached. Returns matching ids + * (only newly-discovered ones — already-hydrated matches are found by + * the normal synchronous search). */ + scanDescriptions: (query: string) => Promise> removeNode: (id: string) => void toggleCollapse: (id: string) => void @@ -1302,6 +1317,7 @@ export const useDiagramStore = create()( activeViewId: null, defaultPositions: initDefaultPositions, defaultViewport: persisted?.defaultViewport ?? null, + pendingBodyNodeIds: {}, rfNodes: deriveRFNodes(initNodes), rfEdges: deriveRFEdges(initNodes, initRelations), selectedNodeId: null, @@ -1543,6 +1559,48 @@ export const useDiagramStore = create()( } }, + hydrateNode(id) { + if (!get().pendingBodyNodeIds[id]) return + const activeId = documents.getActiveId() + if (!activeId) return + documents.hydrateNodeBody(activeId, id).then((body) => { + // The node (or the whole doc) may have changed/closed while the + // read was in flight — re-check before merging. + if (!get().pendingBodyNodeIds[id]) return + set((state) => { + delete state.pendingBodyNodeIds[id] + const node = state.c4Nodes[id] as (C4Node & Record) | undefined + if (node && body !== undefined) node.description = body + }) + }) + }, + + async scanDescriptions(query) { + const q = query.trim().toLowerCase() + const matches = new Set() + if (!q) return matches + const activeId = documents.getActiveId() + const pendingIds = Object.keys(get().pendingBodyNodeIds) + if (!activeId || pendingIds.length === 0) return matches + const results = await Promise.all(pendingIds.map(async (id) => ({ + id, + body: await documents.hydrateNodeBody(activeId, id), + }))) + set((state) => { + for (const { id, body } of results) { + if (body === undefined) continue + if (!state.pendingBodyNodeIds[id]) continue // resolved another way meanwhile + if (body.toLowerCase().includes(q)) { + matches.add(id) + const node = state.c4Nodes[id] as (C4Node & Record) | undefined + if (node) node.description = body + delete state.pendingBodyNodeIds[id] + } + } + }) + return matches + }, + removeNode(id) { get()._pushUndo() get()._markMilestoneEdit() @@ -4157,7 +4215,16 @@ export const useDiagramStore = create()( const defaultPos = data.defaultPositions ?? snapshotPositions(nodes) const defaultVP: { x: number; y: number; zoom: number } | null = data.defaultViewport ?? null + // Lazily-loaded md-folder docs report which node bodies weren't read + // into `data` — track them so `hydrateNode` knows what to fetch. + const activeDocId = documents.getActiveId() + const pendingBody: Record = {} + if (activeDocId) { + for (const id of documents.getPendingBodyNodeIds(activeDocId)) pendingBody[id] = true + } + set((state) => { + state.pendingBodyNodeIds = pendingBody as any state.c4Nodes = nodes as any state.c4Relations = relations as any state.sequences = sequences as any diff --git a/src/renderer/src/store/documentStore.ts b/src/renderer/src/store/documentStore.ts index 83256f2..e364f61 100644 --- a/src/renderer/src/store/documentStore.ts +++ b/src/renderer/src/store/documentStore.ts @@ -9,15 +9,17 @@ // (plain functions + zustand store) so the diagram store can wire into it. import { create } from 'zustand' -import type { DiagramData } from '../types/c4' +import type { DiagramData, C4Node } from '../types/c4' import { serializeToMdFolder, deserializeFromMdFolder, + extractNodeBody, } from '../persist/mdFolder' import { webFolderSupported, pickWebDirectory, readFolderFromHandle, + readOneFileFromHandle, writeFolderToHandle, verifyPermission, saveHandle, @@ -36,6 +38,12 @@ const LS_LEGACY_KEY = 'radical-diagram-v1' * user's files with an empty/stale model after a reload. */ const connectedWebFolders = new Set() +/** Md-folder docs (source==='md') loaded lazily: docId → nodeId → relative + * .md file path, for node bodies not read into memory at load time. Static + * once populated — "already hydrated" is tracked by the caller (diagramStore), + * not here; re-fetching a path here is idempotent, just redundant I/O. */ +const mdBodyPaths = new Map>() + export type DocumentSource = 'ls' | 'fs' | 'md' export interface DocumentMeta { @@ -121,6 +129,49 @@ function defaultNameFromFolder(folderPath: string): string { return base || folderPath } +/** Fetch one md-folder node's body from disk (Electron file or web handle), + * without caching it anywhere — the caller decides what to do with it. */ +async function fetchNodeBody(id: string, nodeId: string): Promise { + const meta = readIndex().docs.find(d => d.id === id) + if (!meta) return undefined + const relPath = mdBodyPaths.get(id)?.[nodeId] + if (!relPath) return undefined + try { + if (meta.folderPath && window.electronAPI?.readFile) { + const res = await window.electronAPI.readFile(`${meta.folderPath}/${relPath}`) + if (!res.success || res.content === undefined) return undefined + return extractNodeBody(res.content) + } + if (!window.electronAPI?.readFile && webFolderSupported()) { + const handle = await loadHandle(id) + if (!handle) return undefined + const content = await readOneFileFromHandle(handle, relPath) + return extractNodeBody(content) + } + } catch { return undefined } + return undefined +} + +/** For an md-folder save: return `data.nodes`, with any not-yet-hydrated + * node's `description` filled in by a transient re-read of its file. Never + * mutates `data` or caches into the live store — purely for serialization, + * so a node the user never opened can't have its saved content clobbered + * with an empty description. */ +async function nodesForMdSave(id: string, data: DiagramData): Promise { + const pending = mdBodyPaths.get(id) + if (!pending || Object.keys(pending).length === 0) return data.nodes + let patched: C4Node[] | null = null + for (let i = 0; i < data.nodes.length; i++) { + const node = data.nodes[i] + if (node.description !== undefined || !(node.id in pending)) continue + const body = await fetchNodeBody(id, node.id) + if (body === undefined) continue + if (!patched) patched = data.nodes.slice() + patched[i] = { ...node, description: body } + } + return patched ?? data.nodes +} + /** Browser-only file picker used when running outside Electron. Resolves with * `{ name, content }` for the chosen file, or `null` if the user cancels. * Implemented via a transient that we click(). */ @@ -238,10 +289,23 @@ export interface DocumentsAPI { * De-dupes by path. Content is NOT loaded here. */ createMdDocument(folderPath: string): DocumentMeta - /** Read the payload for a document. Async because FS reads cross IPC. */ + /** Read the payload for a document. Async because FS reads cross IPC. + * For md-folder docs, node bodies (descriptions) are loaded lazily — see + * `getPendingBodyNodeIds` / `hydrateNodeBody`. */ loadDocument(id: string): Promise - /** Persist new payload under an existing document. */ + /** Node ids whose body (.md file) was not read into `loadDocument`'s + * result — empty for non-md docs or once nothing is pending. */ + getPendingBodyNodeIds(id: string): string[] + + /** Fetch one node's body on demand (md-folder docs only). Pure fetch — + * does not cache or mutate any store; the caller merges the result. */ + hydrateNodeBody(id: string, nodeId: string): Promise + + /** Persist new payload under an existing document. For md-folder docs, + * any node whose body was never hydrated is re-read from disk just for + * serialization (never cached into the live model) so an unopened + * node's saved content is never clobbered with an empty description. */ saveDocument(id: string, data: DiagramData): Promise /** Update the display name. (Does NOT rename files on disk.) */ @@ -378,7 +442,11 @@ export const documents: DocumentsAPI = { const res = await window.electronAPI.readFolder(meta.folderPath) if (!res.success || !res.files) return null if (Object.keys(res.files).length === 0) return null // empty/new folder - try { return deserializeFromMdFolder(res.files) } catch { return null } + try { + const { data, bodyPaths } = deserializeFromMdFolder(res.files, { lazy: true }) + mdBodyPaths.set(id, bodyPaths ?? {}) + return data + } catch { return null } } if (meta.source === 'md' && !window.electronAPI?.readFolder && webFolderSupported()) { const handle = await loadHandle(id) @@ -388,12 +456,22 @@ export const documents: DocumentsAPI = { try { const files = await readFolderFromHandle(handle) if (Object.keys(files).length === 0) return null // empty/new folder - return deserializeFromMdFolder(files) + const { data, bodyPaths } = deserializeFromMdFolder(files, { lazy: true }) + mdBodyPaths.set(id, bodyPaths ?? {}) + return data } catch { return null } } return null }, + getPendingBodyNodeIds(id) { + return Object.keys(mdBodyPaths.get(id) ?? {}) + }, + + async hydrateNodeBody(id, nodeId) { + return fetchNodeBody(id, nodeId) + }, + async saveDocument(id, data) { const idx = readIndex() const meta = idx.docs.find(d => d.id === id) @@ -408,7 +486,8 @@ export const documents: DocumentsAPI = { return } } else if (meta.source === 'md' && meta.folderPath && window.electronAPI?.writeFolder) { - const files = serializeToMdFolder(data, meta.name) + const nodes = await nodesForMdSave(id, data) + const files = serializeToMdFolder({ ...data, nodes }, meta.name) const res = await window.electronAPI.writeFolder(meta.folderPath, files) if (!res.success) { console.warn('[documentStore] folder write failed for', meta.folderPath, res.error) @@ -425,7 +504,8 @@ export const documents: DocumentsAPI = { const handle = await loadHandle(id) if (!handle) return try { - await writeFolderToHandle(handle, serializeToMdFolder(data, meta.name)) + const nodes = await nodesForMdSave(id, data) + await writeFolderToHandle(handle, serializeToMdFolder({ ...data, nodes }, meta.name)) } catch (e) { console.warn('[documentStore] web folder write failed:', e) return diff --git a/tests/lazyNodeHydration.test.ts b/tests/lazyNodeHydration.test.ts new file mode 100644 index 0000000..dc25160 --- /dev/null +++ b/tests/lazyNodeHydration.test.ts @@ -0,0 +1,100 @@ +import { describe, it, expect, beforeEach, vi } from 'vitest' +import { documents } from '../src/renderer/src/store/documentStore' +import { useDiagramStore } from '../src/renderer/src/store/diagramStore' +import type { DiagramData } from '../src/renderer/src/types/c4' + +class MemLS { + private map = new Map() + getItem(k: string): string | null { return this.map.has(k) ? this.map.get(k)! : null } + setItem(k: string, v: string): void { this.map.set(k, String(v)) } + removeItem(k: string): void { this.map.delete(k) } + clear(): void { this.map.clear() } + key(i: number): string | null { return [...this.map.keys()][i] ?? null } + get length(): number { return this.map.size } +} + +function installFolderApi(): { disk: Record } { + const disk: Record = {} + ;(globalThis as any).window.electronAPI = { + openFolder: vi.fn(async () => ({ success: true, folderPath: '/tmp/model', files: { ...disk } })), + pickFolder: vi.fn(async () => ({ success: true, folderPath: '/tmp/model' })), + readFolder: vi.fn(async () => ({ success: true, files: { ...disk } })), + readFile: vi.fn(async (filePath: string) => { + const rel = filePath.replace(/^\/tmp\/model\//, '') + return rel in disk ? { success: true, content: disk[rel] } : { success: false } + }), + writeFolder: vi.fn(async (_path: string, files: Record) => { + for (const k of Object.keys(disk)) delete disk[k] + Object.assign(disk, files) + return { success: true } + }), + } + return { disk } +} + +const SAMPLE: DiagramData = { + nodes: [ + { id: 'sys1', type: 'system', label: 'Payments', collapsed: false, x: 0, y: 0, width: 400, height: 300, description: 'Money mover' }, + { id: 'sys2', type: 'system', label: 'Ledger', collapsed: false, x: 0, y: 0, width: 400, height: 300, description: 'Keeps the books' }, + ], + relations: [], +} + +const flush = () => new Promise((r) => setTimeout(r, 0)) + +describe('lazy node body hydration (md-folder docs)', () => { + beforeEach(() => { + ;(globalThis as any).localStorage = new MemLS() + if (!(globalThis as any).crypto?.randomUUID) { + ;(globalThis as any).crypto = { randomUUID: () => 'id-' + Math.random().toString(36).slice(2) } + } + delete (globalThis as any).window.electronAPI + }) + + async function loadLazyMdDoc(): Promise { + installFolderApi() + const ls = documents.createLSDocument('Draft', SAMPLE) + const meta = await documents.saveAsFolder(ls.id, SAMPLE) + await documents.saveDocument(meta!.id, SAMPLE) + documents.setActiveId(meta!.id) + const data = await documents.loadDocument(meta!.id) + useDiagramStore.getState().loadDiagram(data!) + return meta!.id + } + + it('loadDiagram marks lazily-loaded nodes pending, with no description in memory yet', async () => { + await loadLazyMdDoc() + const state = useDiagramStore.getState() + expect(state.pendingBodyNodeIds.sys1).toBe(true) + expect(state.pendingBodyNodeIds.sys2).toBe(true) + expect(state.c4Nodes.sys1.description).toBeUndefined() + }) + + it('hydrateNode fetches and merges the body, clearing the pending flag, without touching undo', async () => { + await loadLazyMdDoc() + const before = useDiagramStore.getState().canUndo + useDiagramStore.getState().hydrateNode('sys1') + await flush() + const state = useDiagramStore.getState() + expect(state.c4Nodes.sys1.description).toBe('Money mover') + expect(state.pendingBodyNodeIds.sys1).toBeUndefined() + // Not a user edit — must not push undo history. + expect(state.canUndo).toBe(before) + // The other lazy node is untouched. + expect(state.c4Nodes.sys2.description).toBeUndefined() + expect(state.pendingBodyNodeIds.sys2).toBe(true) + }) + + it('scanDescriptions finds a match in an unopened node and merges it in; non-matches stay pending', async () => { + await loadLazyMdDoc() + const matches = await useDiagramStore.getState().scanDescriptions('books') + expect(matches.has('sys2')).toBe(true) + expect(matches.has('sys1')).toBe(false) + const state = useDiagramStore.getState() + expect(state.c4Nodes.sys2.description).toBe('Keeps the books') + expect(state.pendingBodyNodeIds.sys2).toBeUndefined() + // sys1 didn't match — read but discarded, still pending, not cached. + expect(state.c4Nodes.sys1.description).toBeUndefined() + expect(state.pendingBodyNodeIds.sys1).toBe(true) + }) +}) diff --git a/tests/mdFolderDocument.test.ts b/tests/mdFolderDocument.test.ts index c0611f2..37687d6 100644 --- a/tests/mdFolderDocument.test.ts +++ b/tests/mdFolderDocument.test.ts @@ -19,6 +19,10 @@ function installFolderApi(): { disk: Record } { openFolder: vi.fn(async () => ({ success: true, folderPath: '/tmp/model', files: { ...disk } })), pickFolder: vi.fn(async () => ({ success: true, folderPath: '/tmp/model' })), readFolder: vi.fn(async () => ({ success: true, files: { ...disk } })), + readFile: vi.fn(async (filePath: string) => { + const rel = filePath.replace(/^\/tmp\/model\//, '') + return rel in disk ? { success: true, content: disk[rel] } : { success: false } + }), writeFolder: vi.fn(async (_path: string, files: Record) => { // Emulate the main-process prune-and-write: replace managed files. for (const k of Object.keys(disk)) delete disk[k] @@ -67,7 +71,7 @@ describe('documents — md-folder backend', () => { expect(Object.keys(disk).some((p) => p.startsWith('nodes/event-sourcing'))).toBe(true) }) - it('round-trips through save + load on the md backend', async () => { + it('round-trips through save + load on the md backend, loading node bodies lazily', async () => { installFolderApi() const ls = documents.createLSDocument('Draft', SAMPLE) const meta = await documents.saveAsFolder(ls.id, SAMPLE) @@ -76,7 +80,33 @@ describe('documents — md-folder backend', () => { expect(loaded).not.toBeNull() const byId = Object.fromEntries(loaded!.nodes.map((n) => [n.id, n])) expect(byId.sys1.label).toBe('Payments') - expect(byId.sys1.description).toBe('Money mover') expect((byId.adr1 as unknown as Record).status).toBe('accepted') + + // The body isn't read into memory at load time... + expect(byId.sys1.description).toBeUndefined() + expect(documents.getPendingBodyNodeIds(meta!.id)).toContain('sys1') + + // ...but can be fetched on demand. + const body = await documents.hydrateNodeBody(meta!.id, 'sys1') + expect(body).toBe('Money mover') + }) + + it('does not lose an unopened node\'s description on save (lazy round-trip safety)', async () => { + installFolderApi() + const ls = documents.createLSDocument('Draft', SAMPLE) + const meta = await documents.saveAsFolder(ls.id, SAMPLE) + await documents.saveDocument(meta!.id, SAMPLE) + + // Load lazily and save straight back WITHOUT ever hydrating sys1's body — + // this is the scenario that would silently wipe unopened content if + // saveDocument didn't transiently re-read it first. + const loaded = await documents.loadDocument(meta!.id) + expect(loaded!.nodes.find((n) => n.id === 'sys1')!.description).toBeUndefined() + await documents.saveDocument(meta!.id, loaded!) + + const reloaded = await documents.loadDocument(meta!.id) + const body = await documents.hydrateNodeBody(meta!.id, 'sys1') + expect(body).toBe('Money mover') + expect(reloaded!.nodes.find((n) => n.id === 'adr1')).toBeDefined() }) }) diff --git a/tests/mdFolderPersistence.test.ts b/tests/mdFolderPersistence.test.ts index df89203..7d5e80e 100644 --- a/tests/mdFolderPersistence.test.ts +++ b/tests/mdFolderPersistence.test.ts @@ -72,7 +72,7 @@ describe('md-folder persistence', () => { it('round-trips losslessly', () => { const files = serializeToMdFolder(sample, 'Demo') - const back = deserializeFromMdFolder(files) + const { data: back } = deserializeFromMdFolder(files) const byId = (d: DiagramData): Record => Object.fromEntries(d.nodes.map((n) => [n.id, n])) @@ -92,7 +92,7 @@ describe('md-folder persistence', () => { it('preserves custom metamodel property types and multiline prose', () => { const files = serializeToMdFolder(sample) - const back = deserializeFromMdFolder(files) + const { data: back } = deserializeFromMdFolder(files) const adr = back.nodes.find((n) => n.id === 'adr1') as unknown as Record expect(adr.status).toBe('accepted') expect(adr.context).toBe('We need an audit trail.\nMultiple regulators require it.') @@ -107,7 +107,7 @@ describe('md-folder persistence', () => { nodes: [node({ id: 'n1', type: 'adr', label: '123', ...({ date: '2026-01-01', ref: '007' } as Record) })], relations: [], } - const back = deserializeFromMdFolder(serializeToMdFolder(data)) + const { data: back } = deserializeFromMdFolder(serializeToMdFolder(data)) const n = back.nodes[0] as unknown as Record expect(n.label).toBe('123') expect(n.ref).toBe('007')