From 3941118d72056bef233cab8520aa25695c7ddf7d Mon Sep 17 00:00:00 2001 From: Ali Turki Date: Sat, 18 Jul 2026 15:07:40 +0800 Subject: [PATCH 1/3] fix(library): catch-up rescans, modify-event rescans, resilient watches Rescan the active root on launch, workspace switch, and window focus (stale-while-revalidate over the scan cache), so files changed while the app was closed or a workspace was inactive appear without a manual refresh. Fold modify events into the rescan path so in-place edits refresh titles, tags, and search, and treat the watcher's other/any rescan sentinels as rescan triggers instead of dropping them. Attach fs watches with bounded retries and a warning on failure, and add explicit fs scope entries for the registry directory: the unix scope glob requires a literal leading dot, so $HOME/** never matched ~/.docsreader and the registry watch failed silently on every launch. Fixes #18, fixes #19, fixes #22 --- src-tauri/capabilities/default.json | 4 + src/hooks/useLibrary.test.ts | 122 +++++++++++++- src/hooks/useLibrary.ts | 247 ++++++++++++++++------------ 3 files changed, 267 insertions(+), 106 deletions(-) diff --git a/src-tauri/capabilities/default.json b/src-tauri/capabilities/default.json index 8d8062e..59f78b7 100644 --- a/src-tauri/capabilities/default.json +++ b/src-tauri/capabilities/default.json @@ -15,6 +15,8 @@ "allow": [ { "path": "$HOME" }, { "path": "$HOME/**" }, + { "path": "$HOME/.docsreader" }, + { "path": "$HOME/.docsreader/**" }, { "path": "$DESKTOP" }, { "path": "$DESKTOP/**" }, { "path": "$DOCUMENT" }, @@ -84,6 +86,8 @@ "allow": [ { "path": "$HOME" }, { "path": "$HOME/**" }, + { "path": "$HOME/.docsreader" }, + { "path": "$HOME/.docsreader/**" }, { "path": "$DESKTOP" }, { "path": "$DESKTOP/**" }, { "path": "$DOCUMENT" }, diff --git a/src/hooks/useLibrary.test.ts b/src/hooks/useLibrary.test.ts index 3a1f2d9..a2340fa 100644 --- a/src/hooks/useLibrary.test.ts +++ b/src/hooks/useLibrary.test.ts @@ -1,6 +1,8 @@ import { renderHook, waitFor } from "@testing-library/react"; -import { vi, describe, it, expect, beforeEach } from "vitest"; +import { vi, describe, it, expect, beforeEach, afterEach } from "vitest"; +import { watch, type WatchEvent } from "@tauri-apps/plugin-fs"; +import { scanDirectory } from "@/lib/scan"; import type { RegistryWorkspace } from "@/lib/workspaces"; let registry: RegistryWorkspace[] = []; @@ -145,3 +147,121 @@ describe("useLibrary registry sync", () => { expect(dismissed).not.toContain("/manual/only"); }); }); + +// Past DEBOUNCE_MS in useLibrary.ts, so a scheduled rescan fires. +const PAST_DEBOUNCE_MS = 700; +const REGISTRY_DIR = "/home/u/.docsreader"; +const WATCH_OK = async () => () => {}; + +async function workspaceWatchCallback(root: string) { + await waitFor(() => + expect( + vi.mocked(watch).mock.calls.some(([watched]) => watched === root) + ).toBe(true) + ); + const call = vi.mocked(watch).mock.calls.find(([watched]) => watched === root); + if (!call) throw new Error(`no watch attached for ${root}`); + return call[1]; +} + +function fireWatchEvent( + callback: (event: WatchEvent) => void, + type: WatchEvent["type"], + paths: string[] +) { + callback({ type, paths, attrs: {} }); +} + +describe("useLibrary stale-while-revalidate rescans", () => { + beforeEach(() => { + registry = []; + storedRoots = []; + dismissed = []; + vi.clearAllMocks(); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it("rescans the initial root after startup hydration", async () => { + storedRoots = ["/manual/folder"]; + await mount(); + await waitFor(() => + expect(scanDirectory).toHaveBeenCalledWith( + "/manual/folder", + expect.any(Function) + ) + ); + }); + + it("rescans a root when it is selected", async () => { + storedRoots = ["/first", "/second"]; + const hook = await mount(); + vi.mocked(scanDirectory).mockClear(); + await hook.result.current.selectRoot("/second"); + await waitFor(() => + expect(scanDirectory).toHaveBeenCalledWith("/second", expect.any(Function)) + ); + }); + + it("schedules a rescan when a file is modified in place", async () => { + storedRoots = ["/manual/folder"]; + await mount(); + const callback = await workspaceWatchCallback("/manual/folder"); + vi.mocked(scanDirectory).mockClear(); + vi.useFakeTimers(); + fireWatchEvent(callback, { modify: { kind: "data", mode: "content" } }, [ + "/manual/folder/doc.md", + ]); + await vi.advanceTimersByTimeAsync(PAST_DEBOUNCE_MS); + expect(scanDirectory).toHaveBeenCalledWith( + "/manual/folder", + expect.any(Function) + ); + }); + + it("schedules a rescan for an overflow sentinel event", async () => { + storedRoots = ["/manual/folder"]; + await mount(); + const callback = await workspaceWatchCallback("/manual/folder"); + vi.mocked(scanDirectory).mockClear(); + vi.useFakeTimers(); + fireWatchEvent(callback, "other", []); + await vi.advanceTimersByTimeAsync(PAST_DEBOUNCE_MS); + expect(scanDirectory).toHaveBeenCalledWith( + "/manual/folder", + expect.any(Function) + ); + }); +}); + +describe("useLibrary watch resilience", () => { + beforeEach(() => { + registry = []; + storedRoots = []; + dismissed = []; + vi.clearAllMocks(); + }); + + it("retries a failing watch and warns after the final attempt", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + vi.mocked(watch).mockRejectedValue(new Error("forbidden path")); + try { + await mount(); + await waitFor(() => expect(warn).toHaveBeenCalled(), { timeout: 4000 }); + const registryAttempts = vi + .mocked(watch) + .mock.calls.filter(([watched]) => watched === REGISTRY_DIR); + expect(registryAttempts).toHaveLength(3); + expect(warn).toHaveBeenCalledWith( + expect.stringContaining("registry watch failed"), + REGISTRY_DIR, + expect.any(Error) + ); + } finally { + warn.mockRestore(); + vi.mocked(watch).mockImplementation(WATCH_OK); + } + }); +}); diff --git a/src/hooks/useLibrary.ts b/src/hooks/useLibrary.ts index d6198fc..caa5da6 100644 --- a/src/hooks/useLibrary.ts +++ b/src/hooks/useLibrary.ts @@ -1,6 +1,11 @@ import { useCallback, useEffect, useRef, useState } from "react"; import { open } from "@tauri-apps/plugin-dialog"; -import { watch, type UnwatchFn } from "@tauri-apps/plugin-fs"; +import { + watch, + type DebouncedWatchOptions, + type UnwatchFn, + type WatchEvent, +} from "@tauri-apps/plugin-fs"; import { scanDirectory, type ScanProgress, @@ -53,6 +58,12 @@ const DEBOUNCE_MS = 600; const MIN_RESCAN_INTERVAL_MS = 2000; // Coalescing window the fs watcher applies before delivering events. const WATCH_DELAY_MS = 200; +// Attaching a watch can fail transiently (e.g. the directory is created +// moments after setup, as ~/.docsreader is on the first agent write), so +// setup retries with capped exponential backoff before giving up. +const WATCH_ATTACH_MAX_ATTEMPTS = 3; +const WATCH_ATTACH_BASE_BACKOFF_MS = 500; +const WATCH_ATTACH_MAX_BACKOFF_MS = 2000; // Mirror of the Rust scanner's SKIP_DIRS in src-tauri/core/src/scan.rs. Any // directory segment in this set, OR any segment that starts with a dot @@ -119,6 +130,54 @@ function isManifestPath(eventPath: string, root: string): boolean { return MANIFEST_BASENAMES.some((name) => norm === `${r}/${name}`); } +// Attaches an fs watch with bounded retries so a transient setup failure +// does not silently leave the path unwatched. Returns a cancel function +// that stops pending retries and detaches an attached watch. +function watchWithRetry( + path: string, + onEvent: (event: WatchEvent) => void, + options: DebouncedWatchOptions, + label: string +): () => void { + let cancelled = false; + let unwatch: UnwatchFn | undefined; + let retryTimer: ReturnType | undefined; + + const attempt = async (attemptNumber: number) => { + try { + const unwatchFn = await watch(path, onEvent, options); + if (cancelled) { + void unwatchFn(); + return; + } + unwatch = unwatchFn; + } catch (err) { + if (cancelled) return; + if (attemptNumber >= WATCH_ATTACH_MAX_ATTEMPTS) { + console.warn( + `${label} watch failed after ${WATCH_ATTACH_MAX_ATTEMPTS} attempts`, + path, + err + ); + return; + } + const backoff = Math.min( + WATCH_ATTACH_BASE_BACKOFF_MS * 2 ** (attemptNumber - 1), + WATCH_ATTACH_MAX_BACKOFF_MS + ); + retryTimer = setTimeout(() => void attempt(attemptNumber + 1), backoff); + } + }; + + void attempt(1); + + return () => { + cancelled = true; + if (retryTimer) clearTimeout(retryTimer); + if (unwatch) void unwatch(); + }; +} + export function useLibrary(): Library { const [roots, setRoots] = useState([]); const [activeRoot, setActiveRoot] = useState(); @@ -242,12 +301,15 @@ export function useLibrary(): Library { if (stored.length > 0) { const initial = stored.includes(last ?? "") ? (last as string) : stored[0]; setActiveRoot(initial); + // Stale-while-revalidate: show the cache immediately, then rescan + // to pick up files changed while the app was closed. await hydrateFromCache(initial); + void rescan(initial); } setHydrated(true); await reconcileRegistry(); })(); - }, [hydrateFromCache, reconcileRegistry]); + }, [hydrateFromCache, reconcileRegistry, rescan]); const addRoot = useCallback( async (path: string) => { @@ -306,15 +368,22 @@ export function useLibrary(): Library { async (path: string | undefined) => { setActiveRoot(path); await saveLastSelected(path); - if (path && !scans[path]) await hydrateFromCache(path); + if (!path) return; + if (!scans[path]) await hydrateFromCache(path); + // Background workspaces have no watcher, so whatever is on screen may + // be stale; always revalidate on selection. + void rescan(path); }, - [scans, hydrateFromCache] + [scans, hydrateFromCache, rescan] ); // Workspace-level watcher: re-scan the active root when files or - // folders are created, removed, or renamed anywhere inside it. - // Modify-only events for individual files are handled per-tab in - // useTabs, so they're ignored here to avoid redundant scans. + // folders are created, removed, renamed, or modified anywhere inside + // it. Modify events matter because agents (MCP update_doc) rewrite + // files in place, which changes scan-time extraction (titles, tags, + // search text) without any create/remove. A window-focus rescan covers + // changes made while the window was in the background and events were + // missed. // // Two filters protect against runaway work: // 1. Events whose path lies inside a hidden or known-noisy @@ -329,9 +398,7 @@ export function useLibrary(): Library { useEffect(() => { if (!activeRoot) return; let cancelled = false; - let unwatch: UnwatchFn | undefined; let debounceTimer: ReturnType | undefined; - let gitDebounceTimer: ReturnType | undefined; let lastRescanAt = 0; const scheduleRescan = () => { @@ -346,101 +413,73 @@ export function useLibrary(): Library { }, wait); }; - // Lighter than scheduleRescan: refreshes only the workspace's git - // status without re-walking the file tree. Used when a modify - // event lands on a file inside the workspace but not the manifest - - // the file set is unchanged but the git modified/clean state may - // have flipped. - const scheduleGitRefresh = () => { - if (cancelled) return; - if (gitDebounceTimer) clearTimeout(gitDebounceTimer); - gitDebounceTimer = setTimeout(() => { - if (cancelled) return; - void fetchGitStatus(activeRoot).then((gitStatus) => { - if (cancelled) return; - setScans((s) => { - const prev = s[activeRoot]; - if (!prev) return s; - return { ...s, [activeRoot]: { ...prev, gitStatus } }; - }); - }); - }, DEBOUNCE_MS); - }; + const onWatchEvent = (event: WatchEvent) => { + const kind = describeEventKind(event.type); + const paths = Array.isArray(event.paths) ? event.paths : []; + + // Manifest events bypass the dotfile-skip filter, which would + // otherwise drop events for the marker file because of its + // leading dot. + const manifestTouched = paths.some((p) => isManifestPath(p, activeRoot)); + if ( + manifestTouched && + (kind === "create" || + kind === "remove" || + kind === "rename" || + kind === "modify") + ) { + scheduleRescan(); + return; + } - void (async () => { - try { - const unwatchFn = await watch( - activeRoot, - (event) => { - const kind = describeEventKind(event.type); - const paths = Array.isArray(event.paths) ? event.paths : []; - - // Manifest events bypass both the modify-skip and the - // dotfile-skip filters. The dotfile filter would otherwise - // drop create/remove/rename of the marker file because of its - // leading dot, and the modify-skip would drop in-place edits. - const manifestTouched = paths.some((p) => - isManifestPath(p, activeRoot) - ); - if ( - manifestTouched && - (kind === "create" || - kind === "remove" || - kind === "rename" || - kind === "modify") - ) { - scheduleRescan(); - return; - } - - // Modify events on regular workspace files: skip the rescan - // (file set hasn't changed) but refresh git status so the - // file-tree decorations stay live. - if (kind === "modify") { - const someRelevant = - paths.length === 0 || - paths.some((p) => !isSkippedWatchPath(p, activeRoot)); - if (someRelevant) scheduleGitRefresh(); - return; - } - - if (kind !== "create" && kind !== "remove" && kind !== "rename") { - return; - } - // event.paths can contain multiple paths for batched events. - // Only schedule a rescan if at least one path is not skipped. - const someRelevant = - paths.length === 0 || - paths.some((p) => !isSkippedWatchPath(p, activeRoot)); - if (someRelevant) scheduleRescan(); - }, - { recursive: true, delayMs: WATCH_DELAY_MS } - ); - if (cancelled) { - void unwatchFn(); - return; - } - unwatch = unwatchFn; - } catch (err) { - console.error("workspace watch failed", err); + // notify reports FSEvents queue overflow via its rescan sentinel, + // which describeEventKind surfaces as "other"/"any" without useful + // paths. Events were dropped, so rescan unconditionally. + if (kind === "other" || kind === "any") { + scheduleRescan(); + return; } - })(); + if ( + kind !== "create" && + kind !== "remove" && + kind !== "rename" && + kind !== "modify" + ) { + return; + } + // event.paths can contain multiple paths for batched events. + // Only schedule a rescan if at least one path is not skipped. + const someRelevant = + paths.length === 0 || + paths.some((p) => !isSkippedWatchPath(p, activeRoot)); + if (someRelevant) scheduleRescan(); + }; + + const stopWatch = watchWithRetry( + activeRoot, + onWatchEvent, + { recursive: true, delayMs: WATCH_DELAY_MS }, + "workspace" + ); + + window.addEventListener("focus", scheduleRescan); return () => { cancelled = true; if (debounceTimer) clearTimeout(debounceTimer); - if (gitDebounceTimer) clearTimeout(gitDebounceTimer); - if (unwatch) void unwatch(); + window.removeEventListener("focus", scheduleRescan); + stopWatch(); }; }, [activeRoot]); // Registry watcher: when an agent creates a workspace while the app is // open, ~/.docsreader/workspaces.json changes and the new workspace - // appears without a restart. A window-focus pass is the fallback for the - // rare case the directory did not exist when the watch was set up. + // appears without a restart. The watch attaches with retries (the + // directory may not exist until the first agent write); the window-focus + // pass remains the fallback if it never attaches. useEffect(() => { let cancelled = false; - let unwatch: UnwatchFn | undefined; + let stopWatch: (() => void) | undefined; let timer: ReturnType | undefined; const scheduleReconcile = () => { @@ -452,22 +491,20 @@ export function useLibrary(): Library { }; void (async () => { + let dir: string; try { - const dir = await registryDir(); - const unwatchFn = await watch(dir, scheduleReconcile, { - recursive: false, - delayMs: WATCH_DELAY_MS, - }); - if (cancelled) { - void unwatchFn(); - return; - } - unwatch = unwatchFn; + dir = await registryDir(); } catch (err) { - // The registry dir may not exist until the first agent write; the - // focus fallback and next launch still pick new workspaces up. - console.debug("registry watch not active yet", err); + console.warn("registry dir lookup failed", err); + return; } + if (cancelled) return; + stopWatch = watchWithRetry( + dir, + scheduleReconcile, + { recursive: false, delayMs: WATCH_DELAY_MS }, + "registry" + ); })(); window.addEventListener("focus", scheduleReconcile); @@ -475,7 +512,7 @@ export function useLibrary(): Library { cancelled = true; if (timer) clearTimeout(timer); window.removeEventListener("focus", scheduleReconcile); - if (unwatch) void unwatch(); + if (stopWatch) stopWatch(); }; }, [reconcileRegistry]); From 856e94dbfce1b0d0ef74c05caab297bda652ad5b Mon Sep 17 00:00:00 2001 From: Ali Turki Date: Sat, 18 Jul 2026 15:07:40 +0800 Subject: [PATCH 2/3] fix(scan): reject reserved phase names, follow symlinks, count skipped files Reject phase names that collide with the scanner's pruned directory names (build, dist, target, venv): docs written there succeeded via MCP but could never appear in the GUI. Validation reads the same SKIP_DIRS list the scanner uses, so the two cannot drift. Follow symlinks when scanning (walkdir detects loops and reports them as errors), and count walker errors, unreadable files, and oversize files into a new skipped field on ScanResult instead of dropping them silently. The explorer footer shows the skipped count when nonzero. Fixes #20, fixes #21 --- src-tauri/core/src/scan.rs | 88 +++++++++++++++++++-- src-tauri/core/src/write.rs | 46 +++++++++++ src/components/explorer/ExplorerSidebar.tsx | 6 ++ src/lib/scan.ts | 2 + 4 files changed, 137 insertions(+), 5 deletions(-) diff --git a/src-tauri/core/src/scan.rs b/src-tauri/core/src/scan.rs index 3a42b9b..a25c986 100644 --- a/src-tauri/core/src/scan.rs +++ b/src-tauri/core/src/scan.rs @@ -1,7 +1,7 @@ use std::fs::File; use std::io::Read; use std::path::Path; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; @@ -51,6 +51,11 @@ pub struct ScanResult { pub root: String, pub files: Vec, pub truncated: bool, + // Files the scan could not include: unreadable entries, permission-denied + // subtrees, symlink loops, and files over MAX_FILE_BYTES. Default keeps + // cached results from older versions deserializable. + #[serde(default)] + pub skipped: usize, #[serde(default, skip_serializing_if = "Option::is_none")] pub marker: Option, } @@ -125,6 +130,12 @@ fn is_skipped_dir(name: &str) -> bool { SKIP_DIRS.contains(&name) } +/// True when a directory name collides with an entry the scanner prunes; +/// docs placed under such a folder would never appear in the GUI. +pub fn is_reserved_dir_name(name: &str) -> bool { + SKIP_DIRS.iter().any(|d| d.eq_ignore_ascii_case(name)) +} + pub(crate) fn is_markdown(name: &str) -> bool { let lower = name.to_ascii_lowercase(); lower.ends_with(".md") || lower.ends_with(".markdown") || lower.ends_with(".mdx") @@ -149,9 +160,10 @@ pub fn run_scan(progress: &dyn ScanProgressSink, path: String) -> Result = Vec::new(); let mut truncated = false; + let skipped = AtomicUsize::new(0); let walker = WalkDir::new(root_path) - .follow_links(false) + .follow_links(true) .into_iter() .filter_entry(|e| { if e.depth() == 0 { @@ -165,7 +177,16 @@ pub fn run_scan(progress: &dyn ScanProgressSink, path: String) -> Result e, + // Permission-denied subtrees and symlink loops arrive as error + // entries; count them so the GUI can say files were left out. + Err(_) => { + skipped.fetch_add(1, Ordering::Relaxed); + continue; + } + }; if entry.file_type().is_dir() { dirs_visited.fetch_add(1, Ordering::Relaxed); maybe_emit_walk_progress( @@ -192,6 +213,7 @@ pub fn run_scan(progress: &dyn ScanProgressSink, path: String) -> Result MAX_FILE_BYTES { + skipped.fetch_add(1, Ordering::Relaxed); continue; } } @@ -209,7 +231,10 @@ pub fn run_scan(progress: &dyn ScanProgressSink, path: String) -> Result = entries .par_iter() .filter_map(|entry| { - let metadata = entry.metadata().ok()?; + let Ok(metadata) = entry.metadata() else { + skipped.fetch_add(1, Ordering::Relaxed); + return None; + }; let size = metadata.len(); let modified = metadata .modified() @@ -217,7 +242,10 @@ pub fn run_scan(progress: &dyn ScanProgressSink, path: String) -> Result Result Result<(), CoreError> { ), ); } + // A phase folder named like a pruned directory (build, dist, ...) would + // make its docs invisible to the GUI while writes still report success. + if crate::scan::is_reserved_dir_name(phase) { + return Err(CoreError::new( + ErrorCode::InvalidInput, + format!("phase {phase:?} is a reserved name"), + ) + .with_recovery(format!( + "choose a different phase name, e.g. \"{phase}-phase\"" + ))); + } Ok(()) } @@ -563,6 +574,41 @@ mod tests { let _ = std::fs::remove_dir_all(&root); } + #[tokio::test] + async fn rejects_reserved_phase_names() { + let root = test_dir("reserved_phase"); + for phase in ["build", "dist", "target", "venv"] { + let err = write_doc_core( + &root, + &NewDoc { + phase: Some(phase), + ..NewDoc::new("Doc", "x", DocStatus::Research) + }, + ) + .await + .unwrap_err(); + assert_eq!(err.code, ErrorCode::InvalidInput, "phase {phase:?}"); + assert!( + err.message.contains("reserved"), + "message says why: {}", + err.message + ); + assert!(err.recovery.is_some(), "suggests picking another name"); + } + + let ok = write_doc_core( + &root, + &NewDoc { + phase: Some("discovery"), + ..NewDoc::new("Doc", "x", DocStatus::Research) + }, + ) + .await + .unwrap(); + assert_eq!(ok.rel_path, "research/discovery/doc.md"); + let _ = std::fs::remove_dir_all(&root); + } + #[tokio::test] async fn move_unknown_slug_is_doc_not_found() { let root = test_dir("nomove"); diff --git a/src/components/explorer/ExplorerSidebar.tsx b/src/components/explorer/ExplorerSidebar.tsx index 5075271..76bde5c 100644 --- a/src/components/explorer/ExplorerSidebar.tsx +++ b/src/components/explorer/ExplorerSidebar.tsx @@ -321,6 +321,7 @@ function ExplorerFooter({ } const total = activeScan.result.files.length; const visible = total - hiddenCount; + const skipped = activeScan.result.skipped ?? 0; return ( @@ -328,6 +329,11 @@ function ExplorerFooter({ {activeScan.result.truncated && " (50k cap)"} {matchCount !== visible && ` ยท ${matchCount} match`} + {skipped > 0 && ( + + {skipped} skipped + + )} {hiddenCount > 0 && (