From 69420a2b937fe06e88dd41b20971695dd4b5a667 Mon Sep 17 00:00:00 2001 From: Shengyu Fu Date: Sat, 29 Aug 2026 00:08:33 -0500 Subject: [PATCH 01/16] Fix watcher reconciliation regressions Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- README.md | 7 + tgrep-cli/src/serve.rs | 247 ++++++++++++++---- tgrep-cli/tests/watcher_watch_registration.rs | 110 +++++++- tgrep-core/src/git_index.rs | 46 +++- tgrep-core/src/gitignore.rs | 34 ++- tgrep-core/src/hybrid.rs | 5 + tgrep-core/src/reader.rs | 15 ++ 7 files changed, 396 insertions(+), 68 deletions(-) diff --git a/README.md b/README.md index bfcd77a..4ea329f 100644 --- a/README.md +++ b/README.md @@ -306,6 +306,13 @@ gives up waiting after four hours so a continuously busy server still reconciles. On an unchanged tree it finds nothing and leaves the index alone. `--no-watch` turns it off along with the watcher. +On Linux and Android, tgrep registers only the non-ignored directories with +inotify, avoiding watch-descriptor growth beneath ignored trees. This guarantee +is backend-specific: the implementation intentionally keeps one recursive +`ReadDirectoryChangesW` root subscription on Windows and one root FSEvents +stream on macOS, where ignored events are filtered after delivery and ignored +descendants remain watched. kqueue and `PollWatcher` are not covered. + ### Search ```bash diff --git a/tgrep-cli/src/serve.rs b/tgrep-cli/src/serve.rs index a2272b0..59cb2c9 100644 --- a/tgrep-cli/src/serve.rs +++ b/tgrep-cli/src/serve.rs @@ -52,6 +52,10 @@ const WATCHER_QUEUE_CAP: usize = 16_384; /// reconciling stale check runs. const WATCHER_IDLE_POLL: Duration = Duration::from_secs(1); +/// Git's index is hidden from the repository watcher. Poll its metadata only +/// when the case-insensitive tracked-file exemption is active. +const TRACKED_INDEX_POLL: Duration = Duration::from_secs(2); + /// How long a file the watcher never heard about can stay wrong in the index. /// /// Every mutation the index takes after the initial build arrives as an OS @@ -290,6 +294,9 @@ struct ServerState { ignore_rules_dirty: std::sync::atomic::AtomicBool, /// Ensures a burst of ignore-file events uses at most one refresh worker. ignore_refresh_scheduled: std::sync::atomic::AtomicBool, + /// Last observed identity of the worktree-specific Git index. `None` means + /// the published matcher has no case-insensitive tracked-file exemption. + tracked_index_identity: Mutex>, /// Set when events were lost, cleared by the next subscription sync, which /// then re-issues every subscription instead of trusting its own records. /// @@ -648,6 +655,7 @@ pub fn run(root: &Path, index_path: Option<&Path>, options: ServeOptions<'_>) -> gitignore_pending: std::sync::atomic::AtomicBool::new(!no_watch && !no_ignore), ignore_rules_dirty: std::sync::atomic::AtomicBool::new(false), ignore_refresh_scheduled: std::sync::atomic::AtomicBool::new(false), + tracked_index_identity: Mutex::new(None), watch_resubscribe: std::sync::atomic::AtomicBool::new(false), ignore_sources: RwLock::new(Vec::new()), ignore_source_stamps: RwLock::new(IgnoreStamps::new()), @@ -1008,7 +1016,23 @@ fn publish_ignore_matcher( *state.ignore_source_stamps.write().unwrap() = stamps; *state.ignore_sources.write().unwrap() = sources; - *state.gitignore.write().unwrap() = matcher; + // Keep the matcher and identity state in one critical section. Preserve an + // active baseline across refreshes: if the index changes while a refresh is + // walking, the next poll must still see that change rather than having this + // publish silently advance past it. + let mut published = state.gitignore.write().unwrap(); + let mut identity = state.tracked_index_identity.lock().unwrap(); + let current_identity = matcher + .as_ref() + .and_then(tgrep_core::gitignore::IgnoreMatcher::tracked_index_identity); + match (&*identity, current_identity) { + (_, None) => *identity = None, + (None, Some(current)) => *identity = Some(current), + (Some(_), Some(_)) => {} + } + *published = matcher; + drop(identity); + drop(published); state.gitignore_pending.store(false, Ordering::SeqCst); // New rules mean a different set of directories worth hearing about: // a tightened rule releases the subscriptions under it, and a relaxed @@ -1865,7 +1889,21 @@ fn start_file_watcher(state: Arc, root: &Path, queue_cap: usize) -> if std::thread::Builder::new() .name("tgrep-watcher".into()) .spawn(move || { + let mut last_tracked_index_poll = Instant::now(); loop { + if last_tracked_index_poll.elapsed() >= TRACKED_INDEX_POLL { + last_tracked_index_poll = Instant::now(); + if tracked_index_changed(&worker_state) { + eprintln!("[trace] Git index changed; reconciling tracked-file exemptions"); + worker_state + .ignore_rules_dirty + .store(true, Ordering::SeqCst); + schedule_ignore_rules_refresh( + Arc::clone(&worker_state), + worker_root.clone(), + ); + } + } match rx.recv_timeout(WATCHER_IDLE_POLL) { Ok(event) => handle_fs_event(&worker_state, &worker_root, &event), // A quiet interval means the burst has drained, so this is @@ -1919,6 +1957,26 @@ fn start_file_watcher(state: Arc, root: &Path, queue_cap: usize) -> true } +/// Detect a tracked-file exemption change without subscribing to `.git`. +/// +/// The metadata probe is enabled only while the published matcher actually +/// uses the exemption. Updating the observed identity before scheduling makes +/// a burst one dirty signal; the existing refresh scheduler coalesces it with +/// any ignore-source changes and serializes the full stale reconciliation. +fn tracked_index_changed(state: &ServerState) -> bool { + let matcher = state.gitignore.read().unwrap(); + let current = matcher + .as_ref() + .and_then(tgrep_core::gitignore::IgnoreMatcher::tracked_index_identity); + let mut observed = state.tracked_index_identity.lock().unwrap(); + let changed = matches!( + (observed.as_ref(), current.as_ref()), + (Some(previous), Some(current)) if previous != current + ); + *observed = current; + changed +} + /// Decide whether the file watcher should skip a path entirely. /// /// Mirrors the file walker's hidden-path, `--exclude` directory filtering, @@ -2039,10 +2097,10 @@ fn is_ignore_rules_file(root: &Path, path: &Path) -> bool { /// repo whose ignored build output exhausts the budget makes `watch()` return /// an error and the server loses its watcher entirely. /// -/// ReadDirectoryChangesW (Windows) and FSEvents (macOS) subscribe once for the -/// whole subtree, so there is no per-directory registration to withhold. On -/// those platforms filtering on delivery is the only lever available, and -/// [`should_skip_watcher_path`] remains it. +/// This implementation deliberately keeps one recursive root subscription on +/// Windows (`ReadDirectoryChangesW`) and one root stream on macOS (FSEvents). +/// Ignored events are filtered after delivery there, so ignored descendants are +/// not unwatched even though the design avoids per-directory descriptor growth. /// /// Deliberately limited to the backends we can exercise in CI. kqueue and /// `PollWatcher` are per-path too, but nothing here builds or tests them. @@ -2112,6 +2170,17 @@ struct WatchRegistry { watched: std::collections::HashSet, } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum TraversalCompleteness { + Complete, + Incomplete, +} + +struct WatchableDirs { + dirs: std::collections::HashSet, + completeness: TraversalCompleteness, +} + impl WatchRegistry { /// Subscribe to every directory in `desired` that is not already /// subscribed, leaving existing subscriptions alone. @@ -2283,22 +2352,27 @@ impl WatchRegistry { /// would pay forty thousand of them for a doubt only overflow raises. /// /// Returns `(added, removed)`. Only for a set that describes the whole - /// tree — anything absent from `desired` is unsubscribed. To subscribe to - /// a subtree without disturbing the rest, use [`Self::add_all`]. + /// tree. `completeness` makes its authority explicit: a complete set prunes + /// anything absent from `desired`, while an incomplete traversal only adds + /// directories it proved desirable. To subscribe to a subtree without + /// disturbing the rest, use [`Self::add_all`]. fn sync( &mut self, desired: &std::collections::HashSet, + completeness: TraversalCompleteness, force: bool, ) -> (Vec, usize) { - let stale: Vec = self.watched.difference(desired).cloned().collect(); let mut removed = 0; - for dir in stale { - // Best effort. inotify drops a descriptor by itself when the - // directory is deleted, so "not found" is an expected outcome - // here, not an error worth reporting. - let _ = self.watcher.unwatch(&dir); - self.watched.remove(&dir); - removed += 1; + if completeness == TraversalCompleteness::Complete { + let stale: Vec = self.watched.difference(desired).cloned().collect(); + for dir in stale { + // Best effort. inotify drops a descriptor by itself when the + // directory is deleted, so "not found" is an expected outcome + // here, not an error worth reporting. + let _ = self.watcher.unwatch(&dir); + self.watched.remove(&dir); + removed += 1; + } } // Shallowest first, because `desired` is a `HashSet` and hands its @@ -2343,28 +2417,43 @@ impl WatchRegistry { /// Symlinked directories are not descended into, matching the walker (the /// `ignore` crate does not follow links by default). That also keeps a /// symlink cycle from turning this into an infinite walk. +/// +/// Any failed listing, entry read, or type query marks the result incomplete. +/// Its proven directories remain useful for adding subscriptions, but its +/// omissions must not be used to remove existing ones. fn watchable_dirs( root: &Path, start: &Path, exclude_dirs: &[String], gitignore: Option<&tgrep_core::gitignore::IgnoreMatcher>, -) -> std::collections::HashSet { +) -> WatchableDirs { let mut found = std::collections::HashSet::new(); + let mut completeness = TraversalCompleteness::Complete; found.insert(start.to_path_buf()); let mut stack = vec![start.to_path_buf()]; while let Some(dir) = stack.pop() { let Ok(entries) = std::fs::read_dir(&dir) else { // An unreadable directory is not a reason to abandon the rest of - // the tree; the periodic reconcile is what catches what we miss. + // the tree, but it makes the result unsafe for pruning. + completeness = TraversalCompleteness::Incomplete; continue; }; - for entry in entries.flatten() { - if !entry.file_type().is_ok_and(|t| t.is_dir()) { + for entry in entries { + let Ok(entry) = entry else { + completeness = TraversalCompleteness::Incomplete; + continue; + }; + let Ok(file_type) = entry.file_type() else { + completeness = TraversalCompleteness::Incomplete; + continue; + }; + if !file_type.is_dir() { continue; } let path = entry.path(); let Ok(rel) = path.strip_prefix(root) else { + completeness = TraversalCompleteness::Incomplete; continue; }; let rel = rel.to_string_lossy().replace('\\', "/"); @@ -2375,7 +2464,10 @@ fn watchable_dirs( found.insert(path); } } - found + WatchableDirs { + dirs: found, + completeness, + } } /// The directories that must be subscribed to for the ignore sources @@ -2458,13 +2550,13 @@ fn sync_watch_registrations(state: &ServerState, root: &Path) -> (Vec, }; if !state.no_ignore { let sources = state.ignore_sources.read().unwrap(); - desired.extend(ignore_target_dirs(root, &sources)); + desired.dirs.extend(ignore_target_dirs(root, &sources)); } - let total = desired.len(); + let total = desired.dirs.len(); // Consumed here, so one overflow buys one forced pass rather than making // every later reconcile re-register the whole tree. let force = state.watch_resubscribe.swap(false, Ordering::SeqCst); - let (added, removed) = registry.sync(&desired, force); + let (added, removed) = registry.sync(&desired.dirs, desired.completeness, force); if !added.is_empty() || removed > 0 { eprintln!( "[trace] watcher subscriptions: {total} directories \ @@ -3563,19 +3655,10 @@ fn lock_reindex(state: &ServerState) -> std::sync::MutexGuard<'_, ()> { /// Drop everything the index holds for a path. /// -/// The delete is not conditional on a stamp entry. `ServerState` accepts an -/// empty stamp map when `filestamps.json` is missing or unreadable, and the -/// reader can still hold the path in that state, so keying the delete on the -/// stamp alone would leave content the walk now rejects searchable. The stamp -/// removal is best-effort; the index and cache deletes always happen. -/// -/// The cost of that is a tombstone in the overlay for paths that were never -/// indexed — `live::delete_file` records one either way — and this runs for -/// every ineligible file a recovery scan walks past, which at startup is every -/// binary asset in the repository. An existing tombstone is therefore taken as -/// proof there is nothing left to do, which bounds that to one per distinct -/// path. The trace line, which is the part that would be actively misleading, -/// stays conditional on there having been something to drop. +/// A stamp is evidence, but not a precondition: `filestamps.json` may be missing +/// while the active reader still holds the path. Conversely, a rejected file +/// absent from the reader, live overlay, and stamps was never indexed, so +/// recording a tombstone for it would dirty the overlay for no state change. /// /// The caller must already hold `snapshot_gate` and `reindex_lock`. The lock is /// the caller's rather than this function's because `reindex_file` calls in @@ -3589,16 +3672,14 @@ fn drop_indexed_file(state: &ServerState, rel_path: &str, reason: &str) { .is_some(); { let index = state.index.read().unwrap(); - if index.live.has_path(rel_path) || had_stamp { - eprintln!("[trace] reindex: dropped {rel_path} ({reason})"); - } else if index.live.is_deleted(rel_path) { - // Already tombstoned, so there is nothing to record and no reason - // to dirty the overlay again. This is the repeat case: a recovery - // scan or a chatty editor can bring the same rejected path back - // here any number of times. + if index.live.is_deleted(rel_path) { + return; + } + if !had_stamp && !index.live.has_path(rel_path) && !index.reader_has_path(rel_path) { return; } } + eprintln!("[trace] reindex: dropped {rel_path} ({reason})"); state.index.write().unwrap().live.delete_file(rel_path); if let Ok(mut cache) = state.cache.write() { cache.pop(rel_path); @@ -5846,6 +5927,7 @@ mod tests { gitignore_pending: std::sync::atomic::AtomicBool::new(true), ignore_rules_dirty: std::sync::atomic::AtomicBool::new(false), ignore_refresh_scheduled: std::sync::atomic::AtomicBool::new(false), + tracked_index_identity: Mutex::new(None), watch_resubscribe: std::sync::atomic::AtomicBool::new(false), ignore_sources: RwLock::new(Vec::new()), ignore_source_stamps: RwLock::new(IgnoreStamps::new()), @@ -6077,7 +6159,7 @@ mod tests { } #[test] - fn watch_registry_add_all_is_additive_but_sync_prunes() { + fn incomplete_watch_sync_preserves_existing_subscriptions_until_a_complete_pass() { // These two must not be confused. `watch_new_subtree` learns only // about the subtree that just appeared, so if it went through `sync` // every directory outside that subtree would look stale and the server @@ -6115,8 +6197,19 @@ mod tests { "add_all dropped a subscription outside the set it was given" ); - // `sync`, by contrast, is authoritative over the whole tree. - let (added, removed) = registry.sync(&[c.clone()].into_iter().collect(), false); + // A traversal that missed entries is additive only: absence from its + // partial result is not evidence that an existing watch became stale. + let partial: std::collections::HashSet = std::iter::once(c.clone()).collect(); + let (added, removed) = registry.sync(&partial, TraversalCompleteness::Incomplete, false); + assert_eq!((added.len(), removed), (0, 0)); + assert_eq!( + registry.watched, + [a.clone(), b.clone(), c.clone()].into_iter().collect(), + "an incomplete desired set retired valid existing subscriptions" + ); + + // A later complete traversal is authoritative and may prune them. + let (added, removed) = registry.sync(&partial, TraversalCompleteness::Complete, false); assert_eq!((added.len(), removed), (0, 2)); assert_eq!(registry.watched, [c].into_iter().collect()); } @@ -6221,6 +6314,7 @@ mod tests { let dirs = watchable_dirs(root, root, &exclude, Some(&gi)); let rel: std::collections::HashSet = dirs + .dirs .iter() .map(|p| { p.strip_prefix(root) @@ -6265,6 +6359,7 @@ mod tests { let dirs = watchable_dirs(root, root, &[], None); let rel: std::collections::HashSet = dirs + .dirs .iter() .map(|p| { p.strip_prefix(root) @@ -6297,6 +6392,7 @@ mod tests { let gi = tgrep_core::gitignore::build_matcher(root).expect("matcher should build"); let dirs = watchable_dirs(root, &root.join("src/fresh"), &[], Some(&gi)); let rel: std::collections::HashSet = dirs + .dirs .iter() .map(|p| { p.strip_prefix(root) @@ -6323,6 +6419,59 @@ mod tests { ); } + #[test] + fn rejected_never_indexed_file_does_not_dirty_the_overlay() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + let rejected = root.join("asset.png"); + std::fs::write(&rejected, "not indexed despite textual contents\n").unwrap(); + + let _gate = state.snapshot_gate.read().unwrap(); + reindex_file(&state, &rejected, "asset.png"); + + let index = state.index.read().unwrap(); + assert_eq!( + index.live.dirty_count(), + 0, + "a path absent from reader, overlay, and stamps must not create a tombstone" + ); + assert!(!index.live.is_deleted("asset.png")); + } + + #[test] + fn reader_path_without_a_stamp_is_still_tombstoned() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + let indexed = root.join("legacy.rs"); + std::fs::write(&indexed, "fn legacy_reader_entry() {}\n").unwrap(); + builder::build_index_for_files(&root, &index_dir, std::slice::from_ref(&indexed), 1024) + .unwrap(); + *state.index.write().unwrap() = HybridIndex::open(&index_dir, &root).unwrap(); + assert!( + state.file_stamps.read().unwrap().is_empty(), + "fixture requires a reader entry with no stamp" + ); + assert!( + state.index.read().unwrap().reader_has_path("legacy.rs"), + "fixture did not put the path in the active reader" + ); + + let _gate = state.snapshot_gate.read().unwrap(); + let _reindex = lock_reindex(&state); + drop_indexed_file(&state, "legacy.rs", "test rejection"); + + let index = state.index.read().unwrap(); + assert!( + index.live.is_deleted("legacy.rs"), + "reader membership must remain sufficient evidence to tombstone" + ); + assert_eq!(index.live.dirty_count(), 1); + } + /// A transient stat failure is not a deletion. `Path::exists` said it was, /// which meant one `EACCES` — or a Windows sharing violation from a build /// holding the file open — tombstoned content that was still valid, and @@ -7818,7 +7967,7 @@ mod tests { ] .into_iter() .collect(); - let (added, _removed) = registry.sync(&desired, false); + let (added, _removed) = registry.sync(&desired, TraversalCompleteness::Complete, false); assert_eq!(added.len(), 4); let depths: Vec = added.iter().map(|d| d.components().count()).collect(); @@ -7854,13 +8003,13 @@ mod tests { std::fs::remove_dir(&gone).unwrap(); let desired: std::collections::HashSet = std::iter::once(gone.clone()).collect(); - registry.sync(&desired, false); + registry.sync(&desired, TraversalCompleteness::Complete, false); assert!( registry.is_watched(&gone), "the fixture must reproduce the poisoned entry, or it proves nothing" ); - registry.sync(&desired, true); + registry.sync(&desired, TraversalCompleteness::Complete, true); assert!( !registry.is_watched(&gone), "a forced sync must re-issue the subscription and drop the entry \ diff --git a/tgrep-cli/tests/watcher_watch_registration.rs b/tgrep-cli/tests/watcher_watch_registration.rs index e8e765e..be2f5c2 100644 --- a/tgrep-cli/tests/watcher_watch_registration.rs +++ b/tgrep-cli/tests/watcher_watch_registration.rs @@ -1,4 +1,5 @@ -//! The watcher must not take OS subscriptions for trees it is going to ignore. +//! On Linux and Android, the watcher must not take OS subscriptions for trees +//! it is going to ignore. //! //! On Linux `notify` has no recursive inotify mode: `RecursiveMode::Recursive` //! walks the tree and spends one watch descriptor per directory. Ignored build @@ -14,9 +15,13 @@ //! while asserting the watcher still works — a watcher that registered nothing //! at all would trivially satisfy the count. //! -//! Windows (ReadDirectoryChangesW) and macOS (FSEvents) subscribe once for the -//! whole subtree, so there is no per-directory registration to withhold and -//! nothing here applies; the count assertion is Linux-only for that reason. +//! This implementation intentionally uses one recursive +//! `ReadDirectoryChangesW` root subscription on Windows and one root FSEvents +//! stream on macOS. Ignored events are filtered after delivery there, so the +//! implementation avoids per-directory descriptor growth but does not leave +//! ignored descendants unwatched. kqueue and `PollWatcher` are not covered. +//! The descriptor-count assertion is Linux-only; Android has the same selective +//! registration design but is not exercised by this test target. use std::fs; use std::io::{BufRead, BufReader, Write}; @@ -111,6 +116,15 @@ fn wait_for_no_match(port: u16, pattern: &str, timeout: Duration) -> bool { } } +fn git(root: &Path, args: &[&str]) { + let status = Command::new("git") + .current_dir(root) + .args(args) + .status() + .expect("failed to run git"); + assert!(status.success(), "git {args:?} failed"); +} + /// Read `(pid, port)` once the server is accepting connections. fn wait_for_server(index_dir: &Path) -> (u32, u16) { let serve_json = index_dir.join("serve.json"); @@ -174,6 +188,7 @@ fn watcher_does_not_subscribe_to_gitignored_directories() { fs::create_dir_all(&sub).unwrap(); fs::write(sub.join("artifact.txt"), "ignored_build_output\n").unwrap(); } + for i in 0..SOURCE_DIRS { let sub = root.join("src").join(format!("pkg{i}")); fs::create_dir_all(&sub).unwrap(); @@ -244,6 +259,93 @@ fn watcher_does_not_subscribe_to_gitignored_directories() { ); } +/// The Git index is hidden and deliberately not watched, but it changes which +/// case-insensitively ignored paths Git considers tracked. The long-lived +/// watcher must reconcile both directions from the index identity alone. +#[test] +fn watcher_reconciles_forced_add_and_rm_cached_inside_an_ignored_tree() { + let dir = TempDir::new().unwrap(); + let root = dir.path(); + let index_dir = root.join(".tgrep_test_index"); + + git(root, &["init", "--quiet"]); + git(root, &["config", "core.ignorecase", "true"]); + fs::write(root.join(".gitignore"), "IGNORED/\n").unwrap(); + fs::create_dir_all(root.join("src")).unwrap(); + fs::write( + root.join("src").join("lib.rs"), + "fn seeded() { let normal_source_marker = 1; }\n", + ) + .unwrap(); + fs::create_dir_all(root.join("ignored")).unwrap(); + fs::write( + root.join("ignored").join("tracked.rs"), + "fn forced() { let forced_tracked_marker = 2; }\n", + ) + .unwrap(); + git(root, &["add", "--", ".gitignore", "src/lib.rs"]); + + let status = Command::new(tgrep_bin()) + .args([ + "index", + root.to_str().unwrap(), + "--index-path", + index_dir.to_str().unwrap(), + ]) + .status() + .expect("failed to run tgrep index"); + assert!(status.success(), "initial index build failed"); + + let child = Command::new(tgrep_bin()) + .args([ + "serve", + "--index-path", + index_dir.to_str().unwrap(), + root.to_str().unwrap(), + ]) + .stderr(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .spawn() + .expect("failed to start tgrep serve"); + let _server = ServerGuard { child }; + + let (_pid, port) = wait_for_server(&index_dir); + fs::write( + root.join("src").join("watcher-ready.rs"), + "fn ready() { let watcher_ready_marker = 3; }\n", + ) + .unwrap(); + assert!( + wait_for_match(port, "watcher_ready_marker", Duration::from_secs(30)), + "watcher did not become ready" + ); + assert_eq!(search_matches(port, "forced_tracked_marker"), 0); + + // Only .git/index changes: the file already exists under an unsubscribed + // ignored directory, so no ordinary watcher event can rescue this. + git(root, &["add", "-f", "--", "ignored/tracked.rs"]); + assert!( + wait_for_match(port, "forced_tracked_marker", Duration::from_secs(60)), + "git add -f did not restore the tracked file through reconciliation" + ); + + git( + root, + &[ + "rm", + "--cached", + "--force", + "--quiet", + "--", + "ignored/tracked.rs", + ], + ); + assert!( + wait_for_no_match(port, "forced_tracked_marker", Duration::from_secs(60)), + "git rm --cached did not prune stale indexed content" + ); +} + /// New directories still have to be picked up. /// /// Non-recursive subscriptions are not extended by notify, so a directory diff --git a/tgrep-core/src/git_index.rs b/tgrep-core/src/git_index.rs index 2b01e6e..3d8d441 100644 --- a/tgrep-core/src/git_index.rs +++ b/tgrep-core/src/git_index.rs @@ -103,6 +103,23 @@ pub(crate) fn git_dir(repo_root: &Path) -> Option { }) } +/// The repository metadata directory shared by all linked worktrees. +pub(crate) fn common_git_dir(git_dir: &Path) -> PathBuf { + let Ok(target) = std::fs::read_to_string(git_dir.join("commondir")) else { + return git_dir.to_path_buf(); + }; + let target = target.trim(); + if target.is_empty() { + return git_dir.to_path_buf(); + } + let path = Path::new(target); + if path.is_absolute() { + path.to_path_buf() + } else { + git_dir.join(path) + } +} + /// Whether the repository compares paths without regard to case. /// /// git writes `core.ignorecase = true` into the repository's own config when it @@ -112,7 +129,7 @@ pub fn ignores_case(repo_root: &Path) -> bool { let Some(git_dir) = git_dir(repo_root) else { return false; }; - let Ok(config) = std::fs::read_to_string(git_dir.join("config")) else { + let Ok(config) = std::fs::read_to_string(common_git_dir(&git_dir).join("config")) else { return false; }; let mut in_core = false; @@ -453,6 +470,33 @@ mod tests { assert!(tracked.contains("worktree/file.rs")); } + #[test] + fn linked_worktree_uses_the_common_config_and_its_own_index() { + let tmp = tempfile::tempdir().expect("tempdir"); + let common = tmp.path().join("repo.git"); + let worktree_git = common.join("worktrees").join("topic"); + std::fs::create_dir_all(&worktree_git).expect("mkdir"); + std::fs::write(common.join("config"), "[core]\n\tignorecase = true\n").expect("write"); + std::fs::write(worktree_git.join("commondir"), "../..\n").expect("write"); + std::fs::write(worktree_git.join("index"), v2_index(&["worktree/only.rs"])).expect("write"); + + let worktree = tmp.path().join("worktree"); + std::fs::create_dir_all(&worktree).expect("mkdir"); + std::fs::write( + worktree.join(".git"), + format!("gitdir: {}\n", worktree_git.display()), + ) + .expect("write"); + + assert!(ignores_case(&worktree)); + assert_eq!(index_path(&worktree), Some(worktree_git.join("index"))); + assert!( + load_tracked(&worktree) + .expect("loads worktree index") + .contains("worktree/only.rs") + ); + } + #[test] fn a_directory_without_git_reports_nothing() { let tmp = tempfile::tempdir().expect("tempdir"); diff --git a/tgrep-core/src/gitignore.rs b/tgrep-core/src/gitignore.rs index 173dea4..36d75b1 100644 --- a/tgrep-core/src/gitignore.rs +++ b/tgrep-core/src/gitignore.rs @@ -308,6 +308,17 @@ impl IgnoreMatcher { .as_ref() .is_some_and(|ignorecase| ignorecase.excludes(&self.root.join(rel), is_dir)) } + + /// Identity of the Git index behind the tracked-file exemption. + /// + /// `None` means this matcher has no case-insensitive exemption, so callers + /// need not poll anything. The identity itself can represent a missing or + /// unreadable index, since either transition changes what can be exempted. + pub fn tracked_index_identity(&self) -> Option { + self.ignorecase + .as_ref() + .map(CaseInsensitiveIgnore::tracked_index_identity) + } } /// Return `rel` relative to `dir`, or `None` when `rel` is not beneath it. @@ -468,6 +479,10 @@ struct TrackedCache { tracked: Option, } +/// Metadata identity of the worktree-specific Git index. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TrackedIndexIdentity(Option<(std::time::SystemTime, u64)>); + /// Modification time and length of the index, which is what identifies it. /// /// git replaces the index by renaming `index.lock` over it, so any rewrite @@ -551,6 +566,10 @@ impl CaseInsensitiveIgnore { self.tracked_hides(&relative, is_dir) } + fn tracked_index_identity(&self) -> TrackedIndexIdentity { + TrackedIndexIdentity(index_identity(&self.repo_root)) + } + /// Whether the tracked-file set leaves `relative` hidden. /// /// `false` when the index cannot be read: with no way to tell tracked from @@ -626,20 +645,7 @@ fn git_repo_root(root: &Path) -> Option<&Path> { /// the next stale check evicts. pub fn repo_exclude_path(root: &Path) -> Option { let git_dir = crate::git_index::git_dir(git_repo_root(root)?)?; - let common = match std::fs::read_to_string(git_dir.join("commondir")) { - Ok(target) => { - let target = target.trim(); - let path = Path::new(target); - if target.is_empty() { - git_dir - } else if path.is_absolute() { - path.to_path_buf() - } else { - git_dir.join(path) - } - } - Err(_) => git_dir, - }; + let common = crate::git_index::common_git_dir(&git_dir); let path = common.join("info").join("exclude"); path.is_file().then_some(path) } diff --git a/tgrep-core/src/hybrid.rs b/tgrep-core/src/hybrid.rs index bf96237..22114d2 100644 --- a/tgrep-core/src/hybrid.rs +++ b/tgrep-core/src/hybrid.rs @@ -293,6 +293,11 @@ impl HybridIndex { self.reader().all_paths().iter().cloned().collect() } + /// Whether the active on-disk reader contains `path`. + pub fn reader_has_path(&self, path: &str) -> bool { + self.reader().contains_path(path) + } + /// The reader paths `keep` accepts. /// /// For callers that want a few of them — everything under one directory, diff --git a/tgrep-core/src/reader.rs b/tgrep-core/src/reader.rs index 396fa34..eb00cc0 100644 --- a/tgrep-core/src/reader.rs +++ b/tgrep-core/src/reader.rs @@ -14,6 +14,9 @@ pub struct IndexReader { lookup: Option, postings: Option, file_paths: Vec, + /// File IDs sorted by path, for exact membership checks without duplicating + /// every path string in a second collection. + path_order: Vec, num_entries: usize, } @@ -120,11 +123,14 @@ impl IndexReader { for (id, path) in file_entries { file_paths[id as usize] = path; } + let mut path_order: Vec = (0..file_paths.len()).collect(); + path_order.sort_unstable_by(|&a, &b| file_paths[a].cmp(&file_paths[b])); Ok(Self { lookup, postings, file_paths, + path_order, num_entries, }) } @@ -137,6 +143,7 @@ impl IndexReader { lookup: None, postings: None, file_paths: Vec::new(), + path_order: Vec::new(), num_entries: 0, } } @@ -146,6 +153,7 @@ impl IndexReader { self.lookup = None; self.postings = None; self.file_paths.clear(); + self.path_order.clear(); self.num_entries = 0; } @@ -179,6 +187,12 @@ impl IndexReader { self.file_paths.get(file_id as usize).map(|s| s.as_str()) } + pub fn contains_path(&self, path: &str) -> bool { + self.path_order + .binary_search_by(|&id| self.file_paths[id].as_str().cmp(path)) + .is_ok() + } + /// Total number of indexed files. pub fn num_files(&self) -> usize { self.file_paths.len() @@ -555,6 +569,7 @@ mod tests { lookup: opened.lookup, postings: opened.postings, file_paths: opened.file_paths, + path_order: opened.path_order, num_entries: 0, }; assert!( From 260545265997a0e37c6a2fa58305811daa17d8f0 Mon Sep 17 00:00:00 2001 From: Shengyu Fu Date: Sat, 29 Aug 2026 00:25:52 -0500 Subject: [PATCH 02/16] Preserve watcher repair signals Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tgrep-cli/src/serve.rs | 158 ++++++++++++++++++++++++++++++------ tgrep-core/src/git_index.rs | 27 ++++++ tgrep-core/src/gitignore.rs | 54 +++++++++--- 3 files changed, 202 insertions(+), 37 deletions(-) diff --git a/tgrep-cli/src/serve.rs b/tgrep-cli/src/serve.rs index 59cb2c9..32033d9 100644 --- a/tgrep-cli/src/serve.rs +++ b/tgrep-cli/src/serve.rs @@ -294,9 +294,9 @@ struct ServerState { ignore_rules_dirty: std::sync::atomic::AtomicBool, /// Ensures a burst of ignore-file events uses at most one refresh worker. ignore_refresh_scheduled: std::sync::atomic::AtomicBool, - /// Last observed identity of the worktree-specific Git index. `None` means - /// the published matcher has no case-insensitive tracked-file exemption. - tracked_index_identity: Mutex>, + /// Last observed tracked-path membership. `None` means the published + /// matcher has no case-insensitive tracked-file exemption. + tracked_membership: Mutex>, /// Set when events were lost, cleared by the next subscription sync, which /// then re-issues every subscription instead of trusting its own records. /// @@ -655,7 +655,7 @@ pub fn run(root: &Path, index_path: Option<&Path>, options: ServeOptions<'_>) -> gitignore_pending: std::sync::atomic::AtomicBool::new(!no_watch && !no_ignore), ignore_rules_dirty: std::sync::atomic::AtomicBool::new(false), ignore_refresh_scheduled: std::sync::atomic::AtomicBool::new(false), - tracked_index_identity: Mutex::new(None), + tracked_membership: Mutex::new(None), watch_resubscribe: std::sync::atomic::AtomicBool::new(false), ignore_sources: RwLock::new(Vec::new()), ignore_source_stamps: RwLock::new(IgnoreStamps::new()), @@ -1021,17 +1021,19 @@ fn publish_ignore_matcher( // walking, the next poll must still see that change rather than having this // publish silently advance past it. let mut published = state.gitignore.write().unwrap(); - let mut identity = state.tracked_index_identity.lock().unwrap(); - let current_identity = matcher - .as_ref() - .and_then(tgrep_core::gitignore::IgnoreMatcher::tracked_index_identity); - match (&*identity, current_identity) { - (_, None) => *identity = None, - (None, Some(current)) => *identity = Some(current), + let mut membership = state.tracked_membership.lock().unwrap(); + let current_membership = state.watch_enabled.then(|| { + matcher + .as_ref() + .and_then(tgrep_core::gitignore::IgnoreMatcher::tracked_membership_fingerprint) + }); + match (&*membership, current_membership.flatten()) { + (_, None) => *membership = None, + (None, Some(current)) => *membership = Some(current), (Some(_), Some(_)) => {} } *published = matcher; - drop(identity); + drop(membership); drop(published); state.gitignore_pending.store(false, Ordering::SeqCst); // New rules mean a different set of directories worth hearing about: @@ -1893,8 +1895,10 @@ fn start_file_watcher(state: Arc, root: &Path, queue_cap: usize) -> loop { if last_tracked_index_poll.elapsed() >= TRACKED_INDEX_POLL { last_tracked_index_poll = Instant::now(); - if tracked_index_changed(&worker_state) { - eprintln!("[trace] Git index changed; reconciling tracked-file exemptions"); + if tracked_membership_changed(&worker_state) { + eprintln!( + "[trace] Git tracked paths changed; reconciling tracked-file exemptions" + ); worker_state .ignore_rules_dirty .store(true, Ordering::SeqCst); @@ -1960,15 +1964,17 @@ fn start_file_watcher(state: Arc, root: &Path, queue_cap: usize) -> /// Detect a tracked-file exemption change without subscribing to `.git`. /// /// The metadata probe is enabled only while the published matcher actually -/// uses the exemption. Updating the observed identity before scheduling makes +/// uses the exemption. Index metadata only decides when to reparse its paths; +/// an unchanged membership fingerprint does not schedule a repository walk. +/// Updating the observed membership before scheduling makes /// a burst one dirty signal; the existing refresh scheduler coalesces it with /// any ignore-source changes and serializes the full stale reconciliation. -fn tracked_index_changed(state: &ServerState) -> bool { +fn tracked_membership_changed(state: &ServerState) -> bool { let matcher = state.gitignore.read().unwrap(); let current = matcher .as_ref() - .and_then(tgrep_core::gitignore::IgnoreMatcher::tracked_index_identity); - let mut observed = state.tracked_index_identity.lock().unwrap(); + .and_then(tgrep_core::gitignore::IgnoreMatcher::tracked_membership_fingerprint); + let mut observed = state.tracked_membership.lock().unwrap(); let changed = matches!( (observed.as_ref(), current.as_ref()), (Some(previous), Some(current)) if previous != current @@ -2176,6 +2182,22 @@ enum TraversalCompleteness { Incomplete, } +/// Consume one overflow repair request only when this traversal can complete it. +/// +/// Clearing before the sync means an overflow that arrives during a complete +/// pass sets the flag for another pass. Re-arming an incomplete pass preserves +/// the original request without overwriting a concurrent `true`. +fn take_force_resubscribe( + pending: &std::sync::atomic::AtomicBool, + completeness: TraversalCompleteness, +) -> bool { + let force = pending.swap(false, Ordering::SeqCst); + if force && completeness == TraversalCompleteness::Incomplete { + pending.store(true, Ordering::SeqCst); + } + force +} + struct WatchableDirs { dirs: std::collections::HashSet, completeness: TraversalCompleteness, @@ -2553,9 +2575,9 @@ fn sync_watch_registrations(state: &ServerState, root: &Path) -> (Vec, desired.dirs.extend(ignore_target_dirs(root, &sources)); } let total = desired.dirs.len(); - // Consumed here, so one overflow buys one forced pass rather than making - // every later reconcile re-register the whole tree. - let force = state.watch_resubscribe.swap(false, Ordering::SeqCst); + // An incomplete traversal cannot prove that omitted recorded watches are + // live, so it does not get to consume the overflow repair request. + let force = take_force_resubscribe(&state.watch_resubscribe, desired.completeness); let (added, removed) = registry.sync(&desired.dirs, desired.completeness, force); if !added.is_empty() || removed > 0 { eprintln!( @@ -5927,7 +5949,7 @@ mod tests { gitignore_pending: std::sync::atomic::AtomicBool::new(true), ignore_rules_dirty: std::sync::atomic::AtomicBool::new(false), ignore_refresh_scheduled: std::sync::atomic::AtomicBool::new(false), - tracked_index_identity: Mutex::new(None), + tracked_membership: Mutex::new(None), watch_resubscribe: std::sync::atomic::AtomicBool::new(false), ignore_sources: RwLock::new(Vec::new()), ignore_source_stamps: RwLock::new(IgnoreStamps::new()), @@ -6200,7 +6222,17 @@ mod tests { // A traversal that missed entries is additive only: absence from its // partial result is not evidence that an existing watch became stale. let partial: std::collections::HashSet = std::iter::once(c.clone()).collect(); - let (added, removed) = registry.sync(&partial, TraversalCompleteness::Incomplete, false); + let pending = std::sync::atomic::AtomicBool::new(true); + let force = take_force_resubscribe(&pending, TraversalCompleteness::Incomplete); + assert!( + force, + "the incomplete pass must still attempt known entries" + ); + assert!( + pending.load(Ordering::SeqCst), + "an incomplete pass must leave forced resubscription pending" + ); + let (added, removed) = registry.sync(&partial, TraversalCompleteness::Incomplete, force); assert_eq!((added.len(), removed), (0, 0)); assert_eq!( registry.watched, @@ -6209,7 +6241,16 @@ mod tests { ); // A later complete traversal is authoritative and may prune them. - let (added, removed) = registry.sync(&partial, TraversalCompleteness::Complete, false); + let force = take_force_resubscribe(&pending, TraversalCompleteness::Complete); + assert!( + force, + "the next complete pass must inherit the force request" + ); + assert!( + !pending.load(Ordering::SeqCst), + "a complete forced pass consumes the request" + ); + let (added, removed) = registry.sync(&partial, TraversalCompleteness::Complete, force); assert_eq!((added.len(), removed), (0, 2)); assert_eq!(registry.watched, [c].into_iter().collect()); } @@ -6472,6 +6513,75 @@ mod tests { assert_eq!(index.live.dirty_count(), 1); } + #[test] + fn content_only_git_index_rewrite_does_not_request_reconciliation() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let git = |args: &[&str]| { + let status = std::process::Command::new("git") + .current_dir(&root) + .args(args) + .status() + .expect("run git"); + assert!(status.success(), "git {args:?} failed"); + }; + + git(&["init", "--quiet"]); + git(&["config", "core.ignorecase", "true"]); + std::fs::write(root.join(".gitignore"), "IGNORED/\n").unwrap(); + std::fs::create_dir_all(root.join("src")).unwrap(); + std::fs::write(root.join("src/lib.rs"), "fn original() {}\n").unwrap(); + std::fs::create_dir_all(root.join("ignored")).unwrap(); + std::fs::write(root.join("ignored/forced.rs"), "fn forced() {}\n").unwrap(); + git(&["add", "--", ".gitignore", "src/lib.rs"]); + + let state = test_server_state(&root, &root.join(".tgrep")); + let matcher = + tgrep_core::gitignore::build_matcher(&root).expect("case-insensitive matcher"); + let baseline = matcher + .tracked_membership_fingerprint() + .expect("tracked exemption active"); + *state.gitignore.write().unwrap() = Some(matcher); + *state.tracked_membership.lock().unwrap() = Some(baseline); + + std::fs::write(root.join("src/lib.rs"), "fn content_changed() {}\n").unwrap(); + git(&["add", "--", "src/lib.rs"]); + assert!( + !tracked_membership_changed(&state), + "rewriting index metadata with the same tracked paths must not request a full scan" + ); + + std::fs::write(root.join("src/new.rs"), "fn newly_tracked() {}\n").unwrap(); + git(&["add", "--", "src/new.rs"]); + assert!( + !tracked_membership_changed(&state), + "tracked membership outside the exemption set must not request a full scan" + ); + + git(&["add", "-f", "--", "ignored/forced.rs"]); + assert!( + tracked_membership_changed(&state), + "adding a tracked-path exemption must request reconciliation" + ); + assert!( + !tracked_membership_changed(&state), + "an observed membership change must be coalesced" + ); + + git(&[ + "rm", + "--cached", + "--quiet", + "--force", + "--", + "ignored/forced.rs", + ]); + assert!( + tracked_membership_changed(&state), + "removing a tracked-path exemption must request reconciliation" + ); + } + /// A transient stat failure is not a deletion. `Path::exists` said it was, /// which meant one `EACCES` — or a Windows sharing violation from a build /// holding the file open — tombstoned content that was still valid, and diff --git a/tgrep-core/src/git_index.rs b/tgrep-core/src/git_index.rs index 3d8d441..3202e1c 100644 --- a/tgrep-core/src/git_index.rs +++ b/tgrep-core/src/git_index.rs @@ -72,6 +72,25 @@ impl TrackedFiles { // matched — a handful per walk, not one per entry. self.paths.iter().any(|p| p.starts_with(prefix.as_str())) } + + pub(crate) fn fingerprint_matching( + &self, + mut include: impl FnMut(&str) -> bool, + ) -> (usize, u64, u64) { + let mut count = 0; + let mut xor = 0; + let mut sum = 0u64; + for path in &self.paths { + if !include(path) { + continue; + } + let hash = membership_hash(path); + count += 1; + xor ^= hash; + sum = sum.wrapping_add(hash); + } + (count, xor, sum) + } } fn normalise(path: &str) -> String { @@ -80,6 +99,14 @@ fn normalise(path: &str) -> String { .to_ascii_lowercase() } +fn membership_hash(path: &str) -> u64 { + path.as_bytes() + .iter() + .fold(0xcbf29ce484222325, |hash, byte| { + (hash ^ u64::from(*byte)).wrapping_mul(0x100000001b3) + }) +} + /// Locate the directory holding a repository's `.git` metadata. /// /// `.git` is usually a directory, but is a file holding `gitdir: ` in a diff --git a/tgrep-core/src/gitignore.rs b/tgrep-core/src/gitignore.rs index 36d75b1..02f6356 100644 --- a/tgrep-core/src/gitignore.rs +++ b/tgrep-core/src/gitignore.rs @@ -309,15 +309,16 @@ impl IgnoreMatcher { .is_some_and(|ignorecase| ignorecase.excludes(&self.root.join(rel), is_dir)) } - /// Identity of the Git index behind the tracked-file exemption. + /// Fingerprint of the tracked paths behind the tracked-file exemption. /// /// `None` means this matcher has no case-insensitive exemption, so callers - /// need not poll anything. The identity itself can represent a missing or - /// unreadable index, since either transition changes what can be exempted. - pub fn tracked_index_identity(&self) -> Option { + /// need not poll anything. Index metadata is used internally to avoid + /// reparsing an unchanged index, but only path membership contributes to + /// this value. + pub fn tracked_membership_fingerprint(&self) -> Option { self.ignorecase .as_ref() - .map(CaseInsensitiveIgnore::tracked_index_identity) + .map(CaseInsensitiveIgnore::tracked_membership_fingerprint) } } @@ -477,11 +478,13 @@ struct TrackedCache { /// an unreadable index is not retried on every path. loaded_from: Option>, tracked: Option, + /// Fingerprint of only the tracked paths this matcher would otherwise hide. + fingerprint: Option<(usize, u64, u64)>, } -/// Metadata identity of the worktree-specific Git index. +/// Compact, order-independent identity of the tracked exemption set. #[derive(Clone, Debug, PartialEq, Eq)] -pub struct TrackedIndexIdentity(Option<(std::time::SystemTime, u64)>); +pub struct TrackedMembershipFingerprint(Option<(usize, u64, u64)>); /// Modification time and length of the index, which is what identifies it. /// @@ -566,8 +569,17 @@ impl CaseInsensitiveIgnore { self.tracked_hides(&relative, is_dir) } - fn tracked_index_identity(&self) -> TrackedIndexIdentity { - TrackedIndexIdentity(index_identity(&self.repo_root)) + fn tracked_membership_fingerprint(&self) -> TrackedMembershipFingerprint { + let identity = index_identity(&self.repo_root); + { + let cache = self.tracked.read().unwrap(); + if cache.loaded_from.as_ref() == Some(&identity) { + return TrackedMembershipFingerprint(cache.fingerprint); + } + } + let mut cache = self.tracked.write().unwrap(); + self.reload_tracked_if_needed(&mut cache, identity); + TrackedMembershipFingerprint(cache.fingerprint) } /// Whether the tracked-file set leaves `relative` hidden. @@ -584,13 +596,29 @@ impl CaseInsensitiveIgnore { } let mut cache = self.tracked.write().unwrap(); // Another thread may have reloaded it while this one waited. - if cache.loaded_from.as_ref() != Some(&identity) { - cache.tracked = crate::git_index::load_tracked(&self.repo_root); - cache.loaded_from = Some(identity); - } + self.reload_tracked_if_needed(&mut cache, identity); Self::hides(cache.tracked.as_ref(), relative, is_dir) } + fn reload_tracked_if_needed( + &self, + cache: &mut TrackedCache, + identity: Option<(std::time::SystemTime, u64)>, + ) { + if cache.loaded_from.as_ref() == Some(&identity) { + return; + } + cache.tracked = crate::git_index::load_tracked(&self.repo_root); + cache.fingerprint = cache.tracked.as_ref().map(|tracked| { + tracked.fingerprint_matching(|path| { + self.matcher + .matched_path_or_any_parents(Path::new(path), false) + .is_ignore() + }) + }); + cache.loaded_from = Some(identity); + } + fn hides( tracked: Option<&crate::git_index::TrackedFiles>, relative: &str, From f0a31b58a581fea972b92898d263a83acc595228 Mon Sep 17 00:00:00 2001 From: Shengyu Fu Date: Sat, 29 Aug 2026 00:40:05 -0500 Subject: [PATCH 03/16] Retry stale refresh across index generations Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tgrep-cli/src/serve.rs | 265 ++++++++++++++++++++++++++++++++---- tgrep-core/src/gitignore.rs | 56 ++++++-- 2 files changed, 285 insertions(+), 36 deletions(-) diff --git a/tgrep-cli/src/serve.rs b/tgrep-cli/src/serve.rs index 32033d9..4e6232c 100644 --- a/tgrep-cli/src/serve.rs +++ b/tgrep-cli/src/serve.rs @@ -478,8 +478,20 @@ struct ServerState { /// Milliseconds since `started` at the last search request, used by the /// periodic reconcile to stay out of the way of a server in active use. last_search_ms: std::sync::atomic::AtomicU64, + #[cfg(test)] + stale_refresh_hook: Mutex>, } +#[cfg(test)] +#[derive(Clone, Copy)] +enum StaleRefreshPhase { + BeforeWalk, + AfterMatcherPublish, +} + +#[cfg(test)] +type StaleRefreshHook = Arc; + impl ServerState { fn note_search(&self) { self.last_search_ms @@ -681,6 +693,8 @@ pub fn run(root: &Path, index_path: Option<&Path>, options: ServeOptions<'_>) -> unreadable: RwLock::new(std::collections::HashMap::new()), started: serve_start, last_search_ms: std::sync::atomic::AtomicU64::new(0), + #[cfg(test)] + stale_refresh_hook: Mutex::new(None), }); // Bind TCP listener on a random port @@ -1983,6 +1997,28 @@ fn tracked_membership_changed(state: &ServerState) -> bool { changed } +fn tracked_index_generation( + state: &ServerState, +) -> Option { + if !state.watch_enabled { + return None; + } + state + .gitignore + .read() + .unwrap() + .as_ref() + .and_then(tgrep_core::gitignore::IgnoreMatcher::tracked_index_generation) +} + +#[cfg(test)] +fn run_stale_refresh_hook(state: &ServerState, phase: StaleRefreshPhase) { + let hook = state.stale_refresh_hook.lock().unwrap().clone(); + if let Some(hook) = hook { + hook(phase); + } +} + /// Decide whether the file watcher should skip a path entirely. /// /// Mirrors the file walker's hidden-path, `--exclude` directory filtering, @@ -4707,12 +4743,15 @@ fn background_refresh_stale( index_dir: &Path, compare_index_membership: bool, ) -> bool { - let _refresh = state.stale_refresh_lock.lock().unwrap(); + let refresh = state.stale_refresh_lock.lock().unwrap(); // Keep watcher/auto-save mutations out for the complete walk → matcher → // merge → recovery cycle. Search queries do not take this gate and remain // available. Held here rather than inside so the recovery scan below is // still covered by it. - let _gate = state.snapshot_gate.write().unwrap(); + let gate = state.snapshot_gate.write().unwrap(); + let generation_before = tracked_index_generation(state); + #[cfg(test)] + run_stale_refresh_hook(state, StaleRefreshPhase::BeforeWalk); let mut newly_watched = Vec::new(); // Before the walk, not after the subscriptions: this bounds the window the @@ -4739,6 +4778,19 @@ fn background_refresh_stale( if ok { reindex_files_in(state, root, &newly_watched, since); } + let generation_changed = + generation_before.is_some() && generation_before != tracked_index_generation(state); + drop(gate); + drop(refresh); + if generation_changed { + // The walk and matcher may have consumed an intermediate tracked set + // even when an A→B→A transition restored the semantic fingerprint. + // One serialized corrective pass makes the final generation + // authoritative; the existing scheduler coalesces repeated changes. + eprintln!("[trace] Git index changed during stale reconciliation; scheduling a retry"); + state.ignore_rules_dirty.store(true, Ordering::SeqCst); + schedule_ignore_rules_refresh(Arc::clone(state), root.to_path_buf()); + } ok } @@ -4795,6 +4847,8 @@ fn refresh_stale_locked( ), || build_stale_matcher(state, root, &walk), ); + #[cfg(test)] + run_stale_refresh_hook(state, StaleRefreshPhase::AfterMatcherPublish); if walk.skipped_error > 0 { eprintln!( @@ -5975,9 +6029,19 @@ mod tests { unreadable: RwLock::new(std::collections::HashMap::new()), started: Instant::now(), last_search_ms: std::sync::atomic::AtomicU64::new(0), + stale_refresh_hook: Mutex::new(None), }) } + fn test_git(root: &Path, args: &[&str]) { + let status = std::process::Command::new("git") + .current_dir(root) + .args(args) + .status() + .expect("run git"); + assert!(status.success(), "git {args:?} failed"); + } + /// A binary marker's offset is a position in the file, not in the repaired /// text. /// @@ -6517,23 +6581,15 @@ mod tests { fn content_only_git_index_rewrite_does_not_request_reconciliation() { let tmp = TempDir::new().unwrap(); let root = tmp.path().to_path_buf(); - let git = |args: &[&str]| { - let status = std::process::Command::new("git") - .current_dir(&root) - .args(args) - .status() - .expect("run git"); - assert!(status.success(), "git {args:?} failed"); - }; - git(&["init", "--quiet"]); - git(&["config", "core.ignorecase", "true"]); + test_git(&root, &["init", "--quiet"]); + test_git(&root, &["config", "core.ignorecase", "true"]); std::fs::write(root.join(".gitignore"), "IGNORED/\n").unwrap(); std::fs::create_dir_all(root.join("src")).unwrap(); std::fs::write(root.join("src/lib.rs"), "fn original() {}\n").unwrap(); std::fs::create_dir_all(root.join("ignored")).unwrap(); std::fs::write(root.join("ignored/forced.rs"), "fn forced() {}\n").unwrap(); - git(&["add", "--", ".gitignore", "src/lib.rs"]); + test_git(&root, &["add", "--", ".gitignore", "src/lib.rs"]); let state = test_server_state(&root, &root.join(".tgrep")); let matcher = @@ -6545,20 +6601,20 @@ mod tests { *state.tracked_membership.lock().unwrap() = Some(baseline); std::fs::write(root.join("src/lib.rs"), "fn content_changed() {}\n").unwrap(); - git(&["add", "--", "src/lib.rs"]); + test_git(&root, &["add", "--", "src/lib.rs"]); assert!( !tracked_membership_changed(&state), "rewriting index metadata with the same tracked paths must not request a full scan" ); std::fs::write(root.join("src/new.rs"), "fn newly_tracked() {}\n").unwrap(); - git(&["add", "--", "src/new.rs"]); + test_git(&root, &["add", "--", "src/new.rs"]); assert!( !tracked_membership_changed(&state), "tracked membership outside the exemption set must not request a full scan" ); - git(&["add", "-f", "--", "ignored/forced.rs"]); + test_git(&root, &["add", "-f", "--", "ignored/forced.rs"]); assert!( tracked_membership_changed(&state), "adding a tracked-path exemption must request reconciliation" @@ -6568,20 +6624,181 @@ mod tests { "an observed membership change must be coalesced" ); - git(&[ - "rm", - "--cached", - "--quiet", - "--force", - "--", - "ignored/forced.rs", - ]); + test_git( + &root, + &[ + "rm", + "--cached", + "--quiet", + "--force", + "--", + "ignored/forced.rs", + ], + ); assert!( tracked_membership_changed(&state), "removing a tracked-path exemption must request reconciliation" ); } + #[test] + fn git_index_aba_during_refresh_schedules_a_corrective_pass() { + use std::sync::Barrier; + use std::sync::atomic::AtomicUsize; + + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + test_git(&root, &["init", "--quiet"]); + test_git(&root, &["config", "core.ignorecase", "true"]); + std::fs::write(root.join(".gitignore"), "IGNORED/\n").unwrap(); + std::fs::create_dir_all(root.join("src")).unwrap(); + std::fs::write(root.join("src/lib.rs"), "fn ordinary() {}\n").unwrap(); + let ignored = root.join("ignored"); + std::fs::create_dir_all(&ignored).unwrap(); + std::fs::write(ignored.join("forced.rs"), "fn aba_marker() {}\n").unwrap(); + test_git(&root, &["add", "--", ".gitignore", "src/lib.rs"]); + + let state = test_server_state(&root, &root.join(".tgrep")); + state.gitignore_pending.store(false, Ordering::SeqCst); + let matcher = + tgrep_core::gitignore::build_matcher(&root).expect("case-insensitive matcher"); + let baseline = matcher + .tracked_membership_fingerprint() + .expect("tracked exemption active"); + *state.gitignore.write().unwrap() = Some(matcher); + *state.tracked_membership.lock().unwrap() = Some(baseline); + + if PER_DIRECTORY_WATCHES { + let watcher = notify::recommended_watcher(|_: notify::Result| {}).unwrap(); + *state.watch_registry.lock().unwrap() = Some(WatchRegistry { + watcher, + root: root.clone(), + watched: std::iter::once(root.clone()).collect(), + }); + } + + let before_walk_entered = Arc::new(Barrier::new(2)); + let before_walk_release = Arc::new(Barrier::new(2)); + let after_publish_entered = Arc::new(Barrier::new(2)); + let after_publish_release = Arc::new(Barrier::new(2)); + let passes = Arc::new(AtomicUsize::new(0)); + let hook: StaleRefreshHook = { + let before_walk_entered = Arc::clone(&before_walk_entered); + let before_walk_release = Arc::clone(&before_walk_release); + let after_publish_entered = Arc::clone(&after_publish_entered); + let after_publish_release = Arc::clone(&after_publish_release); + let passes = Arc::clone(&passes); + Arc::new(move |phase| match phase { + StaleRefreshPhase::BeforeWalk => { + if passes.fetch_add(1, Ordering::SeqCst) == 0 { + before_walk_entered.wait(); + before_walk_release.wait(); + } + } + StaleRefreshPhase::AfterMatcherPublish => { + if passes.load(Ordering::SeqCst) == 1 { + after_publish_entered.wait(); + after_publish_release.wait(); + } + } + }) + }; + *state.stale_refresh_hook.lock().unwrap() = Some(hook); + + let refresh_state = Arc::clone(&state); + let refresh_root = root.clone(); + let index_dir = state.index_dir.clone(); + let refresh = thread::spawn(move || { + background_refresh_stale(&refresh_state, &refresh_root, &index_dir, true) + }); + + // A: untracked and hidden when the pass captures its generation. + before_walk_entered.wait(); + // B: visible while both the walk and published matcher make decisions. + test_git(&root, &["add", "-f", "--", "ignored/forced.rs"]); + before_walk_release.wait(); + after_publish_entered.wait(); + if PER_DIRECTORY_WATCHES { + assert!( + state + .watch_registry + .lock() + .unwrap() + .as_ref() + .unwrap() + .watched + .contains(&ignored), + "the intermediate matcher must subscribe the tracked ignored directory" + ); + } + // A again before the pass commits what it learned from B. + test_git( + &root, + &[ + "rm", + "--cached", + "--quiet", + "--force", + "--", + "ignored/forced.rs", + ], + ); + after_publish_release.wait(); + assert!( + refresh.join().unwrap(), + "the raced pass itself should finish" + ); + + let deadline = Instant::now() + Duration::from_secs(10); + while (passes.load(Ordering::SeqCst) < 2 + || state.ignore_refresh_scheduled.load(Ordering::SeqCst) + || state.ignore_rules_dirty.load(Ordering::SeqCst)) + && Instant::now() < deadline + { + thread::sleep(Duration::from_millis(10)); + } + assert!( + passes.load(Ordering::SeqCst) >= 2, + "the generation change must schedule a corrective pass" + ); + assert!( + !state.ignore_refresh_scheduled.load(Ordering::SeqCst) + && !state.ignore_rules_dirty.load(Ordering::SeqCst), + "the corrective pass did not finish" + ); + + let matcher = state.gitignore.read().unwrap(); + assert!( + matcher + .as_ref() + .unwrap() + .is_ignored(Path::new("ignored"), true), + "the final matcher must restore the final untracked state" + ); + drop(matcher); + let index = state.index.read().unwrap(); + assert!( + !index.live.has_path("ignored/forced.rs") + && (!index.reader_has_path("ignored/forced.rs") + || index.live.is_deleted("ignored/forced.rs")), + "the corrective pass must remove content indexed from the intermediate state" + ); + drop(index); + if PER_DIRECTORY_WATCHES { + assert!( + !state + .watch_registry + .lock() + .unwrap() + .as_ref() + .unwrap() + .watched + .contains(&ignored), + "the corrective matcher must retire the intermediate subscription" + ); + } + } + /// A transient stat failure is not a deletion. `Path::exists` said it was, /// which meant one `EACCES` — or a Windows sharing violation from a build /// holding the file open — tombstoned content that was still valid, and diff --git a/tgrep-core/src/gitignore.rs b/tgrep-core/src/gitignore.rs index 02f6356..4fa0d69 100644 --- a/tgrep-core/src/gitignore.rs +++ b/tgrep-core/src/gitignore.rs @@ -320,6 +320,13 @@ impl IgnoreMatcher { .as_ref() .map(CaseInsensitiveIgnore::tracked_membership_fingerprint) } + + /// Cheap generation token for the worktree-specific Git index. + pub fn tracked_index_generation(&self) -> Option { + self.ignorecase + .as_ref() + .map(CaseInsensitiveIgnore::tracked_index_generation) + } } /// Return `rel` relative to `dir`, or `None` when `rel` is not beneath it. @@ -476,7 +483,7 @@ pub struct CaseInsensitiveIgnore { struct TrackedCache { /// `None` until something is loaded; `Some` even when the load failed, so /// an unreadable index is not retried on every path. - loaded_from: Option>, + loaded_from: Option>, tracked: Option, /// Fingerprint of only the tracked paths this matcher would otherwise hide. fingerprint: Option<(usize, u64, u64)>, @@ -486,15 +493,40 @@ struct TrackedCache { #[derive(Clone, Debug, PartialEq, Eq)] pub struct TrackedMembershipFingerprint(Option<(usize, u64, u64)>); -/// Modification time and length of the index, which is what identifies it. +/// Metadata generation of the worktree-specific Git index. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TrackedIndexGeneration(Option); + +#[derive(Clone, Debug, PartialEq, Eq)] +struct IndexIdentity { + modified: std::time::SystemTime, + created: Option, + len: u64, + #[cfg(unix)] + device: u64, + #[cfg(unix)] + inode: u64, +} + +/// Cheap metadata identity of the index file generation. /// /// git replaces the index by renaming `index.lock` over it, so any rewrite -/// lands as a new mtime — the same pair git's own racy-index handling relies -/// on. Two rewrites inside one filesystem mtime tick that leave the length -/// unchanged are the gap, and the periodic reconcile is what closes it. -fn index_identity(repo_root: &Path) -> Option<(std::time::SystemTime, u64)> { +/// normally lands as a new file instance as well as a new mtime. Creation time +/// covers that replacement on Windows, and device/inode covers it on Unix, +/// including an A→B→A rewrite inside one filesystem mtime tick. +fn index_identity(repo_root: &Path) -> Option { let meta = std::fs::metadata(crate::git_index::index_path(repo_root)?).ok()?; - Some((meta.modified().ok()?, meta.len())) + #[cfg(unix)] + use std::os::unix::fs::MetadataExt; + Some(IndexIdentity { + modified: meta.modified().ok()?, + created: meta.created().ok(), + len: meta.len(), + #[cfg(unix)] + device: meta.dev(), + #[cfg(unix)] + inode: meta.ino(), + }) } impl CaseInsensitiveIgnore { @@ -582,6 +614,10 @@ impl CaseInsensitiveIgnore { TrackedMembershipFingerprint(cache.fingerprint) } + fn tracked_index_generation(&self) -> TrackedIndexGeneration { + TrackedIndexGeneration(index_identity(&self.repo_root)) + } + /// Whether the tracked-file set leaves `relative` hidden. /// /// `false` when the index cannot be read: with no way to tell tracked from @@ -600,11 +636,7 @@ impl CaseInsensitiveIgnore { Self::hides(cache.tracked.as_ref(), relative, is_dir) } - fn reload_tracked_if_needed( - &self, - cache: &mut TrackedCache, - identity: Option<(std::time::SystemTime, u64)>, - ) { + fn reload_tracked_if_needed(&self, cache: &mut TrackedCache, identity: Option) { if cache.loaded_from.as_ref() == Some(&identity) { return; } From 3c807884fae417f0350507424fcde15d7cb47af4 Mon Sep 17 00:00:00 2001 From: Shengyu Fu Date: Sat, 29 Aug 2026 01:10:15 -0500 Subject: [PATCH 04/16] Freeze tracked membership during reconciliation Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 772a4aa3-4cba-4ac4-aeca-dd184614127a --- tgrep-cli/src/serve.rs | 316 +++++++++++++++++++++++++++--------- tgrep-core/src/builder.rs | 51 ++++-- tgrep-core/src/gitignore.rs | 142 ++++++++-------- tgrep-core/src/walker.rs | 86 +++++++--- 4 files changed, 409 insertions(+), 186 deletions(-) diff --git a/tgrep-cli/src/serve.rs b/tgrep-cli/src/serve.rs index 4e6232c..a67daca 100644 --- a/tgrep-cli/src/serve.rs +++ b/tgrep-cli/src/serve.rs @@ -828,17 +828,19 @@ fn build_stale_matcher( state: &ServerState, root: &Path, walk: &tgrep_core::walker::MetaWalkResult, + ignorecase: Option>, ) -> Option { if state.no_ignore { return None; } let start = Instant::now(); - let matcher = tgrep_core::walker::build_gitignore_matcher_from_files( + let matcher = tgrep_core::walker::build_gitignore_matcher_from_files_with_ignorecase( root, &walk.gitignore_files, &walk.ignore_files, state.no_require_git, + ignorecase, ); let has_matcher = matcher.is_some(); eprintln!( @@ -1030,22 +1032,14 @@ fn publish_ignore_matcher( *state.ignore_source_stamps.write().unwrap() = stamps; *state.ignore_sources.write().unwrap() = sources; - // Keep the matcher and identity state in one critical section. Preserve an - // active baseline across refreshes: if the index changes while a refresh is - // walking, the next poll must still see that change rather than having this - // publish silently advance past it. + // Keep the matcher and semantic baseline in one critical section. The + // matcher's tracked exemption is immutable, so this baseline describes the + // exact decisions the stale walk and watcher publication made. let mut published = state.gitignore.write().unwrap(); let mut membership = state.tracked_membership.lock().unwrap(); - let current_membership = state.watch_enabled.then(|| { - matcher - .as_ref() - .and_then(tgrep_core::gitignore::IgnoreMatcher::tracked_membership_fingerprint) - }); - match (&*membership, current_membership.flatten()) { - (_, None) => *membership = None, - (None, Some(current)) => *membership = Some(current), - (Some(_), Some(_)) => {} - } + *membership = matcher + .as_ref() + .and_then(tgrep_core::gitignore::IgnoreMatcher::tracked_membership_fingerprint); *published = matcher; drop(membership); drop(published); @@ -1987,7 +1981,7 @@ fn tracked_membership_changed(state: &ServerState) -> bool { let matcher = state.gitignore.read().unwrap(); let current = matcher .as_ref() - .and_then(tgrep_core::gitignore::IgnoreMatcher::tracked_membership_fingerprint); + .and_then(tgrep_core::gitignore::IgnoreMatcher::current_tracked_membership_fingerprint); let mut observed = state.tracked_membership.lock().unwrap(); let changed = matches!( (observed.as_ref(), current.as_ref()), @@ -1997,20 +1991,6 @@ fn tracked_membership_changed(state: &ServerState) -> bool { changed } -fn tracked_index_generation( - state: &ServerState, -) -> Option { - if !state.watch_enabled { - return None; - } - state - .gitignore - .read() - .unwrap() - .as_ref() - .and_then(tgrep_core::gitignore::IgnoreMatcher::tracked_index_generation) -} - #[cfg(test)] fn run_stale_refresh_hook(state: &ServerState, phase: StaleRefreshPhase) { let hook = state.stale_refresh_hook.lock().unwrap().clone(); @@ -4749,7 +4729,13 @@ fn background_refresh_stale( // available. Held here rather than inside so the recovery scan below is // still covered by it. let gate = state.snapshot_gate.write().unwrap(); - let generation_before = tracked_index_generation(state); + // One immutable tracked-file exemption is shared by the walk and the + // matcher it publishes. Git-index rewrites cannot change ignore decisions + // halfway through this pass. + let ignorecase = (!state.no_ignore) + .then(|| tgrep_core::gitignore::CaseInsensitiveIgnore::new(root, true, true, true)) + .flatten() + .map(Arc::new); #[cfg(test)] run_stale_refresh_hook(state, StaleRefreshPhase::BeforeWalk); @@ -4764,6 +4750,7 @@ fn background_refresh_stale( index_dir, compare_index_membership, &mut newly_watched, + ignorecase, ); // Directories that were not subscribed while the walk ran could not report @@ -4778,16 +4765,16 @@ fn background_refresh_stale( if ok { reindex_files_in(state, root, &newly_watched, since); } - let generation_changed = - generation_before.is_some() && generation_before != tracked_index_generation(state); + // Compare semantics, not index metadata. A→B→A needs no correction because + // this pass used A throughout, while A→B schedules exactly one coalesced + // refresh. Content-only staging cannot create an immediate refresh loop. + let membership_changed = tracked_membership_changed(state); drop(gate); drop(refresh); - if generation_changed { - // The walk and matcher may have consumed an intermediate tracked set - // even when an A→B→A transition restored the semantic fingerprint. - // One serialized corrective pass makes the final generation - // authoritative; the existing scheduler coalesces repeated changes. - eprintln!("[trace] Git index changed during stale reconciliation; scheduling a retry"); + if membership_changed { + eprintln!( + "[trace] Git tracked paths changed during stale reconciliation; scheduling a retry" + ); state.ignore_rules_dirty.store(true, Ordering::SeqCst); schedule_ignore_rules_refresh(Arc::clone(state), root.to_path_buf()); } @@ -4800,6 +4787,7 @@ fn refresh_stale_locked( index_dir: &Path, compare_index_membership: bool, newly_watched: &mut Vec, + ignorecase: Option>, ) -> bool { use tgrep_core::meta; use tgrep_core::walker; @@ -4810,7 +4798,7 @@ fn refresh_stale_locked( // Walk first. This single traversal feeds both the stale diff and the // watcher's ignore matcher, and it must run before the early returns below // so the matcher can be published on every path out of this function. - let walk = walker::walk_file_metadata( + let walk = walker::walk_file_metadata_with_ignorecase( root, &walker::MetaWalkOptions { exclude_dirs: state.exclude_dirs.clone(), @@ -4818,6 +4806,7 @@ fn refresh_stale_locked( no_require_git: state.no_require_git, max_file_size: state.max_file_size, }, + ignorecase.clone(), ); let walk_ms = start.elapsed().as_millis(); @@ -4845,7 +4834,7 @@ fn refresh_stale_locked( &walk.ignore_files, state.no_require_git, ), - || build_stale_matcher(state, root, &walk), + || build_stale_matcher(state, root, &walk, ignorecase), ); #[cfg(test)] run_stale_refresh_hook(state, StaleRefreshPhase::AfterMatcherPublish); @@ -5002,6 +4991,10 @@ fn reset_to_empty_index(state: &ServerState, root: &Path, index_dir: &Path) { /// caller to fall back. fn bootstrap_index_build(state: &Arc, root: &Path, index_dir: &Path) -> bool { let start = Instant::now(); + let ignorecase = (!state.no_ignore) + .then(|| tgrep_core::gitignore::CaseInsensitiveIgnore::new(root, true, true, true)) + .flatten() + .map(Arc::new); // Anchors the recovery window at the start of the build's traversal, which // is the point from which writes could be missed: nothing under `root` is // subscribed yet, and the walk below has not reached most of it. Taking it @@ -5015,7 +5008,7 @@ fn bootstrap_index_build(state: &Arc, root: &Path, index_dir: &Path // a kernel high-water mark) covers the whole of it. Unlike the incremental // path below, nothing here polls memory on its own. let sampler = crate::mem::PrivatePeakSampler::start(); - let outcome = match builder::build_index_with_options( + let outcome = match builder::build_index_with_options_and_ignorecase( root, Some(index_dir), &builder::BuildOptions { @@ -5032,6 +5025,7 @@ fn bootstrap_index_build(state: &Arc, root: &Path, index_dir: &Path strategy: builder::IndexStrategy::External, buffer_bytes: builder::DEFAULT_INDEX_BUFFER_BYTES, }, + ignorecase.clone(), ) { Ok(outcome) => outcome, Err(e) => { @@ -5091,7 +5085,7 @@ fn bootstrap_index_build(state: &Arc, root: &Path, index_dir: &Path ), } let mut newly_watched = Vec::new(); - if state.watch_enabled && !state.no_ignore { + if !state.no_ignore { let t_gi = Instant::now(); // "Newly watched" here is every directory in the repository, and the // build's walk ran before any of them were subscribed. Deferred rather @@ -5108,11 +5102,12 @@ fn bootstrap_index_build(state: &Arc, root: &Path, index_dir: &Path state.no_require_git, ), || { - tgrep_core::walker::build_gitignore_matcher_from_files( + tgrep_core::walker::build_gitignore_matcher_from_files_with_ignorecase( root, &outcome.gitignore_files, &outcome.ignore_files, state.no_require_git, + ignorecase, ) }, ); @@ -5133,6 +5128,9 @@ fn bootstrap_index_build(state: &Arc, root: &Path, index_dir: &Path state.indexing.store(false, Ordering::SeqCst); drop(gate); + if tracked_membership_changed(state) { + state.ignore_rules_dirty.store(true, Ordering::SeqCst); + } let elapsed = start.elapsed().as_secs_f64(); drop(sampler); @@ -5193,26 +5191,31 @@ fn background_index_build(state: &Arc, root: &Path, index_dir: &Pat // Phase 1: Walk file paths (no content reads) let t_walk = Instant::now(); + let ignorecase = (!state.no_ignore) + .then(|| tgrep_core::gitignore::CaseInsensitiveIgnore::new(root, true, true, true)) + .flatten() + .map(Arc::new); // The recovery window opens with this traversal, not with the subscriptions // it later feeds: a nested `.ignore` written after the walk read its parent // directory but before the matcher is published is invisible to both, and a // timestamp taken any later would date it as already accounted for. let since = SystemTime::now(); - let walk = walker::walk_dir( + let walk = walker::walk_dir_with_ignorecase( root, &WalkOptions { include_hidden: false, no_ignore: state.no_ignore, no_require_git: state.no_require_git, max_file_size: state.max_file_size, - collect_gitignore_files: state.watch_enabled && !state.no_ignore, + collect_gitignore_files: !state.no_ignore, exclude_dirs: state.exclude_dirs.clone(), ..Default::default() }, + ignorecase.clone(), ); let mut newly_watched = Vec::new(); - if state.watch_enabled && !state.no_ignore { + if !state.no_ignore { let start = Instant::now(); // Subscriptions are taken here, partway through the build, so files // written to a directory the walk has already passed are in neither @@ -5229,11 +5232,12 @@ fn background_index_build(state: &Arc, root: &Path, index_dir: &Pat state.no_require_git, ), || { - walker::build_gitignore_matcher_from_files( + walker::build_gitignore_matcher_from_files_with_ignorecase( root, &walk.gitignore_files, &walk.ignore_files, state.no_require_git, + ignorecase, ) }, ); @@ -5490,6 +5494,9 @@ fn background_index_build(state: &Arc, root: &Path, index_dir: &Pat index.live.shrink_to_fit(); } + if tracked_membership_changed(state) { + state.ignore_rules_dirty.store(true, Ordering::SeqCst); + } if state.ignore_rules_dirty.load(Ordering::SeqCst) { schedule_ignore_rules_refresh(Arc::clone(state), root.to_path_buf()); } @@ -6642,7 +6649,7 @@ mod tests { } #[test] - fn git_index_aba_during_refresh_schedules_a_corrective_pass() { + fn first_matcher_publication_uses_one_snapshot_across_git_index_aba() { use std::sync::Barrier; use std::sync::atomic::AtomicUsize; @@ -6658,15 +6665,13 @@ mod tests { std::fs::write(ignored.join("forced.rs"), "fn aba_marker() {}\n").unwrap(); test_git(&root, &["add", "--", ".gitignore", "src/lib.rs"]); - let state = test_server_state(&root, &root.join(".tgrep")); + let mut state = test_server_state(&root, &root.join(".tgrep")); + Arc::get_mut(&mut state).unwrap().watch_enabled = false; state.gitignore_pending.store(false, Ordering::SeqCst); - let matcher = - tgrep_core::gitignore::build_matcher(&root).expect("case-insensitive matcher"); - let baseline = matcher - .tracked_membership_fingerprint() - .expect("tracked exemption active"); - *state.gitignore.write().unwrap() = Some(matcher); - *state.tracked_membership.lock().unwrap() = Some(baseline); + assert!( + state.gitignore.read().unwrap().is_none(), + "the race must cover first matcher publication" + ); if PER_DIRECTORY_WATCHES { let watcher = notify::recommended_watcher(|_: notify::Result| {}).unwrap(); @@ -6712,15 +6717,25 @@ mod tests { background_refresh_stale(&refresh_state, &refresh_root, &index_dir, true) }); - // A: untracked and hidden when the pass captures its generation. + // A: untracked and hidden when the pass captures its immutable set. before_walk_entered.wait(); - // B: visible while both the walk and published matcher make decisions. + // B: the index changes, but this pass must continue using A throughout. test_git(&root, &["add", "-f", "--", "ignored/forced.rs"]); before_walk_release.wait(); after_publish_entered.wait(); + assert!( + state + .gitignore + .read() + .unwrap() + .as_ref() + .unwrap() + .is_ignored(Path::new("ignored"), true), + "the published matcher must use the same A snapshot as the walk" + ); if PER_DIRECTORY_WATCHES { assert!( - state + !state .watch_registry .lock() .unwrap() @@ -6728,10 +6743,11 @@ mod tests { .unwrap() .watched .contains(&ignored), - "the intermediate matcher must subscribe the tracked ignored directory" + "the B transition must not change subscriptions mid-pass" ); } - // A again before the pass commits what it learned from B. + // A again: the pass already represents the final membership, so no + // corrective pass is necessary. test_git( &root, &[ @@ -6749,22 +6765,15 @@ mod tests { "the raced pass itself should finish" ); - let deadline = Instant::now() + Duration::from_secs(10); - while (passes.load(Ordering::SeqCst) < 2 - || state.ignore_refresh_scheduled.load(Ordering::SeqCst) - || state.ignore_rules_dirty.load(Ordering::SeqCst)) - && Instant::now() < deadline - { - thread::sleep(Duration::from_millis(10)); - } + thread::sleep(Duration::from_millis(100)); assert!( - passes.load(Ordering::SeqCst) >= 2, - "the generation change must schedule a corrective pass" + passes.load(Ordering::SeqCst) == 1, + "A→B→A should not need a retry when the first pass used A throughout" ); assert!( !state.ignore_refresh_scheduled.load(Ordering::SeqCst) && !state.ignore_rules_dirty.load(Ordering::SeqCst), - "the corrective pass did not finish" + "the semantic baseline returned to A" ); let matcher = state.gitignore.read().unwrap(); @@ -6781,7 +6790,7 @@ mod tests { !index.live.has_path("ignored/forced.rs") && (!index.reader_has_path("ignored/forced.rs") || index.live.is_deleted("ignored/forced.rs")), - "the corrective pass must remove content indexed from the intermediate state" + "the immutable A walk must not index content from intermediate B" ); drop(index); if PER_DIRECTORY_WATCHES { @@ -6794,7 +6803,164 @@ mod tests { .unwrap() .watched .contains(&ignored), - "the corrective matcher must retire the intermediate subscription" + "the immutable A matcher must leave the ignored tree unsubscribed" + ); + } + } + + #[test] + fn content_only_index_churn_during_refresh_does_not_chain_reconciles() { + use std::sync::atomic::AtomicUsize; + + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + test_git(&root, &["init", "--quiet"]); + test_git(&root, &["config", "core.ignorecase", "true"]); + std::fs::write(root.join(".gitignore"), "IGNORED/\n").unwrap(); + std::fs::create_dir_all(root.join("src")).unwrap(); + std::fs::write(root.join("src/lib.rs"), "fn original() {}\n").unwrap(); + test_git(&root, &["add", "--", ".gitignore", "src/lib.rs"]); + + let state = test_server_state(&root, &root.join(".tgrep")); + state.gitignore_pending.store(false, Ordering::SeqCst); + let passes = Arc::new(AtomicUsize::new(0)); + let hook: StaleRefreshHook = { + let root = root.clone(); + let passes = Arc::clone(&passes); + Arc::new(move |phase| match phase { + StaleRefreshPhase::BeforeWalk => { + passes.fetch_add(1, Ordering::SeqCst); + } + StaleRefreshPhase::AfterMatcherPublish => { + let pass = passes.load(Ordering::SeqCst); + if pass <= 4 { + std::fs::write( + root.join("src/lib.rs"), + format!("fn content_only_{pass}() {{}}\n"), + ) + .unwrap(); + test_git(&root, &["add", "--", "src/lib.rs"]); + } + } + }) + }; + *state.stale_refresh_hook.lock().unwrap() = Some(hook); + + assert!(background_refresh_stale( + &state, + &root, + &state.index_dir, + true + )); + thread::sleep(Duration::from_millis(250)); + + assert_eq!( + passes.load(Ordering::SeqCst), + 1, + "metadata churn with unchanged relevant membership must not chain full scans" + ); + assert!( + !state.ignore_refresh_scheduled.load(Ordering::SeqCst) + && !state.ignore_rules_dirty.load(Ordering::SeqCst) + ); + } + + #[test] + fn membership_change_during_refresh_schedules_one_corrective_pass() { + use std::sync::Barrier; + use std::sync::atomic::AtomicUsize; + + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + test_git(&root, &["init", "--quiet"]); + test_git(&root, &["config", "core.ignorecase", "true"]); + std::fs::write(root.join(".gitignore"), "IGNORED/\n").unwrap(); + let ignored = root.join("ignored"); + std::fs::create_dir_all(&ignored).unwrap(); + std::fs::write(ignored.join("forced.rs"), "fn newly_exempt() {}\n").unwrap(); + test_git(&root, &["add", "--", ".gitignore"]); + + let state = test_server_state(&root, &root.join(".tgrep")); + state.gitignore_pending.store(false, Ordering::SeqCst); + if PER_DIRECTORY_WATCHES { + let watcher = notify::recommended_watcher(|_: notify::Result| {}).unwrap(); + *state.watch_registry.lock().unwrap() = Some(WatchRegistry { + watcher, + root: root.clone(), + watched: std::iter::once(root.clone()).collect(), + }); + } + + let before_walk_entered = Arc::new(Barrier::new(2)); + let before_walk_release = Arc::new(Barrier::new(2)); + let passes = Arc::new(AtomicUsize::new(0)); + let hook: StaleRefreshHook = { + let before_walk_entered = Arc::clone(&before_walk_entered); + let before_walk_release = Arc::clone(&before_walk_release); + let passes = Arc::clone(&passes); + Arc::new(move |phase| { + if matches!(phase, StaleRefreshPhase::BeforeWalk) + && passes.fetch_add(1, Ordering::SeqCst) == 0 + { + before_walk_entered.wait(); + before_walk_release.wait(); + } + }) + }; + *state.stale_refresh_hook.lock().unwrap() = Some(hook); + + let refresh_state = Arc::clone(&state); + let refresh_root = root.clone(); + let index_dir = state.index_dir.clone(); + let refresh = thread::spawn(move || { + background_refresh_stale(&refresh_state, &refresh_root, &index_dir, true) + }); + + before_walk_entered.wait(); + test_git(&root, &["add", "-f", "--", "ignored/forced.rs"]); + before_walk_release.wait(); + assert!(refresh.join().unwrap()); + + let deadline = Instant::now() + Duration::from_secs(10); + while (passes.load(Ordering::SeqCst) < 2 + || state.ignore_refresh_scheduled.load(Ordering::SeqCst) + || state.ignore_rules_dirty.load(Ordering::SeqCst)) + && Instant::now() < deadline + { + thread::sleep(Duration::from_millis(10)); + } + + assert_eq!( + passes.load(Ordering::SeqCst), + 2, + "a semantic A→B change must schedule exactly one corrective pass" + ); + assert!( + !state + .gitignore + .read() + .unwrap() + .as_ref() + .unwrap() + .is_ignored(Path::new("ignored"), true) + ); + let index = state.index.read().unwrap(); + assert!( + index.reader_has_path("ignored/forced.rs") || index.live.has_path("ignored/forced.rs"), + "the corrective pass must index the newly exempt file" + ); + drop(index); + if PER_DIRECTORY_WATCHES { + assert!( + state + .watch_registry + .lock() + .unwrap() + .as_ref() + .unwrap() + .watched + .contains(&ignored), + "the corrective pass must subscribe the newly exempt directory" ); } } diff --git a/tgrep-core/src/builder.rs b/tgrep-core/src/builder.rs index 0b636e8..c073659 100644 --- a/tgrep-core/src/builder.rs +++ b/tgrep-core/src/builder.rs @@ -380,6 +380,17 @@ pub fn build_index_with_options( root: &Path, index_dir: Option<&Path>, opts: &BuildOptions, +) -> Result { + build_index_with_options_and_ignorecase(root, index_dir, opts, None) +} + +/// Build an index using an immutable tracked-file exemption that the caller +/// can also use for watcher matcher publication. +pub fn build_index_with_options_and_ignorecase( + root: &Path, + index_dir: Option<&Path>, + opts: &BuildOptions, + ignorecase: Option>, ) -> Result { let include_hidden = opts.include_hidden; let no_ignore = opts.no_ignore; @@ -395,18 +406,34 @@ pub fn build_index_with_options( if let Some(hint) = gitignore_gate_hint(&root, opts) { eprintln!("{hint}"); } - let walk = walker::walk_dir( - &root, - &walker::WalkOptions { - include_hidden, - no_ignore, - no_require_git: opts.no_require_git, - collect_gitignore_files: opts.collect_gitignore_files, - exclude_dirs: exclude_dirs.to_vec(), - max_file_size: opts.max_file_size, - ..Default::default() - }, - ); + let walk = if let Some(ignorecase) = ignorecase { + walker::walk_dir_with_ignorecase( + &root, + &walker::WalkOptions { + include_hidden, + no_ignore, + no_require_git: opts.no_require_git, + collect_gitignore_files: opts.collect_gitignore_files, + exclude_dirs: exclude_dirs.to_vec(), + max_file_size: opts.max_file_size, + ..Default::default() + }, + Some(ignorecase), + ) + } else { + walker::walk_dir( + &root, + &walker::WalkOptions { + include_hidden, + no_ignore, + no_require_git: opts.no_require_git, + collect_gitignore_files: opts.collect_gitignore_files, + exclude_dirs: exclude_dirs.to_vec(), + max_file_size: opts.max_file_size, + ..Default::default() + }, + ) + }; eprintln!( "Found {} text files ({} binary skipped, {} too large, {} errors)", walk.files.len(), diff --git a/tgrep-core/src/gitignore.rs b/tgrep-core/src/gitignore.rs index 4fa0d69..34885e2 100644 --- a/tgrep-core/src/gitignore.rs +++ b/tgrep-core/src/gitignore.rs @@ -106,7 +106,7 @@ pub struct IgnoreMatcher { /// enlistment it left the watcher subscribing to, and indexing, a 13.4 GiB /// build artifact the walk had already excluded — and every event under it /// re-added a file the next stale check then evicted. - ignorecase: Option, + ignorecase: Option>, } impl IgnoreMatcher { @@ -141,7 +141,7 @@ impl IgnoreMatcher { ancestors: Vec<(String, IgnoreKind, Gitignore)>, repo_exclude: Option<(String, Gitignore)>, global: Gitignore, - ignorecase: Option, + ignorecase: Option>, ) -> Option { let mut nested: Vec = nested .into_iter() @@ -318,14 +318,17 @@ impl IgnoreMatcher { pub fn tracked_membership_fingerprint(&self) -> Option { self.ignorecase .as_ref() - .map(CaseInsensitiveIgnore::tracked_membership_fingerprint) + .map(|ignorecase| ignorecase.tracked_membership_fingerprint()) } - /// Cheap generation token for the worktree-specific Git index. - pub fn tracked_index_generation(&self) -> Option { + /// Fingerprint of the tracked exemption set in the current Git index. + /// + /// Unlike [`Self::tracked_membership_fingerprint`], this probes the current + /// index without changing the immutable set used by this matcher. + pub fn current_tracked_membership_fingerprint(&self) -> Option { self.ignorecase .as_ref() - .map(CaseInsensitiveIgnore::tracked_index_generation) + .map(|ignorecase| ignorecase.current_tracked_membership_fingerprint()) } } @@ -463,40 +466,29 @@ pub fn build_p4ignore_matcher(root: &Path) -> Option { pub struct CaseInsensitiveIgnore { matcher: Gitignore, repo_root: std::path::PathBuf, - /// Loaded on the first path this matcher actually claims, which for most - /// repositories is never. Reading it costs 163 ms and ~30 MB on a - /// 299k-file repository, and nothing at all if no rule ever matches. - /// - /// Reloaded when the index it was read from changes. A walk builds this - /// matcher, uses it and drops it, so a snapshot would do; the file watcher - /// holds one for the life of the server, and there a `git add -f` or a - /// `git rm --cached` rewrites only `.git/index` — which is hidden, so no - /// ignore source changes and nothing republishes the matcher. Frozen, the - /// exemption would keep answering from the tracked set as it stood at - /// startup, and the watcher would disagree with a fresh walk about which - /// files exist until the hourly reconcile. - tracked: std::sync::RwLock, + /// Immutable for the lifetime of this matcher. A stale reconciliation + /// shares one matcher between its walk and watcher publication, so every + /// decision in the pass uses the same tracked-file membership. + tracked: Option, + fingerprint: TrackedMembershipFingerprint, + /// Separate metadata-keyed cache for polling the current semantic + /// membership. It never participates in ignore decisions. + current: std::sync::RwLock, } -/// The tracked-file set, together with the identity of the index it came from. +/// A metadata-keyed semantic fingerprint used only for change polling. #[derive(Default)] struct TrackedCache { - /// `None` until something is loaded; `Some` even when the load failed, so - /// an unreadable index is not retried on every path. + /// `None` until something is loaded; `Some` even when the probe failed, so + /// an unreadable index is not retried on every poll. loaded_from: Option>, - tracked: Option, - /// Fingerprint of only the tracked paths this matcher would otherwise hide. - fingerprint: Option<(usize, u64, u64)>, + fingerprint: TrackedMembershipFingerprint, } /// Compact, order-independent identity of the tracked exemption set. -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct TrackedMembershipFingerprint(Option<(usize, u64, u64)>); -/// Metadata generation of the worktree-specific Git index. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct TrackedIndexGeneration(Option); - #[derive(Clone, Debug, PartialEq, Eq)] struct IndexIdentity { modified: std::time::SystemTime, @@ -577,10 +569,18 @@ impl CaseInsensitiveIgnore { if matcher.is_empty() { return None; } + let identity = index_identity(&repo_root); + let tracked = crate::git_index::load_tracked(&repo_root); + let fingerprint = Self::fingerprint(&matcher, tracked.as_ref()); Some(Self { matcher, repo_root, - tracked: std::sync::RwLock::new(TrackedCache::default()), + tracked, + fingerprint: fingerprint.clone(), + current: std::sync::RwLock::new(TrackedCache { + loaded_from: Some(identity), + fingerprint, + }), }) } @@ -596,59 +596,43 @@ impl CaseInsensitiveIgnore { { return false; } - // Only now is the index worth reading. let relative = relative.to_string_lossy(); - self.tracked_hides(&relative, is_dir) + Self::hides(self.tracked.as_ref(), &relative, is_dir) } fn tracked_membership_fingerprint(&self) -> TrackedMembershipFingerprint { - let identity = index_identity(&self.repo_root); - { - let cache = self.tracked.read().unwrap(); - if cache.loaded_from.as_ref() == Some(&identity) { - return TrackedMembershipFingerprint(cache.fingerprint); - } - } - let mut cache = self.tracked.write().unwrap(); - self.reload_tracked_if_needed(&mut cache, identity); - TrackedMembershipFingerprint(cache.fingerprint) + self.fingerprint.clone() } - fn tracked_index_generation(&self) -> TrackedIndexGeneration { - TrackedIndexGeneration(index_identity(&self.repo_root)) - } - - /// Whether the tracked-file set leaves `relative` hidden. - /// - /// `false` when the index cannot be read: with no way to tell tracked from - /// untracked, excluding could hide real source, so it declines instead. - fn tracked_hides(&self, relative: &str, is_dir: bool) -> bool { + fn current_tracked_membership_fingerprint(&self) -> TrackedMembershipFingerprint { let identity = index_identity(&self.repo_root); { - let cache = self.tracked.read().unwrap(); + let cache = self.current.read().unwrap(); if cache.loaded_from.as_ref() == Some(&identity) { - return Self::hides(cache.tracked.as_ref(), relative, is_dir); + return cache.fingerprint.clone(); } } - let mut cache = self.tracked.write().unwrap(); - // Another thread may have reloaded it while this one waited. - self.reload_tracked_if_needed(&mut cache, identity); - Self::hides(cache.tracked.as_ref(), relative, is_dir) - } - - fn reload_tracked_if_needed(&self, cache: &mut TrackedCache, identity: Option) { + let mut cache = self.current.write().unwrap(); if cache.loaded_from.as_ref() == Some(&identity) { - return; + return cache.fingerprint.clone(); } - cache.tracked = crate::git_index::load_tracked(&self.repo_root); - cache.fingerprint = cache.tracked.as_ref().map(|tracked| { + let tracked = crate::git_index::load_tracked(&self.repo_root); + cache.fingerprint = Self::fingerprint(&self.matcher, tracked.as_ref()); + cache.loaded_from = Some(identity); + cache.fingerprint.clone() + } + + fn fingerprint( + matcher: &Gitignore, + tracked: Option<&crate::git_index::TrackedFiles>, + ) -> TrackedMembershipFingerprint { + TrackedMembershipFingerprint(tracked.map(|tracked| { tracked.fingerprint_matching(|path| { - self.matcher + matcher .matched_path_or_any_parents(Path::new(path), false) .is_ignore() }) - }); - cache.loaded_from = Some(identity); + })) } fn hides( @@ -758,6 +742,24 @@ pub fn matcher_from_ignore_paths_with_options( gitignore_files: &[std::path::PathBuf], ignore_files: &[std::path::PathBuf], no_require_git: bool, +) -> Option { + matcher_from_ignore_paths_with_options_and_ignorecase( + root, + gitignore_files, + ignore_files, + no_require_git, + CaseInsensitiveIgnore::new(root, true, true, true).map(std::sync::Arc::new), + ) +} + +/// Build a matcher using an immutable case-insensitive tracked-file snapshot +/// shared with the walk that discovered `gitignore_files`. +pub fn matcher_from_ignore_paths_with_options_and_ignorecase( + root: &Path, + gitignore_files: &[std::path::PathBuf], + ignore_files: &[std::path::PathBuf], + no_require_git: bool, + ignorecase: Option>, ) -> Option { use ignore::gitignore::GitignoreBuilder; @@ -846,12 +848,6 @@ pub fn matcher_from_ignore_paths_with_options( } else { GitignoreBuilder::new(root).build().ok()? }; - // The same narrowing `walker::walk_dir` and `walk_file_metadata` apply as a - // `filter_entry`. Serving takes no `--no-ignore-vcs` / `--no-ignore-exclude` - // / `--no-ignore-parent`, so the flags the walk was built with are the - // defaults; `no_ignore` is handled by the caller, which does not build a - // matcher at all in that case. - let ignorecase = CaseInsensitiveIgnore::new(root, true, true, true); IgnoreMatcher::with_all_sources( local, true, diff --git a/tgrep-core/src/walker.rs b/tgrep-core/src/walker.rs index cc5be0b..4fe886c 100644 --- a/tgrep-core/src/walker.rs +++ b/tgrep-core/src/walker.rs @@ -216,7 +216,7 @@ fn git_ignorecase_filter( no_ignore_vcs: bool, no_ignore_exclude: bool, no_ignore_parent: bool, -) -> Option { +) -> Option> { (!no_ignore) .then(|| { crate::gitignore::CaseInsensitiveIgnore::new( @@ -227,6 +227,7 @@ fn git_ignorecase_filter( ) }) .flatten() + .map(std::sync::Arc::new) } /// Walk a directory tree, respecting .gitignore rules (unless disabled). @@ -236,6 +237,25 @@ fn git_ignorecase_filter( /// detection is deferred to the caller (which reads the file anyway), /// avoiding an extra 8KB read per file during the walk. pub fn walk_dir(root: &Path, opts: &WalkOptions) -> WalkResult { + let ignorecase = git_ignorecase_filter( + root, + opts.no_ignore, + opts.no_ignore_vcs, + opts.no_ignore_exclude, + opts.no_ignore_parent, + ); + walk_dir_with_ignorecase(root, opts, ignorecase) +} + +/// Walk files using a supplied immutable tracked-file exemption snapshot. +/// +/// Callers that build a point-query matcher from the result can share this +/// value so the walk and publication make every decision from one snapshot. +pub fn walk_dir_with_ignorecase( + root: &Path, + opts: &WalkOptions, + ignorecase: Option>, +) -> WalkResult { let files = std::sync::Mutex::new(Vec::new()); let gitignore_files = std::sync::Mutex::new(Vec::new()); let ignore_files = std::sync::Mutex::new(Vec::new()); @@ -254,14 +274,6 @@ pub fn walk_dir(root: &Path, opts: &WalkOptions) -> WalkResult { .flatten() .map(std::sync::Arc::new); let p4ignore_root = root.clone(); - let ignorecase = git_ignorecase_filter( - &root, - opts.no_ignore, - opts.no_ignore_vcs, - opts.no_ignore_exclude, - opts.no_ignore_parent, - ); - let mut builder = WalkBuilder::new(&root); builder .hidden(!include_hidden) @@ -410,6 +422,24 @@ pub fn build_gitignore_matcher_from_files( ) } +/// Build a point-query matcher sharing the immutable tracked-file exemption +/// used by the walk that discovered these ignore files. +pub fn build_gitignore_matcher_from_files_with_ignorecase( + root: &Path, + gitignore_files: &[PathBuf], + ignore_files: &[PathBuf], + no_require_git: bool, + ignorecase: Option>, +) -> Option { + crate::gitignore::matcher_from_ignore_paths_with_options_and_ignorecase( + root, + gitignore_files, + ignore_files, + no_require_git, + ignorecase, + ) +} + /// Filesystem metadata for a single file (no content read). pub struct FileMeta { pub relative_path: String, @@ -479,6 +509,17 @@ impl Default for MetaWalkOptions { /// explicitly via [`crate::gitignore::ignore_files_in`], which also catches /// ignore files that their own rules would have filtered out of the walk. pub fn walk_file_metadata(root: &Path, opts: &MetaWalkOptions) -> MetaWalkResult { + let ignorecase = git_ignorecase_filter(root, opts.no_ignore, false, false, false); + walk_file_metadata_with_ignorecase(root, opts, ignorecase) +} + +/// Walk metadata using an immutable case-insensitive tracked-file exemption +/// that can also be shared with the resulting point-query matcher. +pub fn walk_file_metadata_with_ignorecase( + root: &Path, + opts: &MetaWalkOptions, + ignorecase: Option>, +) -> MetaWalkResult { let no_ignore = opts.no_ignore; let max_file_size = opts.max_file_size; let results = std::sync::Mutex::new(Vec::new()); @@ -491,11 +532,6 @@ pub fn walk_file_metadata(root: &Path, opts: &MetaWalkOptions) -> MetaWalkResult .flatten() .map(std::sync::Arc::new); let match_root = root.to_path_buf(); - // `MetaWalkOptions` carries no finer ignore flags, and the builder below - // enables gitignore and exclude together on `!no_ignore`. Passing `false` - // for both mirrors that exactly, which is what keeps this walk and - // `walk_dir` agreeing about the tree under `serve`. - let ignorecase = git_ignorecase_filter(root, no_ignore, false, false, false); let root = root.to_path_buf(); let walker = WalkBuilder::new(&root) @@ -1472,14 +1508,11 @@ mod tests { assert!(!matcher.is_ignored(Path::new("src/Gone.TXT"), false)); } - /// The exemption is answered from a cached read of `.git/index`. A walk - /// builds this matcher and drops it, so a snapshot would do — but the file - /// watcher holds one for the life of the server, and `git add -f` rewrites - /// only the index, which is hidden. No ignore source changes, nothing - /// republishes the matcher, and a frozen cache would keep hiding a file - /// that git now tracks until the hourly reconcile. + /// A published matcher keeps an immutable tracked-file snapshot. Polling + /// detects semantic changes separately so event filtering cannot change + /// halfway through a reconciliation pass. #[test] - fn the_tracked_exemption_reloads_when_the_git_index_changes() { + fn the_tracked_exemption_snapshot_is_immutable_and_detects_changes() { let dir = ignorecase_fixture(true, &["src/main.rs", "src/Kept.TXT"]); let root = dir.path(); let matcher = crate::gitignore::matcher_from_ignore_paths( @@ -1494,16 +1527,17 @@ mod tests { fake_git_repo(root, true, &["src/main.rs", "src/Kept.TXT", "src/Gone.TXT"]); - assert!( - !matcher.is_ignored(Path::new("src/Gone.TXT"), false), - "a file git now tracks must stop being hidden without rebuilding \ - the matcher" + assert!(matcher.is_ignored(Path::new("src/Gone.TXT"), false)); + assert_ne!( + matcher.tracked_membership_fingerprint(), + matcher.current_tracked_membership_fingerprint(), + "polling must detect a tracked exemption without mutating decisions" ); // And back: `git rm --cached` is the same problem in reverse, where a // stale cache keeps indexing a file the walk has started hiding. fake_git_repo(root, true, &["src/main.rs"]); - assert!(matcher.is_ignored(Path::new("src/Kept.TXT"), false)); + assert!(!matcher.is_ignored(Path::new("src/Kept.TXT"), false)); } #[test] From e7e8d66d4f60443e79d533c9556c95294e5fa491 Mon Sep 17 00:00:00 2001 From: Shengyu Fu Date: Sat, 29 Aug 2026 02:09:37 -0500 Subject: [PATCH 05/16] Fix reload snapshot integration Route RPC reloads through coordinated tracked-membership snapshots while preserving live public matcher behavior. Stage and atomically publish reload generations with rollback and cache invalidation guards. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 772a4aa3-4cba-4ac4-aeca-dd184614127a --- tgrep-cli/src/serve.rs | 903 ++++++++++++++++++++++++++++-------- tgrep-core/src/gitignore.rs | 104 ++++- tgrep-core/src/walker.rs | 48 +- 3 files changed, 835 insertions(+), 220 deletions(-) diff --git a/tgrep-cli/src/serve.rs b/tgrep-cli/src/serve.rs index a67daca..241f563 100644 --- a/tgrep-cli/src/serve.rs +++ b/tgrep-cli/src/serve.rs @@ -270,6 +270,7 @@ impl ContentCache { struct ServerState { index: RwLock, cache: RwLock, + cache_generation: std::sync::atomic::AtomicU64, root: PathBuf, watcher_active: std::sync::atomic::AtomicBool, /// True while the initial index build is in progress. @@ -658,6 +659,7 @@ pub fn run(root: &Path, index_path: Option<&Path>, options: ServeOptions<'_>) -> CACHE_MAX_BYTES, CACHE_MAX_ENTRY_BYTES, )), + cache_generation: std::sync::atomic::AtomicU64::new(0), root: root.clone(), watcher_active: std::sync::atomic::AtomicBool::new(false), indexing: std::sync::atomic::AtomicBool::new(needs_build), @@ -1066,7 +1068,7 @@ fn publish_ignore_matcher( newly_watched } -fn handle_connection(stream: TcpStream, state: &ServerState) -> Result<()> { +fn handle_connection(stream: TcpStream, state: &Arc) -> Result<()> { let mut reader = BufReader::new(stream.try_clone()?); let mut writer = stream; @@ -1081,7 +1083,7 @@ fn handle_connection(stream: TcpStream, state: &ServerState) -> Result<()> { Ok(()) } -fn process_request(request: &str, state: &ServerState) -> String { +fn process_request(request: &str, state: &Arc) -> String { let req: serde_json::Value = match serde_json::from_str(request) { Ok(v) => v, Err(e) => { @@ -1481,6 +1483,7 @@ fn handle_search( }) .collect() } else { + let cache_generation = state.cache_generation.load(Ordering::SeqCst); // Phase 1: read-lock to find cache hits (peek avoids write-lock need) let mut hit_keys: Vec = Vec::new(); let mut hits: Vec<(String, Arc)> = Vec::with_capacity(candidate_info.len()); @@ -1512,17 +1515,7 @@ fn handle_search( // Phase 3: single write-lock to promote hits and insert misses if !hit_keys.is_empty() || !disk_results.is_empty() { - let mut cache = state.cache.write().unwrap(); - // Promote hit entries so LRU recency stays accurate - for key in &hit_keys { - cache.touch(key); - } - // Insert disk results, re-checking for races with other threads - for (rel_path, content) in &disk_results { - if cache.peek(rel_path).is_none() { - cache.put(rel_path.clone(), Arc::clone(content)); - } - } + update_content_cache(state, cache_generation, &hit_keys, &disk_results); } // Combine hits and disk results, preserving candidate order @@ -1588,6 +1581,43 @@ fn handle_search( json_rpc_result(id, result) } +fn update_content_cache( + state: &ServerState, + expected_generation: u64, + hit_keys: &[String], + disk_results: &[(String, Arc)], +) { + let mut cache = state.cache.write().unwrap(); + // Index mutations invalidate cached paths and advance this generation under + // the same lock. Earlier disk reads may serve their in-flight query, but + // must not repopulate stale bytes for later searches. + if state.cache_generation.load(Ordering::SeqCst) != expected_generation { + return; + } + for key in hit_keys { + cache.touch(key); + } + for (rel_path, content) in disk_results { + if cache.peek(rel_path).is_none() { + cache.put(rel_path.clone(), Arc::clone(content)); + } + } +} + +fn invalidate_cached_paths<'a>(state: &ServerState, paths: impl IntoIterator) { + let Ok(mut cache) = state.cache.write() else { + return; + }; + let mut invalidated = false; + for path in paths { + cache.pop(path); + invalidated = true; + } + if invalidated { + state.cache_generation.fetch_add(1, Ordering::SeqCst); + } +} + fn search_file_matches( rel_path: &str, file: &DecodedFile, @@ -1761,37 +1791,91 @@ fn handle_status(id: Option, state: &ServerState) -> String { json_rpc_result(id, result) } -fn handle_reload(id: Option, state: &ServerState) -> String { +fn handle_reload(id: Option, state: &Arc) -> String { let index_dir = state.index_dir.clone(); + while state.indexing.load(Ordering::SeqCst) || state.flushing.load(Ordering::SeqCst) { + thread::sleep(Duration::from_millis(50)); + } + let refresh = state.stale_refresh_lock.lock().unwrap(); + let gate = state.snapshot_gate.write().unwrap(); + let ignorecase = frozen_tracked_membership(state, &state.root); + #[cfg(test)] + run_stale_refresh_hook(state, StaleRefreshPhase::BeforeWalk); + let since = SystemTime::now(); - // Rebuild from disk. Uses the options form rather than `build_index` so the - // rebuild keeps the ignore semantics the server started with — a reload that - // silently changed them would rewrite the index against different rules. - if let Err(e) = builder::build_index_with_options( + // Rebuild the index and watcher matcher from the same immutable tracked + // membership. The snapshot gate keeps watcher/auto-save mutations out until + // both are published. + let staging_dir = index_dir.join(".reload-build"); + let _ = std::fs::remove_dir_all(&staging_dir); + let outcome = match builder::build_index_with_options_and_ignorecase( &state.root, - Some(&index_dir), + Some(&staging_dir), &builder::BuildOptions { no_ignore: state.no_ignore, no_require_git: state.no_require_git, max_file_size: state.max_file_size, exclude_dirs: state.exclude_dirs.clone(), + collect_gitignore_files: !state.no_ignore, ..Default::default() }, + ignorecase.clone(), ) { - return json_rpc_error(id, -32000, &format!("rebuild failed: {e}")); - } + Ok(outcome) => outcome, + Err(e) => { + let _ = std::fs::remove_dir_all(&staging_dir); + return json_rpc_error(id, -32000, &format!("rebuild failed: {e}")); + } + }; - // Reopen index - match HybridIndex::open(&index_dir, &state.root) { - Ok(new_index) => { - let mut index = state.index.write().unwrap(); - *index = new_index; - let mut cache = state.cache.write().unwrap(); - cache.clear(); - json_rpc_result(id, serde_json::json!({"status": "reloaded"})) + if !publish_reloaded_index(state, &index_dir, &staging_dir, outcome.num_files) { + return json_rpc_error(id, -32000, "rebuild publication failed"); + } + let indexed = outcome.num_files as u64; + state.index_total.store(indexed, Ordering::Relaxed); + state.index_progress.store(indexed, Ordering::Relaxed); + match tgrep_core::meta::read_filestamps(&index_dir) { + Ok(stamps) => *state.file_stamps.write().unwrap() = stamps, + Err(e) => { + eprintln!("[trace] warning: reload could not read file stamps ({e})"); + state.file_stamps.write().unwrap().clear(); } - Err(e) => json_rpc_error(id, -32000, &format!("reopen failed: {e}")), } + + let newly_watched = if state.no_ignore { + Vec::new() + } else { + publish_ignore_matcher( + state, + &state.root, + ignore_sources_of( + &state.root, + &outcome.gitignore_files, + &outcome.ignore_files, + state.no_require_git, + ), + || { + tgrep_core::walker::build_gitignore_matcher_from_files_with_ignorecase( + &state.root, + &outcome.gitignore_files, + &outcome.ignore_files, + state.no_require_git, + ignorecase, + ) + }, + ) + }; + #[cfg(test)] + run_stale_refresh_hook(state, StaleRefreshPhase::AfterMatcherPublish); + let membership_changed = tracked_membership_changed(state); + drop(gate); + drop(refresh); + + if state.watch_enabled { + spawn_recovery_scan(state, &state.root, newly_watched, since); + } + schedule_tracked_membership_correction(state, &state.root, membership_changed); + json_rpc_result(id, serde_json::json!({"status": "reloaded"})) } fn start_file_watcher(state: Arc, root: &Path, queue_cap: usize) -> bool { @@ -1903,7 +1987,8 @@ fn start_file_watcher(state: Arc, root: &Path, queue_cap: usize) -> loop { if last_tracked_index_poll.elapsed() >= TRACKED_INDEX_POLL { last_tracked_index_poll = Instant::now(); - if tracked_membership_changed(&worker_state) { + let changed = poll_tracked_membership_changed(&worker_state); + if changed { eprintln!( "[trace] Git tracked paths changed; reconciling tracked-file exemptions" ); @@ -1991,6 +2076,34 @@ fn tracked_membership_changed(state: &ServerState) -> bool { changed } +fn poll_tracked_membership_changed(state: &ServerState) -> bool { + // A reconcile holds the write side across snapshot, walk and publication. + // Waiting here prevents a poll from committing transient A→B→A membership. + let _gate = state.snapshot_gate.read().unwrap(); + tracked_membership_changed(state) +} + +fn frozen_tracked_membership( + state: &ServerState, + root: &Path, +) -> Option> { + (!state.no_ignore) + .then(|| { + tgrep_core::gitignore::CaseInsensitiveIgnore::frozen_snapshot(root, true, true, true) + }) + .flatten() + .map(Arc::new) +} + +fn schedule_tracked_membership_correction(state: &Arc, root: &Path, changed: bool) { + if !changed { + return; + } + eprintln!("[trace] Git tracked paths changed during reconciliation; scheduling a retry"); + state.ignore_rules_dirty.store(true, Ordering::SeqCst); + schedule_ignore_rules_refresh(Arc::clone(state), root.to_path_buf()); +} + #[cfg(test)] fn run_stale_refresh_hook(state: &ServerState, phase: StaleRefreshPhase) { let hook = state.stale_refresh_hook.lock().unwrap().clone(); @@ -3028,9 +3141,7 @@ fn sweep_removed_files( } state.index.write().unwrap().live.delete_file(rel); state.file_stamps.write().unwrap().remove(rel); - if let Ok(mut cache) = state.cache.write() { - cache.pop(rel); - } + invalidate_cached_paths(state, std::iter::once(rel.as_str())); dropped += 1; } if dropped > 0 { @@ -3615,9 +3726,7 @@ fn handle_fs_event(state: &Arc, root: &Path, event: &Event) { // is processed atomically with respect to flush/auto-save. state.index.write().unwrap().live.delete_file(&rel_path); state.file_stamps.write().unwrap().remove(&rel_path); - if let Ok(mut cache) = state.cache.write() { - cache.pop(&rel_path); - } + invalidate_cached_paths(state, std::iter::once(rel_path.as_str())); continue; } @@ -3719,9 +3828,7 @@ fn drop_indexed_file(state: &ServerState, rel_path: &str, reason: &str) { } eprintln!("[trace] reindex: dropped {rel_path} ({reason})"); state.index.write().unwrap().live.delete_file(rel_path); - if let Ok(mut cache) = state.cache.write() { - cache.pop(rel_path); - } + invalidate_cached_paths(state, std::iter::once(rel_path)); } /// What an event's stat result says about the path it named. @@ -4163,9 +4270,7 @@ fn reindex_file(state: &ServerState, path: &Path, rel_path: &str) { .write() .unwrap() .insert(rel_path.to_string(), current); - if let Ok(mut cache) = state.cache.write() { - cache.pop(rel_path); - } + invalidate_cached_paths(state, std::iter::once(rel_path)); } /// Whether a scheduled reconcile should run now. @@ -4732,10 +4837,7 @@ fn background_refresh_stale( // One immutable tracked-file exemption is shared by the walk and the // matcher it publishes. Git-index rewrites cannot change ignore decisions // halfway through this pass. - let ignorecase = (!state.no_ignore) - .then(|| tgrep_core::gitignore::CaseInsensitiveIgnore::new(root, true, true, true)) - .flatten() - .map(Arc::new); + let ignorecase = frozen_tracked_membership(state, root); #[cfg(test)] run_stale_refresh_hook(state, StaleRefreshPhase::BeforeWalk); @@ -4771,13 +4873,7 @@ fn background_refresh_stale( let membership_changed = tracked_membership_changed(state); drop(gate); drop(refresh); - if membership_changed { - eprintln!( - "[trace] Git tracked paths changed during stale reconciliation; scheduling a retry" - ); - state.ignore_rules_dirty.store(true, Ordering::SeqCst); - schedule_ignore_rules_refresh(Arc::clone(state), root.to_path_buf()); - } + schedule_tracked_membership_correction(state, root, membership_changed); ok } @@ -4943,11 +5039,14 @@ fn refresh_stale_locked( return false; } - if let Ok(mut cache) = state.cache.write() { - for path in changed.iter().chain(added.iter()).chain(deleted.iter()) { - cache.pop(path); - } - } + invalidate_cached_paths( + state, + changed + .iter() + .chain(added.iter()) + .chain(deleted.iter()) + .map(String::as_str), + ); true } @@ -4963,8 +5062,11 @@ fn reset_to_empty_index(state: &ServerState, root: &Path, index_dir: &Path) { } match HybridIndex::open(index_dir, root) { Ok(empty) => { - *state.index.write().unwrap() = empty; - state.cache.write().unwrap().clear(); + let mut index = state.index.write().unwrap(); + let mut cache = state.cache.write().unwrap(); + *index = empty; + cache.clear(); + state.cache_generation.fetch_add(1, Ordering::SeqCst); } Err(e) => eprintln!("[trace] warning: could not reopen an empty index ({e})"), } @@ -4991,10 +5093,7 @@ fn reset_to_empty_index(state: &ServerState, root: &Path, index_dir: &Path) { /// caller to fall back. fn bootstrap_index_build(state: &Arc, root: &Path, index_dir: &Path) -> bool { let start = Instant::now(); - let ignorecase = (!state.no_ignore) - .then(|| tgrep_core::gitignore::CaseInsensitiveIgnore::new(root, true, true, true)) - .flatten() - .map(Arc::new); + let ignorecase = frozen_tracked_membership(state, root); // Anchors the recovery window at the start of the build's traversal, which // is the point from which writes could be missed: nothing under `root` is // subscribed yet, and the walk below has not reached most of it. Taking it @@ -5056,8 +5155,13 @@ fn bootstrap_index_build(state: &Arc, root: &Path, index_dir: &Path } }; let indexed = opened.num_files() as u64; - *state.index.write().unwrap() = opened; - state.cache.write().unwrap().clear(); + { + let mut index = state.index.write().unwrap(); + let mut cache = state.cache.write().unwrap(); + *index = opened; + cache.clear(); + state.cache_generation.fetch_add(1, Ordering::SeqCst); + } state.index_total.store(indexed, Ordering::Relaxed); state.index_progress.store(indexed, Ordering::Relaxed); @@ -5128,9 +5232,8 @@ fn bootstrap_index_build(state: &Arc, root: &Path, index_dir: &Path state.indexing.store(false, Ordering::SeqCst); drop(gate); - if tracked_membership_changed(state) { - state.ignore_rules_dirty.store(true, Ordering::SeqCst); - } + let membership_changed = tracked_membership_changed(state); + schedule_tracked_membership_correction(state, root, membership_changed); let elapsed = start.elapsed().as_secs_f64(); drop(sampler); @@ -5191,10 +5294,7 @@ fn background_index_build(state: &Arc, root: &Path, index_dir: &Pat // Phase 1: Walk file paths (no content reads) let t_walk = Instant::now(); - let ignorecase = (!state.no_ignore) - .then(|| tgrep_core::gitignore::CaseInsensitiveIgnore::new(root, true, true, true)) - .flatten() - .map(Arc::new); + let ignorecase = frozen_tracked_membership(state, root); // The recovery window opens with this traversal, not with the subscriptions // it later feeds: a nested `.ignore` written after the walk read its parent // directory but before the matcher is published is invisible to both, and a @@ -5494,9 +5594,8 @@ fn background_index_build(state: &Arc, root: &Path, index_dir: &Pat index.live.shrink_to_fit(); } - if tracked_membership_changed(state) { - state.ignore_rules_dirty.store(true, Ordering::SeqCst); - } + let membership_changed = tracked_membership_changed(state); + schedule_tracked_membership_correction(state, root, membership_changed); if state.ignore_rules_dirty.load(Ordering::SeqCst) { schedule_ignore_rules_refresh(Arc::clone(state), root.to_path_buf()); } @@ -5617,124 +5716,217 @@ fn publish_staged_index( // background-build / watcher reindex flush) cannot interleave renames // or swap readers out of order. Searches do not take this lock. let _publish = state.publish_lock.lock().unwrap(); - if let Err(e) = move_staged_files(staging_dir, index_dir) { - eprintln!("[trace] warning: flush move failed: {e}"); - let _ = std::fs::remove_dir_all(staging_dir); + let mut moved = match move_staged_files(staging_dir, index_dir) { + Ok(moved) => moved, + Err(e) => { + eprintln!("[trace] warning: flush move failed: {e}"); + return false; + } + }; + let Some((new_reader, reader_files, reader_trigrams)) = + open_published_reader(index_dir, num_files) + else { + match moved.rollback() { + Ok(()) => { + let _ = std::fs::remove_dir_all(staging_dir); + } + Err(e) => { + eprintln!("[trace] warning: failed to roll back index publication: {e}"); + } + } return false; + }; + + // Atomic swap — no outer write lock required. + state.index.read().unwrap().swap_reader(new_reader); + // Brief write lock for in-memory overlay maintenance only. + { + let mut index = state.index.write().unwrap(); + index.prune_persisted_entries(); + index.live.reset_dirty_count(); } + moved.commit(); + eprintln!( + "[trace] flush: reader reopened ({reader_files} files, \ + {reader_trigrams} trigrams), overlay pruned" + ); + let _ = std::fs::remove_dir_all(staging_dir); + true +} - // Open the new reader. The publish mutex is intentionally still held - // here so that move + open + swap form an atomic publish unit (no other - // publisher can interleave a rename or swap a competing reader between - // these steps). The server-wide `state.index` RwLock is NOT taken, so - // search queries continue to be served by the previous reader (whose - // `Arc` they hold) throughout this call. - // - // On Windows, NTFS metadata for a recently-renamed file can transiently - // appear stale (zero-length), causing IndexReader::open to create a - // degenerate reader with files but no trigrams. We retry a few times - // with a short backoff to ride out the transient. - let pruned = 'open: { - const READER_OPEN_RETRIES: u32 = 5; - const READER_OPEN_BACKOFF: Duration = Duration::from_millis(200); - - for attempt in 0..READER_OPEN_RETRIES { - match tgrep_core::reader::IndexReader::open(index_dir) { - Ok(new_reader) => { - let reader_files = new_reader.num_files(); - let reader_trigrams = new_reader.num_trigrams(); - - if new_reader.is_degenerate() { - eprintln!( - "[trace] warning: reader has {reader_files} files but 0 trigrams \ - (attempt {}/{READER_OPEN_RETRIES}, likely stale NTFS metadata)", - attempt + 1 - ); - if attempt + 1 < READER_OPEN_RETRIES { - thread::sleep(READER_OPEN_BACKOFF * (attempt + 1)); - continue; - } - eprintln!( - "[trace] warning: degenerate reader persists after \ - {READER_OPEN_RETRIES} attempts, keeping live overlay as fallback" - ); - break 'open false; - } +fn publish_reloaded_index( + state: &ServerState, + index_dir: &Path, + staging_dir: &Path, + num_files: usize, +) -> bool { + let _publish = state.publish_lock.lock().unwrap(); + let mut moved = match move_staged_files(staging_dir, index_dir) { + Ok(moved) => moved, + Err(e) => { + eprintln!("[trace] warning: reload move failed: {e}"); + return false; + } + }; + let Some((new_reader, reader_files, reader_trigrams)) = + open_published_reader(index_dir, num_files) + else { + match moved.rollback() { + Ok(()) => { + let _ = std::fs::remove_dir_all(staging_dir); + } + Err(e) => { + eprintln!("[trace] warning: failed to roll back reload publication: {e}"); + } + } + return false; + }; - // Validate + warm the lookup mmap before swapping the - // reader in. This catches corruption (unsorted lookup - // table, out-of-bounds posting offsets) and, as a - // side-effect, pages in every byte of lookup.bin so that - // subsequent binary searches never hit cold mmap pages - // — preventing the zero-candidate failure observed on - // Windows after flush. - if let Err(msg) = new_reader.validate_lookup() { - eprintln!( - "[trace] warning: reader validation failed \ - (attempt {}/{READER_OPEN_RETRIES}): {msg}", - attempt + 1 - ); - if attempt + 1 < READER_OPEN_RETRIES { - thread::sleep(READER_OPEN_BACKOFF * (attempt + 1)); - continue; - } - eprintln!( - "[trace] warning: reader validation failed after \ - {READER_OPEN_RETRIES} attempts, keeping live overlay" - ); - break 'open false; - } + // Searches take the outer index lock before consulting the cache. Holding + // both in that order makes the complete reload visible as one generation. + { + let mut index = state.index.write().unwrap(); + let mut cache = state.cache.write().unwrap(); + index.swap_reader(new_reader); + let mut reconciled = index.live.overlay_paths(); + reconciled.extend(index.live.tombstone_paths()); + index.live.clear_reconciled_paths(&reconciled); + index.live.reset_dirty_count(); + cache.clear(); + state.cache_generation.fetch_add(1, Ordering::SeqCst); + } + moved.commit(); + eprintln!( + "[trace] reload: reader reopened ({reader_files} files, \ + {reader_trigrams} trigrams), overlay and cache cleared" + ); + let _ = std::fs::remove_dir_all(staging_dir); + true +} - if reader_files >= num_files { - // Atomic swap — no outer write lock required. - state.index.read().unwrap().swap_reader(new_reader); - // Brief write lock for in-memory overlay maintenance only. - { - let mut index = state.index.write().unwrap(); - index.prune_persisted_entries(); - index.live.reset_dirty_count(); - } - eprintln!( - "[trace] flush: reader reopened ({reader_files} files, \ - {reader_trigrams} trigrams), overlay pruned" - ); - break 'open true; - } else { - eprintln!( - "[trace] warning: reader has {reader_files} files \ - (expected {num_files}), keeping live overlay as fallback" - ); - break 'open false; - } - } - Err(e) => { - if attempt + 1 < READER_OPEN_RETRIES { - eprintln!( - "[trace] warning: reader open failed (attempt {}/{READER_OPEN_RETRIES}): {e}", - attempt + 1 - ); - thread::sleep(READER_OPEN_BACKOFF * (attempt + 1)); - continue; - } +fn open_published_reader( + index_dir: &Path, + num_files: usize, +) -> Option<(tgrep_core::reader::IndexReader, usize, usize)> { + const READER_OPEN_RETRIES: u32 = 5; + const READER_OPEN_BACKOFF: Duration = Duration::from_millis(200); + + for attempt in 0..READER_OPEN_RETRIES { + match tgrep_core::reader::IndexReader::open(index_dir) { + Ok(new_reader) => { + let reader_files = new_reader.num_files(); + let reader_trigrams = new_reader.num_trigrams(); + if new_reader.is_degenerate() { + eprintln!( + "[trace] warning: reader has {reader_files} files but 0 trigrams \ + (attempt {}/{READER_OPEN_RETRIES}, likely stale NTFS metadata)", + attempt + 1 + ); + } else if let Err(msg) = new_reader.validate_lookup() { + eprintln!( + "[trace] warning: reader validation failed \ + (attempt {}/{READER_OPEN_RETRIES}): {msg}", + attempt + 1 + ); + } else if reader_files >= num_files { + return Some((new_reader, reader_files, reader_trigrams)); + } else { eprintln!( - "[trace] warning: failed to reopen reader after flush: {e}, \ - live overlay retained" + "[trace] warning: reader has {reader_files} files \ + (expected {num_files}), keeping live overlay as fallback" ); - break 'open false; + return None; } } + Err(e) => { + eprintln!( + "[trace] warning: reader open failed (attempt {}/{READER_OPEN_RETRIES}): {e}", + attempt + 1 + ); + } } - false - }; - let _ = std::fs::remove_dir_all(staging_dir); - pruned + if attempt + 1 < READER_OPEN_RETRIES { + thread::sleep(READER_OPEN_BACKOFF * (attempt + 1)); + } + } + eprintln!( + "[trace] warning: failed to validate reader after \ + {READER_OPEN_RETRIES} attempts, keeping the previous reader" + ); + None } -/// Move index files from staging to the target directory. +const INDEX_FILE_NAMES: &[&str] = &[ + "index.bin", + "lookup.bin", + "files.bin", + "filestamps.json", + "meta.json", +]; + +struct StagedFileMove { + staging: PathBuf, + target: PathBuf, + backed_up: Vec<&'static str>, + published: Vec<&'static str>, + finished: bool, +} + +impl StagedFileMove { + fn backup_path(&self, name: &str) -> PathBuf { + self.staging.join(format!(".previous-{name}")) + } + + fn rollback(&mut self) -> std::io::Result<()> { + if self.finished { + return Ok(()); + } + let mut first_error = None; + for name in self.backed_up.iter().rev() { + if let Err(e) = publish_file(&self.backup_path(name), &self.target.join(name)) + && first_error.is_none() + { + first_error = Some(e); + } + } + for name in self.published.iter().rev() { + if self.backed_up.contains(name) { + continue; + } + let published = self.target.join(name); + if published.exists() + && let Err(e) = std::fs::remove_file(published) + && first_error.is_none() + { + first_error = Some(e); + } + } + if let Some(e) = first_error { + return Err(e); + } + self.finished = true; + Ok(()) + } + + fn commit(&mut self) { + self.finished = true; + } +} + +impl Drop for StagedFileMove { + fn drop(&mut self) { + if let Err(e) = self.rollback() { + eprintln!("[trace] warning: failed to roll back staged index files: {e}"); + } + } +} + +/// Move index files from staging to the target directory, retaining backups +/// until the caller validates and commits the new reader. /// -/// Files are published in a fixed order, with `meta.json` last. This is only a -/// convention for publication layout; it does not provide atomic publish -/// semantics or reader-side validation by itself. +/// Files are published in a fixed order, with `meta.json` last. Existing files +/// are retained in staging until reader validation succeeds, so dropping the +/// returned transaction rolls back a partial or rejected publication. /// /// Performance note: this function runs under the server's `publish_lock` /// (which serializes concurrent publishers) but does NOT take the @@ -5746,24 +5938,29 @@ fn publish_staged_index( /// created next to the target (same parent) so cross-volume cases should /// not arise; if rename truly fails, the error is surfaced rather than /// silently falling back to a slow copy (see `publish_file`). -fn move_staged_files(staging: &Path, target: &Path) -> std::io::Result<()> { +fn move_staged_files(staging: &Path, target: &Path) -> std::io::Result { std::fs::create_dir_all(target)?; - // Data files first, meta last. - for name in &[ - "index.bin", - "lookup.bin", - "files.bin", - "filestamps.json", - "meta.json", - ] { + let mut moved = StagedFileMove { + staging: staging.to_path_buf(), + target: target.to_path_buf(), + backed_up: Vec::new(), + published: Vec::new(), + finished: false, + }; + for &name in INDEX_FILE_NAMES { let src = staging.join(name); let dst = target.join(name); if !src.exists() { continue; } + if dst.exists() { + publish_file(&dst, &moved.backup_path(name))?; + moved.backed_up.push(name); + } publish_file(&src, &dst)?; + moved.published.push(name); } - Ok(()) + Ok(moved) } /// Publish a single staged file at `src` to `dst`. @@ -6003,6 +6200,7 @@ mod tests { CACHE_MAX_BYTES, CACHE_MAX_ENTRY_BYTES, )), + cache_generation: std::sync::atomic::AtomicU64::new(0), root: root.to_path_buf(), watcher_active: std::sync::atomic::AtomicBool::new(false), indexing: std::sync::atomic::AtomicBool::new(false), @@ -6965,6 +7163,307 @@ mod tests { } } + #[test] + fn rpc_reload_uses_one_snapshot_across_git_index_aba() { + use std::sync::Barrier; + use std::sync::atomic::AtomicUsize; + + let tmp = TempDir::new().unwrap(); + let root = std::fs::canonicalize(tmp.path()).unwrap(); + test_git(&root, &["init", "--quiet"]); + test_git(&root, &["config", "core.ignorecase", "true"]); + std::fs::write(root.join(".gitignore"), "IGNORED/\n").unwrap(); + let ignored = root.join("ignored"); + std::fs::create_dir_all(&ignored).unwrap(); + std::fs::write(ignored.join("forced.rs"), "fn reload_aba() {}\n").unwrap(); + test_git(&root, &["add", "--", ".gitignore"]); + + let mut state = test_server_state(&root, &root.join(".tgrep")); + Arc::get_mut(&mut state).unwrap().watch_enabled = false; + state.gitignore_pending.store(false, Ordering::SeqCst); + + let before_walk_entered = Arc::new(Barrier::new(2)); + let before_walk_release = Arc::new(Barrier::new(2)); + let after_publish_entered = Arc::new(Barrier::new(2)); + let after_publish_release = Arc::new(Barrier::new(2)); + let passes = Arc::new(AtomicUsize::new(0)); + let hook: StaleRefreshHook = { + let before_walk_entered = Arc::clone(&before_walk_entered); + let before_walk_release = Arc::clone(&before_walk_release); + let after_publish_entered = Arc::clone(&after_publish_entered); + let after_publish_release = Arc::clone(&after_publish_release); + let passes = Arc::clone(&passes); + Arc::new(move |phase| match phase { + StaleRefreshPhase::BeforeWalk => { + if passes.fetch_add(1, Ordering::SeqCst) == 0 { + before_walk_entered.wait(); + before_walk_release.wait(); + } + } + StaleRefreshPhase::AfterMatcherPublish => { + if passes.load(Ordering::SeqCst) == 1 { + after_publish_entered.wait(); + after_publish_release.wait(); + } + } + }) + }; + *state.stale_refresh_hook.lock().unwrap() = Some(hook); + + let reload_state = Arc::clone(&state); + let reload = thread::spawn(move || handle_reload(None, &reload_state)); + before_walk_entered.wait(); + test_git(&root, &["add", "-f", "--", "ignored/forced.rs"]); + before_walk_release.wait(); + after_publish_entered.wait(); + test_git( + &root, + &[ + "rm", + "--cached", + "--quiet", + "--force", + "--", + "ignored/forced.rs", + ], + ); + after_publish_release.wait(); + + let response = reload.join().unwrap(); + assert!(response.contains("\"status\":\"reloaded\""), "{response}"); + thread::sleep(Duration::from_millis(100)); + assert_eq!( + passes.load(Ordering::SeqCst), + 1, + "A→B→A must not need a correction when reload used A throughout" + ); + assert!( + state + .gitignore + .read() + .unwrap() + .as_ref() + .unwrap() + .is_ignored(Path::new("ignored"), true) + ); + let index = state.index.read().unwrap(); + assert!( + !index.reader_has_path("ignored/forced.rs") + && !index.live.has_path("ignored/forced.rs"), + "reload index and matcher must both represent final A" + ); + assert!( + !state.ignore_refresh_scheduled.load(Ordering::SeqCst) + && !state.ignore_rules_dirty.load(Ordering::SeqCst) + ); + } + + #[test] + fn rpc_reload_membership_change_schedules_one_corrective_pass() { + use std::sync::Barrier; + use std::sync::atomic::AtomicUsize; + + let tmp = TempDir::new().unwrap(); + let root = std::fs::canonicalize(tmp.path()).unwrap(); + test_git(&root, &["init", "--quiet"]); + test_git(&root, &["config", "core.ignorecase", "true"]); + std::fs::write(root.join(".gitignore"), "IGNORED/\n").unwrap(); + std::fs::create_dir_all(root.join("ignored")).unwrap(); + std::fs::write( + root.join("ignored/forced.rs"), + "fn reload_membership_change() {}\n", + ) + .unwrap(); + test_git(&root, &["add", "--", ".gitignore"]); + + let mut state = test_server_state(&root, &root.join(".tgrep")); + Arc::get_mut(&mut state).unwrap().watch_enabled = false; + state.gitignore_pending.store(false, Ordering::SeqCst); + let before_walk_entered = Arc::new(Barrier::new(2)); + let before_walk_release = Arc::new(Barrier::new(2)); + let passes = Arc::new(AtomicUsize::new(0)); + let hook: StaleRefreshHook = { + let before_walk_entered = Arc::clone(&before_walk_entered); + let before_walk_release = Arc::clone(&before_walk_release); + let passes = Arc::clone(&passes); + Arc::new(move |phase| { + if matches!(phase, StaleRefreshPhase::BeforeWalk) + && passes.fetch_add(1, Ordering::SeqCst) == 0 + { + before_walk_entered.wait(); + before_walk_release.wait(); + } + }) + }; + *state.stale_refresh_hook.lock().unwrap() = Some(hook); + + let reload_state = Arc::clone(&state); + let reload = thread::spawn(move || handle_reload(None, &reload_state)); + before_walk_entered.wait(); + test_git(&root, &["add", "-f", "--", "ignored/forced.rs"]); + before_walk_release.wait(); + let response = reload.join().unwrap(); + assert!(response.contains("\"status\":\"reloaded\""), "{response}"); + + let deadline = Instant::now() + Duration::from_secs(10); + while (passes.load(Ordering::SeqCst) < 2 + || state.ignore_refresh_scheduled.load(Ordering::SeqCst) + || state.ignore_rules_dirty.load(Ordering::SeqCst)) + && Instant::now() < deadline + { + thread::sleep(Duration::from_millis(10)); + } + + assert_eq!( + passes.load(Ordering::SeqCst), + 2, + "A→B during reload must schedule exactly one corrective pass" + ); + assert!( + !state + .gitignore + .read() + .unwrap() + .as_ref() + .unwrap() + .is_ignored(Path::new("ignored"), true) + ); + let index = state.index.read().unwrap(); + assert!( + index.reader_has_path("ignored/forced.rs") || index.live.has_path("ignored/forced.rs"), + "the correction must make reload's index agree with final B" + ); + } + + #[test] + fn rpc_reload_build_failure_preserves_active_index() { + let tmp = TempDir::new().unwrap(); + let root = std::fs::canonicalize(tmp.path()).unwrap(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + state + .index + .write() + .unwrap() + .live + .upsert_file("active.rs", b"fn active_before_reload() {}\n"); + + std::fs::write(index_dir.join(".reload-build"), b"blocks staging directory").unwrap(); + let response = handle_reload(None, &state); + + assert!(response.contains("rebuild failed"), "{response}"); + assert!( + state.index.read().unwrap().live.has_path("active.rs"), + "a failed staged build must not disturb the active index" + ); + } + + #[test] + fn rpc_reload_waits_for_initial_build() { + use std::sync::mpsc::{RecvTimeoutError, channel}; + + let tmp = TempDir::new().unwrap(); + let root = std::fs::canonicalize(tmp.path()).unwrap(); + let state = test_server_state(&root, &root.join(".tgrep")); + state.indexing.store(true, Ordering::SeqCst); + let (entered_tx, entered_rx) = channel(); + *state.stale_refresh_hook.lock().unwrap() = Some(Arc::new(move |phase| { + if matches!(phase, StaleRefreshPhase::BeforeWalk) { + entered_tx.send(()).unwrap(); + } + })); + + let reload_state = Arc::clone(&state); + let reload = thread::spawn(move || handle_reload(None, &reload_state)); + assert!(matches!( + entered_rx.recv_timeout(Duration::from_millis(150)), + Err(RecvTimeoutError::Timeout) + )); + + state.indexing.store(false, Ordering::SeqCst); + entered_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + let response = reload.join().unwrap(); + assert!(response.contains("\"status\":\"reloaded\""), "{response}"); + } + + #[test] + fn pre_reload_disk_read_cannot_repopulate_content_cache() { + let tmp = TempDir::new().unwrap(); + let root = std::fs::canonicalize(tmp.path()).unwrap(); + let state = test_server_state(&root, &root.join(".tgrep")); + let before_reload = state.cache_generation.load(Ordering::SeqCst); + let stale = vec![( + "stale.rs".to_string(), + Arc::new(DecodedFile::new( + b"old content".to_vec(), + tgrep_core::encoding::EncodingMode::Auto, + )), + )]; + + invalidate_cached_paths(&state, std::iter::once("stale.rs")); + update_content_cache(&state, before_reload, &[], &stale); + assert!( + state.cache.read().unwrap().peek("stale.rs").is_none(), + "a disk read from the previous index generation must not refill the cache" + ); + + let current = state.cache_generation.load(Ordering::SeqCst); + update_content_cache(&state, current, &[], &stale); + assert!(state.cache.read().unwrap().peek("stale.rs").is_some()); + } + + #[test] + fn tracked_membership_poll_waits_for_reconcile_publication() { + use std::sync::mpsc::{RecvTimeoutError, channel}; + + let tmp = TempDir::new().unwrap(); + let root = std::fs::canonicalize(tmp.path()).unwrap(); + test_git(&root, &["init", "--quiet"]); + test_git(&root, &["config", "core.ignorecase", "true"]); + std::fs::write(root.join(".gitignore"), "IGNORED/\n").unwrap(); + std::fs::create_dir_all(root.join("ignored")).unwrap(); + std::fs::write(root.join("ignored/forced.rs"), "fn transient() {}\n").unwrap(); + test_git(&root, &["add", "--", ".gitignore"]); + + let state = test_server_state(&root, &root.join(".tgrep")); + state.gitignore_pending.store(false, Ordering::SeqCst); + assert!(background_refresh_stale( + &state, + &root, + &state.index_dir, + true + )); + + let gate = state.snapshot_gate.write().unwrap(); + test_git(&root, &["add", "-f", "--", "ignored/forced.rs"]); + let (result_tx, result_rx) = channel(); + let poll_state = Arc::clone(&state); + let poll = thread::spawn(move || { + result_tx + .send(poll_tracked_membership_changed(&poll_state)) + .unwrap(); + }); + assert!(matches!( + result_rx.recv_timeout(Duration::from_millis(150)), + Err(RecvTimeoutError::Timeout) + )); + + test_git( + &root, + &[ + "rm", + "--cached", + "--quiet", + "--force", + "--", + "ignored/forced.rs", + ], + ); + drop(gate); + assert!(!result_rx.recv_timeout(Duration::from_secs(5)).unwrap()); + poll.join().unwrap(); + } + /// A transient stat failure is not a deletion. `Path::exists` said it was, /// which meant one `EACCES` — or a Windows sharing violation from a build /// holding the file open — tombstoned content that was still valid, and @@ -8915,7 +9414,8 @@ mod tests { write_file(&staging.join(name), name.as_bytes()); } write_file(&staging.join("ignored.txt"), b"nope"); - move_staged_files(&staging, &target).unwrap(); + let mut moved = move_staged_files(&staging, &target).unwrap(); + moved.commit(); for name in ["index.bin", "lookup.bin", "files.bin", "meta.json"] { assert_eq!(std::fs::read(target.join(name)).unwrap(), name.as_bytes()); assert!( @@ -8928,4 +9428,21 @@ mod tests { "unknown files should be left alone" ); } + + #[test] + fn staged_file_move_rolls_back_replaced_files() { + let tmp = TempDir::new().unwrap(); + let staging = tmp.path().join("staging"); + let target = tmp.path().join("target"); + std::fs::create_dir_all(&staging).unwrap(); + std::fs::create_dir_all(&target).unwrap(); + write_file(&staging.join("index.bin"), b"new"); + write_file(&target.join("index.bin"), b"old"); + + let mut moved = move_staged_files(&staging, &target).unwrap(); + assert_eq!(std::fs::read(target.join("index.bin")).unwrap(), b"new"); + moved.rollback().unwrap(); + + assert_eq!(std::fs::read(target.join("index.bin")).unwrap(), b"old"); + } } diff --git a/tgrep-core/src/gitignore.rs b/tgrep-core/src/gitignore.rs index 34885e2..a3e9cbd 100644 --- a/tgrep-core/src/gitignore.rs +++ b/tgrep-core/src/gitignore.rs @@ -466,22 +466,30 @@ pub fn build_p4ignore_matcher(root: &Path) -> Option { pub struct CaseInsensitiveIgnore { matcher: Gitignore, repo_root: std::path::PathBuf, - /// Immutable for the lifetime of this matcher. A stale reconciliation - /// shares one matcher between its walk and watcher publication, so every - /// decision in the pass uses the same tracked-file membership. - tracked: Option, - fingerprint: TrackedMembershipFingerprint, + tracked: TrackedMembership, /// Separate metadata-keyed cache for polling the current semantic - /// membership. It never participates in ignore decisions. + /// membership of a frozen matcher. It never participates in decisions. current: std::sync::RwLock, } -/// A metadata-keyed semantic fingerprint used only for change polling. +enum TrackedMembership { + /// Existing public behavior: reload after Git index identity changes. + Live(std::sync::RwLock), + /// Reconciliation behavior: one immutable set shared by a walk and the + /// point-query matcher published from that walk. + Frozen { + tracked: Option, + fingerprint: TrackedMembershipFingerprint, + }, +} + +/// A metadata-keyed tracked set and semantic fingerprint. #[derive(Default)] struct TrackedCache { /// `None` until something is loaded; `Some` even when the probe failed, so /// an unreadable index is not retried on every poll. loaded_from: Option>, + tracked: Option, fingerprint: TrackedMembershipFingerprint, } @@ -532,11 +540,39 @@ impl CaseInsensitiveIgnore { /// `--no-ignore-vcs` the case-sensitive pass lets everything through, which /// would leave this the only thing excluding — the exact opposite of what /// the flag asks for. + /// + /// The tracked-file exemption remains live: a long-lived matcher reloads it + /// after the Git index changes. pub fn new( root: &Path, use_gitignore: bool, use_exclude: bool, use_parents: bool, + ) -> Option { + Self::build(root, use_gitignore, use_exclude, use_parents, false) + } + + /// Build an immutable tracked-membership snapshot for a coordinated walk + /// and matcher publication. + /// + /// Ordinary point-query callers should use [`Self::new`], which preserves + /// live Git-index reload behavior. Server reconciliation shares this value + /// through an `Arc` so one pass cannot mix different tracked sets. + pub fn frozen_snapshot( + root: &Path, + use_gitignore: bool, + use_exclude: bool, + use_parents: bool, + ) -> Option { + Self::build(root, use_gitignore, use_exclude, use_parents, true) + } + + fn build( + root: &Path, + use_gitignore: bool, + use_exclude: bool, + use_parents: bool, + frozen: bool, ) -> Option { use ignore::gitignore::GitignoreBuilder; @@ -569,18 +605,21 @@ impl CaseInsensitiveIgnore { if matcher.is_empty() { return None; } - let identity = index_identity(&repo_root); - let tracked = crate::git_index::load_tracked(&repo_root); - let fingerprint = Self::fingerprint(&matcher, tracked.as_ref()); + let tracked = if frozen { + let tracked = crate::git_index::load_tracked(&repo_root); + let fingerprint = Self::fingerprint(&matcher, tracked.as_ref()); + TrackedMembership::Frozen { + tracked, + fingerprint, + } + } else { + TrackedMembership::Live(std::sync::RwLock::new(TrackedCache::default())) + }; Some(Self { matcher, repo_root, tracked, - fingerprint: fingerprint.clone(), - current: std::sync::RwLock::new(TrackedCache { - loaded_from: Some(identity), - fingerprint, - }), + current: std::sync::RwLock::new(TrackedCache::default()), }) } @@ -597,14 +636,35 @@ impl CaseInsensitiveIgnore { return false; } let relative = relative.to_string_lossy(); - Self::hides(self.tracked.as_ref(), &relative, is_dir) + match &self.tracked { + TrackedMembership::Live(cache) => { + let identity = index_identity(&self.repo_root); + let mut cache = cache.write().unwrap(); + self.reload_tracked_if_needed(&mut cache, identity); + Self::hides(cache.tracked.as_ref(), &relative, is_dir) + } + TrackedMembership::Frozen { tracked, .. } => { + Self::hides(tracked.as_ref(), &relative, is_dir) + } + } } fn tracked_membership_fingerprint(&self) -> TrackedMembershipFingerprint { - self.fingerprint.clone() + match &self.tracked { + TrackedMembership::Live(cache) => { + let identity = index_identity(&self.repo_root); + let mut cache = cache.write().unwrap(); + self.reload_tracked_if_needed(&mut cache, identity); + cache.fingerprint.clone() + } + TrackedMembership::Frozen { fingerprint, .. } => fingerprint.clone(), + } } fn current_tracked_membership_fingerprint(&self) -> TrackedMembershipFingerprint { + if matches!(&self.tracked, TrackedMembership::Live(_)) { + return self.tracked_membership_fingerprint(); + } let identity = index_identity(&self.repo_root); { let cache = self.current.read().unwrap(); @@ -618,10 +678,20 @@ impl CaseInsensitiveIgnore { } let tracked = crate::git_index::load_tracked(&self.repo_root); cache.fingerprint = Self::fingerprint(&self.matcher, tracked.as_ref()); + cache.tracked = tracked; cache.loaded_from = Some(identity); cache.fingerprint.clone() } + fn reload_tracked_if_needed(&self, cache: &mut TrackedCache, identity: Option) { + if cache.loaded_from.as_ref() == Some(&identity) { + return; + } + cache.tracked = crate::git_index::load_tracked(&self.repo_root); + cache.fingerprint = Self::fingerprint(&self.matcher, cache.tracked.as_ref()); + cache.loaded_from = Some(identity); + } + fn fingerprint( matcher: &Gitignore, tracked: Option<&crate::git_index::TrackedFiles>, diff --git a/tgrep-core/src/walker.rs b/tgrep-core/src/walker.rs index 4fe886c..a3da75a 100644 --- a/tgrep-core/src/walker.rs +++ b/tgrep-core/src/walker.rs @@ -1508,11 +1508,8 @@ mod tests { assert!(!matcher.is_ignored(Path::new("src/Gone.TXT"), false)); } - /// A published matcher keeps an immutable tracked-file snapshot. Polling - /// detects semantic changes separately so event filtering cannot change - /// halfway through a reconciliation pass. #[test] - fn the_tracked_exemption_snapshot_is_immutable_and_detects_changes() { + fn public_matcher_reloads_tracked_exemptions_after_index_changes() { let dir = ignorecase_fixture(true, &["src/main.rs", "src/Kept.TXT"]); let root = dir.path(); let matcher = crate::gitignore::matcher_from_ignore_paths( @@ -1527,17 +1524,48 @@ mod tests { fake_git_repo(root, true, &["src/main.rs", "src/Kept.TXT", "src/Gone.TXT"]); + assert!( + !matcher.is_ignored(Path::new("src/Gone.TXT"), false), + "the legacy public matcher must observe a newly tracked exemption" + ); + + fake_git_repo(root, true, &["src/main.rs"]); + assert!( + matcher.is_ignored(Path::new("src/Kept.TXT"), false), + "the legacy public matcher must observe a removed exemption" + ); + } + + /// Reconciliation deliberately opts into the opposite behavior: the walk + /// and matcher publication must not change answers halfway through a pass. + #[test] + fn frozen_tracked_exemption_is_immutable_and_detects_changes() { + let dir = ignorecase_fixture(true, &["src/main.rs", "src/Kept.TXT"]); + let root = dir.path(); + let ignorecase = + crate::gitignore::CaseInsensitiveIgnore::frozen_snapshot(root, true, true, true) + .map(std::sync::Arc::new); + let matcher = crate::gitignore::matcher_from_ignore_paths_with_options_and_ignorecase( + root, + std::slice::from_ref(&root.join(".gitignore")), + &[], + false, + ignorecase, + ) + .expect("the fixture has rules"); + assert!(matcher.is_ignored(Path::new("src/Gone.TXT"), false)); + fake_git_repo(root, true, &["src/main.rs", "src/Kept.TXT", "src/Gone.TXT"]); + + assert!( + matcher.is_ignored(Path::new("src/Gone.TXT"), false), + "a frozen matcher must preserve the pass snapshot" + ); assert_ne!( matcher.tracked_membership_fingerprint(), matcher.current_tracked_membership_fingerprint(), - "polling must detect a tracked exemption without mutating decisions" + "semantic polling must detect changes without mutating decisions" ); - - // And back: `git rm --cached` is the same problem in reverse, where a - // stale cache keeps indexing a file the walk has started hiding. - fake_git_repo(root, true, &["src/main.rs"]); - assert!(!matcher.is_ignored(Path::new("src/Kept.TXT"), false)); } #[test] From 1d8f67da97e465f7127ea9fe7811e0152ea8ba44 Mon Sep 17 00:00:00 2001 From: Shengyu Fu Date: Sat, 29 Aug 2026 02:58:39 -0500 Subject: [PATCH 06/16] Bind file stamps to indexed bytes Validate stable owned reads before publishing stamps across full builds, stale deltas, startup recovery, reload, and watcher updates. Retry incomplete startup reconciliation and preserve public builder APIs and stamp compatibility. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 772a4aa3-4cba-4ac4-aeca-dd184614127a --- tgrep-cli/src/serve.rs | 314 ++++++++++++++++++------------ tgrep-core/src/builder.rs | 389 +++++++++++++++++++++++--------------- tgrep-core/src/meta.rs | 33 ++-- tgrep-core/src/walker.rs | 11 +- 4 files changed, 448 insertions(+), 299 deletions(-) diff --git a/tgrep-cli/src/serve.rs b/tgrep-cli/src/serve.rs index 241f563..a632311 100644 --- a/tgrep-cli/src/serve.rs +++ b/tgrep-cli/src/serve.rs @@ -481,6 +481,8 @@ struct ServerState { last_search_ms: std::sync::atomic::AtomicU64, #[cfg(test)] stale_refresh_hook: Mutex>, + #[cfg(test)] + after_file_read_hook: Mutex>, } #[cfg(test)] @@ -697,6 +699,8 @@ pub fn run(root: &Path, index_path: Option<&Path>, options: ServeOptions<'_>) -> last_search_ms: std::sync::atomic::AtomicU64::new(0), #[cfg(test)] stale_refresh_hook: Mutex::new(None), + #[cfg(test)] + after_file_read_hook: Mutex::new(None), }); // Bind TCP listener on a random port @@ -1800,6 +1804,10 @@ fn handle_reload(id: Option, state: &Arc) -> Str let gate = state.snapshot_gate.write().unwrap(); let ignorecase = frozen_tracked_membership(state, &state.root); #[cfg(test)] + let after_file_read = state.after_file_read_hook.lock().unwrap().clone(); + #[cfg(not(test))] + let after_file_read = None; + #[cfg(test)] run_stale_refresh_hook(state, StaleRefreshPhase::BeforeWalk); let since = SystemTime::now(); @@ -1808,7 +1816,7 @@ fn handle_reload(id: Option, state: &Arc) -> Str // both are published. let staging_dir = index_dir.join(".reload-build"); let _ = std::fs::remove_dir_all(&staging_dir); - let outcome = match builder::build_index_with_options_and_ignorecase( + let observed = match builder::build_index_with_options_and_ignorecase_observed( &state.root, Some(&staging_dir), &builder::BuildOptions { @@ -1820,6 +1828,7 @@ fn handle_reload(id: Option, state: &Arc) -> Str ..Default::default() }, ignorecase.clone(), + after_file_read, ) { Ok(outcome) => outcome, Err(e) => { @@ -1827,6 +1836,18 @@ fn handle_reload(id: Option, state: &Arc) -> Str return json_rpc_error(id, -32000, &format!("rebuild failed: {e}")); } }; + if !observed.unreadable.is_empty() { + let _ = std::fs::remove_dir_all(&staging_dir); + return json_rpc_error( + id, + -32000, + &format!( + "rebuild could not capture {} file(s) at a stable version", + observed.unreadable.len() + ), + ); + } + let outcome = observed.outcome; if !publish_reloaded_index(state, &index_dir, &staging_dir, outcome.num_files) { return json_rpc_error(id, -32000, "rebuild publication failed"); @@ -4147,8 +4168,6 @@ fn read_within_limit(file: &mut std::fs::File, limit: Option, capacity: usi /// The caller must hold `snapshot_gate`: the read, the commit, and the stamp /// update have to be atomic with respect to a flush or auto-save. fn reindex_file(state: &ServerState, path: &Path, rel_path: &str) { - use tgrep_core::meta::FileStamp; - // Against other indexers, not against searches. The gate above is held for // read, so without this a recovery scan and the watcher worker can both be // here for the same path, both read, and the one that read the *older* @@ -4182,15 +4201,7 @@ fn reindex_file(state: &ServerState, path: &Path, rel_path: &str) { let Ok(meta) = file.metadata() else { return; }; - let current = FileStamp { - mtime: meta - .modified() - .ok() - .and_then(|t| t.duration_since(std::time::SystemTime::UNIX_EPOCH).ok()) - .map(|d| d.as_secs()) - .unwrap_or(0), - size: meta.len(), - }; + let current = tgrep_core::meta::file_stamp(&meta); // The rules `walk_file_metadata` applies, and for the same reason: the walk // is authoritative about what belongs in the index, so anything it rejects @@ -4254,6 +4265,38 @@ fn reindex_file(state: &ServerState, path: &Path, rel_path: &str) { } else { Some(tgrep_core::live::LiveIndex::compute_trigram_masks(&text)) }; + let original_stable = file + .metadata() + .ok() + .map(|meta| tgrep_core::meta::file_stamp(&meta)) + .as_ref() + == Some(¤t); + let reopened_matches = open_within_root(&state.root, path) + .ok() + .and_then(|mut verify| { + (tgrep_core::meta::file_stamp(&verify.metadata().ok()?) == current).then(|| { + let verified = match read_within_limit( + &mut verify, + state.max_file_size, + current.size.min(1 << 20) as usize, + ) { + CappedRead::Data(verified) => verified, + CappedRead::TooLarge | CappedRead::Failed => return false, + }; + verified == data + && verify + .metadata() + .ok() + .map(|meta| tgrep_core::meta::file_stamp(&meta)) + .as_ref() + == Some(¤t) + }) + }) + .unwrap_or(false); + if !original_stable || !reopened_matches { + eprintln!("[trace] reindex: {rel_path} changed while being read; deferring"); + return; + } eprintln!("[trace] reindex: modified {rel_path}"); // Gate held by the caller — the commit + stamp update is processed @@ -4420,36 +4463,6 @@ fn create_empty_index(index_dir: &Path) -> Result<()> { Ok(()) } -/// Stamps for the walked files that are actually in the index. -/// -/// The build stamps its work from a *second* traversal, taken after the content -/// walk that fed the index, so a file created between the two appears here and -/// nowhere else. Publishing a stamp for it would be a lie the rest of the -/// server believes: `reindex_file` returns early when the stamp already matches -/// what is on disk, and the periodic reconcile runs with -/// `compare_index_membership` off, so it compares stamps alone too — the file -/// would stay unsearchable until something changed it again. Withholding the -/// stamp instead makes the very next event or scan treat it as new, which is -/// what it is. -fn stamps_for_index_members( - files: Vec, - indexed: &std::collections::HashSet, -) -> std::collections::HashMap { - files - .into_iter() - .filter(|fm| indexed.contains(&fm.relative_path)) - .map(|fm| { - ( - fm.relative_path, - tgrep_core::meta::FileStamp { - mtime: fm.mtime, - size: fm.size, - }, - ) - }) - .collect() -} - /// Drop the stamps the build has no right to publish, because an event for /// those paths arrived while it ran. /// @@ -4640,7 +4653,7 @@ fn stream_merge_stale_changes( let result = (|| -> Result { let build = || { - builder::build_index_for_files( + builder::build_index_for_files_observed( root, &delta_dir, &files, @@ -4655,7 +4668,14 @@ fn stream_merge_stale_changes( Ok(pool) => pool.install(build)?, Err(_) => build()?, }; - let delta_count = outcome.indexed; + let delta_count = outcome.outcome.indexed; + // Candidate stamps from the metadata walk are only observations made + // before the delta read. Replace them with stamps validated around the + // exact reads that produced this delta. + for path in &desired_paths { + published_stamps.remove(path); + } + published_stamps.extend(outcome.stamps); // Withhold stamps for files the delta could not read. A published stamp // means "indexed at this version", so keeping one for a skipped file @@ -4672,22 +4692,22 @@ fn stream_merge_stale_changes( for path in changed.iter().chain(added).chain(deleted) { memo.remove(path); } - for path in &outcome.unreadable { + for path in &outcome.outcome.unreadable { let rel = path .strip_prefix(root) .unwrap_or(path) .to_string_lossy() .replace('\\', "/"); - if let Some(stamp) = published_stamps.remove(&rel) { + if let Some(stamp) = stamps.get(&rel).cloned() { memo.insert(rel, stamp); } } } - if !outcome.unreadable.is_empty() { + if !outcome.outcome.unreadable.is_empty() { eprintln!( "[trace] {} file(s) were unreadable during the delta build; \ their stamps are withheld so a later reconcile retries them", - outcome.unreadable.len() + outcome.outcome.unreadable.len() ); } @@ -4877,6 +4897,23 @@ fn background_refresh_stale( ok } +fn reconcile_incomplete_build( + state: &Arc, + root: &Path, + index_dir: &Path, + reason: &str, +) { + let mut retry_delay = Duration::from_secs(1); + while !background_refresh_stale(state, root, index_dir, true) { + eprintln!( + "[trace] {reason}: reconciliation failed; retrying in {:.0}s", + retry_delay.as_secs_f64() + ); + thread::sleep(retry_delay); + retry_delay = (retry_delay * 2).min(Duration::from_secs(30)); + } +} + fn refresh_stale_locked( state: &Arc, root: &Path, @@ -5107,7 +5144,7 @@ fn bootstrap_index_build(state: &Arc, root: &Path, index_dir: &Path // a kernel high-water mark) covers the whole of it. Unlike the incremental // path below, nothing here polls memory on its own. let sampler = crate::mem::PrivatePeakSampler::start(); - let outcome = match builder::build_index_with_options_and_ignorecase( + let observed = match builder::build_index_with_options_and_ignorecase_observed( root, Some(index_dir), &builder::BuildOptions { @@ -5125,6 +5162,7 @@ fn bootstrap_index_build(state: &Arc, root: &Path, index_dir: &Path buffer_bytes: builder::DEFAULT_INDEX_BUFFER_BYTES, }, ignorecase.clone(), + None, ) { Ok(outcome) => outcome, Err(e) => { @@ -5136,6 +5174,8 @@ fn bootstrap_index_build(state: &Arc, root: &Path, index_dir: &Path return false; } }; + let omitted = observed.unreadable.len(); + let outcome = observed.outcome; // Publish under the snapshot gate, and clear `indexing` before releasing // it. `handle_fs_event` only skips while `indexing` is true, so flipping @@ -5248,6 +5288,9 @@ fn bootstrap_index_build(state: &Arc, root: &Path, index_dir: &Path if state.ignore_rules_dirty.load(Ordering::SeqCst) { schedule_ignore_rules_refresh(Arc::clone(state), root.to_path_buf()); } + if omitted > 0 { + reconcile_incomplete_build(state, root, index_dir, "bootstrap"); + } true } @@ -5411,34 +5454,45 @@ fn background_index_build(state: &Arc, root: &Path, index_dir: &Pat } let mut incremental_flushes = 0u32; + let prior_stamps = state.file_stamps.read().unwrap().clone(); + let mut stable_stamps: std::collections::HashMap = + prior_stamps + .iter() + .filter(|(path, _)| skip_paths.contains(path.as_str())) + .map(|(path, stamp)| (path.clone(), stamp.clone())) + .collect(); + let mut unstable_paths = Vec::new(); for (batch_idx, batch) in new_files.chunks(BATCH_SIZE).enumerate() { // Parallel: read files + extract trigrams (no locks held). Run inside // the bounded pool when available so indexing CPU stays capped. let extract = || { batch .par_iter() - .filter_map(|path| { - let data = std::fs::read(path).ok()?; - let data = tgrep_core::encoding::decode_for_index(&data); - if tgrep_core::trigram::is_binary(&data) { - return None; - } + .map(|path| { let rel_path = path .strip_prefix(root) .unwrap_or(path) .to_string_lossy() .replace('\\', "/"); - - let mut trigrams = tgrep_core::trigram::extract(&data); - let lower = data.to_ascii_lowercase(); - if lower != *data { - trigrams.extend(tgrep_core::trigram::extract(&lower)); + match builder::extract_file_for_live(path) { + Ok((trigrams, stamp)) => Ok((rel_path, trigrams, stamp)), + Err(error) => { + eprintln!("tgrep: skipping {}: {error}", path.display()); + Err(path.clone()) + } } - Some((rel_path, trigrams)) }) - .collect::)>>() + .collect::>() }; - let batch_results: Vec<(String, Vec)> = match &index_pool { + type LiveBuildExtraction = std::result::Result< + ( + String, + Option, + tgrep_core::meta::FileStamp, + ), + PathBuf, + >; + let batch_results: Vec = match &index_pool { Some(pool) => pool.install(extract), None => extract(), }; @@ -5446,8 +5500,18 @@ fn background_index_build(state: &Arc, root: &Path, index_dir: &Pat // Sequential: insert into LiveIndex (brief write lock per batch) { let mut index = state.index.write().unwrap(); - for (rel_path, trigrams) in batch_results { - index.live.upsert_file_with_trigrams(&rel_path, trigrams); + for result in batch_results { + let (rel_path, trigrams, stamp) = match result { + Ok(extracted) => extracted, + Err(path) => { + unstable_paths.push(path); + continue; + } + }; + stable_stamps.insert(rel_path.clone(), stamp); + if let Some(trigrams) = trigrams { + index.live.commit_upsert(&rel_path, trigrams); + } } } @@ -5507,31 +5571,6 @@ fn background_index_build(state: &Arc, root: &Path, index_dir: &Pat start.elapsed().as_secs_f64() ); - // Walk filesystem metadata BEFORE the flush so we can publish the - // resulting per-file stamps atomically with the index files. Writing - // them after a successful flush would leave a multi-minute window where - // the index looks fully published but `filestamps.json` is missing — a - // server kill in that window disables incremental stale detection on - // the next start. - let walk_meta = tgrep_core::walker::walk_file_metadata( - root, - &tgrep_core::walker::MetaWalkOptions { - exclude_dirs: state.exclude_dirs.clone(), - no_ignore: state.no_ignore, - no_require_git: state.no_require_git, - max_file_size: state.max_file_size, - }, - ); - let stamps: std::collections::HashMap = { - let indexed = { - let index = state.index.read().unwrap(); - let mut paths = index.reader_paths(); - paths.extend(index.live.overlay_paths()); - paths - }; - stamps_for_index_members(walk_meta.files, &indexed) - }; - // The in-memory build is done — surface "complete" in status now even // though the final disk flush below can take minutes for very large // repos. Set `flushing` *before* clearing `indexing` so the auto-save @@ -5564,7 +5603,7 @@ fn background_index_build(state: &Arc, root: &Path, index_dir: &Pat // // Minus whatever changed underneath the walk, which the stamps would // otherwise describe as indexed when the index holds the older bytes. - *state.file_stamps.write().unwrap() = withhold_stamps_for_deferred(state, root, stamps); + *state.file_stamps.write().unwrap() = withhold_stamps_for_deferred(state, root, stable_stamps); state.indexing.store(false, Ordering::SeqCst); // Final flush to disk for the bulk build. Use the same streaming @@ -5599,6 +5638,9 @@ fn background_index_build(state: &Arc, root: &Path, index_dir: &Pat if state.ignore_rules_dirty.load(Ordering::SeqCst) { schedule_ignore_rules_refresh(Arc::clone(state), root.to_path_buf()); } + if seeded_count > 0 || !unstable_paths.is_empty() { + reconcile_incomplete_build(state, root, index_dir, "background index"); + } } /// Memory-bounded append-only flush used during the initial bulk build. @@ -6235,6 +6277,8 @@ mod tests { started: Instant::now(), last_search_ms: std::sync::atomic::AtomicU64::new(0), stale_refresh_hook: Mutex::new(None), + #[cfg(test)] + after_file_read_hook: Mutex::new(None), }) } @@ -7386,6 +7430,61 @@ mod tests { assert!(response.contains("\"status\":\"reloaded\""), "{response}"); } + #[test] + fn rpc_reload_retries_a_file_changed_after_its_bytes_are_read() { + use std::sync::Barrier; + use std::sync::atomic::AtomicBool; + + let tmp = TempDir::new().unwrap(); + let root = std::fs::canonicalize(tmp.path()).unwrap(); + let path = root.join("race.rs"); + std::fs::write(&path, "fn before_marker() {}\n").unwrap(); + let mut state = test_server_state(&root, &root.join(".tgrep")); + Arc::get_mut(&mut state).unwrap().watch_enabled = false; + + let read_entered = Arc::new(Barrier::new(2)); + let read_release = Arc::new(Barrier::new(2)); + let paused = Arc::new(AtomicBool::new(false)); + *state.after_file_read_hook.lock().unwrap() = Some({ + let path = path.clone(); + let read_entered = Arc::clone(&read_entered); + let read_release = Arc::clone(&read_release); + let paused = Arc::clone(&paused); + Arc::new(move |read_path| { + if read_path == path && !paused.swap(true, Ordering::SeqCst) { + read_entered.wait(); + read_release.wait(); + } + }) + }); + + let reload_state = Arc::clone(&state); + let reload = thread::spawn(move || handle_reload(None, &reload_state)); + read_entered.wait(); + std::fs::write(&path, "fn after__marker() {}\n").unwrap(); + read_release.wait(); + let response = reload.join().unwrap(); + assert!(response.contains("\"status\":\"reloaded\""), "{response}"); + + let final_query = process_request( + r#"{"jsonrpc":"2.0","id":1,"method":"search","params":{"pattern":"after__marker"}}"#, + &state, + ); + assert!(final_query.contains("race.rs"), "{final_query}"); + let stale_query = process_request( + r#"{"jsonrpc":"2.0","id":2,"method":"search","params":{"pattern":"before_marker"}}"#, + &state, + ); + assert!(stale_query.contains("\"num_matches\":0"), "{stale_query}"); + assert_eq!( + state.file_stamps.read().unwrap().get("race.rs"), + std::fs::metadata(&path) + .ok() + .map(|meta| tgrep_core::meta::file_stamp(&meta)) + .as_ref() + ); + } + #[test] fn pre_reload_disk_read_cannot_repopulate_content_cache() { let tmp = TempDir::new().unwrap(); @@ -8813,39 +8912,6 @@ mod tests { ); } - /// A stamp is a claim about the index, not about the filesystem. - /// - /// The build stamps from a second traversal that runs after the one that - /// fed the index, so a file created between them is on disk and in no - /// index. Stamping it makes every later check agree it is up to date: - /// `reindex_file` returns early on a matching stamp, and the periodic - /// reconcile compares stamps alone. The file would never be searchable. - #[test] - fn stamps_are_published_only_for_what_the_build_indexed() { - use tgrep_core::walker::FileMeta; - - let files = vec![ - FileMeta { - relative_path: "indexed.rs".to_string(), - mtime: 1, - size: 10, - }, - FileMeta { - relative_path: "arrived_between_the_walks.rs".to_string(), - mtime: 2, - size: 20, - }, - ]; - let indexed = std::iter::once("indexed.rs".to_string()).collect(); - - let stamps = stamps_for_index_members(files, &indexed); - assert!(stamps.contains_key("indexed.rs")); - assert!( - !stamps.contains_key("arrived_between_the_walks.rs"), - "a file the build never indexed must not be stamped as if it had been" - ); - } - /// The matcher reads its sources itself, inside the build. A replace that /// lands while it is reading leaves it enforcing the old rules, and stamps /// taken afterwards describe the new file — so pathname, timestamp and diff --git a/tgrep-core/src/builder.rs b/tgrep-core/src/builder.rs index c073659..b1615ac 100644 --- a/tgrep-core/src/builder.rs +++ b/tgrep-core/src/builder.rs @@ -31,12 +31,6 @@ const LOOKUP_WRITE_CHUNK_ENTRIES: usize = 4096; /// larger budget that scales with the thread pool was tried and rejected: it was /// both slower *and* hungrier there, because the kernel's large files are a rare /// minority and the extra headroom bought no throughput. -/// -/// Files large enough to be mapped are charged [`MAPPED_BATCH_CHARGE`] instead -/// of their length, so a tree that is mostly oversized files still fills the -/// pool. Raising this budget would trade memory on *every* tree for that; -/// charging mapped bytes honestly costs nothing on trees like the kernel, where -/// only 110 of 94,747 files are mapped at all. const INDEX_BUILD_BATCH_BYTES: u64 = 64 * 1024 * 1024; /// Default arena budget for [`IndexStrategy::External`] before spilling. @@ -171,6 +165,13 @@ pub struct BuildOutcome { pub ignore_files: Vec, } +/// Additional reconciliation evidence for coordinated server builds. +#[doc(hidden)] +pub struct ObservedBuildOutcome { + pub outcome: BuildOutcome, + pub unreadable: Vec, +} + /// Build a trigram index for all text files under `root`. pub fn build_index( root: &Path, @@ -218,47 +219,14 @@ fn gitignore_gate_hint(root: &Path, opts: &BuildOptions) -> Option { )) } -/// Nominal heap charge for a file large enough to be memory-mapped. -/// -/// A mapped file's bytes never reach the heap, so charging it its own length -/// against the heap budget is charging for memory the build does not allocate. -/// What it does cost is its extracted trigram map, and that is bounded by the -/// number of *distinct* trigrams — at most 16.7M, and in practice saturating -/// long before file size does. Saturating the charge keeps that real cost -/// bounded while letting a batch hold enough large files to fill the pool. -const MAPPED_BATCH_CHARGE: u64 = 2 * 1024 * 1024; - -/// Ceiling on mapped bytes one batch may put in flight. -/// -/// Mapped pages are file-backed and reclaimable, so they are not the same -/// liability as heap, but they are still resident: a batch of large files maps -/// all of them at once, and the process working set follows. This is the second -/// half of the bound — the heap budget alone would let a batch of 32 MiB files -/// map 64 MiB *per worker*. -const MAPPED_BATCH_BYTES: u64 = 256 * 1024 * 1024; - -/// What one file costs a batch, split by where the bytes live. +/// What one owned file buffer costs a batch. #[derive(Clone, Copy, Default)] struct BatchCharge { - /// Charge against the heap budget: the read buffer for a small file, or the - /// saturating stand-in for a mapped file's extracted trigram map. heap: u64, - /// Charge against the mapped budget; zero for a file that is read. - mapped: u64, } fn batch_charge(size: u64) -> BatchCharge { - if size >= MMAP_MIN_BYTES { - BatchCharge { - heap: size.min(MAPPED_BATCH_CHARGE), - mapped: size, - } - } else { - BatchCharge { - heap: size, - mapped: 0, - } - } + BatchCharge { heap: size } } /// Charge for a file whose size could not be read. @@ -269,11 +237,9 @@ fn batch_charge(size: u64) -> BatchCharge { /// losing the bound precisely for the file whose size is unknown. const UNKNOWN_SIZE_CHARGE: BatchCharge = BatchCharge { heap: INDEX_BUILD_BATCH_BYTES, - mapped: MAPPED_BATCH_BYTES, }; -/// Split a file list into batches bounded by file count, heap bytes and mapped -/// bytes, given each file's charge in walk order. +/// Split a file list into batches bounded by file count and heap bytes. /// /// A file whose charge alone exceeds a budget forms a batch of its own rather /// than being split, so the bound is "one budget, plus at most one oversized @@ -282,19 +248,14 @@ fn batch_ranges(charges: &[BatchCharge], budget: u64) -> Vec= INDEX_BUILD_BATCH_SIZE - || heap.saturating_add(charge.heap) > budget - || mapped.saturating_add(charge.mapped) > MAPPED_BATCH_BYTES; + let full = i - start >= INDEX_BUILD_BATCH_SIZE || heap.saturating_add(charge.heap) > budget; if i > start && full { ranges.push(start..i); start = i; heap = 0; - mapped = 0; } heap = heap.saturating_add(charge.heap); - mapped = mapped.saturating_add(charge.mapped); } if start < charges.len() { ranges.push(start..charges.len()); @@ -316,63 +277,151 @@ fn batch_sizes_and_charges(files: &[std::path::PathBuf]) -> (Vec, Vec), - Mapped(memmap2::Mmap), +type ExtractedFile = (String, trigram::TrigramMaskMap, meta::FileStamp); + +#[doc(hidden)] +pub type BuildReadObserver = std::sync::Arc; + +#[derive(Debug, Clone, PartialEq, Eq)] +struct FileVersion { + stamp: meta::FileStamp, + created_nanos: Option, + #[cfg(unix)] + device: u64, + #[cfg(unix)] + inode: u64, + #[cfg(unix)] + change_seconds: i64, + #[cfg(unix)] + change_nanos: i64, } -type ExtractedFile = (String, trigram::TrigramMaskMap); +fn file_version(metadata: &std::fs::Metadata) -> FileVersion { + #[cfg(unix)] + use std::os::unix::fs::MetadataExt; + FileVersion { + stamp: meta::file_stamp(metadata), + created_nanos: metadata + .created() + .ok() + .and_then(|time| time.duration_since(std::time::UNIX_EPOCH).ok()) + .map(|duration| duration.as_nanos()), + #[cfg(unix)] + device: metadata.dev(), + #[cfg(unix)] + inode: metadata.ino(), + #[cfg(unix)] + change_seconds: metadata.ctime(), + #[cfg(unix)] + change_nanos: metadata.ctime_nsec(), + } +} -impl std::ops::Deref for FileBytes { - type Target = [u8]; +/// Read owned bytes so concurrent truncation cannot invalidate memory while +/// extraction is in progress. +fn read_for_index( + path: &Path, + _size: u64, +) -> std::io::Result<(Vec, std::fs::File, FileVersion)> { + let mut file = std::fs::File::open(path)?; + let before = file_version(&file.metadata()?); + let path_before = file_version(&std::fs::metadata(path)?); + if before != path_before { + return Err(std::io::Error::new( + std::io::ErrorKind::Interrupted, + "file was replaced while opening it", + )); + } + use std::io::Read; + let mut bytes = Vec::with_capacity(usize::try_from(before.stamp.size).unwrap_or(0)); + file.read_to_end(&mut bytes)?; + Ok((bytes, file, before)) +} - fn deref(&self) -> &[u8] { - match self { - FileBytes::Read(bytes) => bytes, - FileBytes::Mapped(map) => map, +fn path_still_has_bytes( + path: &Path, + expected_version: &FileVersion, + expected: &[u8], +) -> std::io::Result { + use std::io::Read; + + let mut current = std::fs::File::open(path)?; + if file_version(¤t.metadata()?) != *expected_version { + return Ok(false); + } + let mut offset = 0; + let mut buffer = [0u8; 64 * 1024]; + loop { + let read = current.read(&mut buffer)?; + if read == 0 { + break; } + if expected.get(offset..offset + read) != Some(&buffer[..read]) { + return Ok(false); + } + offset += read; } + Ok(offset == expected.len() + && file_version(¤t.metadata()?) == *expected_version + && file_version(&std::fs::metadata(path)?) == *expected_version) } -/// Get a file's bytes for indexing, mapping it when it is large enough to be -/// worth avoiding the copy. -/// -/// Falls back to reading whenever mapping is unavailable, so a filesystem that -/// cannot map still indexes correctly. -fn read_for_index(path: &Path, size: u64) -> std::io::Result { - if size >= MMAP_MIN_BYTES { - // SAFETY: the map is read-only and owned by the returned value, which - // is dropped once the file's trigrams have been extracted. A concurrent - // truncation would be undefined behaviour; that is inherent to mapping - // a file being indexed, and is the same exposure the search path - // accepts. - let mapped = - std::fs::File::open(path).and_then(|file| unsafe { memmap2::Mmap::map(&file) }); - if let Ok(map) = mapped { - return Ok(FileBytes::Mapped(map)); +fn extract_stable_file( + path: &Path, + size: u64, + after_read: Option<&(dyn Fn(&Path) + Send + Sync)>, +) -> std::io::Result<(trigram::TrigramMaskMap, meta::FileStamp, bool)> { + const ATTEMPTS: usize = 3; + let mut last_error = None; + for _ in 0..ATTEMPTS { + let (data, file, before) = match read_for_index(path, size) { + Ok(read) => read, + Err(error) => { + last_error = Some(error); + continue; + } + }; + let text = crate::encoding::decode_for_index(&data); + let binary = trigram::is_binary(&text); + let per_tri = if binary { + Default::default() + } else { + trigram::extract_merged_masks(&text) + }; + if let Some(after_read) = after_read { + after_read(path); } + let after_file = file.metadata().map(|metadata| file_version(&metadata)); + if after_file.as_ref().is_ok_and(|version| version == &before) + && path_still_has_bytes(path, &before, &data).unwrap_or(false) + { + return Ok((per_tri, before.stamp, binary)); + } + last_error = Some(std::io::Error::new( + std::io::ErrorKind::Interrupted, + "file changed while it was being indexed", + )); } - std::fs::read(path).map(FileBytes::Read) + Err(last_error.unwrap_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::Interrupted, + "file could not be read at one stable version", + ) + })) +} + +/// Read one file at a metadata-stable version for the server's in-heap build. +/// +/// The returned stamp was validated after trigram extraction and therefore +/// describes the same version as `trigrams`. `None` trigrams means that stable +/// version was binary by content. +#[doc(hidden)] +pub fn extract_file_for_live( + path: &Path, +) -> std::io::Result<(Option, meta::FileStamp)> { + let size = std::fs::metadata(path)?.len(); + let (trigrams, stamp, binary) = extract_stable_file(path, size, None)?; + Ok(((!binary).then_some(trigrams), stamp)) } /// Build a trigram index, choosing how postings are accumulated. @@ -392,6 +441,20 @@ pub fn build_index_with_options_and_ignorecase( opts: &BuildOptions, ignorecase: Option>, ) -> Result { + build_index_with_options_and_ignorecase_observed(root, index_dir, opts, ignorecase, None) + .map(|observed| observed.outcome) +} + +/// Testable form of [`build_index_with_options_and_ignorecase`] that observes +/// the interval between extraction and metadata validation. +#[doc(hidden)] +pub fn build_index_with_options_and_ignorecase_observed( + root: &Path, + index_dir: Option<&Path>, + opts: &BuildOptions, + ignorecase: Option>, + after_file_read: Option, +) -> Result { let include_hidden = opts.include_hidden; let no_ignore = opts.no_ignore; let exclude_dirs = opts.exclude_dirs.as_slice(); @@ -455,6 +518,8 @@ pub fn build_index_with_options_and_ignorecase( // bounding each batch by cumulative bytes caps how much raw file content is // resident at once, since the whole batch is read concurrently. let mut file_id_map: Vec<(u32, String)> = Vec::with_capacity(walk.files.len()); + let mut stamps = std::collections::HashMap::with_capacity(walk.files.len()); + let mut unreadable = Vec::new(); let mut sink = match opts.strategy { IndexStrategy::InMemory => PostingSink::InMemory(Vec::new()), IndexStrategy::External => { @@ -470,27 +535,38 @@ pub fn build_index_with_options_and_ignorecase( for range in batch_ranges(&charges, INDEX_BUILD_BATCH_BYTES) { let batch = &walk.files[range.clone()]; let batch_sizes = &sizes[range]; - let batch_data: Vec = batch + let batch_data: Vec> = batch .par_iter() .zip(batch_sizes.par_iter()) - .filter_map(|(path, &size)| { - let data = read_for_index(path, size).ok()?; - let text = crate::encoding::decode_for_index(&data); - if trigram::is_binary(&text) { - binary_skipped.fetch_add(1, std::sync::atomic::Ordering::Relaxed); - return None; - } + .map(|(path, &size)| { let rel = path .strip_prefix(&root) .unwrap_or(path) .to_string_lossy() .replace('\\', "/"); - let per_tri = trigram::extract_merged_masks(&text); - Some((rel, per_tri)) + match extract_stable_file(path, size, after_file_read.as_deref()) { + Ok((per_tri, stamp, binary)) => Ok(((rel, per_tri, stamp), binary)), + Err(error) => { + eprintln!("tgrep: skipping {}: {error}", path.display()); + Err(path.clone()) + } + } }) .collect(); - for (path, per_tri) in batch_data { + for entry in batch_data { + let ((path, per_tri, stamp), binary) = match entry { + Ok(extracted) => extracted, + Err(path) => { + unreadable.push(path); + continue; + } + }; + stamps.insert(path.clone(), stamp); + if binary { + binary_skipped.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + continue; + } let file_id = file_id_map.len() as u32; file_id_map.push((file_id, path)); sink.push_file(file_id, per_tri)?; @@ -528,23 +604,19 @@ pub fn build_index_with_options_and_ignorecase( } } - // Write per-file stamps for ALL walked files (including those later - // rejected as binary-by-content) so the stale check on next startup - // won't re-process unchanged files that aren't in the index. - let all_walked: Vec = walk - .files - .iter() - .filter_map(|p| p.strip_prefix(&root).ok()) - .map(|p| p.to_string_lossy().replace('\\', "/")) - .collect(); - let stamps = meta::collect_filestamps(&root, &all_walked); + // Every stamp came from the same stable read that produced the indexed + // bytes (or proved a file binary). Unreadable/unstable paths stay unstamped + // so a serialized reconciliation must revisit them. meta::write_filestamps(&stamps, &index_dir)?; eprintln!("Index built successfully at {}", index_dir.display()); - Ok(BuildOutcome { - num_files: file_id_map.len(), - gitignore_files, - ignore_files, + Ok(ObservedBuildOutcome { + outcome: BuildOutcome { + num_files: file_id_map.len(), + gitignore_files, + ignore_files, + }, + unreadable, }) } @@ -568,12 +640,20 @@ pub struct FileDeltaOutcome { pub unreadable: Vec, } +/// Additional read-bound stamps for coordinated stale-delta publication. +#[doc(hidden)] +pub struct ObservedFileDeltaOutcome { + pub outcome: FileDeltaOutcome, + pub stamps: std::collections::HashMap, +} + /// Build an external-sort index for an exact list of absolute file paths. /// /// This is used by `tgrep serve` to build a bounded delta for files that are /// absent from an otherwise-complete index. It deliberately skips the directory -/// walk and filestamp write: the caller already has both from its stale-state -/// scan and publishes the complete stamp set with the merged index. +/// walk and filestamp write: the caller has the complete candidate set from its +/// stale-state scan, then replaces candidate stamps with the read-bound stamps +/// returned here before publishing the merged index. /// /// Files that become unreadable or binary after the stale scan are omitted /// rather than fatal, matching a normal index build. Aborting instead would be @@ -587,6 +667,19 @@ pub fn build_index_for_files( files: &[std::path::PathBuf], buffer_bytes: usize, ) -> Result { + build_index_for_files_observed(root, index_dir, files, buffer_bytes) + .map(|observed| observed.outcome) +} + +/// Testable/coordinated form of [`build_index_for_files`] that returns the +/// read-bound stamps needed for atomic stale publication. +#[doc(hidden)] +pub fn build_index_for_files_observed( + root: &Path, + index_dir: &Path, + files: &[std::path::PathBuf], + buffer_bytes: usize, +) -> Result { let input_root = root; let root = std::fs::canonicalize(root)?; std::fs::create_dir_all(index_dir)?; @@ -595,43 +688,44 @@ pub fn build_index_for_files( let mut file_id_map: Vec<(u32, String)> = Vec::with_capacity(files.len()); let mut sorter = ExternalSorter::new(index_dir, buffer_bytes); let mut unreadable: Vec = Vec::new(); + let mut stamps = std::collections::HashMap::with_capacity(files.len()); for range in batch_ranges(&charges, INDEX_BUILD_BATCH_BYTES) { let batch = &files[range.clone()]; let batch_sizes = &sizes[range]; - let batch_data: Vec> = batch + let batch_data: Vec> = batch .par_iter() .zip(batch_sizes.par_iter()) .filter_map(|(path, &size)| { - let data = match read_for_index(path, size) { - Ok(data) => data, + let (per_tri, stamp, binary) = match extract_stable_file(path, size, None) { + Ok(extracted) => extracted, Err(error) => { eprintln!("tgrep: skipping {}: {error}", path.display()); return Some(Err(path.clone())); } }; - let text = crate::encoding::decode_for_index(&data); - if trigram::is_binary(&text) { - return None; - } let rel = path .strip_prefix(&root) .or_else(|_| path.strip_prefix(input_root)) .unwrap_or(path) .to_string_lossy() .replace('\\', "/"); - Some(Ok((rel, trigram::extract_merged_masks(&text)))) + Some(Ok(((rel, per_tri, stamp), binary))) }) .collect(); for entry in batch_data { - let (path, per_tri) = match entry { + let ((path, per_tri, stamp), binary) = match entry { Ok(extracted) => extracted, Err(skipped) => { unreadable.push(skipped); continue; } }; + stamps.insert(path.clone(), stamp); + if binary { + continue; + } let file_id = u32::try_from(file_id_map.len()).map_err(|_| { Error::IndexCorrupted("file count exceeds the u32 file-id limit".into()) })?; @@ -649,9 +743,12 @@ pub fn build_index_for_files( trigram_count, Some(true), )?; - Ok(FileDeltaOutcome { - indexed: file_id_map.len(), - unreadable, + Ok(ObservedFileDeltaOutcome { + outcome: FileDeltaOutcome { + indexed: file_id_map.len(), + unreadable, + }, + stamps, }) } @@ -1325,39 +1422,27 @@ mod tests { #[test] fn batches_are_bounded_by_cumulative_heap_bytes() { - // Files below the mapping threshold are read, so their whole length is - // heap: five 900 KiB reads against a 2 MiB budget fit two at a time. + // Five 900 KiB reads against a 2 MiB budget fit two at a time. let charges = charges(&[900 * 1024; 5]); assert_eq!(batch_ranges(&charges, 2 * MB), vec![0..2, 2..4, 4..5]); } #[test] - fn large_mapped_files_still_fill_a_batch() { - // Regression: charging a mapped file its own length against the heap - // budget made a batch of large files degenerate to one or two, leaving - // every worker but one idle. Mapped bytes never reach the heap, so only - // the mapped budget bounds these: 256 MiB / 32 MiB is eight per batch. + fn large_owned_files_respect_the_heap_budget() { let charges = charges(&[32 * MB; 16]); assert_eq!( batch_ranges(&charges, INDEX_BUILD_BATCH_BYTES), - vec![0..8, 8..16] + vec![0..2, 2..4, 4..6, 6..8, 8..10, 10..12, 12..14, 14..16] ); } #[test] - fn mapped_bytes_in_flight_stay_bounded() { - // The other half of the bound: a batch may not map more than the mapped - // budget, whatever mix of sizes it is handed. + fn owned_bytes_in_flight_stay_bounded() { let sizes: Vec = (0..200).map(|i| (i % 9 + 1) * 8 * MB).collect(); let charges = charges(&sizes); for range in batch_ranges(&charges, INDEX_BUILD_BATCH_BYTES) { let single = range.end - range.start == 1; - let mapped: u64 = charges[range.clone()].iter().map(|c| c.mapped).sum(); let heap: u64 = charges[range.clone()].iter().map(|c| c.heap).sum(); - assert!( - single || mapped <= MAPPED_BATCH_BYTES, - "batch {range:?} maps {mapped} bytes" - ); assert!( single || heap <= INDEX_BUILD_BATCH_BYTES, "batch {range:?} reads {heap} bytes" diff --git a/tgrep-core/src/meta.rs b/tgrep-core/src/meta.rs index 6bb40ae..98a1340 100644 --- a/tgrep-core/src/meta.rs +++ b/tgrep-core/src/meta.rs @@ -68,6 +68,21 @@ pub struct FileStamp { pub size: u64, } +/// Convert filesystem metadata into the persisted stamp used by change +/// detection. +pub fn file_stamp(metadata: &std::fs::Metadata) -> FileStamp { + let mtime = metadata + .modified() + .ok() + .and_then(|t| t.duration_since(SystemTime::UNIX_EPOCH).ok()) + .map(|d| d.as_secs()) + .unwrap_or(0); + FileStamp { + mtime, + size: metadata.len(), + } +} + /// Write per-file stamps to `filestamps.json` in the index directory. pub fn write_filestamps(stamps: &HashMap, index_dir: &Path) -> Result<()> { let path = index_dir.join(FILESTAMPS_FILENAME); @@ -95,21 +110,9 @@ pub fn collect_filestamps(root: &Path, paths: &[String]) -> HashMap limit) { return ignore::WalkState::Continue; } - let mtime = meta - .modified() - .ok() - .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()) - .map(|d| d.as_secs()) - .unwrap_or(0); + let stamp = crate::meta::file_stamp(&meta); results.lock().unwrap().push(FileMeta { relative_path: rel_path, - mtime, - size: meta.len(), + mtime: stamp.mtime, + size: stamp.size, }); ignore::WalkState::Continue From 49a66de3d20ae83a7712d485ea24dacf6e635ba8 Mon Sep 17 00:00:00 2001 From: Shengyu Fu Date: Sat, 29 Aug 2026 04:01:05 -0500 Subject: [PATCH 07/16] Fix watcher stamp race without global rereads Restore the mmap-backed builder and bounded batch accounting, then repair reload and startup publication through deferred event replay, no-watch catch-up, and stable containment-safe concrete event reads. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 772a4aa3-4cba-4ac4-aeca-dd184614127a --- tgrep-cli/src/serve.rs | 1220 +++++++++++++++++++++++++++++-------- tgrep-core/src/builder.rs | 389 +++++------- tgrep-core/src/meta.rs | 33 +- tgrep-core/src/walker.rs | 11 +- 4 files changed, 1146 insertions(+), 507 deletions(-) diff --git a/tgrep-cli/src/serve.rs b/tgrep-cli/src/serve.rs index a632311..e7cd54b 100644 --- a/tgrep-cli/src/serve.rs +++ b/tgrep-cli/src/serve.rs @@ -481,14 +481,14 @@ struct ServerState { last_search_ms: std::sync::atomic::AtomicU64, #[cfg(test)] stale_refresh_hook: Mutex>, - #[cfg(test)] - after_file_read_hook: Mutex>, } #[cfg(test)] #[derive(Clone, Copy)] enum StaleRefreshPhase { BeforeWalk, + AfterBuildBeforeStampPublish, + AfterConcreteRead, AfterMatcherPublish, } @@ -699,8 +699,6 @@ pub fn run(root: &Path, index_path: Option<&Path>, options: ServeOptions<'_>) -> last_search_ms: std::sync::atomic::AtomicU64::new(0), #[cfg(test)] stale_refresh_hook: Mutex::new(None), - #[cfg(test)] - after_file_read_hook: Mutex::new(None), }); // Bind TCP listener on a random port @@ -1802,12 +1800,15 @@ fn handle_reload(id: Option, state: &Arc) -> Str } let refresh = state.stale_refresh_lock.lock().unwrap(); let gate = state.snapshot_gate.write().unwrap(); + { + let _deferred = match state.deferred_events.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + }; + state.indexing.store(true, Ordering::SeqCst); + } let ignorecase = frozen_tracked_membership(state, &state.root); #[cfg(test)] - let after_file_read = state.after_file_read_hook.lock().unwrap().clone(); - #[cfg(not(test))] - let after_file_read = None; - #[cfg(test)] run_stale_refresh_hook(state, StaleRefreshPhase::BeforeWalk); let since = SystemTime::now(); @@ -1816,7 +1817,7 @@ fn handle_reload(id: Option, state: &Arc) -> Str // both are published. let staging_dir = index_dir.join(".reload-build"); let _ = std::fs::remove_dir_all(&staging_dir); - let observed = match builder::build_index_with_options_and_ignorecase_observed( + let outcome = match builder::build_index_with_options_and_ignorecase( &state.root, Some(&staging_dir), &builder::BuildOptions { @@ -1828,40 +1829,65 @@ fn handle_reload(id: Option, state: &Arc) -> Str ..Default::default() }, ignorecase.clone(), - after_file_read, ) { Ok(outcome) => outcome, Err(e) => { let _ = std::fs::remove_dir_all(&staging_dir); + state.indexing.store(false, Ordering::SeqCst); + drop(gate); + drop(refresh); + replay_deferred_events(state, &state.root); + schedule_pending_ignore_refresh(state); return json_rpc_error(id, -32000, &format!("rebuild failed: {e}")); } }; - if !observed.unreadable.is_empty() { + #[cfg(test)] + run_stale_refresh_hook(state, StaleRefreshPhase::AfterBuildBeforeStampPublish); + + // Freeze the deferred buffer through publication. Events already captured + // have their stamps withheld; callbacks arriving now wait on this lock, + // then observe `indexing == false` and process normally after the gate drops. + let deferred = match state.deferred_events.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + }; + let stamps = match tgrep_core::meta::read_filestamps(&staging_dir) { + Ok(stamps) if state.watch_enabled => { + withhold_stamps_for_deferred_snapshot(&state.root, stamps, deferred.as_ref()) + } + // Without a watcher there is no event to identify a file that changed + // between extraction and the builder's later stamp collection. Publish + // no claims from that racy window; the serialized catch-up below then + // treats every path as changed and rebuilds it once. + Ok(_) => std::collections::HashMap::new(), + Err(e) => { + eprintln!("[trace] warning: reload could not read file stamps ({e})"); + std::collections::HashMap::new() + } + }; + if let Err(e) = tgrep_core::meta::write_filestamps(&stamps, &staging_dir) { let _ = std::fs::remove_dir_all(&staging_dir); - return json_rpc_error( - id, - -32000, - &format!( - "rebuild could not capture {} file(s) at a stable version", - observed.unreadable.len() - ), - ); + state.indexing.store(false, Ordering::SeqCst); + drop(deferred); + drop(gate); + drop(refresh); + replay_deferred_events(state, &state.root); + schedule_pending_ignore_refresh(state); + return json_rpc_error(id, -32000, &format!("could not stage rebuild stamps: {e}")); } - let outcome = observed.outcome; - if !publish_reloaded_index(state, &index_dir, &staging_dir, outcome.num_files) { + state.indexing.store(false, Ordering::SeqCst); + drop(deferred); + drop(gate); + drop(refresh); + replay_deferred_events(state, &state.root); + schedule_pending_ignore_refresh(state); return json_rpc_error(id, -32000, "rebuild publication failed"); } let indexed = outcome.num_files as u64; state.index_total.store(indexed, Ordering::Relaxed); state.index_progress.store(indexed, Ordering::Relaxed); - match tgrep_core::meta::read_filestamps(&index_dir) { - Ok(stamps) => *state.file_stamps.write().unwrap() = stamps, - Err(e) => { - eprintln!("[trace] warning: reload could not read file stamps ({e})"); - state.file_stamps.write().unwrap().clear(); - } - } + *state.file_stamps.write().unwrap() = stamps; let newly_watched = if state.no_ignore { Vec::new() @@ -1888,10 +1914,29 @@ fn handle_reload(id: Option, state: &Arc) -> Str }; #[cfg(test)] run_stale_refresh_hook(state, StaleRefreshPhase::AfterMatcherPublish); - let membership_changed = tracked_membership_changed(state); + let mut membership_changed = tracked_membership_changed(state); + state.indexing.store(false, Ordering::SeqCst); + drop(deferred); drop(gate); drop(refresh); + replay_deferred_events(state, &state.root); + schedule_pending_ignore_refresh(state); + if !state.watch_enabled { + #[cfg(test)] + if membership_changed { + run_stale_refresh_hook(state, StaleRefreshPhase::BeforeWalk); + } + let caught_up = catch_up_unwatched_build(state, &state.root, &state.index_dir); + membership_changed = tracked_membership_changed(state); + if !caught_up { + return json_rpc_error( + id, + -32000, + "rebuild catch-up did not complete; reload was not authoritative", + ); + } + } if state.watch_enabled { spawn_recovery_scan(state, &state.root, newly_watched, since); } @@ -2977,7 +3022,7 @@ fn reindex_files_in(state: &Arc, root: &Path, dirs: &[PathBuf], sin should_skip_watcher_path(&rel, &state.exclude_dirs, gitignore.as_ref()) }; if !skip { - reindex_file(state, &path, &rel); + reindex_file(state, &path, &rel, false); } } if listing_complete { @@ -3328,7 +3373,7 @@ fn watch_new_subtree(state: &Arc, root: &Path, dir: &Path) { } for (path, rel) in &files { - reindex_file(state, path, rel); + reindex_file(state, path, rel, true); } } @@ -3383,6 +3428,12 @@ fn schedule_ignore_rules_refresh(state: Arc, root: PathBuf) { }); } +fn schedule_pending_ignore_refresh(state: &Arc) { + if state.ignore_rules_dirty.load(Ordering::SeqCst) { + schedule_ignore_rules_refresh(Arc::clone(state), state.root.clone()); + } +} + /// Remember the paths in an event that arrived mid-build, for replay once the /// build publishes. Returns whether it did — a `false` means the build finished /// underneath us and the caller should handle the event normally. @@ -3493,7 +3544,30 @@ fn replay_deferred_events(state: &Arc, root: &Path) { let start = Instant::now(); let count = paths.len(); + let mut missing_subtree = false; for (path, introduces_dir) in paths { + if !is_real_dir(&path) + && path.strip_prefix(root).is_ok_and(|rel| { + let prefix = format!("{}/", rel.to_string_lossy().replace('\\', "/")); + let index = state.index.read().unwrap(); + index + .reader_paths() + .iter() + .any(|indexed| indexed.starts_with(&prefix)) + || index + .live + .overlay_paths() + .iter() + .any(|indexed| indexed.starts_with(&prefix)) + }) + { + // A single directory removal/rename event names no descendants. + // Replaying it as a file event would drop only the directory path, + // leaving every indexed child searchable. One coalesced full pass + // supplies the missing subtree membership evidence. + missing_subtree = true; + continue; + } let kind = if introduces_dir { EventKind::Create(notify::event::CreateKind::Any) } else { @@ -3511,6 +3585,10 @@ fn replay_deferred_events(state: &Arc, root: &Path) { }, ); } + if missing_subtree { + state.ignore_rules_dirty.store(true, Ordering::SeqCst); + schedule_ignore_rules_refresh(Arc::clone(state), root.to_path_buf()); + } eprintln!( "[trace] watcher: replayed {count} change(s) deferred during the initial build in {:.1}ms", start.elapsed().as_secs_f64() * 1000.0 @@ -3805,7 +3883,7 @@ fn handle_fs_event(state: &Arc, root: &Path, event: &Event) { continue; } - reindex_file(state, path, &rel_path); + reindex_file(state, path, &rel_path, true); } } @@ -4167,7 +4245,9 @@ fn read_within_limit(file: &mut std::fs::File, limit: Option, capacity: usi /// /// The caller must hold `snapshot_gate`: the read, the commit, and the stamp /// update have to be atomic with respect to a flush or auto-save. -fn reindex_file(state: &ServerState, path: &Path, rel_path: &str) { +fn reindex_file(state: &Arc, path: &Path, rel_path: &str, force: bool) { + use tgrep_core::meta::FileStamp; + // Against other indexers, not against searches. The gate above is held for // read, so without this a recovery scan and the watcher worker can both be // here for the same path, both read, and the one that read the *older* @@ -4195,13 +4275,27 @@ fn reindex_file(state: &ServerState, path: &Path, rel_path: &str) { // because a build held the file open for a moment. The stale path // already treats unreadable files this way, keeping what it has and // retrying later, and the watcher should not disagree with it. + if force { + retry_failed_forced_reindex(state, rel_path, "the file could not be opened"); + } return; } }; let Ok(meta) = file.metadata() else { + if force { + retry_failed_forced_reindex(state, rel_path, "the opened file could not be inspected"); + } return; }; - let current = tgrep_core::meta::file_stamp(&meta); + let mut current = FileStamp { + mtime: meta + .modified() + .ok() + .and_then(|t| t.duration_since(std::time::SystemTime::UNIX_EPOCH).ok()) + .map(|d| d.as_secs()) + .unwrap_or(0), + size: meta.len(), + }; // The rules `walk_file_metadata` applies, and for the same reason: the walk // is authoritative about what belongs in the index, so anything it rejects @@ -4233,7 +4327,7 @@ fn reindex_file(state: &ServerState, path: &Path, rel_path: &str) { return; } - if state.file_stamps.read().unwrap().get(rel_path) == Some(¤t) { + if !force && state.file_stamps.read().unwrap().get(rel_path) == Some(¤t) { return; } @@ -4246,7 +4340,7 @@ fn reindex_file(state: &ServerState, path: &Path, rel_path: &str) { // From the handle, not the path: re-opening here is what would let a // symlink take the place of the file we just approved. let mut file = file; - let data = match read_within_limit( + let mut data = match read_within_limit( &mut file, state.max_file_size, current.size.min(1 << 20) as usize, @@ -4256,8 +4350,93 @@ fn reindex_file(state: &ServerState, path: &Path, rel_path: &str) { drop_indexed_file(state, rel_path, "grew past the size limit while being read"); return; } - CappedRead::Failed => return, + CappedRead::Failed => { + if force { + retry_failed_forced_reindex(state, rel_path, "the file could not be read"); + } + return; + } }; + if force { + #[cfg(test)] + run_stale_refresh_hook(state, StaleRefreshPhase::AfterConcreteRead); + // A concrete event is stronger evidence than the persisted stamp: + // FileStamp intentionally stores only whole-second mtime plus size for + // compatibility, so an equal-length rewrite can compare equal. Read a + // second containment-safe handle and require two consecutive byte + // snapshots to agree. This also avoids committing a mixed version when + // a writer changes the file while the first read is in progress. + let mut stable = false; + for _ in 0..2 { + let mut verify = match open_within_root(&state.root, path) { + Ok(file) => file, + Err(e) if proves_ineligible(&e) => { + drop_indexed_file(state, rel_path, "no longer eligible"); + return; + } + Err(_) => { + retry_failed_forced_reindex( + state, + rel_path, + "the verification handle could not be opened", + ); + return; + } + }; + let Ok(meta) = verify.metadata() else { + retry_failed_forced_reindex( + state, + rel_path, + "the verification handle could not be inspected", + ); + return; + }; + let verify_current = FileStamp { + mtime: meta + .modified() + .ok() + .and_then(|t| t.duration_since(std::time::SystemTime::UNIX_EPOCH).ok()) + .map(|d| d.as_secs()) + .unwrap_or(0), + size: meta.len(), + }; + if !meta.is_file() + || tgrep_core::walker::is_binary_extension(path) + || state + .max_file_size + .is_some_and(|limit| verify_current.size > limit) + { + drop_indexed_file(state, rel_path, "no longer eligible"); + return; + } + let verified = match read_within_limit( + &mut verify, + state.max_file_size, + verify_current.size.min(1 << 20) as usize, + ) { + CappedRead::Data(data) => data, + CappedRead::TooLarge => { + drop_indexed_file(state, rel_path, "grew past the size limit while being read"); + return; + } + CappedRead::Failed => { + retry_failed_forced_reindex(state, rel_path, "the verification read failed"); + return; + } + }; + current = verify_current; + if data == verified { + data = verified; + stable = true; + break; + } + data = verified; + } + if !stable { + retry_failed_forced_reindex(state, rel_path, "the file kept changing while read"); + return; + } + } let text = tgrep_core::encoding::decode_for_index(&data); let is_binary = tgrep_core::trigram::is_binary(&text); let per_tri = if is_binary { @@ -4265,38 +4444,6 @@ fn reindex_file(state: &ServerState, path: &Path, rel_path: &str) { } else { Some(tgrep_core::live::LiveIndex::compute_trigram_masks(&text)) }; - let original_stable = file - .metadata() - .ok() - .map(|meta| tgrep_core::meta::file_stamp(&meta)) - .as_ref() - == Some(¤t); - let reopened_matches = open_within_root(&state.root, path) - .ok() - .and_then(|mut verify| { - (tgrep_core::meta::file_stamp(&verify.metadata().ok()?) == current).then(|| { - let verified = match read_within_limit( - &mut verify, - state.max_file_size, - current.size.min(1 << 20) as usize, - ) { - CappedRead::Data(verified) => verified, - CappedRead::TooLarge | CappedRead::Failed => return false, - }; - verified == data - && verify - .metadata() - .ok() - .map(|meta| tgrep_core::meta::file_stamp(&meta)) - .as_ref() - == Some(¤t) - }) - }) - .unwrap_or(false); - if !original_stable || !reopened_matches { - eprintln!("[trace] reindex: {rel_path} changed while being read; deferring"); - return; - } eprintln!("[trace] reindex: modified {rel_path}"); // Gate held by the caller — the commit + stamp update is processed @@ -4316,6 +4463,22 @@ fn reindex_file(state: &ServerState, path: &Path, rel_path: &str) { invalidate_cached_paths(state, std::iter::once(rel_path)); } +fn retry_failed_forced_reindex(state: &Arc, rel_path: &str, reason: &str) { + // In-memory stamps override the persisted map during stale comparison. + // A sentinel therefore records "this path must be read" without rewriting + // filestamps.json for a transient event failure. + state.file_stamps.write().unwrap().insert( + rel_path.to_string(), + tgrep_core::meta::FileStamp { + mtime: u64::MAX, + size: u64::MAX, + }, + ); + eprintln!("[trace] warning: {rel_path} was not reindexed because {reason}; scheduling a retry"); + state.ignore_rules_dirty.store(true, Ordering::SeqCst); + schedule_ignore_rules_refresh(Arc::clone(state), state.root.clone()); +} + /// Whether a scheduled reconcile should run now. /// /// Split out from the loop so the schedule can be exercised without waiting @@ -4463,6 +4626,36 @@ fn create_empty_index(index_dir: &Path) -> Result<()> { Ok(()) } +/// Stamps for the walked files that are actually in the index. +/// +/// The build stamps its work from a *second* traversal, taken after the content +/// walk that fed the index, so a file created between the two appears here and +/// nowhere else. Publishing a stamp for it would be a lie the rest of the +/// server believes: `reindex_file` returns early when the stamp already matches +/// what is on disk, and the periodic reconcile runs with +/// `compare_index_membership` off, so it compares stamps alone too — the file +/// would stay unsearchable until something changed it again. Withholding the +/// stamp instead makes the very next event or scan treat it as new, which is +/// what it is. +fn stamps_for_index_members( + files: Vec, + indexed: &std::collections::HashSet, +) -> std::collections::HashMap { + files + .into_iter() + .filter(|fm| indexed.contains(&fm.relative_path)) + .map(|fm| { + ( + fm.relative_path, + tgrep_core::meta::FileStamp { + mtime: fm.mtime, + size: fm.size, + }, + ) + }) + .collect() +} + /// Drop the stamps the build has no right to publish, because an event for /// those paths arrived while it ran. /// @@ -4492,29 +4685,54 @@ fn create_empty_index(index_dir: &Path) -> Result<()> { fn withhold_stamps_for_deferred( state: &ServerState, root: &Path, - mut stamps: std::collections::HashMap, + stamps: std::collections::HashMap, ) -> std::collections::HashMap { let deferred = match state.deferred_events.lock() { Ok(guard) => guard, Err(poisoned) => poisoned.into_inner(), }; - let Some(paths) = deferred.as_ref() else { + withhold_stamps_for_deferred_snapshot(root, stamps, deferred.as_ref()) +} + +fn withhold_stamps_for_deferred_snapshot( + root: &Path, + mut stamps: std::collections::HashMap, + paths: Option<&std::collections::HashMap>, +) -> std::collections::HashMap { + let Some(paths) = paths else { eprintln!( "[trace] warning: too many changes during the initial build to say which files the \ walk raced; publishing no stamps so the reconcile re-reads them" ); return std::collections::HashMap::new(); }; - let mut withheld = 0usize; - for path in paths.keys() { + let mut exact = std::collections::HashSet::new(); + let mut directories = std::collections::HashSet::new(); + for (path, introduces_dir) in paths { let Ok(rel) = path.strip_prefix(root) else { continue; }; let rel = rel.to_string_lossy().replace('\\', "/"); - if stamps.remove(&rel).is_some() { - withheld += 1; + exact.insert(rel.clone()); + if *introduces_dir { + directories.insert(rel); } } + let before = stamps.len(); + stamps.retain(|rel, _| { + if exact.contains(rel) { + return false; + } + let mut ancestor = rel.as_str(); + while let Some((parent, _)) = ancestor.rsplit_once('/') { + if directories.contains(parent) { + return false; + } + ancestor = parent; + } + true + }); + let withheld = before - stamps.len(); if withheld > 0 { eprintln!( "[trace] watcher: {withheld} file(s) changed during the initial build; their stamps \ @@ -4653,7 +4871,7 @@ fn stream_merge_stale_changes( let result = (|| -> Result { let build = || { - builder::build_index_for_files_observed( + builder::build_index_for_files( root, &delta_dir, &files, @@ -4668,14 +4886,7 @@ fn stream_merge_stale_changes( Ok(pool) => pool.install(build)?, Err(_) => build()?, }; - let delta_count = outcome.outcome.indexed; - // Candidate stamps from the metadata walk are only observations made - // before the delta read. Replace them with stamps validated around the - // exact reads that produced this delta. - for path in &desired_paths { - published_stamps.remove(path); - } - published_stamps.extend(outcome.stamps); + let delta_count = outcome.indexed; // Withhold stamps for files the delta could not read. A published stamp // means "indexed at this version", so keeping one for a skipped file @@ -4692,22 +4903,22 @@ fn stream_merge_stale_changes( for path in changed.iter().chain(added).chain(deleted) { memo.remove(path); } - for path in &outcome.outcome.unreadable { + for path in &outcome.unreadable { let rel = path .strip_prefix(root) .unwrap_or(path) .to_string_lossy() .replace('\\', "/"); - if let Some(stamp) = stamps.get(&rel).cloned() { + if let Some(stamp) = published_stamps.remove(&rel) { memo.insert(rel, stamp); } } } - if !outcome.outcome.unreadable.is_empty() { + if !outcome.unreadable.is_empty() { eprintln!( "[trace] {} file(s) were unreadable during the delta build; \ their stamps are withheld so a later reconcile retries them", - outcome.outcome.unreadable.len() + outcome.unreadable.len() ); } @@ -4873,6 +5084,7 @@ fn background_refresh_stale( compare_index_membership, &mut newly_watched, ignorecase, + true, ); // Directories that were not subscribed while the walk ran could not report @@ -4897,21 +5109,23 @@ fn background_refresh_stale( ok } -fn reconcile_incomplete_build( - state: &Arc, - root: &Path, - index_dir: &Path, - reason: &str, -) { - let mut retry_delay = Duration::from_secs(1); - while !background_refresh_stale(state, root, index_dir, true) { - eprintln!( - "[trace] {reason}: reconciliation failed; retrying in {:.0}s", - retry_delay.as_secs_f64() - ); - thread::sleep(retry_delay); - retry_delay = (retry_delay * 2).min(Duration::from_secs(30)); - } +fn catch_up_unwatched_build(state: &Arc, root: &Path, index_dir: &Path) -> bool { + let refresh = state.stale_refresh_lock.lock().unwrap(); + let gate = state.snapshot_gate.write().unwrap(); + let ignorecase = frozen_tracked_membership(state, root); + let mut ignored_watches = Vec::new(); + let caught_up = refresh_stale_locked( + state, + root, + index_dir, + true, + &mut ignored_watches, + ignorecase, + false, + ); + drop(gate); + drop(refresh); + caught_up } fn refresh_stale_locked( @@ -4921,6 +5135,7 @@ fn refresh_stale_locked( compare_index_membership: bool, newly_watched: &mut Vec, ignorecase: Option>, + run_test_hooks: bool, ) -> bool { use tgrep_core::meta; use tgrep_core::walker; @@ -4970,7 +5185,11 @@ fn refresh_stale_locked( || build_stale_matcher(state, root, &walk, ignorecase), ); #[cfg(test)] - run_stale_refresh_hook(state, StaleRefreshPhase::AfterMatcherPublish); + if run_test_hooks { + run_stale_refresh_hook(state, StaleRefreshPhase::AfterMatcherPublish); + } + #[cfg(not(test))] + let _ = run_test_hooks; if walk.skipped_error > 0 { eprintln!( @@ -5144,7 +5363,7 @@ fn bootstrap_index_build(state: &Arc, root: &Path, index_dir: &Path // a kernel high-water mark) covers the whole of it. Unlike the incremental // path below, nothing here polls memory on its own. let sampler = crate::mem::PrivatePeakSampler::start(); - let observed = match builder::build_index_with_options_and_ignorecase_observed( + let outcome = match builder::build_index_with_options_and_ignorecase( root, Some(index_dir), &builder::BuildOptions { @@ -5162,7 +5381,6 @@ fn bootstrap_index_build(state: &Arc, root: &Path, index_dir: &Path buffer_bytes: builder::DEFAULT_INDEX_BUFFER_BYTES, }, ignorecase.clone(), - None, ) { Ok(outcome) => outcome, Err(e) => { @@ -5174,8 +5392,8 @@ fn bootstrap_index_build(state: &Arc, root: &Path, index_dir: &Path return false; } }; - let omitted = observed.unreadable.len(); - let outcome = observed.outcome; + #[cfg(test)] + run_stale_refresh_hook(state, StaleRefreshPhase::AfterBuildBeforeStampPublish); // Publish under the snapshot gate, and clear `indexing` before releasing // it. `handle_fs_event` only skips while `indexing` is true, so flipping @@ -5215,19 +5433,33 @@ fn bootstrap_index_build(state: &Arc, root: &Path, index_dir: &Path // and its walk handed back the .gitignore / .ignore paths. Building the // matcher from those is what keeps this cheap — `gitignore::build_matcher` // would rewalk the whole tree, which cost 49 s on a 289k-file repo. - match tgrep_core::meta::read_filestamps(index_dir) { + let stamps = match tgrep_core::meta::read_filestamps(index_dir) { // Minus the paths whose events arrived while the build ran: the builder // read those files at some point during its walk and stamped what it // saw, so for anything written afterwards the stamp describes bytes the // index does not hold. See `withhold_stamps_for_deferred`. - Ok(stamps) => { - *state.file_stamps.write().unwrap() = withhold_stamps_for_deferred(state, root, stamps) + Ok(stamps) if state.watch_enabled => withhold_stamps_for_deferred(state, root, stamps), + Ok(_) => std::collections::HashMap::new(), + Err(e) => { + eprintln!( + "[trace] warning: could not load file stamps ({e}); \ + the watcher may reindex on spurious events" + ); + std::collections::HashMap::new() } - Err(e) => eprintln!( - "[trace] warning: could not load file stamps ({e}); \ - the watcher may reindex on spurious events" - ), + }; + if !state.watch_enabled + && let Err(e) = tgrep_core::meta::write_filestamps(&stamps, index_dir) + { + drop(gate); + eprintln!( + "[trace] warning: could not prepare unwatched bootstrap catch-up ({e}); \ + falling back to the in-heap build" + ); + reset_to_empty_index(state, root, index_dir); + return false; } + *state.file_stamps.write().unwrap() = stamps; let mut newly_watched = Vec::new(); if !state.no_ignore { let t_gi = Instant::now(); @@ -5270,8 +5502,21 @@ fn bootstrap_index_build(state: &Arc, root: &Path, index_dir: &Path spawn_recovery_scan(state, root, newly_watched, since); } - state.indexing.store(false, Ordering::SeqCst); + if state.watch_enabled { + state.indexing.store(false, Ordering::SeqCst); + } drop(gate); + if !state.watch_enabled { + if !catch_up_unwatched_build(state, root, index_dir) { + eprintln!( + "[trace] warning: unwatched bootstrap catch-up was incomplete; \ + falling back to the in-heap build" + ); + reset_to_empty_index(state, root, index_dir); + return false; + } + state.indexing.store(false, Ordering::SeqCst); + } let membership_changed = tracked_membership_changed(state); schedule_tracked_membership_correction(state, root, membership_changed); @@ -5288,9 +5533,6 @@ fn bootstrap_index_build(state: &Arc, root: &Path, index_dir: &Path if state.ignore_rules_dirty.load(Ordering::SeqCst) { schedule_ignore_rules_refresh(Arc::clone(state), root.to_path_buf()); } - if omitted > 0 { - reconcile_incomplete_build(state, root, index_dir, "bootstrap"); - } true } @@ -5454,45 +5696,34 @@ fn background_index_build(state: &Arc, root: &Path, index_dir: &Pat } let mut incremental_flushes = 0u32; - let prior_stamps = state.file_stamps.read().unwrap().clone(); - let mut stable_stamps: std::collections::HashMap = - prior_stamps - .iter() - .filter(|(path, _)| skip_paths.contains(path.as_str())) - .map(|(path, stamp)| (path.clone(), stamp.clone())) - .collect(); - let mut unstable_paths = Vec::new(); for (batch_idx, batch) in new_files.chunks(BATCH_SIZE).enumerate() { // Parallel: read files + extract trigrams (no locks held). Run inside // the bounded pool when available so indexing CPU stays capped. let extract = || { batch .par_iter() - .map(|path| { + .filter_map(|path| { + let data = std::fs::read(path).ok()?; + let data = tgrep_core::encoding::decode_for_index(&data); + if tgrep_core::trigram::is_binary(&data) { + return None; + } let rel_path = path .strip_prefix(root) .unwrap_or(path) .to_string_lossy() .replace('\\', "/"); - match builder::extract_file_for_live(path) { - Ok((trigrams, stamp)) => Ok((rel_path, trigrams, stamp)), - Err(error) => { - eprintln!("tgrep: skipping {}: {error}", path.display()); - Err(path.clone()) - } + + let mut trigrams = tgrep_core::trigram::extract(&data); + let lower = data.to_ascii_lowercase(); + if lower != *data { + trigrams.extend(tgrep_core::trigram::extract(&lower)); } + Some((rel_path, trigrams)) }) - .collect::>() + .collect::)>>() }; - type LiveBuildExtraction = std::result::Result< - ( - String, - Option, - tgrep_core::meta::FileStamp, - ), - PathBuf, - >; - let batch_results: Vec = match &index_pool { + let batch_results: Vec<(String, Vec)> = match &index_pool { Some(pool) => pool.install(extract), None => extract(), }; @@ -5500,18 +5731,8 @@ fn background_index_build(state: &Arc, root: &Path, index_dir: &Pat // Sequential: insert into LiveIndex (brief write lock per batch) { let mut index = state.index.write().unwrap(); - for result in batch_results { - let (rel_path, trigrams, stamp) = match result { - Ok(extracted) => extracted, - Err(path) => { - unstable_paths.push(path); - continue; - } - }; - stable_stamps.insert(rel_path.clone(), stamp); - if let Some(trigrams) = trigrams { - index.live.commit_upsert(&rel_path, trigrams); - } + for (rel_path, trigrams) in batch_results { + index.live.upsert_file_with_trigrams(&rel_path, trigrams); } } @@ -5570,6 +5791,33 @@ fn background_index_build(state: &Arc, root: &Path, index_dir: &Pat incremental_flushes, start.elapsed().as_secs_f64() ); + #[cfg(test)] + run_stale_refresh_hook(state, StaleRefreshPhase::AfterBuildBeforeStampPublish); + + // Walk filesystem metadata BEFORE the flush so we can publish the + // resulting per-file stamps atomically with the index files. Writing + // them after a successful flush would leave a multi-minute window where + // the index looks fully published but `filestamps.json` is missing — a + // server kill in that window disables incremental stale detection on + // the next start. + let walk_meta = tgrep_core::walker::walk_file_metadata( + root, + &tgrep_core::walker::MetaWalkOptions { + exclude_dirs: state.exclude_dirs.clone(), + no_ignore: state.no_ignore, + no_require_git: state.no_require_git, + max_file_size: state.max_file_size, + }, + ); + let stamps: std::collections::HashMap = { + let indexed = { + let index = state.index.read().unwrap(); + let mut paths = index.reader_paths(); + paths.extend(index.live.overlay_paths()); + paths + }; + stamps_for_index_members(walk_meta.files, &indexed) + }; // The in-memory build is done — surface "complete" in status now even // though the final disk flush below can take minutes for very large @@ -5603,8 +5851,14 @@ fn background_index_build(state: &Arc, root: &Path, index_dir: &Pat // // Minus whatever changed underneath the walk, which the stamps would // otherwise describe as indexed when the index holds the older bytes. - *state.file_stamps.write().unwrap() = withhold_stamps_for_deferred(state, root, stable_stamps); - state.indexing.store(false, Ordering::SeqCst); + *state.file_stamps.write().unwrap() = if state.watch_enabled { + withhold_stamps_for_deferred(state, root, stamps) + } else { + std::collections::HashMap::new() + }; + if state.watch_enabled { + state.indexing.store(false, Ordering::SeqCst); + } // Final flush to disk for the bulk build. Use the same streaming // append-only path as incremental flushes so the final complete publish @@ -5623,6 +5877,15 @@ fn background_index_build(state: &Arc, root: &Path, index_dir: &Pat drop(gate); state.flushing.store(false, Ordering::SeqCst); + if !state.watch_enabled { + if !catch_up_unwatched_build(state, root, index_dir) { + eprintln!( + "[trace] warning: unwatched background-build catch-up was incomplete; \ + leaving stamps invalid for the scheduled stale check" + ); + } + state.indexing.store(false, Ordering::SeqCst); + } // Reclaim memory held by the indexing-time live overlay — but only when // the flush actually completed and `prune_persisted_entries` ran. If the @@ -5638,9 +5901,6 @@ fn background_index_build(state: &Arc, root: &Path, index_dir: &Pat if state.ignore_rules_dirty.load(Ordering::SeqCst) { schedule_ignore_rules_refresh(Arc::clone(state), root.to_path_buf()); } - if seeded_count > 0 || !unstable_paths.is_empty() { - reconcile_incomplete_build(state, root, index_dir, "background index"); - } } /// Memory-bounded append-only flush used during the initial bulk build. @@ -6277,8 +6537,6 @@ mod tests { started: Instant::now(), last_search_ms: std::sync::atomic::AtomicU64::new(0), stale_refresh_hook: Mutex::new(None), - #[cfg(test)] - after_file_read_hook: Mutex::new(None), }) } @@ -6783,7 +7041,7 @@ mod tests { std::fs::write(&rejected, "not indexed despite textual contents\n").unwrap(); let _gate = state.snapshot_gate.read().unwrap(); - reindex_file(&state, &rejected, "asset.png"); + reindex_file(&state, &rejected, "asset.png", false); let index = state.index.read().unwrap(); assert_eq!( @@ -6948,6 +7206,8 @@ mod tests { after_publish_release.wait(); } } + StaleRefreshPhase::AfterBuildBeforeStampPublish => {} + StaleRefreshPhase::AfterConcreteRead => {} }) }; *state.stale_refresh_hook.lock().unwrap() = Some(hook); @@ -7084,6 +7344,8 @@ mod tests { test_git(&root, &["add", "--", "src/lib.rs"]); } } + StaleRefreshPhase::AfterBuildBeforeStampPublish => {} + StaleRefreshPhase::AfterConcreteRead => {} }) }; *state.stale_refresh_hook.lock().unwrap() = Some(hook); @@ -7250,6 +7512,8 @@ mod tests { after_publish_release.wait(); } } + StaleRefreshPhase::AfterBuildBeforeStampPublish => {} + StaleRefreshPhase::AfterConcreteRead => {} }) }; *state.stale_refresh_hook.lock().unwrap() = Some(hook); @@ -7380,111 +7644,326 @@ mod tests { } #[test] - fn rpc_reload_build_failure_preserves_active_index() { + fn rpc_reload_without_watcher_repairs_change_after_extraction() { let tmp = TempDir::new().unwrap(); let root = std::fs::canonicalize(tmp.path()).unwrap(); - let index_dir = root.join(".tgrep"); - let state = test_server_state(&root, &index_dir); - state - .index - .write() - .unwrap() - .live - .upsert_file("active.rs", b"fn active_before_reload() {}\n"); + test_git(&root, &["init", "--quiet"]); + let path = root.join("raced.rs"); + std::fs::write(&path, "fn old_reload_marker() {}\n").unwrap(); - std::fs::write(index_dir.join(".reload-build"), b"blocks staging directory").unwrap(); - let response = handle_reload(None, &state); + let mut state = test_server_state(&root, &root.join(".tgrep")); + Arc::get_mut(&mut state).unwrap().watch_enabled = false; + state.gitignore_pending.store(false, Ordering::SeqCst); + let hook_path = path.clone(); + *state.stale_refresh_hook.lock().unwrap() = Some(Arc::new(move |phase| { + if matches!(phase, StaleRefreshPhase::AfterBuildBeforeStampPublish) { + std::fs::write(&hook_path, "fn new_reload_marker() {}\n").unwrap(); + } + })); - assert!(response.contains("rebuild failed"), "{response}"); + let response = handle_reload(None, &state); + assert!(response.contains("\"status\":\"reloaded\""), "{response}"); + let result = handle_search( + None, + &serde_json::json!({"pattern": "new_reload_marker"}), + &state, + ); assert!( - state.index.read().unwrap().live.has_path("active.rs"), - "a failed staged build must not disturb the active index" + result.contains("new_reload_marker"), + "the no-watch catch-up must index the bytes written after extraction: {result}" ); } #[test] - fn rpc_reload_waits_for_initial_build() { - use std::sync::mpsc::{RecvTimeoutError, channel}; - + fn unwatched_external_bootstrap_repairs_change_after_extraction() { let tmp = TempDir::new().unwrap(); let root = std::fs::canonicalize(tmp.path()).unwrap(); - let state = test_server_state(&root, &root.join(".tgrep")); + test_git(&root, &["init", "--quiet"]); + let path = root.join("raced.rs"); + std::fs::write(&path, "fn old_bootstrap_marker() {}\n").unwrap(); + + let mut state = test_server_state(&root, &root.join(".tgrep")); + Arc::get_mut(&mut state).unwrap().watch_enabled = false; state.indexing.store(true, Ordering::SeqCst); - let (entered_tx, entered_rx) = channel(); + state.gitignore_pending.store(false, Ordering::SeqCst); + let hook_path = path.clone(); *state.stale_refresh_hook.lock().unwrap() = Some(Arc::new(move |phase| { - if matches!(phase, StaleRefreshPhase::BeforeWalk) { - entered_tx.send(()).unwrap(); + if matches!(phase, StaleRefreshPhase::AfterBuildBeforeStampPublish) { + std::fs::write(&hook_path, "fn new_bootstrap_marker() {}\n").unwrap(); } })); - let reload_state = Arc::clone(&state); - let reload = thread::spawn(move || handle_reload(None, &reload_state)); - assert!(matches!( - entered_rx.recv_timeout(Duration::from_millis(150)), - Err(RecvTimeoutError::Timeout) - )); - - state.indexing.store(false, Ordering::SeqCst); - entered_rx.recv_timeout(Duration::from_secs(5)).unwrap(); - let response = reload.join().unwrap(); - assert!(response.contains("\"status\":\"reloaded\""), "{response}"); + assert!(bootstrap_index_build(&state, &root, &state.index_dir)); + let result = handle_search( + None, + &serde_json::json!({"pattern": "new_bootstrap_marker"}), + &state, + ); + assert!( + result.contains("new_bootstrap_marker"), + "the no-watch bootstrap catch-up must index the final bytes: {result}" + ); } #[test] - fn rpc_reload_retries_a_file_changed_after_its_bytes_are_read() { - use std::sync::Barrier; - use std::sync::atomic::AtomicBool; - + fn unwatched_resumed_build_repairs_change_after_extraction() { let tmp = TempDir::new().unwrap(); let root = std::fs::canonicalize(tmp.path()).unwrap(); - let path = root.join("race.rs"); - std::fs::write(&path, "fn before_marker() {}\n").unwrap(); + test_git(&root, &["init", "--quiet"]); + std::fs::write(root.join("seeded.rs"), "fn seeded() {}\n").unwrap(); let mut state = test_server_state(&root, &root.join(".tgrep")); Arc::get_mut(&mut state).unwrap().watch_enabled = false; + builder::build_index_with_options( + &root, + Some(&state.index_dir), + &builder::BuildOptions { + no_require_git: false, + ..Default::default() + }, + ) + .unwrap(); + *state.index.write().unwrap() = HybridIndex::open(&state.index_dir, &root).unwrap(); - let read_entered = Arc::new(Barrier::new(2)); - let read_release = Arc::new(Barrier::new(2)); - let paused = Arc::new(AtomicBool::new(false)); - *state.after_file_read_hook.lock().unwrap() = Some({ - let path = path.clone(); - let read_entered = Arc::clone(&read_entered); - let read_release = Arc::clone(&read_release); - let paused = Arc::clone(&paused); - Arc::new(move |read_path| { - if read_path == path && !paused.swap(true, Ordering::SeqCst) { - read_entered.wait(); - read_release.wait(); - } - }) - }); - - let reload_state = Arc::clone(&state); - let reload = thread::spawn(move || handle_reload(None, &reload_state)); - read_entered.wait(); - std::fs::write(&path, "fn after__marker() {}\n").unwrap(); - read_release.wait(); - let response = reload.join().unwrap(); - assert!(response.contains("\"status\":\"reloaded\""), "{response}"); + let path = root.join("raced.rs"); + std::fs::write(&path, "fn old_resumed_marker() {}\n").unwrap(); + state.indexing.store(true, Ordering::SeqCst); + state.gitignore_pending.store(false, Ordering::SeqCst); + let hook_path = path.clone(); + *state.stale_refresh_hook.lock().unwrap() = Some(Arc::new(move |phase| { + if matches!(phase, StaleRefreshPhase::AfterBuildBeforeStampPublish) { + std::fs::write(&hook_path, "fn new_resumed_marker() {}\n").unwrap(); + } + })); - let final_query = process_request( - r#"{"jsonrpc":"2.0","id":1,"method":"search","params":{"pattern":"after__marker"}}"#, + background_index_build(&state, &root, &state.index_dir); + let result = handle_search( + None, + &serde_json::json!({"pattern": "new_resumed_marker"}), &state, ); - assert!(final_query.contains("race.rs"), "{final_query}"); - let stale_query = process_request( - r#"{"jsonrpc":"2.0","id":2,"method":"search","params":{"pattern":"before_marker"}}"#, + assert!( + result.contains("new_resumed_marker"), + "the no-watch resumed-build catch-up must index the final bytes: {result}" + ); + } + + #[test] + fn rpc_reload_replays_event_received_after_extraction() { + let tmp = TempDir::new().unwrap(); + let root = std::fs::canonicalize(tmp.path()).unwrap(); + test_git(&root, &["init", "--quiet"]); + let path = root.join("raced.rs"); + std::fs::write(&path, "fn old_watched_marker() {}\n").unwrap(); + let state = test_server_state(&root, &root.join(".tgrep")); + state.gitignore_pending.store(false, Ordering::SeqCst); + let prior = root.join("prior.rs"); + state + .deferred_events + .lock() + .unwrap() + .as_mut() + .unwrap() + .insert(prior.clone(), false); + + let hook_path = path.clone(); + let hook_state = Arc::clone(&state); + *state.stale_refresh_hook.lock().unwrap() = Some(Arc::new(move |phase| { + if matches!(phase, StaleRefreshPhase::AfterBuildBeforeStampPublish) { + assert!( + hook_state + .deferred_events + .lock() + .unwrap() + .as_ref() + .unwrap() + .contains_key(&prior), + "reload must preserve events awaiting an earlier build replay" + ); + std::fs::write(&hook_path, "fn new_watched_marker() {}\n").unwrap(); + assert!(defer_events_during_build( + &hook_state, + &Event { + kind: EventKind::Modify(notify::event::ModifyKind::Data( + notify::event::DataChange::Any, + )), + paths: vec![hook_path.clone()], + attrs: Default::default(), + } + )); + } + })); + + let response = handle_reload(None, &state); + assert!(response.contains("\"status\":\"reloaded\""), "{response}"); + let result = handle_search( + None, + &serde_json::json!({"pattern": "new_watched_marker"}), &state, ); - assert!(stale_query.contains("\"num_matches\":0"), "{stale_query}"); - assert_eq!( - state.file_stamps.read().unwrap().get("race.rs"), - std::fs::metadata(&path) - .ok() - .map(|meta| tgrep_core::meta::file_stamp(&meta)) - .as_ref() + assert!( + result.contains("new_watched_marker"), + "the replay must force the concrete event despite the later matching stamp: {result}" + ); + } + + #[test] + fn rpc_reload_schedules_ignore_rule_change_received_during_build() { + let tmp = TempDir::new().unwrap(); + let root = std::fs::canonicalize(tmp.path()).unwrap(); + test_git(&root, &["init", "--quiet"]); + let rules = root.join(".gitignore"); + std::fs::write(&rules, "").unwrap(); + std::fs::create_dir(root.join("ignored")).unwrap(); + std::fs::write(root.join("ignored/file.rs"), "fn should_disappear() {}\n").unwrap(); + let state = test_server_state(&root, &root.join(".tgrep")); + state.gitignore_pending.store(false, Ordering::SeqCst); + + let hook_rules = rules.clone(); + let hook_root = root.clone(); + let hook_state = Arc::clone(&state); + *state.stale_refresh_hook.lock().unwrap() = Some(Arc::new(move |phase| { + if matches!(phase, StaleRefreshPhase::AfterBuildBeforeStampPublish) { + std::fs::write(&hook_rules, "ignored/\n").unwrap(); + handle_fs_event( + &hook_state, + &hook_root, + &Event { + kind: EventKind::Modify(notify::event::ModifyKind::Data( + notify::event::DataChange::Any, + )), + paths: vec![hook_rules.clone()], + attrs: Default::default(), + }, + ); + } + })); + + let response = handle_reload(None, &state); + assert!(response.contains("\"status\":\"reloaded\""), "{response}"); + let deadline = Instant::now() + Duration::from_secs(10); + while (state.ignore_refresh_scheduled.load(Ordering::SeqCst) + || state.ignore_rules_dirty.load(Ordering::SeqCst)) + && Instant::now() < deadline + { + thread::sleep(Duration::from_millis(10)); + } + + assert!( + !state.ignore_refresh_scheduled.load(Ordering::SeqCst) + && !state.ignore_rules_dirty.load(Ordering::SeqCst), + "the rule change observed during reload must run a serialized refresh" + ); + let index = state.index.read().unwrap(); + assert!( + !index.reader_has_path("ignored/file.rs") || index.live.is_deleted("ignored/file.rs"), + "the refresh must remove content hidden by the new rule" + ); + } + + #[test] + fn concrete_event_reindexes_equal_length_rewrite_with_matching_stamp() { + let tmp = TempDir::new().unwrap(); + let root = std::fs::canonicalize(tmp.path()).unwrap(); + let path = root.join("same.rs"); + std::fs::write(&path, "fn old_marker() {}\n").unwrap(); + let state = test_server_state(&root, &root.join(".tgrep")); + state.gitignore_pending.store(false, Ordering::SeqCst); + + { + let _gate = state.snapshot_gate.read().unwrap(); + reindex_file(&state, &path, "same.rs", false); + } + std::fs::write(&path, "fn new_marker() {}\n").unwrap(); + let current = tgrep_core::meta::collect_filestamps(&root, &["same.rs".to_string()]) + .remove("same.rs") + .unwrap(); + state + .file_stamps + .write() + .unwrap() + .insert("same.rs".to_string(), current); + + // Mutation control: the speculative path still trusts the persisted + // stamp and therefore leaves the old posting set in place. + { + let _gate = state.snapshot_gate.read().unwrap(); + reindex_file(&state, &path, "same.rs", false); + } + let stale = handle_search(None, &serde_json::json!({"pattern": "new_marker"}), &state); + assert!( + !stale.contains("\"content\":\"fn new_marker"), + "the control must demonstrate that stamp-only filtering misses the rewrite" + ); + + handle_fs_event( + &state, + &root, + &Event { + kind: EventKind::Modify(notify::event::ModifyKind::Data( + notify::event::DataChange::Any, + )), + paths: vec![path], + attrs: Default::default(), + }, + ); + let repaired = handle_search(None, &serde_json::json!({"pattern": "new_marker"}), &state); + assert!( + repaired.contains("\"content\":\"fn new_marker"), + "a concrete event must bypass the coarse matching stamp: {repaired}" ); } + #[test] + fn rpc_reload_build_failure_preserves_active_index() { + let tmp = TempDir::new().unwrap(); + let root = std::fs::canonicalize(tmp.path()).unwrap(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + state + .index + .write() + .unwrap() + .live + .upsert_file("active.rs", b"fn active_before_reload() {}\n"); + + std::fs::write(index_dir.join(".reload-build"), b"blocks staging directory").unwrap(); + let response = handle_reload(None, &state); + + assert!(response.contains("rebuild failed"), "{response}"); + assert!( + state.index.read().unwrap().live.has_path("active.rs"), + "a failed staged build must not disturb the active index" + ); + } + + #[test] + fn rpc_reload_waits_for_initial_build() { + use std::sync::mpsc::{RecvTimeoutError, channel}; + + let tmp = TempDir::new().unwrap(); + let root = std::fs::canonicalize(tmp.path()).unwrap(); + let state = test_server_state(&root, &root.join(".tgrep")); + state.indexing.store(true, Ordering::SeqCst); + let (entered_tx, entered_rx) = channel(); + *state.stale_refresh_hook.lock().unwrap() = Some(Arc::new(move |phase| { + if matches!(phase, StaleRefreshPhase::BeforeWalk) { + entered_tx.send(()).unwrap(); + } + })); + + let reload_state = Arc::clone(&state); + let reload = thread::spawn(move || handle_reload(None, &reload_state)); + assert!(matches!( + entered_rx.recv_timeout(Duration::from_millis(150)), + Err(RecvTimeoutError::Timeout) + )); + + state.indexing.store(false, Ordering::SeqCst); + entered_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + let response = reload.join().unwrap(); + assert!(response.contains("\"status\":\"reloaded\""), "{response}"); + } + #[test] fn pre_reload_disk_read_cannot_repopulate_content_cache() { let tmp = TempDir::new().unwrap(); @@ -8030,6 +8509,148 @@ mod tests { ); } + #[cfg(unix)] + #[test] + fn concrete_event_rejects_file_swapped_to_outside_symlink_between_reads() { + let outside = TempDir::new().unwrap(); + let target = outside.path().join("outside.rs"); + std::fs::write(&target, "fn outside_file_marker() {}\n").unwrap(); + let root_dir = TempDir::new().unwrap(); + let root = std::fs::canonicalize(root_dir.path()).unwrap(); + let path = root.join("raced.rs"); + std::fs::write(&path, "fn inside_file_marker() {}\n").unwrap(); + let state = test_server_state(&root, &root.join(".tgrep")); + state.gitignore_pending.store(false, Ordering::SeqCst); + { + let _gate = state.snapshot_gate.read().unwrap(); + reindex_file(&state, &path, "raced.rs", false); + } + + let swap_path = path.clone(); + let swap_target = target.clone(); + *state.stale_refresh_hook.lock().unwrap() = Some(Arc::new(move |phase| { + if matches!(phase, StaleRefreshPhase::AfterConcreteRead) { + std::fs::remove_file(&swap_path).unwrap(); + std::os::unix::fs::symlink(&swap_target, &swap_path).unwrap(); + } + })); + handle_fs_event( + &state, + &root, + &Event { + kind: EventKind::Modify(notify::event::ModifyKind::Data( + notify::event::DataChange::Any, + )), + paths: vec![path], + attrs: Default::default(), + }, + ); + + let result = handle_search( + None, + &serde_json::json!({"pattern": "outside_file_marker"}), + &state, + ); + assert!( + !result.contains("outside_file_marker"), + "verification must not follow a replacement symlink outside the root" + ); + assert!(state.index.read().unwrap().live.is_deleted("raced.rs")); + } + + #[cfg(unix)] + #[test] + fn concrete_event_rejects_ancestor_swapped_to_outside_symlink_between_reads() { + let outside = TempDir::new().unwrap(); + std::fs::write( + outside.path().join("raced.rs"), + "fn outside_ancestor_marker() {}\n", + ) + .unwrap(); + let root_dir = TempDir::new().unwrap(); + let root = std::fs::canonicalize(root_dir.path()).unwrap(); + let dir = root.join("dir"); + std::fs::create_dir(&dir).unwrap(); + let path = dir.join("raced.rs"); + std::fs::write(&path, "fn inside_ancestor_marker() {}\n").unwrap(); + let state = test_server_state(&root, &root.join(".tgrep")); + state.gitignore_pending.store(false, Ordering::SeqCst); + { + let _gate = state.snapshot_gate.read().unwrap(); + reindex_file(&state, &path, "dir/raced.rs", false); + } + + let swap_dir = dir.clone(); + let moved_dir = root.join("original-dir"); + let outside_dir = outside.path().to_path_buf(); + *state.stale_refresh_hook.lock().unwrap() = Some(Arc::new(move |phase| { + if matches!(phase, StaleRefreshPhase::AfterConcreteRead) { + std::fs::rename(&swap_dir, &moved_dir).unwrap(); + std::os::unix::fs::symlink(&outside_dir, &swap_dir).unwrap(); + } + })); + handle_fs_event( + &state, + &root, + &Event { + kind: EventKind::Modify(notify::event::ModifyKind::Data( + notify::event::DataChange::Any, + )), + paths: vec![path], + attrs: Default::default(), + }, + ); + + let result = handle_search( + None, + &serde_json::json!({"pattern": "outside_ancestor_marker"}), + &state, + ); + assert!( + !result.contains("outside_ancestor_marker"), + "verification must not traverse a replacement ancestor symlink" + ); + assert!(state.index.read().unwrap().live.is_deleted("dir/raced.rs")); + } + + #[test] + fn concrete_event_caps_growth_between_verification_reads() { + let tmp = TempDir::new().unwrap(); + let root = std::fs::canonicalize(tmp.path()).unwrap(); + let path = root.join("grows.rs"); + std::fs::write(&path, "fn small() {}\n").unwrap(); + let mut state = test_server_state(&root, &root.join(".tgrep")); + Arc::get_mut(&mut state).unwrap().max_file_size = Some(64); + state.gitignore_pending.store(false, Ordering::SeqCst); + { + let _gate = state.snapshot_gate.read().unwrap(); + reindex_file(&state, &path, "grows.rs", false); + } + + let grow_path = path.clone(); + *state.stale_refresh_hook.lock().unwrap() = Some(Arc::new(move |phase| { + if matches!(phase, StaleRefreshPhase::AfterConcreteRead) { + std::fs::write(&grow_path, vec![b'x'; 4096]).unwrap(); + } + })); + handle_fs_event( + &state, + &root, + &Event { + kind: EventKind::Modify(notify::event::ModifyKind::Data( + notify::event::DataChange::Any, + )), + paths: vec![path], + attrs: Default::default(), + }, + ); + + assert!( + state.index.read().unwrap().live.is_deleted("grows.rs"), + "growth past the configured limit must be rejected before a second unbounded read" + ); + } + /// Nothing may be resolved that could climb back out of the root. #[test] fn open_within_root_refuses_paths_that_escape_or_are_not_literal() { @@ -8202,6 +8823,51 @@ mod tests { ); } + #[test] + fn a_deferred_directory_rename_reconciles_indexed_descendants() { + let tmp = TempDir::new().unwrap(); + let root = std::fs::canonicalize(tmp.path()).unwrap(); + let dir = root.join("removed"); + let path = dir.join("deep.rs"); + std::fs::create_dir(&dir).unwrap(); + std::fs::write(&path, "fn removed_descendant() {}\n").unwrap(); + let state = test_server_state(&root, &root.join(".tgrep")); + state.gitignore_pending.store(false, Ordering::SeqCst); + { + let _gate = state.snapshot_gate.read().unwrap(); + reindex_file(&state, &path, "removed/deep.rs", false); + } + + state.indexing.store(true, Ordering::SeqCst); + std::fs::rename(&dir, root.join("moved")).unwrap(); + handle_fs_event( + &state, + &root, + &Event { + kind: EventKind::Modify(notify::event::ModifyKind::Name( + notify::event::RenameMode::From, + )), + paths: vec![dir], + attrs: Default::default(), + }, + ); + state.indexing.store(false, Ordering::SeqCst); + replay_deferred_events(&state, &root); + + let deadline = Instant::now() + Duration::from_secs(10); + while (state.ignore_refresh_scheduled.load(Ordering::SeqCst) + || state.ignore_rules_dirty.load(Ordering::SeqCst)) + && Instant::now() < deadline + { + thread::sleep(Duration::from_millis(10)); + } + let index = state.index.read().unwrap(); + assert!( + !index.reader_has_path("removed/deep.rs") && !index.live.has_path("removed/deep.rs"), + "the coalesced reconcile must retire descendants named by no removal event" + ); + } + /// A file that cannot be opened right now is not a file that stopped /// belonging in the index. Evicting on a transient error would drop live /// content because something else held the file open for a moment. @@ -8218,7 +8884,7 @@ mod tests { let path = root.join("locked.rs"); std::fs::write(&path, "fn readable() {}\n").unwrap(); - reindex_file(&state, &path, "locked.rs"); + reindex_file(&state, &path, "locked.rs", false); assert!(state.index.read().unwrap().live.has_path("locked.rs")); std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o000)).unwrap(); @@ -8226,7 +8892,7 @@ mod tests { // Running as root, where the mode is advisory. Nothing to test. return; } - reindex_file(&state, &path, "locked.rs"); + reindex_file(&state, &path, "locked.rs", false); assert!( !state.index.read().unwrap().live.is_deleted("locked.rs"), @@ -8293,7 +8959,7 @@ mod tests { let path = root.join("d").join("a.rs"); std::fs::create_dir(root.join("d")).unwrap(); std::fs::write(&path, "fn a() {}\n").unwrap(); - reindex_file(&state, &path, "d/a.rs"); + reindex_file(&state, &path, "d/a.rs", false); assert!(state.index.read().unwrap().live.has_path("d/a.rs")); // What `reindex_files_in` produces when an entry in `d` failed to @@ -8333,7 +8999,7 @@ mod tests { let path = root.join("d").join("a.rs"); std::fs::create_dir(root.join("d")).unwrap(); std::fs::write(&path, "fn a() {}\n").unwrap(); - reindex_file(&state, &path, "d/a.rs"); + reindex_file(&state, &path, "d/a.rs", false); assert!(state.index.read().unwrap().live.has_path("d/a.rs")); // `d` enumerated cleanly and did not contain `a.rs` at the time — but @@ -8366,7 +9032,7 @@ mod tests { let dir = root.join("d"); std::fs::create_dir(&dir).unwrap(); std::fs::write(dir.join("a.rs"), "fn ours() {}\n").unwrap(); - reindex_file(&state, &dir.join("a.rs"), "d/a.rs"); + reindex_file(&state, &dir.join("a.rs"), "d/a.rs", false); assert!(state.index.read().unwrap().live.has_path("d/a.rs")); // The scan listed the root, did not find `d`, and is about to sweep @@ -8409,7 +9075,7 @@ mod tests { let path = root.join("x.rs"); std::fs::write(&path, "fn was_a_real_file() {}\n").unwrap(); - reindex_file(&state, &path, "x.rs"); + reindex_file(&state, &path, "x.rs", false); assert!(state.index.read().unwrap().live.has_path("x.rs")); std::fs::remove_file(&path).unwrap(); @@ -8450,7 +9116,7 @@ mod tests { let path = root.join("x.rs"); std::fs::write(&path, "fn indexed() {}\n").unwrap(); - reindex_file(&state, &path, "x.rs"); + reindex_file(&state, &path, "x.rs", false); assert!(state.index.read().unwrap().live.has_path("x.rs")); std::fs::remove_file(&path).unwrap(); @@ -8756,7 +9422,7 @@ mod tests { let path = root.join("seeded.rs"); std::fs::write(&path, "fn seeded() {}\n").unwrap(); let gate = state.snapshot_gate.read().unwrap(); - reindex_file(&state, &path, "seeded.rs"); + reindex_file(&state, &path, "seeded.rs", false); assert!(state.index.read().unwrap().live.has_path("seeded.rs")); // Indexed, and searchable, but with nothing in the stamp map to say so @@ -8912,6 +9578,39 @@ mod tests { ); } + /// A stamp is a claim about the index, not about the filesystem. + /// + /// The build stamps from a second traversal that runs after the one that + /// fed the index, so a file created between them is on disk and in no + /// index. Stamping it makes every later check agree it is up to date: + /// `reindex_file` returns early on a matching stamp, and the periodic + /// reconcile compares stamps alone. The file would never be searchable. + #[test] + fn stamps_are_published_only_for_what_the_build_indexed() { + use tgrep_core::walker::FileMeta; + + let files = vec![ + FileMeta { + relative_path: "indexed.rs".to_string(), + mtime: 1, + size: 10, + }, + FileMeta { + relative_path: "arrived_between_the_walks.rs".to_string(), + mtime: 2, + size: 20, + }, + ]; + let indexed = std::iter::once("indexed.rs".to_string()).collect(); + + let stamps = stamps_for_index_members(files, &indexed); + assert!(stamps.contains_key("indexed.rs")); + assert!( + !stamps.contains_key("arrived_between_the_walks.rs"), + "a file the build never indexed must not be stamped as if it had been" + ); + } + /// The matcher reads its sources itself, inside the build. A replace that /// lands while it is reading leaves it enforcing the old rules, and stamps /// taken afterwards describe the new file — so pathname, timestamp and @@ -8962,8 +9661,8 @@ mod tests { std::fs::create_dir_all(&nested).unwrap(); std::fs::write(dir.join("a.rs"), "fn a() {}\n").unwrap(); std::fs::write(nested.join("b.rs"), "fn b() {}\n").unwrap(); - reindex_file(&state, &dir.join("a.rs"), "d/a.rs"); - reindex_file(&state, &nested.join("b.rs"), "d/deep/b.rs"); + reindex_file(&state, &dir.join("a.rs"), "d/a.rs", false); + reindex_file(&state, &nested.join("b.rs"), "d/deep/b.rs", false); assert!(state.index.read().unwrap().live.has_path("d/a.rs")); // Still there, merely unlistable from the scan's point of view: the @@ -9228,6 +9927,14 @@ mod tests { let stamps: std::collections::HashMap = [ ("quiet.rs".to_string(), FileStamp { mtime: 1, size: 10 }), ("racy.rs".to_string(), FileStamp { mtime: 2, size: 20 }), + ( + "moved/deep.rs".to_string(), + FileStamp { mtime: 3, size: 30 }, + ), + ( + "moved-aside.rs".to_string(), + FileStamp { mtime: 4, size: 40 }, + ), ] .into_iter() .collect(); @@ -9239,6 +9946,13 @@ mod tests { .as_mut() .unwrap() .insert(root.join("racy.rs"), false); + state + .deferred_events + .lock() + .unwrap() + .as_mut() + .unwrap() + .insert(root.join("moved"), true); let published = withhold_stamps_for_deferred(&state, &root, stamps.clone()); assert!( @@ -9249,6 +9963,14 @@ mod tests { !published.contains_key("racy.rs"), "a file whose event is waiting to be replayed must not be stamped as indexed" ); + assert!( + !published.contains_key("moved/deep.rs"), + "a directory event must withhold stamps for every descendant the subtree replay covers" + ); + assert!( + published.contains_key("moved-aside.rs"), + "directory-prefix matching must not withhold similarly named siblings" + ); // Overflowed: the buffer names nothing, so nothing in the map can be // told apart from what changed underneath it. @@ -9311,14 +10033,14 @@ mod tests { let path = root.join("grows.rs"); std::fs::write(&path, "fn small_enough() {}\n").unwrap(); - reindex_file(&state, &path, "grows.rs"); + reindex_file(&state, &path, "grows.rs", false); assert!( state.index.read().unwrap().live.has_path("grows.rs"), "a file under the cap should index normally" ); std::fs::write(&path, "x".repeat(4096)).unwrap(); - reindex_file(&state, &path, "grows.rs"); + reindex_file(&state, &path, "grows.rs", false); assert!( state.index.read().unwrap().live.is_deleted("grows.rs"), "content past the cap must not stay searchable" diff --git a/tgrep-core/src/builder.rs b/tgrep-core/src/builder.rs index b1615ac..c073659 100644 --- a/tgrep-core/src/builder.rs +++ b/tgrep-core/src/builder.rs @@ -31,6 +31,12 @@ const LOOKUP_WRITE_CHUNK_ENTRIES: usize = 4096; /// larger budget that scales with the thread pool was tried and rejected: it was /// both slower *and* hungrier there, because the kernel's large files are a rare /// minority and the extra headroom bought no throughput. +/// +/// Files large enough to be mapped are charged [`MAPPED_BATCH_CHARGE`] instead +/// of their length, so a tree that is mostly oversized files still fills the +/// pool. Raising this budget would trade memory on *every* tree for that; +/// charging mapped bytes honestly costs nothing on trees like the kernel, where +/// only 110 of 94,747 files are mapped at all. const INDEX_BUILD_BATCH_BYTES: u64 = 64 * 1024 * 1024; /// Default arena budget for [`IndexStrategy::External`] before spilling. @@ -165,13 +171,6 @@ pub struct BuildOutcome { pub ignore_files: Vec, } -/// Additional reconciliation evidence for coordinated server builds. -#[doc(hidden)] -pub struct ObservedBuildOutcome { - pub outcome: BuildOutcome, - pub unreadable: Vec, -} - /// Build a trigram index for all text files under `root`. pub fn build_index( root: &Path, @@ -219,14 +218,47 @@ fn gitignore_gate_hint(root: &Path, opts: &BuildOptions) -> Option { )) } -/// What one owned file buffer costs a batch. +/// Nominal heap charge for a file large enough to be memory-mapped. +/// +/// A mapped file's bytes never reach the heap, so charging it its own length +/// against the heap budget is charging for memory the build does not allocate. +/// What it does cost is its extracted trigram map, and that is bounded by the +/// number of *distinct* trigrams — at most 16.7M, and in practice saturating +/// long before file size does. Saturating the charge keeps that real cost +/// bounded while letting a batch hold enough large files to fill the pool. +const MAPPED_BATCH_CHARGE: u64 = 2 * 1024 * 1024; + +/// Ceiling on mapped bytes one batch may put in flight. +/// +/// Mapped pages are file-backed and reclaimable, so they are not the same +/// liability as heap, but they are still resident: a batch of large files maps +/// all of them at once, and the process working set follows. This is the second +/// half of the bound — the heap budget alone would let a batch of 32 MiB files +/// map 64 MiB *per worker*. +const MAPPED_BATCH_BYTES: u64 = 256 * 1024 * 1024; + +/// What one file costs a batch, split by where the bytes live. #[derive(Clone, Copy, Default)] struct BatchCharge { + /// Charge against the heap budget: the read buffer for a small file, or the + /// saturating stand-in for a mapped file's extracted trigram map. heap: u64, + /// Charge against the mapped budget; zero for a file that is read. + mapped: u64, } fn batch_charge(size: u64) -> BatchCharge { - BatchCharge { heap: size } + if size >= MMAP_MIN_BYTES { + BatchCharge { + heap: size.min(MAPPED_BATCH_CHARGE), + mapped: size, + } + } else { + BatchCharge { + heap: size, + mapped: 0, + } + } } /// Charge for a file whose size could not be read. @@ -237,9 +269,11 @@ fn batch_charge(size: u64) -> BatchCharge { /// losing the bound precisely for the file whose size is unknown. const UNKNOWN_SIZE_CHARGE: BatchCharge = BatchCharge { heap: INDEX_BUILD_BATCH_BYTES, + mapped: MAPPED_BATCH_BYTES, }; -/// Split a file list into batches bounded by file count and heap bytes. +/// Split a file list into batches bounded by file count, heap bytes and mapped +/// bytes, given each file's charge in walk order. /// /// A file whose charge alone exceeds a budget forms a batch of its own rather /// than being split, so the bound is "one budget, plus at most one oversized @@ -248,14 +282,19 @@ fn batch_ranges(charges: &[BatchCharge], budget: u64) -> Vec= INDEX_BUILD_BATCH_SIZE || heap.saturating_add(charge.heap) > budget; + let full = i - start >= INDEX_BUILD_BATCH_SIZE + || heap.saturating_add(charge.heap) > budget + || mapped.saturating_add(charge.mapped) > MAPPED_BATCH_BYTES; if i > start && full { ranges.push(start..i); start = i; heap = 0; + mapped = 0; } heap = heap.saturating_add(charge.heap); + mapped = mapped.saturating_add(charge.mapped); } if start < charges.len() { ranges.push(start..charges.len()); @@ -277,151 +316,63 @@ fn batch_sizes_and_charges(files: &[std::path::PathBuf]) -> (Vec, Vec; - -#[derive(Debug, Clone, PartialEq, Eq)] -struct FileVersion { - stamp: meta::FileStamp, - created_nanos: Option, - #[cfg(unix)] - device: u64, - #[cfg(unix)] - inode: u64, - #[cfg(unix)] - change_seconds: i64, - #[cfg(unix)] - change_nanos: i64, -} - -fn file_version(metadata: &std::fs::Metadata) -> FileVersion { - #[cfg(unix)] - use std::os::unix::fs::MetadataExt; - FileVersion { - stamp: meta::file_stamp(metadata), - created_nanos: metadata - .created() - .ok() - .and_then(|time| time.duration_since(std::time::UNIX_EPOCH).ok()) - .map(|duration| duration.as_nanos()), - #[cfg(unix)] - device: metadata.dev(), - #[cfg(unix)] - inode: metadata.ino(), - #[cfg(unix)] - change_seconds: metadata.ctime(), - #[cfg(unix)] - change_nanos: metadata.ctime_nsec(), - } +/// Smallest file worth memory-mapping instead of reading. +/// +/// The same tradeoff the search path makes: mapping costs a syscall pair and a +/// page-table setup per file, which is a bad deal across the ~94k mostly-small +/// files of a kernel tree, but above this size it is what stops a large +/// generated header from costing its full size in heap in every worker that +/// touches one. Mapped pages are file-backed, so the kernel can reclaim them +/// under pressure instead of the process holding an allocation it cannot give +/// back. +/// +/// 1 MiB is where the measurement pointed rather than a round guess. In the AMD +/// register headers — the worst case in the kernel tree — an 8 MiB threshold +/// mapped only 11 of 488 files and left 69% of the bytes on the heap, while +/// 1 MiB maps 102 files covering 87% of the bytes. It stays cheap on ordinary +/// trees because it is far above the size of real source: only 110 of the +/// kernel's 94,747 files reach it at all. +const MMAP_MIN_BYTES: u64 = 1024 * 1024; + +/// The bytes of a file to index, either read onto the heap or borrowed from a +/// memory map. +enum FileBytes { + Read(Vec), + Mapped(memmap2::Mmap), } -/// Read owned bytes so concurrent truncation cannot invalidate memory while -/// extraction is in progress. -fn read_for_index( - path: &Path, - _size: u64, -) -> std::io::Result<(Vec, std::fs::File, FileVersion)> { - let mut file = std::fs::File::open(path)?; - let before = file_version(&file.metadata()?); - let path_before = file_version(&std::fs::metadata(path)?); - if before != path_before { - return Err(std::io::Error::new( - std::io::ErrorKind::Interrupted, - "file was replaced while opening it", - )); - } - use std::io::Read; - let mut bytes = Vec::with_capacity(usize::try_from(before.stamp.size).unwrap_or(0)); - file.read_to_end(&mut bytes)?; - Ok((bytes, file, before)) -} +type ExtractedFile = (String, trigram::TrigramMaskMap); -fn path_still_has_bytes( - path: &Path, - expected_version: &FileVersion, - expected: &[u8], -) -> std::io::Result { - use std::io::Read; +impl std::ops::Deref for FileBytes { + type Target = [u8]; - let mut current = std::fs::File::open(path)?; - if file_version(¤t.metadata()?) != *expected_version { - return Ok(false); - } - let mut offset = 0; - let mut buffer = [0u8; 64 * 1024]; - loop { - let read = current.read(&mut buffer)?; - if read == 0 { - break; - } - if expected.get(offset..offset + read) != Some(&buffer[..read]) { - return Ok(false); + fn deref(&self) -> &[u8] { + match self { + FileBytes::Read(bytes) => bytes, + FileBytes::Mapped(map) => map, } - offset += read; } - Ok(offset == expected.len() - && file_version(¤t.metadata()?) == *expected_version - && file_version(&std::fs::metadata(path)?) == *expected_version) } -fn extract_stable_file( - path: &Path, - size: u64, - after_read: Option<&(dyn Fn(&Path) + Send + Sync)>, -) -> std::io::Result<(trigram::TrigramMaskMap, meta::FileStamp, bool)> { - const ATTEMPTS: usize = 3; - let mut last_error = None; - for _ in 0..ATTEMPTS { - let (data, file, before) = match read_for_index(path, size) { - Ok(read) => read, - Err(error) => { - last_error = Some(error); - continue; - } - }; - let text = crate::encoding::decode_for_index(&data); - let binary = trigram::is_binary(&text); - let per_tri = if binary { - Default::default() - } else { - trigram::extract_merged_masks(&text) - }; - if let Some(after_read) = after_read { - after_read(path); - } - let after_file = file.metadata().map(|metadata| file_version(&metadata)); - if after_file.as_ref().is_ok_and(|version| version == &before) - && path_still_has_bytes(path, &before, &data).unwrap_or(false) - { - return Ok((per_tri, before.stamp, binary)); +/// Get a file's bytes for indexing, mapping it when it is large enough to be +/// worth avoiding the copy. +/// +/// Falls back to reading whenever mapping is unavailable, so a filesystem that +/// cannot map still indexes correctly. +fn read_for_index(path: &Path, size: u64) -> std::io::Result { + if size >= MMAP_MIN_BYTES { + // SAFETY: the map is read-only and owned by the returned value, which + // is dropped once the file's trigrams have been extracted. A concurrent + // truncation would be undefined behaviour; that is inherent to mapping + // a file being indexed, and is the same exposure the search path + // accepts. + let mapped = + std::fs::File::open(path).and_then(|file| unsafe { memmap2::Mmap::map(&file) }); + if let Ok(map) = mapped { + return Ok(FileBytes::Mapped(map)); } - last_error = Some(std::io::Error::new( - std::io::ErrorKind::Interrupted, - "file changed while it was being indexed", - )); } - Err(last_error.unwrap_or_else(|| { - std::io::Error::new( - std::io::ErrorKind::Interrupted, - "file could not be read at one stable version", - ) - })) -} - -/// Read one file at a metadata-stable version for the server's in-heap build. -/// -/// The returned stamp was validated after trigram extraction and therefore -/// describes the same version as `trigrams`. `None` trigrams means that stable -/// version was binary by content. -#[doc(hidden)] -pub fn extract_file_for_live( - path: &Path, -) -> std::io::Result<(Option, meta::FileStamp)> { - let size = std::fs::metadata(path)?.len(); - let (trigrams, stamp, binary) = extract_stable_file(path, size, None)?; - Ok(((!binary).then_some(trigrams), stamp)) + std::fs::read(path).map(FileBytes::Read) } /// Build a trigram index, choosing how postings are accumulated. @@ -441,20 +392,6 @@ pub fn build_index_with_options_and_ignorecase( opts: &BuildOptions, ignorecase: Option>, ) -> Result { - build_index_with_options_and_ignorecase_observed(root, index_dir, opts, ignorecase, None) - .map(|observed| observed.outcome) -} - -/// Testable form of [`build_index_with_options_and_ignorecase`] that observes -/// the interval between extraction and metadata validation. -#[doc(hidden)] -pub fn build_index_with_options_and_ignorecase_observed( - root: &Path, - index_dir: Option<&Path>, - opts: &BuildOptions, - ignorecase: Option>, - after_file_read: Option, -) -> Result { let include_hidden = opts.include_hidden; let no_ignore = opts.no_ignore; let exclude_dirs = opts.exclude_dirs.as_slice(); @@ -518,8 +455,6 @@ pub fn build_index_with_options_and_ignorecase_observed( // bounding each batch by cumulative bytes caps how much raw file content is // resident at once, since the whole batch is read concurrently. let mut file_id_map: Vec<(u32, String)> = Vec::with_capacity(walk.files.len()); - let mut stamps = std::collections::HashMap::with_capacity(walk.files.len()); - let mut unreadable = Vec::new(); let mut sink = match opts.strategy { IndexStrategy::InMemory => PostingSink::InMemory(Vec::new()), IndexStrategy::External => { @@ -535,38 +470,27 @@ pub fn build_index_with_options_and_ignorecase_observed( for range in batch_ranges(&charges, INDEX_BUILD_BATCH_BYTES) { let batch = &walk.files[range.clone()]; let batch_sizes = &sizes[range]; - let batch_data: Vec> = batch + let batch_data: Vec = batch .par_iter() .zip(batch_sizes.par_iter()) - .map(|(path, &size)| { + .filter_map(|(path, &size)| { + let data = read_for_index(path, size).ok()?; + let text = crate::encoding::decode_for_index(&data); + if trigram::is_binary(&text) { + binary_skipped.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + return None; + } let rel = path .strip_prefix(&root) .unwrap_or(path) .to_string_lossy() .replace('\\', "/"); - match extract_stable_file(path, size, after_file_read.as_deref()) { - Ok((per_tri, stamp, binary)) => Ok(((rel, per_tri, stamp), binary)), - Err(error) => { - eprintln!("tgrep: skipping {}: {error}", path.display()); - Err(path.clone()) - } - } + let per_tri = trigram::extract_merged_masks(&text); + Some((rel, per_tri)) }) .collect(); - for entry in batch_data { - let ((path, per_tri, stamp), binary) = match entry { - Ok(extracted) => extracted, - Err(path) => { - unreadable.push(path); - continue; - } - }; - stamps.insert(path.clone(), stamp); - if binary { - binary_skipped.fetch_add(1, std::sync::atomic::Ordering::Relaxed); - continue; - } + for (path, per_tri) in batch_data { let file_id = file_id_map.len() as u32; file_id_map.push((file_id, path)); sink.push_file(file_id, per_tri)?; @@ -604,19 +528,23 @@ pub fn build_index_with_options_and_ignorecase_observed( } } - // Every stamp came from the same stable read that produced the indexed - // bytes (or proved a file binary). Unreadable/unstable paths stay unstamped - // so a serialized reconciliation must revisit them. + // Write per-file stamps for ALL walked files (including those later + // rejected as binary-by-content) so the stale check on next startup + // won't re-process unchanged files that aren't in the index. + let all_walked: Vec = walk + .files + .iter() + .filter_map(|p| p.strip_prefix(&root).ok()) + .map(|p| p.to_string_lossy().replace('\\', "/")) + .collect(); + let stamps = meta::collect_filestamps(&root, &all_walked); meta::write_filestamps(&stamps, &index_dir)?; eprintln!("Index built successfully at {}", index_dir.display()); - Ok(ObservedBuildOutcome { - outcome: BuildOutcome { - num_files: file_id_map.len(), - gitignore_files, - ignore_files, - }, - unreadable, + Ok(BuildOutcome { + num_files: file_id_map.len(), + gitignore_files, + ignore_files, }) } @@ -640,20 +568,12 @@ pub struct FileDeltaOutcome { pub unreadable: Vec, } -/// Additional read-bound stamps for coordinated stale-delta publication. -#[doc(hidden)] -pub struct ObservedFileDeltaOutcome { - pub outcome: FileDeltaOutcome, - pub stamps: std::collections::HashMap, -} - /// Build an external-sort index for an exact list of absolute file paths. /// /// This is used by `tgrep serve` to build a bounded delta for files that are /// absent from an otherwise-complete index. It deliberately skips the directory -/// walk and filestamp write: the caller has the complete candidate set from its -/// stale-state scan, then replaces candidate stamps with the read-bound stamps -/// returned here before publishing the merged index. +/// walk and filestamp write: the caller already has both from its stale-state +/// scan and publishes the complete stamp set with the merged index. /// /// Files that become unreadable or binary after the stale scan are omitted /// rather than fatal, matching a normal index build. Aborting instead would be @@ -667,19 +587,6 @@ pub fn build_index_for_files( files: &[std::path::PathBuf], buffer_bytes: usize, ) -> Result { - build_index_for_files_observed(root, index_dir, files, buffer_bytes) - .map(|observed| observed.outcome) -} - -/// Testable/coordinated form of [`build_index_for_files`] that returns the -/// read-bound stamps needed for atomic stale publication. -#[doc(hidden)] -pub fn build_index_for_files_observed( - root: &Path, - index_dir: &Path, - files: &[std::path::PathBuf], - buffer_bytes: usize, -) -> Result { let input_root = root; let root = std::fs::canonicalize(root)?; std::fs::create_dir_all(index_dir)?; @@ -688,44 +595,43 @@ pub fn build_index_for_files_observed( let mut file_id_map: Vec<(u32, String)> = Vec::with_capacity(files.len()); let mut sorter = ExternalSorter::new(index_dir, buffer_bytes); let mut unreadable: Vec = Vec::new(); - let mut stamps = std::collections::HashMap::with_capacity(files.len()); for range in batch_ranges(&charges, INDEX_BUILD_BATCH_BYTES) { let batch = &files[range.clone()]; let batch_sizes = &sizes[range]; - let batch_data: Vec> = batch + let batch_data: Vec> = batch .par_iter() .zip(batch_sizes.par_iter()) .filter_map(|(path, &size)| { - let (per_tri, stamp, binary) = match extract_stable_file(path, size, None) { - Ok(extracted) => extracted, + let data = match read_for_index(path, size) { + Ok(data) => data, Err(error) => { eprintln!("tgrep: skipping {}: {error}", path.display()); return Some(Err(path.clone())); } }; + let text = crate::encoding::decode_for_index(&data); + if trigram::is_binary(&text) { + return None; + } let rel = path .strip_prefix(&root) .or_else(|_| path.strip_prefix(input_root)) .unwrap_or(path) .to_string_lossy() .replace('\\', "/"); - Some(Ok(((rel, per_tri, stamp), binary))) + Some(Ok((rel, trigram::extract_merged_masks(&text)))) }) .collect(); for entry in batch_data { - let ((path, per_tri, stamp), binary) = match entry { + let (path, per_tri) = match entry { Ok(extracted) => extracted, Err(skipped) => { unreadable.push(skipped); continue; } }; - stamps.insert(path.clone(), stamp); - if binary { - continue; - } let file_id = u32::try_from(file_id_map.len()).map_err(|_| { Error::IndexCorrupted("file count exceeds the u32 file-id limit".into()) })?; @@ -743,12 +649,9 @@ pub fn build_index_for_files_observed( trigram_count, Some(true), )?; - Ok(ObservedFileDeltaOutcome { - outcome: FileDeltaOutcome { - indexed: file_id_map.len(), - unreadable, - }, - stamps, + Ok(FileDeltaOutcome { + indexed: file_id_map.len(), + unreadable, }) } @@ -1422,27 +1325,39 @@ mod tests { #[test] fn batches_are_bounded_by_cumulative_heap_bytes() { - // Five 900 KiB reads against a 2 MiB budget fit two at a time. + // Files below the mapping threshold are read, so their whole length is + // heap: five 900 KiB reads against a 2 MiB budget fit two at a time. let charges = charges(&[900 * 1024; 5]); assert_eq!(batch_ranges(&charges, 2 * MB), vec![0..2, 2..4, 4..5]); } #[test] - fn large_owned_files_respect_the_heap_budget() { + fn large_mapped_files_still_fill_a_batch() { + // Regression: charging a mapped file its own length against the heap + // budget made a batch of large files degenerate to one or two, leaving + // every worker but one idle. Mapped bytes never reach the heap, so only + // the mapped budget bounds these: 256 MiB / 32 MiB is eight per batch. let charges = charges(&[32 * MB; 16]); assert_eq!( batch_ranges(&charges, INDEX_BUILD_BATCH_BYTES), - vec![0..2, 2..4, 4..6, 6..8, 8..10, 10..12, 12..14, 14..16] + vec![0..8, 8..16] ); } #[test] - fn owned_bytes_in_flight_stay_bounded() { + fn mapped_bytes_in_flight_stay_bounded() { + // The other half of the bound: a batch may not map more than the mapped + // budget, whatever mix of sizes it is handed. let sizes: Vec = (0..200).map(|i| (i % 9 + 1) * 8 * MB).collect(); let charges = charges(&sizes); for range in batch_ranges(&charges, INDEX_BUILD_BATCH_BYTES) { let single = range.end - range.start == 1; + let mapped: u64 = charges[range.clone()].iter().map(|c| c.mapped).sum(); let heap: u64 = charges[range.clone()].iter().map(|c| c.heap).sum(); + assert!( + single || mapped <= MAPPED_BATCH_BYTES, + "batch {range:?} maps {mapped} bytes" + ); assert!( single || heap <= INDEX_BUILD_BATCH_BYTES, "batch {range:?} reads {heap} bytes" diff --git a/tgrep-core/src/meta.rs b/tgrep-core/src/meta.rs index 98a1340..6bb40ae 100644 --- a/tgrep-core/src/meta.rs +++ b/tgrep-core/src/meta.rs @@ -68,21 +68,6 @@ pub struct FileStamp { pub size: u64, } -/// Convert filesystem metadata into the persisted stamp used by change -/// detection. -pub fn file_stamp(metadata: &std::fs::Metadata) -> FileStamp { - let mtime = metadata - .modified() - .ok() - .and_then(|t| t.duration_since(SystemTime::UNIX_EPOCH).ok()) - .map(|d| d.as_secs()) - .unwrap_or(0); - FileStamp { - mtime, - size: metadata.len(), - } -} - /// Write per-file stamps to `filestamps.json` in the index directory. pub fn write_filestamps(stamps: &HashMap, index_dir: &Path) -> Result<()> { let path = index_dir.join(FILESTAMPS_FILENAME); @@ -110,9 +95,21 @@ pub fn collect_filestamps(root: &Path, paths: &[String]) -> HashMap limit) { return ignore::WalkState::Continue; } - let stamp = crate::meta::file_stamp(&meta); + let mtime = meta + .modified() + .ok() + .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()) + .map(|d| d.as_secs()) + .unwrap_or(0); results.lock().unwrap().push(FileMeta { relative_path: rel_path, - mtime: stamp.mtime, - size: stamp.size, + mtime, + size: meta.len(), }); ignore::WalkState::Continue From 7214a4fcb5e32619ceba4f94c6324fcd6969435d Mon Sep 17 00:00:00 2001 From: Shengyu Fu Date: Sat, 29 Aug 2026 16:50:34 -0500 Subject: [PATCH 08/16] Address watcher follow-up review findings Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tgrep-cli/src/serve.rs | 279 +++++++++++++++++++++++++++++------- tgrep-core/src/builder.rs | 246 ++++++++++++++++++++++++++----- tgrep-core/src/git_index.rs | 117 +++++++++++---- tgrep-core/src/gitignore.rs | 90 ++++++------ tgrep-core/src/hybrid.rs | 5 + tgrep-core/src/live.rs | 23 +++ tgrep-core/src/meta.rs | 33 +++-- tgrep-core/src/reader.rs | 40 ++++++ tgrep-core/src/walker.rs | 25 ++-- 9 files changed, 664 insertions(+), 194 deletions(-) diff --git a/tgrep-cli/src/serve.rs b/tgrep-cli/src/serve.rs index e7cd54b..122a482 100644 --- a/tgrep-cli/src/serve.rs +++ b/tgrep-cli/src/serve.rs @@ -489,6 +489,7 @@ enum StaleRefreshPhase { BeforeWalk, AfterBuildBeforeStampPublish, AfterConcreteRead, + BeforeConcreteCommit, AfterMatcherPublish, } @@ -1928,14 +1929,15 @@ fn handle_reload(id: Option, state: &Arc) -> Str run_stale_refresh_hook(state, StaleRefreshPhase::BeforeWalk); } let caught_up = catch_up_unwatched_build(state, &state.root, &state.index_dir); - membership_changed = tracked_membership_changed(state); if !caught_up { + schedule_tracked_membership_correction(state, &state.root, membership_changed); return json_rpc_error( id, -32000, "rebuild catch-up did not complete; reload was not authoritative", ); } + membership_changed = tracked_membership_changed(state); } if state.watch_enabled { spawn_recovery_scan(state, &state.root, newly_watched, since); @@ -2389,6 +2391,7 @@ fn take_force_resubscribe( let force = pending.swap(false, Ordering::SeqCst); if force && completeness == TraversalCompleteness::Incomplete { pending.store(true, Ordering::SeqCst); + return false; } force } @@ -3548,17 +3551,9 @@ fn replay_deferred_events(state: &Arc, root: &Path) { for (path, introduces_dir) in paths { if !is_real_dir(&path) && path.strip_prefix(root).is_ok_and(|rel| { - let prefix = format!("{}/", rel.to_string_lossy().replace('\\', "/")); + let rel = rel.to_string_lossy().replace('\\', "/"); let index = state.index.read().unwrap(); - index - .reader_paths() - .iter() - .any(|indexed| indexed.starts_with(&prefix)) - || index - .live - .overlay_paths() - .iter() - .any(|indexed| indexed.starts_with(&prefix)) + index.reader_has_descendant_path(&rel) || index.live.has_descendant_path(&rel) }) { // A single directory removal/rename event names no descendants. @@ -4207,6 +4202,39 @@ enum CappedRead { Failed, } +fn file_still_has_bytes( + root: &Path, + path: &Path, + expected_version: &tgrep_core::builder::FileVersion, + expected: &[u8], +) -> std::io::Result { + use std::io::Read; + + let mut current = open_within_root(root, path)?; + if tgrep_core::builder::file_version(¤t.metadata()?) != *expected_version { + return Ok(false); + } + let mut offset = 0; + let mut buffer = [0u8; 64 * 1024]; + loop { + let read = current.read(&mut buffer)?; + if read == 0 { + break; + } + if expected.get(offset..offset + read) != Some(&buffer[..read]) { + return Ok(false); + } + offset += read; + } + if offset != expected.len() + || tgrep_core::builder::file_version(¤t.metadata()?) != *expected_version + { + return Ok(false); + } + let path_current = open_within_root(root, path)?; + Ok(tgrep_core::builder::file_version(&path_current.metadata()?) == *expected_version) +} + /// Reads a file's contents, never pulling in more than one byte past the cap. /// /// The size that qualified this file was stat'd before the read, and appending @@ -4246,8 +4274,6 @@ fn read_within_limit(file: &mut std::fs::File, limit: Option, capacity: usi /// The caller must hold `snapshot_gate`: the read, the commit, and the stamp /// update have to be atomic with respect to a flush or auto-save. fn reindex_file(state: &Arc, path: &Path, rel_path: &str, force: bool) { - use tgrep_core::meta::FileStamp; - // Against other indexers, not against searches. The gate above is held for // read, so without this a recovery scan and the watcher worker can both be // here for the same path, both read, and the one that read the *older* @@ -4287,15 +4313,8 @@ fn reindex_file(state: &Arc, path: &Path, rel_path: &str, force: bo } return; }; - let mut current = FileStamp { - mtime: meta - .modified() - .ok() - .and_then(|t| t.duration_since(std::time::SystemTime::UNIX_EPOCH).ok()) - .map(|d| d.as_secs()) - .unwrap_or(0), - size: meta.len(), - }; + let mut version = tgrep_core::builder::file_version(&meta); + let mut current = version.stamp().clone(); // The rules `walk_file_metadata` applies, and for the same reason: the walk // is authoritative about what belongs in the index, so anything it rejects @@ -4360,12 +4379,10 @@ fn reindex_file(state: &Arc, path: &Path, rel_path: &str, force: bo if force { #[cfg(test)] run_stale_refresh_hook(state, StaleRefreshPhase::AfterConcreteRead); - // A concrete event is stronger evidence than the persisted stamp: - // FileStamp intentionally stores only whole-second mtime plus size for - // compatibility, so an equal-length rewrite can compare equal. Read a - // second containment-safe handle and require two consecutive byte - // snapshots to agree. This also avoids committing a mixed version when - // a writer changes the file while the first read is in progress. + // A concrete event is stronger evidence than the persisted stamp. Read + // containment-safe snapshots until the bytes and full-resolution file + // version agree, then bind the final decision to a fresh read of those + // exact bytes immediately before commit. let mut stable = false; for _ in 0..2 { let mut verify = match open_within_root(&state.root, path) { @@ -4391,15 +4408,8 @@ fn reindex_file(state: &Arc, path: &Path, rel_path: &str, force: bo ); return; }; - let verify_current = FileStamp { - mtime: meta - .modified() - .ok() - .and_then(|t| t.duration_since(std::time::SystemTime::UNIX_EPOCH).ok()) - .map(|d| d.as_secs()) - .unwrap_or(0), - size: meta.len(), - }; + let verify_version = tgrep_core::builder::file_version(&meta); + let verify_current = verify_version.stamp().clone(); if !meta.is_file() || tgrep_core::walker::is_binary_extension(path) || state @@ -4424,13 +4434,23 @@ fn reindex_file(state: &Arc, path: &Path, rel_path: &str, force: bo return; } }; - current = verify_current; - if data == verified { + let handle_stable = verify.metadata().is_ok_and(|metadata| { + tgrep_core::builder::file_version(&metadata) == verify_version + }); + if data == verified + && handle_stable + && file_still_has_bytes(&state.root, path, &verify_version, &verified) + .unwrap_or(false) + { data = verified; + current = verify_current; + version = verify_version; stable = true; break; } data = verified; + current = verify_current; + version = verify_version; } if !stable { retry_failed_forced_reindex(state, rel_path, "the file kept changing while read"); @@ -4444,6 +4464,12 @@ fn reindex_file(state: &Arc, path: &Path, rel_path: &str, force: bo } else { Some(tgrep_core::live::LiveIndex::compute_trigram_masks(&text)) }; + #[cfg(test)] + run_stale_refresh_hook(state, StaleRefreshPhase::BeforeConcreteCommit); + if force && !file_still_has_bytes(&state.root, path, &version, &data).unwrap_or(false) { + retry_failed_forced_reindex(state, rel_path, "the file changed before commit"); + return; + } eprintln!("[trace] reindex: modified {rel_path}"); // Gate held by the caller — the commit + stamp update is processed @@ -5123,8 +5149,10 @@ fn catch_up_unwatched_build(state: &Arc, root: &Path, index_dir: &P ignorecase, false, ); + let membership_changed = tracked_membership_changed(state); drop(gate); drop(refresh); + schedule_tracked_membership_correction(state, root, membership_changed); caught_up } @@ -6184,25 +6212,38 @@ impl StagedFileMove { return Ok(()); } let mut first_error = None; - for name in self.backed_up.iter().rev() { - if let Err(e) = publish_file(&self.backup_path(name), &self.target.join(name)) - && first_error.is_none() + let mut pending_published = Vec::new(); + for name in std::mem::take(&mut self.published).into_iter().rev() { + let published = self.target.join(name); + if let Err(e) = std::fs::remove_file(&published) + && e.kind() != std::io::ErrorKind::NotFound { - first_error = Some(e); + if first_error.is_none() { + first_error = Some(e); + } + pending_published.push(name); } } - for name in self.published.iter().rev() { - if self.backed_up.contains(name) { - continue; - } - let published = self.target.join(name); - if published.exists() - && let Err(e) = std::fs::remove_file(published) - && first_error.is_none() - { - first_error = Some(e); + pending_published.reverse(); + self.published = pending_published; + + let mut pending_backups = Vec::new(); + for name in std::mem::take(&mut self.backed_up).into_iter().rev() { + match publish_file(&self.backup_path(name), &self.target.join(name)) { + Ok(()) => self.published.retain(|published| *published != name), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + self.published.retain(|published| *published != name); + } + Err(e) => { + if first_error.is_none() { + first_error = Some(e); + } + pending_backups.push(name); + } } } + pending_backups.reverse(); + self.backed_up = pending_backups; if let Some(e) = first_error { return Err(e); } @@ -6796,8 +6837,8 @@ mod tests { let pending = std::sync::atomic::AtomicBool::new(true); let force = take_force_resubscribe(&pending, TraversalCompleteness::Incomplete); assert!( - force, - "the incomplete pass must still attempt known entries" + !force, + "the incomplete pass must not re-register known entries" ); assert!( pending.load(Ordering::SeqCst), @@ -7208,6 +7249,7 @@ mod tests { } StaleRefreshPhase::AfterBuildBeforeStampPublish => {} StaleRefreshPhase::AfterConcreteRead => {} + StaleRefreshPhase::BeforeConcreteCommit => {} }) }; *state.stale_refresh_hook.lock().unwrap() = Some(hook); @@ -7346,6 +7388,7 @@ mod tests { } StaleRefreshPhase::AfterBuildBeforeStampPublish => {} StaleRefreshPhase::AfterConcreteRead => {} + StaleRefreshPhase::BeforeConcreteCommit => {} }) }; *state.stale_refresh_hook.lock().unwrap() = Some(hook); @@ -7514,6 +7557,7 @@ mod tests { } StaleRefreshPhase::AfterBuildBeforeStampPublish => {} StaleRefreshPhase::AfterConcreteRead => {} + StaleRefreshPhase::BeforeConcreteCommit => {} }) }; *state.stale_refresh_hook.lock().unwrap() = Some(hook); @@ -7913,6 +7957,55 @@ mod tests { ); } + #[test] + fn concrete_event_rejects_change_after_verification_before_commit() { + let tmp = TempDir::new().unwrap(); + let root = std::fs::canonicalize(tmp.path()).unwrap(); + let path = root.join("raced.rs"); + std::fs::write(&path, "fn old_marker() {}\n").unwrap(); + let state = test_server_state(&root, &root.join(".tgrep")); + state.gitignore_pending.store(false, Ordering::SeqCst); + { + let _gate = state.snapshot_gate.read().unwrap(); + reindex_file(&state, &path, "raced.rs", false); + } + let dirty_before = state.index.read().unwrap().live.dirty_count(); + std::fs::write(&path, "fn first_new_() {}\n").unwrap(); + + state.indexing.store(true, Ordering::SeqCst); + let changed = path.clone(); + *state.stale_refresh_hook.lock().unwrap() = Some(Arc::new(move |phase| { + if matches!(phase, StaleRefreshPhase::BeforeConcreteCommit) { + std::fs::write(&changed, "fn final_new_() {}\n").unwrap(); + } + })); + { + let _gate = state.snapshot_gate.read().unwrap(); + reindex_file(&state, &path, "raced.rs", true); + } + + assert_eq!( + state.index.read().unwrap().live.dirty_count(), + dirty_before, + "bytes invalidated after verification must not be committed" + ); + assert_eq!( + state.file_stamps.read().unwrap().get("raced.rs"), + Some(&tgrep_core::meta::FileStamp { + mtime: u64::MAX, + size: u64::MAX, + }), + "a rejected final version must remain scheduled for correction" + ); + + *state.stale_refresh_hook.lock().unwrap() = None; + state.indexing.store(false, Ordering::SeqCst); + let deadline = Instant::now() + Duration::from_secs(10); + while state.ignore_refresh_scheduled.load(Ordering::SeqCst) && Instant::now() < deadline { + thread::sleep(Duration::from_millis(10)); + } + } + #[test] fn rpc_reload_build_failure_preserves_active_index() { let tmp = TempDir::new().unwrap(); @@ -10233,4 +10326,80 @@ mod tests { assert_eq!(std::fs::read(target.join("index.bin")).unwrap(), b"old"); } + + #[test] + fn rollback_restores_backup_when_published_target_is_already_missing() { + let tmp = TempDir::new().unwrap(); + let staging = tmp.path().join("staging"); + let target = tmp.path().join("target"); + std::fs::create_dir_all(&staging).unwrap(); + std::fs::create_dir_all(&target).unwrap(); + write_file(&staging.join("index.bin"), b"new"); + write_file(&target.join("index.bin"), b"old"); + + let mut moved = move_staged_files(&staging, &target).unwrap(); + std::fs::remove_file(target.join("index.bin")).unwrap(); + moved.rollback().unwrap(); + + assert_eq!(std::fs::read(target.join("index.bin")).unwrap(), b"old"); + } + + #[test] + fn rollback_restores_backup_when_replacement_was_never_published() { + let tmp = TempDir::new().unwrap(); + let staging = tmp.path().join("staging"); + let target = tmp.path().join("target"); + std::fs::create_dir_all(&staging).unwrap(); + std::fs::create_dir_all(&target).unwrap(); + write_file(&staging.join(".previous-index.bin"), b"old"); + + let mut moved = StagedFileMove { + staging, + target: target.clone(), + backed_up: vec!["index.bin"], + published: Vec::new(), + finished: false, + }; + moved.rollback().unwrap(); + + assert_eq!(std::fs::read(target.join("index.bin")).unwrap(), b"old"); + } + + #[test] + fn rollback_retry_does_not_delete_an_already_restored_backup() { + let tmp = TempDir::new().unwrap(); + let staging = tmp.path().join("staging"); + let target = tmp.path().join("target"); + std::fs::create_dir_all(&staging).unwrap(); + std::fs::create_dir_all(&target).unwrap(); + write_file(&staging.join(".previous-index.bin"), b"old-index"); + write_file(&staging.join(".previous-lookup.bin"), b"old-lookup"); + write_file(&target.join("index.bin"), b"new-index"); + std::fs::create_dir(target.join("lookup.bin")).unwrap(); + + let mut moved = StagedFileMove { + staging, + target: target.clone(), + backed_up: vec!["index.bin", "lookup.bin"], + published: vec!["index.bin", "lookup.bin"], + finished: false, + }; + assert!(moved.rollback().is_err()); + assert_eq!( + std::fs::read(target.join("index.bin")).unwrap(), + b"old-index" + ); + + std::fs::remove_dir(target.join("lookup.bin")).unwrap(); + moved.rollback().unwrap(); + + assert_eq!( + std::fs::read(target.join("index.bin")).unwrap(), + b"old-index" + ); + assert_eq!( + std::fs::read(target.join("lookup.bin")).unwrap(), + b"old-lookup" + ); + } } diff --git a/tgrep-core/src/builder.rs b/tgrep-core/src/builder.rs index c073659..56587be 100644 --- a/tgrep-core/src/builder.rs +++ b/tgrep-core/src/builder.rs @@ -38,6 +38,7 @@ const LOOKUP_WRITE_CHUNK_ENTRIES: usize = 4096; /// charging mapped bytes honestly costs nothing on trees like the kernel, where /// only 110 of 94,747 files are mapped at all. const INDEX_BUILD_BATCH_BYTES: u64 = 64 * 1024 * 1024; +const MAX_OWNED_FILE_BYTES: u64 = INDEX_BUILD_BATCH_BYTES; /// Default arena budget for [`IndexStrategy::External`] before spilling. pub use crate::external::DEFAULT_BUFFER_BYTES as DEFAULT_INDEX_BUFFER_BYTES; @@ -343,6 +344,49 @@ enum FileBytes { type ExtractedFile = (String, trigram::TrigramMaskMap); +/// Full-resolution identity used to validate that bytes came from one file +/// version without changing the persisted [`meta::FileStamp`] format. +#[doc(hidden)] +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FileVersion { + stamp: meta::FileStamp, + modified: Option, + created: Option, + #[cfg(unix)] + device: u64, + #[cfg(unix)] + inode: u64, + #[cfg(unix)] + change_seconds: i64, + #[cfg(unix)] + change_nanos: i64, +} + +impl FileVersion { + pub fn stamp(&self) -> &meta::FileStamp { + &self.stamp + } +} + +#[doc(hidden)] +pub fn file_version(metadata: &std::fs::Metadata) -> FileVersion { + #[cfg(unix)] + use std::os::unix::fs::MetadataExt; + FileVersion { + stamp: meta::file_stamp(metadata), + modified: metadata.modified().ok(), + created: metadata.created().ok(), + #[cfg(unix)] + device: metadata.dev(), + #[cfg(unix)] + inode: metadata.ino(), + #[cfg(unix)] + change_seconds: metadata.ctime(), + #[cfg(unix)] + change_nanos: metadata.ctime_nsec(), + } +} + impl std::ops::Deref for FileBytes { type Target = [u8]; @@ -357,8 +401,8 @@ impl std::ops::Deref for FileBytes { /// Get a file's bytes for indexing, mapping it when it is large enough to be /// worth avoiding the copy. /// -/// Falls back to reading whenever mapping is unavailable, so a filesystem that -/// cannot map still indexes correctly. +/// Falls back to an owned read whenever mapping is unavailable, but never lets +/// that fallback allocate beyond the extraction batch's owned-buffer budget. fn read_for_index(path: &Path, size: u64) -> std::io::Result { if size >= MMAP_MIN_BYTES { // SAFETY: the map is read-only and owned by the returned value, which @@ -372,7 +416,36 @@ fn read_for_index(path: &Path, size: u64) -> std::io::Result { return Ok(FileBytes::Mapped(map)); } } - std::fs::read(path).map(FileBytes::Read) + read_owned_for_index(path, size).map(FileBytes::Read) +} + +fn read_owned_for_index(path: &Path, size: u64) -> std::io::Result> { + use std::io::Read; + + if size > MAX_OWNED_FILE_BYTES { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "file is too large for the {} MiB owned-buffer fallback", + MAX_OWNED_FILE_BYTES / (1024 * 1024) + ), + )); + } + let mut file = std::fs::File::open(path)?; + let mut bytes = Vec::with_capacity(usize::try_from(size).unwrap_or(0)); + std::io::Read::by_ref(&mut file) + .take(MAX_OWNED_FILE_BYTES.saturating_add(1)) + .read_to_end(&mut bytes)?; + if bytes.len() as u64 > MAX_OWNED_FILE_BYTES { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "file grew beyond the {} MiB owned-buffer fallback", + MAX_OWNED_FILE_BYTES / (1024 * 1024) + ), + )); + } + Ok(bytes) } /// Build a trigram index, choosing how postings are accumulated. @@ -381,7 +454,12 @@ pub fn build_index_with_options( index_dir: Option<&Path>, opts: &BuildOptions, ) -> Result { - build_index_with_options_and_ignorecase(root, index_dir, opts, None) + let root = std::fs::canonicalize(root)?; + let ignorecase = (!opts.no_ignore) + .then(|| crate::gitignore::CaseInsensitiveIgnore::new(&root, true, true, true)) + .flatten() + .map(std::sync::Arc::new); + build_index_with_options_and_ignorecase(&root, index_dir, opts, ignorecase) } /// Build an index using an immutable tracked-file exemption that the caller @@ -406,34 +484,19 @@ pub fn build_index_with_options_and_ignorecase( if let Some(hint) = gitignore_gate_hint(&root, opts) { eprintln!("{hint}"); } - let walk = if let Some(ignorecase) = ignorecase { - walker::walk_dir_with_ignorecase( - &root, - &walker::WalkOptions { - include_hidden, - no_ignore, - no_require_git: opts.no_require_git, - collect_gitignore_files: opts.collect_gitignore_files, - exclude_dirs: exclude_dirs.to_vec(), - max_file_size: opts.max_file_size, - ..Default::default() - }, - Some(ignorecase), - ) - } else { - walker::walk_dir( - &root, - &walker::WalkOptions { - include_hidden, - no_ignore, - no_require_git: opts.no_require_git, - collect_gitignore_files: opts.collect_gitignore_files, - exclude_dirs: exclude_dirs.to_vec(), - max_file_size: opts.max_file_size, - ..Default::default() - }, - ) - }; + let walk = walker::walk_dir_with_ignorecase( + &root, + &walker::WalkOptions { + include_hidden, + no_ignore, + no_require_git: opts.no_require_git, + collect_gitignore_files: opts.collect_gitignore_files, + exclude_dirs: exclude_dirs.to_vec(), + max_file_size: opts.max_file_size, + ..Default::default() + }, + ignorecase, + ); eprintln!( "Found {} text files ({} binary skipped, {} too large, {} errors)", walk.files.len(), @@ -470,11 +533,23 @@ pub fn build_index_with_options_and_ignorecase( for range in batch_ranges(&charges, INDEX_BUILD_BATCH_BYTES) { let batch = &walk.files[range.clone()]; let batch_sizes = &sizes[range]; + let owned_limit_error = std::sync::Mutex::new(None); let batch_data: Vec = batch .par_iter() .zip(batch_sizes.par_iter()) .filter_map(|(path, &size)| { - let data = read_for_index(path, size).ok()?; + let data = match read_for_index(path, size) { + Ok(data) => data, + Err(error) => { + if error.kind() == std::io::ErrorKind::InvalidData { + let mut first = owned_limit_error.lock().unwrap(); + if first.is_none() { + *first = Some((path.clone(), error)); + } + } + return None; + } + }; let text = crate::encoding::decode_for_index(&data); if trigram::is_binary(&text) { binary_skipped.fetch_add(1, std::sync::atomic::Ordering::Relaxed); @@ -489,6 +564,12 @@ pub fn build_index_with_options_and_ignorecase( Some((rel, per_tri)) }) .collect(); + if let Some((path, error)) = owned_limit_error.into_inner().unwrap() { + return Err(Error::Io(std::io::Error::new( + error.kind(), + format!("{}: {error}", path.display()), + ))); + } for (path, per_tri) in batch_data { let file_id = file_id_map.len() as u32; @@ -1396,6 +1477,105 @@ mod tests { ); } + #[test] + fn owned_read_rejects_files_beyond_the_batch_budget() { + let tmp = tempfile::tempdir().unwrap(); + let path = tmp.path().join("sparse.txt"); + std::fs::File::create(&path) + .unwrap() + .set_len(MAX_OWNED_FILE_BYTES + 1) + .unwrap(); + + let error = read_owned_for_index(&path, MAX_OWNED_FILE_BYTES + 1).unwrap_err(); + assert_eq!(error.kind(), std::io::ErrorKind::InvalidData); + } + + #[test] + fn file_version_distinguishes_subsecond_modified_times() { + let stamp = meta::FileStamp { mtime: 1, size: 10 }; + let base = std::time::UNIX_EPOCH + std::time::Duration::from_secs(1); + let first = FileVersion { + stamp: stamp.clone(), + modified: Some(base + std::time::Duration::from_nanos(100)), + created: Some(base), + #[cfg(unix)] + device: 1, + #[cfg(unix)] + inode: 2, + #[cfg(unix)] + change_seconds: 1, + #[cfg(unix)] + change_nanos: 0, + }; + let second = FileVersion { + modified: Some(base + std::time::Duration::from_nanos(200)), + ..first.clone() + }; + + assert_ne!(first, second); + assert_eq!(first.stamp, second.stamp); + } + + fn write_test_git_index(root: &Path, tracked: &[&str]) { + let git = root.join(".git"); + std::fs::create_dir_all(&git).unwrap(); + std::fs::write(git.join("config"), "[core]\n\tignorecase = true\n").unwrap(); + let mut index = Vec::new(); + index.extend_from_slice(b"DIRC"); + index.extend_from_slice(&2u32.to_be_bytes()); + index.extend_from_slice(&(tracked.len() as u32).to_be_bytes()); + for path in tracked { + let start = index.len(); + index.extend_from_slice(&[0u8; 60]); + index.extend_from_slice(&(path.len() as u16).to_be_bytes()); + index.extend_from_slice(path.as_bytes()); + index.push(0); + while (index.len() - start) % 8 != 0 { + index.push(0); + } + } + std::fs::write(git.join("index"), index).unwrap(); + } + + #[test] + fn default_builder_constructs_snapshot_but_explicit_none_is_preserved() { + let repo = tempfile::tempdir().unwrap(); + let root = repo.path(); + std::fs::create_dir(root.join("ignored")).unwrap(); + std::fs::write(root.join(".gitignore"), "IGNORED/\n").unwrap(); + std::fs::write(root.join("ignored/tracked.rs"), "fn tracked() {}\n").unwrap(); + std::fs::write(root.join("ignored/untracked.rs"), "fn untracked() {}\n").unwrap(); + write_test_git_index(root, &[".gitignore", "ignored/tracked.rs"]); + + let default_index = tempfile::tempdir().unwrap(); + build_index_with_options(root, Some(default_index.path()), &BuildOptions::default()) + .unwrap(); + let default_reader = IndexReader::open(default_index.path()).unwrap(); + assert!( + default_reader.contains_path("ignored/tracked.rs"), + "the default builder must construct the tracked-file exemption" + ); + assert!( + !default_reader.contains_path("ignored/untracked.rs"), + "the default builder must apply case-insensitive root rules" + ); + + let none_index = tempfile::tempdir().unwrap(); + build_index_with_options_and_ignorecase( + root, + Some(none_index.path()), + &BuildOptions::default(), + None, + ) + .unwrap(); + assert!( + IndexReader::open(none_index.path()) + .unwrap() + .contains_path("ignored/untracked.rs"), + "an explicit None must not reconstruct the exemption" + ); + } + #[test] fn batches_cover_every_file_exactly_once() { let sizes: Vec = (0..500).map(|i| (i as u64 % 7) * 9 * MB).collect(); diff --git a/tgrep-core/src/git_index.rs b/tgrep-core/src/git_index.rs index 3202e1c..e65307c 100644 --- a/tgrep-core/src/git_index.rs +++ b/tgrep-core/src/git_index.rs @@ -147,41 +147,74 @@ pub(crate) fn common_git_dir(git_dir: &Path) -> PathBuf { } } -/// Whether the repository compares paths without regard to case. -/// -/// git writes `core.ignorecase = true` into the repository's own config when it -/// detects a case-insensitive filesystem, so the per-repository config is the -/// authoritative place to read it and needs no config-precedence handling. -pub fn ignores_case(repo_root: &Path) -> bool { - let Some(git_dir) = git_dir(repo_root) else { - return false; - }; - let Ok(config) = std::fs::read_to_string(common_git_dir(&git_dir).join("config")) else { - return false; - }; - let mut in_core = false; +fn config_bool(config: &str, section: &str, key: &str) -> Option { + let mut in_section = false; + let mut result = None; for line in config.lines() { let line = line.trim(); - if let Some(section) = line.strip_prefix('[').and_then(|l| l.strip_suffix(']')) { - in_core = section.trim().eq_ignore_ascii_case("core"); + if line.is_empty() || line.starts_with('#') || line.starts_with(';') { continue; } - if !in_core { + if let Some(header) = line + .strip_prefix('[') + .and_then(|line| line.strip_suffix(']')) + { + in_section = header.trim().eq_ignore_ascii_case(section); continue; } - let Some((key, value)) = line.split_once('=') else { + if !in_section { + continue; + } + let (candidate, value) = line + .split_once('=') + .map_or((line, None), |(candidate, value)| (candidate, Some(value))); + if !candidate.trim().eq_ignore_ascii_case(key) { continue; - }; - if key.trim().eq_ignore_ascii_case("ignorecase") { - // git spells boolean true several ways. - let value = value.trim(); - return matches!( - value.to_ascii_lowercase().as_str(), - "true" | "yes" | "on" | "1" - ); } + result = match value.map(str::trim) { + None => Some(true), + Some(value) + if matches!( + value.to_ascii_lowercase().as_str(), + "true" | "yes" | "on" | "1" + ) => + { + Some(true) + } + Some(value) + if matches!( + value.to_ascii_lowercase().as_str(), + "false" | "no" | "off" | "0" | "" + ) => + { + Some(false) + } + Some(_) => None, + }; + } + result +} + +/// Whether the repository compares paths without regard to case. +/// +/// A repository may opt into per-worktree config. In that layout Git reads the +/// common config first, then lets `$GIT_DIR/config.worktree` override it. +pub fn ignores_case(repo_root: &Path) -> bool { + let Some(git_dir) = git_dir(repo_root) else { + return false; + }; + let common = common_git_dir(&git_dir); + let Ok(config) = std::fs::read_to_string(common.join("config")) else { + return false; + }; + let common_ignorecase = config_bool(&config, "core", "ignorecase").unwrap_or(false); + if !config_bool(&config, "extensions", "worktreeConfig").unwrap_or(false) { + return common_ignorecase; } - false + std::fs::read_to_string(git_dir.join("config.worktree")) + .ok() + .and_then(|config| config_bool(&config, "core", "ignorecase")) + .unwrap_or(common_ignorecase) } /// Read the tracked paths from `.git/index`. @@ -503,7 +536,6 @@ mod tests { let common = tmp.path().join("repo.git"); let worktree_git = common.join("worktrees").join("topic"); std::fs::create_dir_all(&worktree_git).expect("mkdir"); - std::fs::write(common.join("config"), "[core]\n\tignorecase = true\n").expect("write"); std::fs::write(worktree_git.join("commondir"), "../..\n").expect("write"); std::fs::write(worktree_git.join("index"), v2_index(&["worktree/only.rs"])).expect("write"); @@ -515,7 +547,38 @@ mod tests { ) .expect("write"); + std::fs::write( + common.join("config"), + "[core]\n\tignorecase = true\n[extensions]\n\tworktreeConfig\n", + ) + .expect("write"); + std::fs::write( + worktree_git.join("config.worktree"), + "[core]\n\tignorecase = false\n", + ) + .expect("write"); + assert!( + !ignores_case(&worktree), + "worktree false must override common true" + ); + + std::fs::write( + common.join("config"), + "[core]\n\tignorecase = false\n[extensions]\n\tworktreeConfig = true\n", + ) + .expect("write"); + std::fs::write( + worktree_git.join("config.worktree"), + "[core]\n\tignorecase = true\n", + ) + .expect("write"); assert!(ignores_case(&worktree)); + + std::fs::write(common.join("config"), "[core]\n\tignorecase = false\n").expect("write"); + assert!( + !ignores_case(&worktree), + "config.worktree is inactive until the extension is enabled" + ); assert_eq!(index_path(&worktree), Some(worktree_git.join("index"))); assert!( load_tracked(&worktree) diff --git a/tgrep-core/src/gitignore.rs b/tgrep-core/src/gitignore.rs index a3e9cbd..0a860db 100644 --- a/tgrep-core/src/gitignore.rs +++ b/tgrep-core/src/gitignore.rs @@ -469,27 +469,29 @@ pub struct CaseInsensitiveIgnore { tracked: TrackedMembership, /// Separate metadata-keyed cache for polling the current semantic /// membership of a frozen matcher. It never participates in decisions. - current: std::sync::RwLock, + current: std::sync::RwLock, } enum TrackedMembership { - /// Existing public behavior: reload after Git index identity changes. - Live(std::sync::RwLock), + /// Ordinary walks load one immutable snapshot only if a matching ignore + /// rule actually needs the tracked-file exemption. + Lazy(std::sync::OnceLock), /// Reconciliation behavior: one immutable set shared by a walk and the /// point-query matcher published from that walk. - Frozen { - tracked: Option, - fingerprint: TrackedMembershipFingerprint, - }, + Frozen(TrackedSnapshot), } -/// A metadata-keyed tracked set and semantic fingerprint. +struct TrackedSnapshot { + tracked: Option, + fingerprint: TrackedMembershipFingerprint, +} + +/// A metadata-keyed semantic fingerprint that does not retain parsed paths. #[derive(Default)] -struct TrackedCache { +struct TrackedFingerprintCache { /// `None` until something is loaded; `Some` even when the probe failed, so /// an unreadable index is not retried on every poll. loaded_from: Option>, - tracked: Option, fingerprint: TrackedMembershipFingerprint, } @@ -541,8 +543,8 @@ impl CaseInsensitiveIgnore { /// would leave this the only thing excluding — the exact opposite of what /// the flag asks for. /// - /// The tracked-file exemption remains live: a long-lived matcher reloads it - /// after the Git index changes. + /// The tracked-file exemption is loaded lazily on the first matching rule + /// and remains immutable for the lifetime of this value. pub fn new( root: &Path, use_gitignore: bool, @@ -555,9 +557,10 @@ impl CaseInsensitiveIgnore { /// Build an immutable tracked-membership snapshot for a coordinated walk /// and matcher publication. /// - /// Ordinary point-query callers should use [`Self::new`], which preserves - /// live Git-index reload behavior. Server reconciliation shares this value - /// through an `Arc` so one pass cannot mix different tracked sets. + /// Ordinary point-query callers should use [`Self::new`], which defers the + /// Git-index read until a matching rule needs it. Server reconciliation + /// shares this eagerly initialized value through an `Arc` so one pass + /// cannot mix different tracked sets and can poll for later changes. pub fn frozen_snapshot( root: &Path, use_gitignore: bool, @@ -606,20 +609,15 @@ impl CaseInsensitiveIgnore { return None; } let tracked = if frozen { - let tracked = crate::git_index::load_tracked(&repo_root); - let fingerprint = Self::fingerprint(&matcher, tracked.as_ref()); - TrackedMembership::Frozen { - tracked, - fingerprint, - } + TrackedMembership::Frozen(Self::load_snapshot(&repo_root, &matcher)) } else { - TrackedMembership::Live(std::sync::RwLock::new(TrackedCache::default())) + TrackedMembership::Lazy(std::sync::OnceLock::new()) }; Some(Self { matcher, repo_root, tracked, - current: std::sync::RwLock::new(TrackedCache::default()), + current: std::sync::RwLock::new(TrackedFingerprintCache::default()), }) } @@ -637,34 +635,31 @@ impl CaseInsensitiveIgnore { } let relative = relative.to_string_lossy(); match &self.tracked { - TrackedMembership::Live(cache) => { - let identity = index_identity(&self.repo_root); - let mut cache = cache.write().unwrap(); - self.reload_tracked_if_needed(&mut cache, identity); - Self::hides(cache.tracked.as_ref(), &relative, is_dir) - } - TrackedMembership::Frozen { tracked, .. } => { - Self::hides(tracked.as_ref(), &relative, is_dir) + TrackedMembership::Lazy(snapshot) => Self::hides( + snapshot + .get_or_init(|| Self::load_snapshot(&self.repo_root, &self.matcher)) + .tracked + .as_ref(), + &relative, + is_dir, + ), + TrackedMembership::Frozen(snapshot) => { + Self::hides(snapshot.tracked.as_ref(), &relative, is_dir) } } } fn tracked_membership_fingerprint(&self) -> TrackedMembershipFingerprint { match &self.tracked { - TrackedMembership::Live(cache) => { - let identity = index_identity(&self.repo_root); - let mut cache = cache.write().unwrap(); - self.reload_tracked_if_needed(&mut cache, identity); - cache.fingerprint.clone() - } - TrackedMembership::Frozen { fingerprint, .. } => fingerprint.clone(), + TrackedMembership::Lazy(snapshot) => snapshot + .get_or_init(|| Self::load_snapshot(&self.repo_root, &self.matcher)) + .fingerprint + .clone(), + TrackedMembership::Frozen(snapshot) => snapshot.fingerprint.clone(), } } fn current_tracked_membership_fingerprint(&self) -> TrackedMembershipFingerprint { - if matches!(&self.tracked, TrackedMembership::Live(_)) { - return self.tracked_membership_fingerprint(); - } let identity = index_identity(&self.repo_root); { let cache = self.current.read().unwrap(); @@ -678,18 +673,17 @@ impl CaseInsensitiveIgnore { } let tracked = crate::git_index::load_tracked(&self.repo_root); cache.fingerprint = Self::fingerprint(&self.matcher, tracked.as_ref()); - cache.tracked = tracked; cache.loaded_from = Some(identity); cache.fingerprint.clone() } - fn reload_tracked_if_needed(&self, cache: &mut TrackedCache, identity: Option) { - if cache.loaded_from.as_ref() == Some(&identity) { - return; + fn load_snapshot(repo_root: &Path, matcher: &Gitignore) -> TrackedSnapshot { + let tracked = crate::git_index::load_tracked(repo_root); + let fingerprint = Self::fingerprint(matcher, tracked.as_ref()); + TrackedSnapshot { + tracked, + fingerprint, } - cache.tracked = crate::git_index::load_tracked(&self.repo_root); - cache.fingerprint = Self::fingerprint(&self.matcher, cache.tracked.as_ref()); - cache.loaded_from = Some(identity); } fn fingerprint( diff --git a/tgrep-core/src/hybrid.rs b/tgrep-core/src/hybrid.rs index 22114d2..5a60168 100644 --- a/tgrep-core/src/hybrid.rs +++ b/tgrep-core/src/hybrid.rs @@ -298,6 +298,11 @@ impl HybridIndex { self.reader().contains_path(path) } + /// Whether the active on-disk reader contains a path below `directory`. + pub fn reader_has_descendant_path(&self, directory: &str) -> bool { + self.reader().has_descendant_path(directory) + } + /// The reader paths `keep` accepts. /// /// For callers that want a few of them — everything under one directory, diff --git a/tgrep-core/src/live.rs b/tgrep-core/src/live.rs index 804e914..0f5ca2e 100644 --- a/tgrep-core/src/live.rs +++ b/tgrep-core/src/live.rs @@ -196,6 +196,17 @@ impl LiveIndex { self.path_to_id.contains_key(path) } + /// Whether the live overlay contains a path strictly below `directory`. + pub fn has_descendant_path(&self, directory: &str) -> bool { + if directory.is_empty() { + return !self.path_to_id.is_empty(); + } + self.path_to_id.keys().any(|path| { + path.strip_prefix(directory) + .is_some_and(|suffix| suffix.starts_with('/')) + }) + } + /// Number of files in the overlay. pub fn num_files(&self) -> usize { self.file_paths.len() @@ -499,6 +510,18 @@ impl LiveIndex { mod tests { use super::*; + #[test] + fn descendant_lookup_is_directory_boundary_aware() { + let mut live = LiveIndex::new(); + live.upsert_file_with_trigrams("src/deep/a.rs", Vec::new()); + live.upsert_file_with_trigrams("src2/not-a-child.rs", Vec::new()); + + assert!(live.has_descendant_path("src")); + assert!(live.has_descendant_path("src/deep")); + assert!(!live.has_descendant_path("sr")); + assert!(!live.has_descendant_path("src/deep/a.rs")); + } + #[test] fn clearing_reconciled_paths_removes_entries_and_tombstones_only_for_them() { let mut live = LiveIndex::new(); diff --git a/tgrep-core/src/meta.rs b/tgrep-core/src/meta.rs index 6bb40ae..98a1340 100644 --- a/tgrep-core/src/meta.rs +++ b/tgrep-core/src/meta.rs @@ -68,6 +68,21 @@ pub struct FileStamp { pub size: u64, } +/// Convert filesystem metadata into the persisted stamp used by change +/// detection. +pub fn file_stamp(metadata: &std::fs::Metadata) -> FileStamp { + let mtime = metadata + .modified() + .ok() + .and_then(|t| t.duration_since(SystemTime::UNIX_EPOCH).ok()) + .map(|d| d.as_secs()) + .unwrap_or(0); + FileStamp { + mtime, + size: metadata.len(), + } +} + /// Write per-file stamps to `filestamps.json` in the index directory. pub fn write_filestamps(stamps: &HashMap, index_dir: &Path) -> Result<()> { let path = index_dir.join(FILESTAMPS_FILENAME); @@ -95,21 +110,9 @@ pub fn collect_filestamps(root: &Path, paths: &[String]) -> HashMap bool { + if directory.is_empty() { + return !self.path_order.is_empty(); + } + let prefix = directory + .as_bytes() + .iter() + .copied() + .chain(std::iter::once(b'/')); + let lower = self.path_order.partition_point(|&id| { + self.file_paths[id] + .as_bytes() + .iter() + .copied() + .cmp(prefix.clone()) + .is_lt() + }); + self.path_order.get(lower).is_some_and(|&id| { + self.file_paths[id] + .strip_prefix(directory) + .is_some_and(|suffix| suffix.starts_with('/')) + }) + } + /// Total number of indexed files. pub fn num_files(&self) -> usize { self.file_paths.len() @@ -532,6 +557,21 @@ mod tests { assert_eq!(reader.file_paths[2], "c.rs"); } + #[test] + fn descendant_lookup_is_sorted_and_directory_boundary_aware() { + let tmp = TempDir::new().unwrap(); + let mut files = ondisk::encode_file_entry(0, "elsewhere.rs").unwrap(); + files.extend_from_slice(&ondisk::encode_file_entry(1, "src/deep/a.rs").unwrap()); + files.extend_from_slice(&ondisk::encode_file_entry(2, "src2/not-a-child.rs").unwrap()); + write_index(tmp.path(), &[], &[], &files); + let reader = IndexReader::open(tmp.path()).unwrap(); + + assert!(reader.has_descendant_path("src")); + assert!(reader.has_descendant_path("src/deep")); + assert!(!reader.has_descendant_path("sr")); + assert!(!reader.has_descendant_path("src2/not-a-child.rs")); + } + #[test] fn is_degenerate_detects_mmap_counter_disagreement() { let tmp = TempDir::new().unwrap(); diff --git a/tgrep-core/src/walker.rs b/tgrep-core/src/walker.rs index a3da75a..12ac65b 100644 --- a/tgrep-core/src/walker.rs +++ b/tgrep-core/src/walker.rs @@ -616,16 +616,11 @@ pub fn walk_file_metadata_with_ignorecase( if max_file_size.is_some_and(|limit| meta.len() > limit) { return ignore::WalkState::Continue; } - let mtime = meta - .modified() - .ok() - .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()) - .map(|d| d.as_secs()) - .unwrap_or(0); + let stamp = crate::meta::file_stamp(&meta); results.lock().unwrap().push(FileMeta { relative_path: rel_path, - mtime, - size: meta.len(), + mtime: stamp.mtime, + size: stamp.size, }); ignore::WalkState::Continue @@ -1509,7 +1504,7 @@ mod tests { } #[test] - fn public_matcher_reloads_tracked_exemptions_after_index_changes() { + fn public_matcher_lazily_freezes_tracked_exemptions_on_first_match() { let dir = ignorecase_fixture(true, &["src/main.rs", "src/Kept.TXT"]); let root = dir.path(); let matcher = crate::gitignore::matcher_from_ignore_paths( @@ -1519,20 +1514,18 @@ mod tests { ) .expect("the fixture has rules"); - // Untracked, and `*.txt` matches it once case is ignored. - assert!(matcher.is_ignored(Path::new("src/Gone.TXT"), false)); - + // Construction alone must not read the index. The first matching query + // observes this update and freezes that membership for later queries. fake_git_repo(root, true, &["src/main.rs", "src/Kept.TXT", "src/Gone.TXT"]); - assert!( !matcher.is_ignored(Path::new("src/Gone.TXT"), false), - "the legacy public matcher must observe a newly tracked exemption" + "the first matching query must initialize from the current index" ); fake_git_repo(root, true, &["src/main.rs"]); assert!( - matcher.is_ignored(Path::new("src/Kept.TXT"), false), - "the legacy public matcher must observe a removed exemption" + !matcher.is_ignored(Path::new("src/Kept.TXT"), false), + "the lazy snapshot must remain immutable after initialization" ); } From fce4140741fbb56df9eb22512a2ec3d2848bdbab Mon Sep 17 00:00:00 2001 From: Shengyu Fu Date: Sat, 29 Aug 2026 18:17:20 -0500 Subject: [PATCH 09/16] Address remaining Copilot review comments Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 96b9f8ae-c5ec-450c-a003-cb9ade61d9df --- tgrep-cli/src/serve.rs | 52 ++++++----- tgrep-core/src/builder.rs | 172 +++++++++++++++++++++++++++++++++--- tgrep-core/src/gitignore.rs | 24 +++-- tgrep-core/src/walker.rs | 30 +++++++ 4 files changed, 230 insertions(+), 48 deletions(-) diff --git a/tgrep-cli/src/serve.rs b/tgrep-cli/src/serve.rs index 122a482..6adee7d 100644 --- a/tgrep-cli/src/serve.rs +++ b/tgrep-cli/src/serve.rs @@ -2772,11 +2772,11 @@ fn sync_watch_registrations(state: &ServerState, root: &Path) -> (Vec, let sources = state.ignore_sources.read().unwrap(); desired.dirs.extend(ignore_target_dirs(root, &sources)); } - let total = desired.dirs.len(); // An incomplete traversal cannot prove that omitted recorded watches are // live, so it does not get to consume the overflow repair request. let force = take_force_resubscribe(&state.watch_resubscribe, desired.completeness); let (added, removed) = registry.sync(&desired.dirs, desired.completeness, force); + let total = registry.watched.len(); if !added.is_empty() || removed > 0 { eprintln!( "[trace] watcher subscriptions: {total} directories \ @@ -4210,29 +4210,34 @@ fn file_still_has_bytes( ) -> std::io::Result { use std::io::Read; - let mut current = open_within_root(root, path)?; - if tgrep_core::builder::file_version(¤t.metadata()?) != *expected_version { - return Ok(false); - } - let mut offset = 0; - let mut buffer = [0u8; 64 * 1024]; - loop { - let read = current.read(&mut buffer)?; - if read == 0 { - break; - } - if expected.get(offset..offset + read) != Some(&buffer[..read]) { + fn matches( + mut file: std::fs::File, + expected_version: &tgrep_core::builder::FileVersion, + expected: &[u8], + ) -> std::io::Result { + if tgrep_core::builder::file_version(&file.metadata()?) != *expected_version { return Ok(false); } - offset += read; + let mut offset = 0; + let mut buffer = [0u8; 64 * 1024]; + loop { + let read = file.read(&mut buffer)?; + if read == 0 { + break; + } + if expected.get(offset..offset + read) != Some(&buffer[..read]) { + return Ok(false); + } + offset += read; + } + Ok(offset == expected.len() + && tgrep_core::builder::file_version(&file.metadata()?) == *expected_version) } - if offset != expected.len() - || tgrep_core::builder::file_version(¤t.metadata()?) != *expected_version - { + + if !matches(open_within_root(root, path)?, expected_version, expected)? { return Ok(false); } - let path_current = open_within_root(root, path)?; - Ok(tgrep_core::builder::file_version(&path_current.metadata()?) == *expected_version) + matches(open_within_root(root, path)?, expected_version, expected) } /// Reads a file's contents, never pulling in more than one byte past the cap. @@ -5907,6 +5912,7 @@ fn background_index_build(state: &Arc, root: &Path, index_dir: &Pat state.flushing.store(false, Ordering::SeqCst); if !state.watch_enabled { if !catch_up_unwatched_build(state, root, index_dir) { + state.ignore_rules_dirty.store(true, Ordering::SeqCst); eprintln!( "[trace] warning: unwatched background-build catch-up was incomplete; \ leaving stamps invalid for the scheduled stale check" @@ -7126,7 +7132,7 @@ mod tests { } #[test] - fn content_only_git_index_rewrite_does_not_request_reconciliation() { + fn content_only_git_index_rewrite_is_ignored_but_membership_changes_reconcile() { let tmp = TempDir::new().unwrap(); let root = tmp.path().to_path_buf(); @@ -7157,9 +7163,13 @@ mod tests { std::fs::write(root.join("src/new.rs"), "fn newly_tracked() {}\n").unwrap(); test_git(&root, &["add", "--", "src/new.rs"]); + assert!( + tracked_membership_changed(&state), + "conservative polling must notice every tracked-membership change" + ); assert!( !tracked_membership_changed(&state), - "tracked membership outside the exemption set must not request a full scan" + "an observed membership change must be coalesced" ); test_git(&root, &["add", "-f", "--", "ignored/forced.rs"]); diff --git a/tgrep-core/src/builder.rs b/tgrep-core/src/builder.rs index 56587be..1a7dd37 100644 --- a/tgrep-core/src/builder.rs +++ b/tgrep-core/src/builder.rs @@ -338,10 +338,52 @@ const MMAP_MIN_BYTES: u64 = 1024 * 1024; /// The bytes of a file to index, either read onto the heap or borrowed from a /// memory map. enum FileBytes { - Read(Vec), + Read { + bytes: Vec, + _permit: OwnedReadPermit, + }, Mapped(memmap2::Mmap), } +struct OwnedReadBudget { + available: std::sync::Mutex, + ready: std::sync::Condvar, +} + +impl OwnedReadBudget { + fn new(bytes: u64) -> std::sync::Arc { + std::sync::Arc::new(Self { + available: std::sync::Mutex::new(bytes), + ready: std::sync::Condvar::new(), + }) + } + + fn acquire(self: &std::sync::Arc, bytes: u64) -> OwnedReadPermit { + let mut available = self.available.lock().unwrap(); + while *available < bytes { + available = self.ready.wait(available).unwrap(); + } + *available -= bytes; + OwnedReadPermit { + budget: std::sync::Arc::clone(self), + bytes, + } + } +} + +struct OwnedReadPermit { + budget: std::sync::Arc, + bytes: u64, +} + +impl Drop for OwnedReadPermit { + fn drop(&mut self) { + let mut available = self.budget.available.lock().unwrap(); + *available += self.bytes; + self.budget.ready.notify_all(); + } +} + type ExtractedFile = (String, trigram::TrigramMaskMap); /// Full-resolution identity used to validate that bytes came from one file @@ -392,7 +434,7 @@ impl std::ops::Deref for FileBytes { fn deref(&self) -> &[u8] { match self { - FileBytes::Read(bytes) => bytes, + FileBytes::Read { bytes, .. } => bytes, FileBytes::Mapped(map) => map, } } @@ -403,7 +445,11 @@ impl std::ops::Deref for FileBytes { /// /// Falls back to an owned read whenever mapping is unavailable, but never lets /// that fallback allocate beyond the extraction batch's owned-buffer budget. -fn read_for_index(path: &Path, size: u64) -> std::io::Result { +fn read_for_index( + path: &Path, + size: u64, + owned_budget: &std::sync::Arc, +) -> std::io::Result { if size >= MMAP_MIN_BYTES { // SAFETY: the map is read-only and owned by the returned value, which // is dropped once the file's trigrams have been extracted. A concurrent @@ -416,7 +462,50 @@ fn read_for_index(path: &Path, size: u64) -> std::io::Result { return Ok(FileBytes::Mapped(map)); } } - read_owned_for_index(path, size).map(FileBytes::Read) + if size > MAX_OWNED_FILE_BYTES { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "file is too large for the {} MiB owned-buffer fallback", + MAX_OWNED_FILE_BYTES / (1024 * 1024) + ), + )); + } + let permit = owned_budget.acquire(size); + match read_owned_for_index(path, size) { + Ok(bytes) => Ok(FileBytes::Read { + bytes, + _permit: permit, + }), + Err(error) + if error.kind() == std::io::ErrorKind::WouldBlock && size < MAX_OWNED_FILE_BYTES => + { + // The file grew after it was stat'd. Release the smaller claim + // before waiting for the full allowance so concurrent growers + // cannot deadlock while each holds part of the batch budget. + drop(permit); + let permit = owned_budget.acquire(MAX_OWNED_FILE_BYTES); + read_owned_for_index(path, MAX_OWNED_FILE_BYTES) + .map(|bytes| FileBytes::Read { + bytes, + _permit: permit, + }) + .map_err(|retry| { + if retry.kind() == std::io::ErrorKind::WouldBlock { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "file grew beyond the {} MiB owned-buffer fallback", + MAX_OWNED_FILE_BYTES / (1024 * 1024) + ), + ) + } else { + retry + } + }) + } + Err(error) => Err(error), + } } fn read_owned_for_index(path: &Path, size: u64) -> std::io::Result> { @@ -434,15 +523,13 @@ fn read_owned_for_index(path: &Path, size: u64) -> std::io::Result> { let mut file = std::fs::File::open(path)?; let mut bytes = Vec::with_capacity(usize::try_from(size).unwrap_or(0)); std::io::Read::by_ref(&mut file) - .take(MAX_OWNED_FILE_BYTES.saturating_add(1)) + .take(size) .read_to_end(&mut bytes)?; - if bytes.len() as u64 > MAX_OWNED_FILE_BYTES { + let mut extra = [0u8; 1]; + if file.read(&mut extra)? != 0 { return Err(std::io::Error::new( - std::io::ErrorKind::InvalidData, - format!( - "file grew beyond the {} MiB owned-buffer fallback", - MAX_OWNED_FILE_BYTES / (1024 * 1024) - ), + std::io::ErrorKind::WouldBlock, + "file grew beyond its owned-buffer reservation", )); } Ok(bytes) @@ -534,11 +621,12 @@ pub fn build_index_with_options_and_ignorecase( let batch = &walk.files[range.clone()]; let batch_sizes = &sizes[range]; let owned_limit_error = std::sync::Mutex::new(None); + let owned_budget = OwnedReadBudget::new(INDEX_BUILD_BATCH_BYTES); let batch_data: Vec = batch .par_iter() .zip(batch_sizes.par_iter()) .filter_map(|(path, &size)| { - let data = match read_for_index(path, size) { + let data = match read_for_index(path, size, &owned_budget) { Ok(data) => data, Err(error) => { if error.kind() == std::io::ErrorKind::InvalidData { @@ -680,11 +768,12 @@ pub fn build_index_for_files( for range in batch_ranges(&charges, INDEX_BUILD_BATCH_BYTES) { let batch = &files[range.clone()]; let batch_sizes = &sizes[range]; + let owned_budget = OwnedReadBudget::new(INDEX_BUILD_BATCH_BYTES); let batch_data: Vec> = batch .par_iter() .zip(batch_sizes.par_iter()) .filter_map(|(path, &size)| { - let data = match read_for_index(path, size) { + let data = match read_for_index(path, size, &owned_budget) { Ok(data) => data, Err(error) => { eprintln!("tgrep: skipping {}: {error}", path.display()); @@ -1490,6 +1579,63 @@ mod tests { assert_eq!(error.kind(), std::io::ErrorKind::InvalidData); } + #[test] + fn owned_read_reports_growth_beyond_its_reservation() { + let tmp = tempfile::tempdir().unwrap(); + let path = tmp.path().join("growing.txt"); + std::fs::write(&path, b"four").unwrap(); + + let error = read_owned_for_index(&path, 2).unwrap_err(); + assert_eq!(error.kind(), std::io::ErrorKind::WouldBlock); + } + + #[test] + fn indexed_read_retries_growth_with_the_full_batch_reservation() { + let tmp = tempfile::tempdir().unwrap(); + let path = tmp.path().join("growing.txt"); + std::fs::write(&path, b"four").unwrap(); + let budget = OwnedReadBudget::new(INDEX_BUILD_BATCH_BYTES); + + let bytes = read_for_index(&path, 2, &budget).unwrap(); + assert_eq!(&*bytes, b"four"); + } + + #[test] + fn owned_read_permit_releases_batch_capacity() { + let budget = OwnedReadBudget::new(10); + let permit = budget.acquire(7); + assert_eq!(*budget.available.lock().unwrap(), 3); + drop(permit); + assert_eq!(*budget.available.lock().unwrap(), 10); + } + + #[test] + fn owned_read_budget_serializes_fallbacks_that_exceed_the_remaining_capacity() { + let budget = OwnedReadBudget::new(10); + let first = budget.acquire(10); + let (started_tx, started_rx) = std::sync::mpsc::channel(); + let (acquired_tx, acquired_rx) = std::sync::mpsc::channel(); + let waiting_budget = std::sync::Arc::clone(&budget); + let waiter = std::thread::spawn(move || { + started_tx.send(()).unwrap(); + let _second = waiting_budget.acquire(10); + acquired_tx.send(()).unwrap(); + }); + + started_rx.recv().unwrap(); + assert!( + acquired_rx + .recv_timeout(std::time::Duration::from_millis(50)) + .is_err(), + "a second fallback must wait rather than exceed the shared budget" + ); + drop(first); + acquired_rx + .recv_timeout(std::time::Duration::from_secs(1)) + .unwrap(); + waiter.join().unwrap(); + } + #[test] fn file_version_distinguishes_subsecond_modified_times() { let stamp = meta::FileStamp { mtime: 1, size: 10 }; diff --git a/tgrep-core/src/gitignore.rs b/tgrep-core/src/gitignore.rs index 0a860db..2901281 100644 --- a/tgrep-core/src/gitignore.rs +++ b/tgrep-core/src/gitignore.rs @@ -609,7 +609,7 @@ impl CaseInsensitiveIgnore { return None; } let tracked = if frozen { - TrackedMembership::Frozen(Self::load_snapshot(&repo_root, &matcher)) + TrackedMembership::Frozen(Self::load_snapshot(&repo_root)) } else { TrackedMembership::Lazy(std::sync::OnceLock::new()) }; @@ -637,7 +637,7 @@ impl CaseInsensitiveIgnore { match &self.tracked { TrackedMembership::Lazy(snapshot) => Self::hides( snapshot - .get_or_init(|| Self::load_snapshot(&self.repo_root, &self.matcher)) + .get_or_init(|| Self::load_snapshot(&self.repo_root)) .tracked .as_ref(), &relative, @@ -652,7 +652,7 @@ impl CaseInsensitiveIgnore { fn tracked_membership_fingerprint(&self) -> TrackedMembershipFingerprint { match &self.tracked { TrackedMembership::Lazy(snapshot) => snapshot - .get_or_init(|| Self::load_snapshot(&self.repo_root, &self.matcher)) + .get_or_init(|| Self::load_snapshot(&self.repo_root)) .fingerprint .clone(), TrackedMembership::Frozen(snapshot) => snapshot.fingerprint.clone(), @@ -672,14 +672,14 @@ impl CaseInsensitiveIgnore { return cache.fingerprint.clone(); } let tracked = crate::git_index::load_tracked(&self.repo_root); - cache.fingerprint = Self::fingerprint(&self.matcher, tracked.as_ref()); + cache.fingerprint = Self::fingerprint(tracked.as_ref()); cache.loaded_from = Some(identity); cache.fingerprint.clone() } - fn load_snapshot(repo_root: &Path, matcher: &Gitignore) -> TrackedSnapshot { + fn load_snapshot(repo_root: &Path) -> TrackedSnapshot { let tracked = crate::git_index::load_tracked(repo_root); - let fingerprint = Self::fingerprint(matcher, tracked.as_ref()); + let fingerprint = Self::fingerprint(tracked.as_ref()); TrackedSnapshot { tracked, fingerprint, @@ -687,16 +687,12 @@ impl CaseInsensitiveIgnore { } fn fingerprint( - matcher: &Gitignore, tracked: Option<&crate::git_index::TrackedFiles>, ) -> TrackedMembershipFingerprint { - TrackedMembershipFingerprint(tracked.map(|tracked| { - tracked.fingerprint_matching(|path| { - matcher - .matched_path_or_any_parents(Path::new(path), false) - .is_ignore() - }) - })) + // Directory visibility depends on every tracked descendant, including + // paths whose final file match is whitelisted. Fingerprint the full set + // so changes that make an ignored ancestor walkable are observable. + TrackedMembershipFingerprint(tracked.map(|tracked| tracked.fingerprint_matching(|_| true))) } fn hides( diff --git a/tgrep-core/src/walker.rs b/tgrep-core/src/walker.rs index 12ac65b..a8b536e 100644 --- a/tgrep-core/src/walker.rs +++ b/tgrep-core/src/walker.rs @@ -1561,6 +1561,36 @@ mod tests { ); } + #[test] + fn tracked_fingerprint_includes_whitelisted_files_under_ignored_directories() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path(); + fake_git_repo(root, true, &["qlogs/kept.rs"]); + fs::write(root.join(".gitignore"), "QLogs/\n!qlogs/kept.rs\n").unwrap(); + fs::create_dir_all(root.join("qlogs")).unwrap(); + fs::write(root.join("qlogs/kept.rs"), "tracked").unwrap(); + + let ignorecase = + crate::gitignore::CaseInsensitiveIgnore::frozen_snapshot(root, true, true, true) + .map(std::sync::Arc::new); + let matcher = crate::gitignore::matcher_from_ignore_paths_with_options_and_ignorecase( + root, + std::slice::from_ref(&root.join(".gitignore")), + &[], + false, + ignorecase, + ) + .expect("the fixture has rules"); + let baseline = matcher.tracked_membership_fingerprint(); + + fake_git_repo(root, true, &[]); + assert_ne!( + baseline, + matcher.current_tracked_membership_fingerprint(), + "polling must notice when an ignored ancestor loses its tracked descendant" + ); + } + #[test] fn no_ignore_turns_the_whole_thing_off() { let dir = ignorecase_fixture(true, &["src/main.rs"]); From 79664fe038068ac43cda49ab8f3987ec4136720f Mon Sep 17 00:00:00 2001 From: Shengyu Fu Date: Sat, 29 Aug 2026 19:05:02 -0500 Subject: [PATCH 10/16] Preserve unlimited large-file indexing Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 96b9f8ae-c5ec-450c-a003-cb9ade61d9df --- tgrep-core/src/builder.rs | 43 +++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/tgrep-core/src/builder.rs b/tgrep-core/src/builder.rs index 1a7dd37..ae6c478 100644 --- a/tgrep-core/src/builder.rs +++ b/tgrep-core/src/builder.rs @@ -340,7 +340,7 @@ const MMAP_MIN_BYTES: u64 = 1024 * 1024; enum FileBytes { Read { bytes: Vec, - _permit: OwnedReadPermit, + _permit: Option, }, Mapped(memmap2::Mmap), } @@ -449,6 +449,7 @@ fn read_for_index( path: &Path, size: u64, owned_budget: &std::sync::Arc, + configured_limit: Option, ) -> std::io::Result { if size >= MMAP_MIN_BYTES { // SAFETY: the map is read-only and owned by the returned value, which @@ -462,17 +463,16 @@ fn read_for_index( return Ok(FileBytes::Mapped(map)); } } - if size > MAX_OWNED_FILE_BYTES { + let owned_limit = configured_limit.unwrap_or(size); + if size > owned_limit { return Err(std::io::Error::new( std::io::ErrorKind::InvalidData, - format!( - "file is too large for the {} MiB owned-buffer fallback", - MAX_OWNED_FILE_BYTES / (1024 * 1024) - ), + format!("file exceeds the configured {} byte limit", owned_limit), )); } - let permit = owned_budget.acquire(size); - match read_owned_for_index(path, size) { + let oversized = size > MAX_OWNED_FILE_BYTES; + let permit = (!oversized).then(|| owned_budget.acquire(size)); + match read_owned_for_index(path, size, owned_limit) { Ok(bytes) => Ok(FileBytes::Read { bytes, _permit: permit, @@ -484,11 +484,12 @@ fn read_for_index( // before waiting for the full allowance so concurrent growers // cannot deadlock while each holds part of the batch budget. drop(permit); - let permit = owned_budget.acquire(MAX_OWNED_FILE_BYTES); - read_owned_for_index(path, MAX_OWNED_FILE_BYTES) + let retry_limit = configured_limit.unwrap_or(MAX_OWNED_FILE_BYTES); + let permit = owned_budget.acquire(MAX_OWNED_FILE_BYTES.min(retry_limit)); + read_owned_for_index(path, MAX_OWNED_FILE_BYTES.min(retry_limit), retry_limit) .map(|bytes| FileBytes::Read { bytes, - _permit: permit, + _permit: Some(permit), }) .map_err(|retry| { if retry.kind() == std::io::ErrorKind::WouldBlock { @@ -508,16 +509,13 @@ fn read_for_index( } } -fn read_owned_for_index(path: &Path, size: u64) -> std::io::Result> { +fn read_owned_for_index(path: &Path, size: u64, owned_limit: u64) -> std::io::Result> { use std::io::Read; - if size > MAX_OWNED_FILE_BYTES { + if size > owned_limit { return Err(std::io::Error::new( std::io::ErrorKind::InvalidData, - format!( - "file is too large for the {} MiB owned-buffer fallback", - MAX_OWNED_FILE_BYTES / (1024 * 1024) - ), + format!("file exceeds the configured {} byte limit", owned_limit), )); } let mut file = std::fs::File::open(path)?; @@ -626,7 +624,7 @@ pub fn build_index_with_options_and_ignorecase( .par_iter() .zip(batch_sizes.par_iter()) .filter_map(|(path, &size)| { - let data = match read_for_index(path, size, &owned_budget) { + let data = match read_for_index(path, size, &owned_budget, opts.max_file_size) { Ok(data) => data, Err(error) => { if error.kind() == std::io::ErrorKind::InvalidData { @@ -773,7 +771,7 @@ pub fn build_index_for_files( .par_iter() .zip(batch_sizes.par_iter()) .filter_map(|(path, &size)| { - let data = match read_for_index(path, size, &owned_budget) { + let data = match read_for_index(path, size, &owned_budget, None) { Ok(data) => data, Err(error) => { eprintln!("tgrep: skipping {}: {error}", path.display()); @@ -1575,7 +1573,8 @@ mod tests { .set_len(MAX_OWNED_FILE_BYTES + 1) .unwrap(); - let error = read_owned_for_index(&path, MAX_OWNED_FILE_BYTES + 1).unwrap_err(); + let error = read_owned_for_index(&path, MAX_OWNED_FILE_BYTES + 1, MAX_OWNED_FILE_BYTES) + .unwrap_err(); assert_eq!(error.kind(), std::io::ErrorKind::InvalidData); } @@ -1585,7 +1584,7 @@ mod tests { let path = tmp.path().join("growing.txt"); std::fs::write(&path, b"four").unwrap(); - let error = read_owned_for_index(&path, 2).unwrap_err(); + let error = read_owned_for_index(&path, 2, 2).unwrap_err(); assert_eq!(error.kind(), std::io::ErrorKind::WouldBlock); } @@ -1596,7 +1595,7 @@ mod tests { std::fs::write(&path, b"four").unwrap(); let budget = OwnedReadBudget::new(INDEX_BUILD_BATCH_BYTES); - let bytes = read_for_index(&path, 2, &budget).unwrap(); + let bytes = read_for_index(&path, 2, &budget, None).unwrap(); assert_eq!(&*bytes, b"four"); } From 4bb61cc67c1786219048293e3e55fafa052b43ad Mon Sep 17 00:00:00 2001 From: Shengyu Fu Date: Sat, 29 Aug 2026 19:07:41 -0500 Subject: [PATCH 11/16] Serialize unlimited fallback reads Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 96b9f8ae-c5ec-450c-a003-cb9ade61d9df --- tgrep-core/src/builder.rs | 102 +++++++++++++++++++++++++------------- 1 file changed, 67 insertions(+), 35 deletions(-) diff --git a/tgrep-core/src/builder.rs b/tgrep-core/src/builder.rs index ae6c478..5aa0629 100644 --- a/tgrep-core/src/builder.rs +++ b/tgrep-core/src/builder.rs @@ -340,12 +340,13 @@ const MMAP_MIN_BYTES: u64 = 1024 * 1024; enum FileBytes { Read { bytes: Vec, - _permit: Option, + _permit: OwnedReadPermit, }, Mapped(memmap2::Mmap), } struct OwnedReadBudget { + capacity: u64, available: std::sync::Mutex, ready: std::sync::Condvar, } @@ -353,20 +354,24 @@ struct OwnedReadBudget { impl OwnedReadBudget { fn new(bytes: u64) -> std::sync::Arc { std::sync::Arc::new(Self { + capacity: bytes, available: std::sync::Mutex::new(bytes), ready: std::sync::Condvar::new(), }) } fn acquire(self: &std::sync::Arc, bytes: u64) -> OwnedReadPermit { + // One oversized fallback may exceed the ordinary budget, but it claims + // the whole budget so no other owned read can overlap it. + let charged = bytes.min(self.capacity); let mut available = self.available.lock().unwrap(); - while *available < bytes { + while *available < charged { available = self.ready.wait(available).unwrap(); } - *available -= bytes; + *available -= charged; OwnedReadPermit { budget: std::sync::Arc::clone(self), - bytes, + bytes: charged, } } } @@ -443,8 +448,9 @@ impl std::ops::Deref for FileBytes { /// Get a file's bytes for indexing, mapping it when it is large enough to be /// worth avoiding the copy. /// -/// Falls back to an owned read whenever mapping is unavailable, but never lets -/// that fallback allocate beyond the extraction batch's owned-buffer budget. +/// Falls back to an owned read whenever mapping is unavailable. Ordinary reads +/// share the batch budget; an unlimited oversized fallback claims it +/// exclusively so at most one such allocation is in flight. fn read_for_index( path: &Path, size: u64, @@ -463,47 +469,54 @@ fn read_for_index( return Ok(FileBytes::Mapped(map)); } } - let owned_limit = configured_limit.unwrap_or(size); + let owned_limit = configured_limit.unwrap_or(u64::MAX); if size > owned_limit { return Err(std::io::Error::new( std::io::ErrorKind::InvalidData, format!("file exceeds the configured {} byte limit", owned_limit), )); } - let oversized = size > MAX_OWNED_FILE_BYTES; - let permit = (!oversized).then(|| owned_budget.acquire(size)); + let permit = owned_budget.acquire(size); match read_owned_for_index(path, size, owned_limit) { Ok(bytes) => Ok(FileBytes::Read { bytes, _permit: permit, }), - Err(error) - if error.kind() == std::io::ErrorKind::WouldBlock && size < MAX_OWNED_FILE_BYTES => - { + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { // The file grew after it was stat'd. Release the smaller claim - // before waiting for the full allowance so concurrent growers + // before waiting for the retry allowance so concurrent growers // cannot deadlock while each holds part of the batch budget. drop(permit); - let retry_limit = configured_limit.unwrap_or(MAX_OWNED_FILE_BYTES); - let permit = owned_budget.acquire(MAX_OWNED_FILE_BYTES.min(retry_limit)); - read_owned_for_index(path, MAX_OWNED_FILE_BYTES.min(retry_limit), retry_limit) - .map(|bytes| FileBytes::Read { - bytes, - _permit: Some(permit), - }) - .map_err(|retry| { - if retry.kind() == std::io::ErrorKind::WouldBlock { - std::io::Error::new( - std::io::ErrorKind::InvalidData, - format!( - "file grew beyond the {} MiB owned-buffer fallback", - MAX_OWNED_FILE_BYTES / (1024 * 1024) - ), - ) - } else { - retry - } - }) + match configured_limit { + Some(limit) => { + let permit = owned_budget.acquire(limit); + read_owned_for_index(path, limit, limit) + .map(|bytes| FileBytes::Read { + bytes, + _permit: permit, + }) + .map_err(|retry| { + if retry.kind() == std::io::ErrorKind::WouldBlock { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("file exceeds the configured {} byte limit", limit), + ) + } else { + retry + } + }) + } + None => { + // No caller limit means no implicit fallback limit either. + // Claiming more than the budget takes it exclusively, so + // this unbounded read cannot overlap another owned buffer. + let permit = owned_budget.acquire(u64::MAX); + read_owned_unbounded(path, size).map(|bytes| FileBytes::Read { + bytes, + _permit: permit, + }) + } + } } Err(error) => Err(error), } @@ -533,6 +546,16 @@ fn read_owned_for_index(path: &Path, size: u64, owned_limit: u64) -> std::io::Re Ok(bytes) } +fn read_owned_unbounded(path: &Path, expected_size: u64) -> std::io::Result> { + use std::io::Read; + + let mut file = std::fs::File::open(path)?; + let initial_capacity = expected_size.min(MAX_OWNED_FILE_BYTES); + let mut bytes = Vec::with_capacity(usize::try_from(initial_capacity).unwrap_or(0)); + file.read_to_end(&mut bytes)?; + Ok(bytes) +} + /// Build a trigram index, choosing how postings are accumulated. pub fn build_index_with_options( root: &Path, @@ -1608,16 +1631,25 @@ mod tests { assert_eq!(*budget.available.lock().unwrap(), 10); } + #[test] + fn oversized_owned_read_claims_the_budget_exclusively() { + let budget = OwnedReadBudget::new(10); + let permit = budget.acquire(100); + assert_eq!(*budget.available.lock().unwrap(), 0); + drop(permit); + assert_eq!(*budget.available.lock().unwrap(), 10); + } + #[test] fn owned_read_budget_serializes_fallbacks_that_exceed_the_remaining_capacity() { let budget = OwnedReadBudget::new(10); - let first = budget.acquire(10); + let first = budget.acquire(100); let (started_tx, started_rx) = std::sync::mpsc::channel(); let (acquired_tx, acquired_rx) = std::sync::mpsc::channel(); let waiting_budget = std::sync::Arc::clone(&budget); let waiter = std::thread::spawn(move || { started_tx.send(()).unwrap(); - let _second = waiting_budget.acquire(10); + let _second = waiting_budget.acquire(100); acquired_tx.send(()).unwrap(); }); From a78317a22d98413a97015cfe191748ffd57b516a Mon Sep 17 00:00:00 2001 From: Shengyu Fu Date: Sat, 29 Aug 2026 22:02:48 -0500 Subject: [PATCH 12/16] Honor max file size before mmap Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 96b9f8ae-c5ec-450c-a003-cb9ade61d9df --- tgrep-core/src/builder.rs | 37 +++++++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/tgrep-core/src/builder.rs b/tgrep-core/src/builder.rs index 5aa0629..9993c0d 100644 --- a/tgrep-core/src/builder.rs +++ b/tgrep-core/src/builder.rs @@ -457,6 +457,13 @@ fn read_for_index( owned_budget: &std::sync::Arc, configured_limit: Option, ) -> std::io::Result { + let owned_limit = configured_limit.unwrap_or(u64::MAX); + if size > owned_limit { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("file exceeds the configured {} byte limit", owned_limit), + )); + } if size >= MMAP_MIN_BYTES { // SAFETY: the map is read-only and owned by the returned value, which // is dropped once the file's trigrams have been extracted. A concurrent @@ -466,16 +473,13 @@ fn read_for_index( let mapped = std::fs::File::open(path).and_then(|file| unsafe { memmap2::Mmap::map(&file) }); if let Ok(map) = mapped { - return Ok(FileBytes::Mapped(map)); + if map.len() as u64 <= owned_limit { + return Ok(FileBytes::Mapped(map)); + } + // The file grew between the walk's stat and the map. Do not let a + // successful mmap bypass a caller-supplied max-file-size limit. } } - let owned_limit = configured_limit.unwrap_or(u64::MAX); - if size > owned_limit { - return Err(std::io::Error::new( - std::io::ErrorKind::InvalidData, - format!("file exceeds the configured {} byte limit", owned_limit), - )); - } let permit = owned_budget.acquire(size); match read_owned_for_index(path, size, owned_limit) { Ok(bytes) => Ok(FileBytes::Read { @@ -1601,6 +1605,23 @@ mod tests { assert_eq!(error.kind(), std::io::ErrorKind::InvalidData); } + #[test] + fn configured_limit_is_checked_before_mmap() { + let tmp = tempfile::tempdir().unwrap(); + let path = tmp.path().join("too-large.txt"); + std::fs::File::create(&path) + .unwrap() + .set_len(MMAP_MIN_BYTES) + .unwrap(); + let budget = OwnedReadBudget::new(INDEX_BUILD_BATCH_BYTES); + + let error = match read_for_index(&path, MMAP_MIN_BYTES, &budget, Some(MMAP_MIN_BYTES - 1)) { + Ok(_) => panic!("configured max-file-size must be checked before mmap"), + Err(error) => error, + }; + assert_eq!(error.kind(), std::io::ErrorKind::InvalidData); + } + #[test] fn owned_read_reports_growth_beyond_its_reservation() { let tmp = tempfile::tempdir().unwrap(); From 108ce828f1dcc01456020fa7f6e6a54109a04305 Mon Sep 17 00:00:00 2001 From: Shengyu Fu Date: Sat, 29 Aug 2026 22:07:47 -0500 Subject: [PATCH 13/16] Fix clippy warning in mmap limit check Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 96b9f8ae-c5ec-450c-a003-cb9ade61d9df --- tgrep-core/src/builder.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tgrep-core/src/builder.rs b/tgrep-core/src/builder.rs index 9993c0d..8dcfdd8 100644 --- a/tgrep-core/src/builder.rs +++ b/tgrep-core/src/builder.rs @@ -472,13 +472,13 @@ fn read_for_index( // accepts. let mapped = std::fs::File::open(path).and_then(|file| unsafe { memmap2::Mmap::map(&file) }); - if let Ok(map) = mapped { - if map.len() as u64 <= owned_limit { - return Ok(FileBytes::Mapped(map)); - } - // The file grew between the walk's stat and the map. Do not let a - // successful mmap bypass a caller-supplied max-file-size limit. + if let Ok(map) = mapped + && map.len() as u64 <= owned_limit + { + return Ok(FileBytes::Mapped(map)); } + // The file grew between the walk's stat and the map. Do not let a + // successful mmap bypass a caller-supplied max-file-size limit. } let permit = owned_budget.acquire(size); match read_owned_for_index(path, size, owned_limit) { From bb5283043caf764d7f15160e9d63429dd5be400c Mon Sep 17 00:00:00 2001 From: Shengyu Fu Date: Sat, 29 Aug 2026 22:16:15 -0500 Subject: [PATCH 14/16] Skip files that race past size limit Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 96b9f8ae-c5ec-450c-a003-cb9ade61d9df --- tgrep-core/src/builder.rs | 59 ++++++++++++++++++++++++++------------- 1 file changed, 39 insertions(+), 20 deletions(-) diff --git a/tgrep-core/src/builder.rs b/tgrep-core/src/builder.rs index 8dcfdd8..9759418 100644 --- a/tgrep-core/src/builder.rs +++ b/tgrep-core/src/builder.rs @@ -317,6 +317,19 @@ fn batch_sizes_and_charges(files: &[std::path::PathBuf]) -> (Vec, Vec, +) -> Vec { + files + .iter() + .filter(|path| !excluded.contains(*path)) + .filter_map(|path| path.strip_prefix(root).ok()) + .map(|path| path.to_string_lossy().replace('\\', "/")) + .collect() +} + /// Smallest file worth memory-mapping instead of reading. /// /// The same tradeoff the search path makes: mapping costs a syscall pair and a @@ -641,11 +654,11 @@ pub fn build_index_with_options_and_ignorecase( // The walk already stats every entry but discards the size, so recover it // here rather than widening WalkResult into the search and serve paths. let (sizes, charges) = batch_sizes_and_charges(&walk.files); + let raced_too_large = std::sync::Mutex::new(std::collections::HashSet::new()); for range in batch_ranges(&charges, INDEX_BUILD_BATCH_BYTES) { let batch = &walk.files[range.clone()]; let batch_sizes = &sizes[range]; - let owned_limit_error = std::sync::Mutex::new(None); let owned_budget = OwnedReadBudget::new(INDEX_BUILD_BATCH_BYTES); let batch_data: Vec = batch .par_iter() @@ -655,10 +668,7 @@ pub fn build_index_with_options_and_ignorecase( Ok(data) => data, Err(error) => { if error.kind() == std::io::ErrorKind::InvalidData { - let mut first = owned_limit_error.lock().unwrap(); - if first.is_none() { - *first = Some((path.clone(), error)); - } + raced_too_large.lock().unwrap().insert(path.clone()); } return None; } @@ -677,12 +687,6 @@ pub fn build_index_with_options_and_ignorecase( Some((rel, per_tri)) }) .collect(); - if let Some((path, error)) = owned_limit_error.into_inner().unwrap() { - return Err(Error::Io(std::io::Error::new( - error.kind(), - format!("{}: {error}", path.display()), - ))); - } for (path, per_tri) in batch_data { let file_id = file_id_map.len() as u32; @@ -722,15 +726,17 @@ pub fn build_index_with_options_and_ignorecase( } } - // Write per-file stamps for ALL walked files (including those later - // rejected as binary-by-content) so the stale check on next startup - // won't re-process unchanged files that aren't in the index. - let all_walked: Vec = walk - .files - .iter() - .filter_map(|p| p.strip_prefix(&root).ok()) - .map(|p| p.to_string_lossy().replace('\\', "/")) - .collect(); + // Write per-file stamps for walked files, including those later rejected + // as binary-by-content. A file that raced past max-file-size is different: + // withholding its stamp leaves it eligible for a later retry. + let raced_too_large = raced_too_large.into_inner().unwrap(); + if !raced_too_large.is_empty() { + eprintln!( + "Skipped {} files that grew past max-file-size during extraction", + raced_too_large.len() + ); + } + let all_walked = walked_paths_for_stamps(&root, &walk.files, &raced_too_large); let stamps = meta::collect_filestamps(&root, &all_walked); meta::write_filestamps(&stamps, &index_dir)?; @@ -1518,6 +1524,19 @@ mod tests { sizes.iter().copied().map(batch_charge).collect() } + #[test] + fn raced_oversized_files_are_excluded_from_published_stamps() { + let root = std::path::PathBuf::from("repo"); + let kept = root.join("kept.rs"); + let skipped = root.join("grew.rs"); + let excluded = std::collections::HashSet::from([skipped.clone()]); + + assert_eq!( + walked_paths_for_stamps(&root, &[kept, skipped], &excluded), + vec!["kept.rs"] + ); + } + #[test] fn batches_are_bounded_by_cumulative_heap_bytes() { // Files below the mapping threshold are read, so their whole length is From 0706f585a23817bf80a9781176c74870da8898e8 Mon Sep 17 00:00:00 2001 From: Shengyu Fu Date: Sun, 30 Aug 2026 06:04:37 -0500 Subject: [PATCH 15/16] Preserve staging after rollback failure Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 96b9f8ae-c5ec-450c-a003-cb9ade61d9df --- tgrep-cli/src/serve.rs | 48 +++++++++++++++++++++++++++++------------- 1 file changed, 33 insertions(+), 15 deletions(-) diff --git a/tgrep-cli/src/serve.rs b/tgrep-cli/src/serve.rs index 6adee7d..d71cb8e 100644 --- a/tgrep-cli/src/serve.rs +++ b/tgrep-cli/src/serve.rs @@ -4900,7 +4900,7 @@ fn stream_merge_stale_changes( let mut published_stamps = stamps.clone(); - let result = (|| -> Result { + let result = (|| -> Result { let build = || { builder::build_index_for_files( root, @@ -4970,7 +4970,7 @@ fn stream_merge_stale_changes( .count(); let expected_files = reader.num_files() - removed_reader_files + delta.num_files(); let published = publish_staged_index(state, index_dir, &staging_dir, expected_files); - if published { + if published.is_published() { // `publish_staged_index` prunes overlay entries represented by the // new reader. Also clear reconciled entries intentionally omitted // (newly ignored/binary/deleted) and old tombstones for files the @@ -4992,13 +4992,17 @@ fn stream_merge_stale_changes( })(); let _ = std::fs::remove_dir_all(&delta_dir); - let _ = std::fs::remove_dir_all(&staging_dir); + if !matches!(&result, Ok(PublishStatus::RollbackFailed)) { + let _ = std::fs::remove_dir_all(&staging_dir); + } else { + eprintln!("[trace] warning: preserving {staging_dir:?} after rollback failure"); + } state.flushing.store(false, Ordering::SeqCst); - if matches!(&result, Ok(true)) { + if matches!(&result, Ok(PublishStatus::Published)) { *state.file_stamps.write().unwrap() = published_stamps; } match result { - Ok(true) => { + Ok(PublishStatus::Published) => { eprintln!( "[trace] {operation}: streamed {} changes into the index in {:.1}s", candidates.len(), @@ -5006,7 +5010,7 @@ fn stream_merge_stale_changes( ); true } - Ok(false) => { + Ok(PublishStatus::Failed) | Ok(PublishStatus::RollbackFailed) => { eprintln!( "[trace] warning: {operation} delta could not be published; \ keeping the old index" @@ -6022,7 +6026,7 @@ fn flush_append_only_overlay_locked( eprintln!("[trace] warning: failed to write staging filestamps: {e}"); } - let pruned = publish_staged_index(state, index_dir, &staging_dir, num_files); + let pruned = publish_staged_index(state, index_dir, &staging_dir, num_files).is_published(); eprintln!( "[trace] append-only flush: {num_files} files on disk (complete={complete}) in {:.1}s", flush_start.elapsed().as_secs_f64() @@ -6040,14 +6044,27 @@ fn flush_append_only_overlay_locked( /// renames or swap readers out of order. `num_files` is the expected on-disk /// file count used to reject a partially-published reader. /// -/// Returns `true` when the swap + prune succeeded, `false` on any failure (the -/// previous reader and the live overlay are retained as the fallback). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum PublishStatus { + Published, + Failed, + RollbackFailed, +} + +impl PublishStatus { + fn is_published(self) -> bool { + matches!(self, Self::Published) + } +} + +/// Returns the publication outcome. A rollback failure preserves the staging +/// directory so its backups remain available for recovery. fn publish_staged_index( state: &ServerState, index_dir: &Path, staging_dir: &Path, num_files: usize, -) -> bool { +) -> PublishStatus { // Held across move + open + swap so concurrent publishers (auto-save / // background-build / watcher reindex flush) cannot interleave renames // or swap readers out of order. Searches do not take this lock. @@ -6056,21 +6073,22 @@ fn publish_staged_index( Ok(moved) => moved, Err(e) => { eprintln!("[trace] warning: flush move failed: {e}"); - return false; + return PublishStatus::Failed; } }; let Some((new_reader, reader_files, reader_trigrams)) = open_published_reader(index_dir, num_files) else { - match moved.rollback() { + return match moved.rollback() { Ok(()) => { let _ = std::fs::remove_dir_all(staging_dir); + PublishStatus::Failed } Err(e) => { eprintln!("[trace] warning: failed to roll back index publication: {e}"); + PublishStatus::RollbackFailed } - } - return false; + }; }; // Atomic swap — no outer write lock required. @@ -6087,7 +6105,7 @@ fn publish_staged_index( {reader_trigrams} trigrams), overlay pruned" ); let _ = std::fs::remove_dir_all(staging_dir); - true + PublishStatus::Published } fn publish_reloaded_index( From 361f7b08d4930ec138e666339d842745b0e8cfb1 Mon Sep 17 00:00:00 2001 From: Shengyu Fu Date: Sun, 30 Aug 2026 06:08:04 -0500 Subject: [PATCH 16/16] Preserve backups after move rollback failure Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 96b9f8ae-c5ec-450c-a003-cb9ade61d9df --- tgrep-cli/src/serve.rs | 94 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 89 insertions(+), 5 deletions(-) diff --git a/tgrep-cli/src/serve.rs b/tgrep-cli/src/serve.rs index d71cb8e..e96aeb0 100644 --- a/tgrep-cli/src/serve.rs +++ b/tgrep-cli/src/serve.rs @@ -6073,7 +6073,11 @@ fn publish_staged_index( Ok(moved) => moved, Err(e) => { eprintln!("[trace] warning: flush move failed: {e}"); - return PublishStatus::Failed; + return if e.rollback_failed() { + PublishStatus::RollbackFailed + } else { + PublishStatus::Failed + }; } }; let Some((new_reader, reader_files, reader_trigrams)) = @@ -6085,6 +6089,7 @@ fn publish_staged_index( PublishStatus::Failed } Err(e) => { + moved.preserve(); eprintln!("[trace] warning: failed to roll back index publication: {e}"); PublishStatus::RollbackFailed } @@ -6130,6 +6135,7 @@ fn publish_reloaded_index( let _ = std::fs::remove_dir_all(staging_dir); } Err(e) => { + moved.preserve(); eprintln!("[trace] warning: failed to roll back reload publication: {e}"); } } @@ -6278,6 +6284,20 @@ impl StagedFileMove { fn commit(&mut self) { self.finished = true; } + + fn preserve(&mut self) { + // Leave every remaining backup and published file exactly where the + // failed rollback left it so an operator or later recovery can use it. + self.finished = true; + } + + fn fail(mut self, publish: std::io::Error) -> MoveStagedFilesError { + let rollback = self.rollback().err(); + if rollback.is_some() { + self.preserve(); + } + MoveStagedFilesError { publish, rollback } + } } impl Drop for StagedFileMove { @@ -6288,6 +6308,34 @@ impl Drop for StagedFileMove { } } +#[derive(Debug)] +struct MoveStagedFilesError { + publish: std::io::Error, + rollback: Option, +} + +impl MoveStagedFilesError { + fn rollback_failed(&self) -> bool { + self.rollback.is_some() + } +} + +impl std::fmt::Display for MoveStagedFilesError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.publish)?; + if let Some(rollback) = &self.rollback { + write!(f, "; rollback also failed: {rollback}")?; + } + Ok(()) + } +} + +impl std::error::Error for MoveStagedFilesError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(&self.publish) + } +} + /// Move index files from staging to the target directory, retaining backups /// until the caller validates and commits the new reader. /// @@ -6305,8 +6353,14 @@ impl Drop for StagedFileMove { /// created next to the target (same parent) so cross-volume cases should /// not arise; if rename truly fails, the error is surfaced rather than /// silently falling back to a slow copy (see `publish_file`). -fn move_staged_files(staging: &Path, target: &Path) -> std::io::Result { - std::fs::create_dir_all(target)?; +fn move_staged_files( + staging: &Path, + target: &Path, +) -> Result { + std::fs::create_dir_all(target).map_err(|publish| MoveStagedFilesError { + publish, + rollback: None, + })?; let mut moved = StagedFileMove { staging: staging.to_path_buf(), target: target.to_path_buf(), @@ -6321,10 +6375,14 @@ fn move_staged_files(staging: &Path, target: &Path) -> std::io::Result