From 139a75197e10d3413ff443ffe06012ab03558448 Mon Sep 17 00:00:00 2001 From: Shengyu Fu Date: Wed, 26 Aug 2026 20:50:43 -0700 Subject: [PATCH 01/28] Refresh the watcher's ignore rules when a `.ignore` file changes `is_ignore_rules_file` decides whether a filesystem event should rebuild and republish the matcher in `ServerState::gitignore`. It recognized `.gitignore` at any depth and root-level `p4ignore.ini`, but not `.ignore`. `.ignore` is a first-class ignore source everywhere else: the walk collects it separately, the matcher applies it, and it even outranks `.gitignore`. Unlike `.gitignore` it is not git-gated, so it is the one source that works outside a repository. Because the event never matched, a `.ignore` written while the server was live never scheduled the refresh, and the write then fell through to the reindex path where `should_skip_watcher_path` drops any dot-prefixed segment. Nothing happened at all: the startup matcher stayed published and files under the newly excluded directory kept being indexed until the hourly reconcile or a restart. Match `.ignore` by file name alongside `.gitignore`, using the existing `tgrep_core::gitignore` filename constants. `p4ignore.ini` stays root-scoped, mirroring the walker. Fixes #104 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tgrep-cli/src/serve.rs | 27 +++++- tgrep-cli/tests/watcher_dot_ignore.rs | 121 ++++++++++++++++++++++---- 2 files changed, 130 insertions(+), 18 deletions(-) diff --git a/tgrep-cli/src/serve.rs b/tgrep-cli/src/serve.rs index 9ab5549..47df660 100644 --- a/tgrep-cli/src/serve.rs +++ b/tgrep-cli/src/serve.rs @@ -1656,9 +1656,24 @@ fn should_skip_watcher_path( false } +/// Whether a changed path is an ignore-rules source, i.e. one whose contents +/// feed the matcher published in `ServerState::gitignore`. +/// +/// `.gitignore` and `.ignore` are matched by file name at any depth, because +/// [`tgrep_core::gitignore::matcher_from_ignore_paths`] anchors nested files of +/// both kinds. `.ignore` must be included even though it is git-agnostic — +/// leaving it out meant a `.ignore` written while the server was live never +/// refreshed the matcher, so the watcher kept indexing files the rule excluded. +/// +/// `p4ignore.ini` stays root-scoped, mirroring the walker, which only reads the +/// root-level file. fn is_ignore_rules_file(root: &Path, path: &Path) -> bool { - path.file_name().and_then(|name| name.to_str()) == Some(".gitignore") - || path == root.join(tgrep_core::gitignore::P4IGNORE_FILENAME) + let name = path.file_name().and_then(|name| name.to_str()); + matches!( + name, + Some(tgrep_core::gitignore::GITIGNORE_FILENAME) + | Some(tgrep_core::gitignore::DOT_IGNORE_FILENAME) + ) || path == root.join(tgrep_core::gitignore::P4IGNORE_FILENAME) } fn schedule_ignore_rules_refresh(state: Arc, root: PathBuf) { @@ -3672,6 +3687,14 @@ mod tests { root, Path::new("workspace/p4ignore.ini") )); + // `.ignore` is a first-class ignore source (and, unlike `.gitignore`, + // applies outside a git repo), so live edits to it must refresh the + // matcher too — at the root and nested. + assert!(is_ignore_rules_file(root, Path::new("workspace/.ignore"))); + assert!(is_ignore_rules_file( + root, + Path::new("workspace/nested/.ignore") + )); assert!(!is_ignore_rules_file( root, Path::new("workspace/nested/p4ignore.ini") diff --git a/tgrep-cli/tests/watcher_dot_ignore.rs b/tgrep-cli/tests/watcher_dot_ignore.rs index 652141c..914dc5d 100644 --- a/tgrep-cli/tests/watcher_dot_ignore.rs +++ b/tgrep-cli/tests/watcher_dot_ignore.rs @@ -101,22 +101,21 @@ fn wait_for_match(port: u16, pattern: &str, timeout: Duration) -> bool { } } -#[test] -fn watcher_honors_dot_ignore_and_still_indexes_new_files() { - let dir = TempDir::new().unwrap(); - let root = dir.path(); - let index_dir = root.join(".tgrep_test_index"); - - // No `.git` here: `.ignore` must apply on its own. - fs::write(root.join(".ignore"), "secret/\n").unwrap(); - fs::create_dir_all(root.join("secret")).unwrap(); - fs::create_dir_all(root.join("src")).unwrap(); - fs::write( - root.join("src").join("lib.rs"), - "fn seeded() { let normal_source_marker = 1; }\n", - ) - .unwrap(); +/// Poll until `pattern` stops being searchable, returning whether it went away. +fn wait_for_no_match(port: u16, pattern: &str, timeout: Duration) -> bool { + let start = Instant::now(); + loop { + if search_matches(port, pattern) == 0 { + return true; + } + if start.elapsed() > timeout { + return false; + } + thread::sleep(Duration::from_millis(100)); + } +} +fn build_index(root: &Path, index_dir: &Path) { let status = Command::new(tgrep_bin()) .args([ "index", @@ -127,7 +126,9 @@ fn watcher_honors_dot_ignore_and_still_indexes_new_files() { .status() .expect("failed to run tgrep index"); assert!(status.success(), "initial index build failed"); +} +fn spawn_server(root: &Path, index_dir: &Path) -> ServerGuard { let child = Command::new(tgrep_bin()) .args([ "serve", @@ -139,7 +140,27 @@ fn watcher_honors_dot_ignore_and_still_indexes_new_files() { .stdout(std::process::Stdio::null()) .spawn() .expect("failed to start tgrep serve"); - let _server = ServerGuard { child }; + ServerGuard { child } +} + +#[test] +fn watcher_honors_dot_ignore_and_still_indexes_new_files() { + let dir = TempDir::new().unwrap(); + let root = dir.path(); + let index_dir = root.join(".tgrep_test_index"); + + // No `.git` here: `.ignore` must apply on its own. + fs::write(root.join(".ignore"), "secret/\n").unwrap(); + fs::create_dir_all(root.join("secret")).unwrap(); + fs::create_dir_all(root.join("src")).unwrap(); + fs::write( + root.join("src").join("lib.rs"), + "fn seeded() { let normal_source_marker = 1; }\n", + ) + .unwrap(); + + build_index(root, &index_dir); + let _server = spawn_server(root, &index_dir); let port = wait_for_port(&index_dir); @@ -176,3 +197,71 @@ fn watcher_honors_dot_ignore_and_still_indexes_new_files() { "watcher indexed a file under a directory excluded by .ignore" ); } + +/// A `.ignore` written *after* the server is live must refresh the ignore +/// rules, exactly as a `.gitignore` write does. +/// +/// The startup matcher is built from the walk that the initial index used, so +/// a `.ignore` that already exists is honored for free — which is what the +/// test above covers. Rules that appear later only take effect if the watcher +/// recognizes the `.ignore` write as an ignore-rules change and schedules the +/// refresh; when it does not, the stale matcher stays published and the +/// already-indexed content under the newly excluded directory stays +/// searchable indefinitely (the periodic reconcile is on an hourly timer). +/// +/// The fixture seeds the ignored file *before* indexing, so the assertion is a +/// transition — searchable, then not — rather than a fixed sleep racing the +/// refresh. +#[test] +fn late_dot_ignore_refreshes_the_watchers_ignore_rules() { + let dir = TempDir::new().unwrap(); + let root = dir.path(); + let index_dir = root.join(".tgrep_test_index"); + + // No `.git` here either: `.ignore` is not git-gated, so this pins the + // refresh path for the one ignore source that works outside a repo. + fs::create_dir_all(root.join("secret")).unwrap(); + fs::create_dir_all(root.join("src")).unwrap(); + fs::write( + root.join("secret").join("creds.txt"), + "late_ignored_leak_marker\n", + ) + .unwrap(); + fs::write( + root.join("src").join("lib.rs"), + "fn seeded() { let normal_source_marker = 1; }\n", + ) + .unwrap(); + + build_index(root, &index_dir); + let _server = spawn_server(root, &index_dir); + + let port = wait_for_port(&index_dir); + + assert!( + wait_for_match(port, "normal_source_marker", Duration::from_secs(30)), + "expected the seeded source file to be searchable" + ); + // Positive control: with no `.ignore` yet, the seeded file under `secret/` + // is legitimately indexed. Without this the assertion below could pass + // simply because the file was never indexed in the first place. + assert!( + wait_for_match(port, "late_ignored_leak_marker", Duration::from_secs(30)), + "expected the file under secret/ to be indexed before any .ignore exists" + ); + thread::sleep(Duration::from_secs(2)); + + fs::write(root.join(".ignore"), "secret/\n").unwrap(); + + assert!( + wait_for_no_match(port, "late_ignored_leak_marker", Duration::from_secs(60)), + "a .ignore written while the server was live never refreshed the ignore \ + rules; content under the newly excluded directory is still searchable" + ); + + // The refresh must not take the rest of the index with it. + assert!( + search_matches(port, "normal_source_marker") > 0, + "the ignore-rules refresh dropped a file that is not ignored" + ); +} From 64ec4c325bf0cc859e31fe797c0ed39d9f97d215 Mon Sep 17 00:00:00 2001 From: Shengyu Fu Date: Wed, 26 Aug 2026 21:13:30 -0700 Subject: [PATCH 02/28] Stop the watcher from subscribing to ignored directories The watcher already discarded events for ignored paths, but only after the OS had delivered them. On Linux that is too late to matter: inotify has no recursive mode, so notify's `RecursiveMode::Recursive` walks the tree and spends one watch descriptor per directory. A repository whose `target/` or `node_modules/` holds most of its directories therefore burns most of the per-user `fs.inotify.max_user_watches` budget on events that are thrown away -- and because notify propagates the first registration failure, exhausting that budget makes `watch()` return an error and the server loses its watcher entirely. Subscribe per directory on inotify backends instead. `watchable_dirs` walks the tree once, pruning ignored, hidden and `--exclude`d subtrees before descending, and `WatchRegistry::sync` reconciles the live subscription set against it. The sync runs whenever the ignore matcher is published, so relaxing a rule subscribes to the tree it used to hide and tightening one drops it. A directory that cannot be subscribed is now reported and skipped rather than taking down the whole watcher. Non-recursive watches are not extended by notify, so a directory that appears at runtime is picked up in `watch_new_subtree`, which also indexes the files already inside it to close the create race. Windows (ReadDirectoryChangesW) and macOS (FSEvents) subscribe once for the whole subtree, so there is no per-directory registration to withhold; they keep the single recursive watch and delivery-time filtering. The behaviour the two paths must share -- new directories get indexed, new directories under an ignored path do not -- is tested everywhere. Along the way, `state.gitignore` had three publish sites and only one of them went through the helper. They are unified behind `publish_ignore_matcher` so the sync hook cannot be missed. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: bcc85b55-e821-447e-b9a9-144b78829ae4 --- tgrep-cli/src/serve.rs | 643 +++++++++++++++--- tgrep-cli/tests/watcher_watch_registration.rs | 308 +++++++++ 2 files changed, 862 insertions(+), 89 deletions(-) create mode 100644 tgrep-cli/tests/watcher_watch_registration.rs diff --git a/tgrep-cli/src/serve.rs b/tgrep-cli/src/serve.rs index 47df660..93fd52f 100644 --- a/tgrep-cli/src/serve.rs +++ b/tgrep-cli/src/serve.rs @@ -296,6 +296,12 @@ struct ServerState { index_total: std::sync::atomic::AtomicU64, /// True when file watching is enabled for this server. watch_enabled: bool, + /// The live watcher and the directories it is subscribed to. + /// + /// Held here rather than by `run` because the subscription set is not + /// fixed: publishing a new ignore matcher renarrows it, and a directory + /// created after startup has to be subscribed as it appears. + watch_registry: Mutex>, /// Directories to exclude from indexing. exclude_dirs: Vec, /// Disable all source-control ignore files for every server discovery path. @@ -570,6 +576,7 @@ pub fn run(root: &Path, index_path: Option<&Path>, options: ServeOptions<'_>) -> index_progress: std::sync::atomic::AtomicU64::new(0), index_total: std::sync::atomic::AtomicU64::new(0), watch_enabled: !no_watch, + watch_registry: Mutex::new(None), exclude_dirs: exclude_dirs.to_vec(), no_ignore, no_require_git, @@ -656,14 +663,13 @@ pub fn run(root: &Path, index_path: Option<&Path>, options: ServeOptions<'_>) -> } // Start file watcher (unless --no-watch) - let _watcher = if no_watch { + if no_watch { eprintln!("[trace] file watcher disabled (--no-watch)"); - None } else { let watcher_state = Arc::clone(&state); let watcher_root = root.clone(); - start_file_watcher(watcher_state, &watcher_root, watcher_queue_cap) - }; + start_file_watcher(watcher_state, &watcher_root, watcher_queue_cap); + } // Set up graceful shutdown let shutdown_index_dir = index_dir.clone(); @@ -744,14 +750,23 @@ fn build_stale_matcher( matcher } -/// Commit matcher and index semantics together while the stale refresh holds -/// `snapshot_gate`. `None` is a legitimate matcher when no rules exist. -fn commit_stale_matcher( +/// Publish a new ignore matcher and bring everything that depends on it up to +/// date. `None` is a legitimate matcher when no rules exist. +/// +/// Callers on the stale path hold `snapshot_gate` for write, which is what +/// makes the matcher swap and the index decisions around it atomic from the +/// watcher's point of view. +fn publish_ignore_matcher( state: &ServerState, + root: &Path, matcher: Option, ) { *state.gitignore.write().unwrap() = matcher; state.gitignore_pending.store(false, Ordering::SeqCst); + // New rules mean a different set of directories worth hearing about: + // a tightened rule releases the subscriptions under it, and a relaxed + // one takes subscriptions for the tree it used to hide. + sync_watch_registrations(state, root); } fn handle_connection(stream: TcpStream, state: &ServerState) -> Result<()> { @@ -1482,11 +1497,7 @@ fn handle_reload(id: Option, state: &ServerState) -> String { } } -fn start_file_watcher( - state: Arc, - root: &Path, - queue_cap: usize, -) -> Option { +fn start_file_watcher(state: Arc, root: &Path, queue_cap: usize) -> bool { use std::sync::mpsc::{RecvTimeoutError, TrySendError}; let root_path = root.to_path_buf(); @@ -1525,19 +1536,46 @@ fn start_file_watcher( Ok(w) => w, Err(e) => { eprintln!("[trace] warning: failed to start file watcher: {e}"); - return None; + return false; } }; - if let Err(e) = watcher.watch(root, RecursiveMode::Recursive) { + // The root is always subscribed. On a whole-subtree backend that single + // recursive subscription is the entire watch set; on a per-directory + // backend it is the anchor, and `sync_watch_registrations` below adds the + // descendants the ignore rules allow. + let root_mode = if PER_DIRECTORY_WATCHES { + RecursiveMode::NonRecursive + } else { + RecursiveMode::Recursive + }; + if let Err(e) = watcher.watch(root, root_mode) { eprintln!("[trace] warning: failed to watch directory: {e}"); - return None; + return false; + } + + *state.watch_registry.lock().unwrap() = Some(WatchRegistry { + watcher, + watched: std::iter::once(root.to_path_buf()).collect(), + }); + + // Subscribing to the descendants needs the ignore matcher, and on a warm + // start it is still being built on another thread. Skipping the sync here + // costs nothing: events are dropped while `gitignore_pending` is set, and + // the publish that clears it runs this same sync. Subscribing first and + // narrowing afterwards would mean briefly holding exactly the watches this + // is meant to avoid — on a repo big enough to exhaust the inotify budget, + // long enough to fail. + if state.gitignore_pending.load(Ordering::SeqCst) { + eprintln!("[trace] watcher subscriptions deferred until the ignore matcher is ready"); + } else { + sync_watch_registrations(&state, root); } let worker_state = Arc::clone(&state); let worker_root = root_path; let worker_index_dir = state.index_dir.clone(); - std::thread::Builder::new() + if std::thread::Builder::new() .name("tgrep-watcher".into()) .spawn(move || { loop { @@ -1579,14 +1617,19 @@ fn start_file_watcher( } } }) - .ok()?; + .is_err() + { + eprintln!("[trace] warning: failed to start the watcher worker thread"); + *state.watch_registry.lock().unwrap() = None; + return false; + } state .watcher_active .store(true, std::sync::atomic::Ordering::Relaxed); eprintln!("[trace] file watcher started"); - Some(watcher) + true } /// Decide whether the file watcher should skip a path entirely. @@ -1617,6 +1660,30 @@ fn should_skip_watcher_path( rel_path: &str, exclude_dirs: &[String], gitignore: Option<&tgrep_core::gitignore::IgnoreMatcher>, +) -> bool { + should_skip_watcher_entry(rel_path, exclude_dirs, gitignore, false) +} + +/// [`should_skip_watcher_path`] for a path known to be a directory. +/// +/// Two rules read differently for a directory. `--exclude` names apply to the +/// final segment as well, because the walker drops the whole subtree when the +/// entry it is looking at *is* the excluded directory. And the gitignore +/// matcher is told it is matching a directory, so a directory-only rule like +/// `build/` matches — as a file path, `build` does not. +fn should_skip_watcher_dir( + rel_path: &str, + exclude_dirs: &[String], + gitignore: Option<&tgrep_core::gitignore::IgnoreMatcher>, +) -> bool { + should_skip_watcher_entry(rel_path, exclude_dirs, gitignore, true) +} + +fn should_skip_watcher_entry( + rel_path: &str, + exclude_dirs: &[String], + gitignore: Option<&tgrep_core::gitignore::IgnoreMatcher>, + is_dir: bool, ) -> bool { // Single streaming pass over path components — no Vec allocation // on the hot watcher path. The hidden-component check applies to @@ -1632,23 +1699,21 @@ fn should_skip_watcher_path( if seg.starts_with('.') { return true; } - // Ancestor (i.e. not the last segment) — apply exclude_dirs. - if segments.peek().is_some() - && !exclude_dirs.is_empty() - && exclude_dirs.iter().any(|d| d == seg) - { + // An ancestor is always a directory; the final segment is one only + // when the caller says so. + let segment_is_dir = segments.peek().is_some() || is_dir; + if segment_is_dir && !exclude_dirs.is_empty() && exclude_dirs.iter().any(|d| d == seg) { return true; } } // Gitignore check (if a matcher is available). if let Some(gi) = gitignore { - // We don't know whether the path is a dir or a file here — for - // the watcher's purposes we treat all events as "file" matches. - // Notify usually fires per-file events anyway, and gitignore - // rules that target dirs would have already skipped the dir's - // contents via `matched_path_or_any_parents`. - if gi.is_ignored(Path::new(rel_path), false) { + // For a file event we don't know whether the path is a dir, so we + // treat it as a file. Notify usually fires per-file events anyway, + // and gitignore rules that target dirs would have already skipped + // the dir's contents via `matched_path_or_any_parents`. + if gi.is_ignored(Path::new(rel_path), is_dir) { return true; } } @@ -1676,6 +1741,238 @@ fn is_ignore_rules_file(root: &Path, path: &Path) -> bool { ) || path == root.join(tgrep_core::gitignore::P4IGNORE_FILENAME) } +/// Whether this platform's `notify` backend takes one OS subscription per +/// directory rather than a single recursive one for the whole tree. +/// +/// inotify has no recursive mode. `RecursiveMode::Recursive` makes notify walk +/// the tree itself and spend one watch descriptor per directory, so every +/// ignored directory costs a descriptor from the per-user +/// `fs.inotify.max_user_watches` budget purely to deliver events we then throw +/// away. Worse, notify's registration loop propagates the first failure, so a +/// repo whose ignored build output exhausts the budget makes `watch()` return +/// an error and the server loses its watcher entirely. +/// +/// ReadDirectoryChangesW (Windows) and FSEvents (macOS) subscribe once for the +/// whole subtree, so there is no per-directory registration to withhold. On +/// those platforms filtering on delivery is the only lever available, and +/// [`should_skip_watcher_path`] remains it. +/// +/// Deliberately limited to the backends we can exercise in CI. kqueue and +/// `PollWatcher` are per-path too, but nothing here builds or tests them. +const PER_DIRECTORY_WATCHES: bool = cfg!(any(target_os = "linux", target_os = "android")); + +/// The watcher plus the set of directories it is currently subscribed to. +/// +/// Only meaningful when [`PER_DIRECTORY_WATCHES`] is true; elsewhere `watched` +/// holds just the root, which is subscribed recursively. +struct WatchRegistry { + watcher: RecommendedWatcher, + watched: std::collections::HashSet, +} + +impl WatchRegistry { + /// Bring the subscription set in line with `desired`, subscribing to + /// directories that are newly relevant and dropping ones that are not. + /// + /// Returns `(added, removed)`. A single directory that cannot be + /// subscribed is reported and skipped rather than failing the sync: the + /// watcher is still useful for everything else, and giving up on the whole + /// tree is exactly the failure mode this registration exists to avoid. + fn sync(&mut self, desired: &std::collections::HashSet) -> (usize, usize) { + let stale: Vec = self.watched.difference(desired).cloned().collect(); + let mut removed = 0; + for dir in stale { + // Best effort. inotify drops a descriptor by itself when the + // directory is deleted, so "not found" is an expected outcome + // here, not an error worth reporting. + let _ = self.watcher.unwatch(&dir); + self.watched.remove(&dir); + removed += 1; + } + + let missing: Vec = desired.difference(&self.watched).cloned().collect(); + let mut added = 0; + let mut failures = 0; + for dir in missing { + match self.watcher.watch(&dir, RecursiveMode::NonRecursive) { + Ok(()) => { + self.watched.insert(dir); + added += 1; + } + Err(e) => { + // One line per sync, not per directory: exhausting the + // inotify budget fails thousands of these at once. + if failures == 0 { + eprintln!( + "[trace] warning: could not watch {}: {e} \ + (continuing with the directories that succeeded)", + dir.display() + ); + } + failures += 1; + } + } + } + if failures > 1 { + eprintln!("[trace] warning: {failures} directories could not be watched"); + } + (added, removed) + } +} + +/// Every directory at or below `start` whose contents the watcher needs to +/// hear about. +/// +/// A per-directory subscription reports the files directly inside it, so the +/// set is "`start`, plus every descendant directory the indexer would walk +/// into". Ignored directories are pruned along with their subtrees, which is +/// the whole point: the tree under `target/` or `node_modules/` is usually +/// most of the directories in a repo. +/// +/// `root` is the repository root and is only used to build the relative paths +/// the ignore rules are written against; `start` is where the walk begins. +/// They differ when a subtree that appeared at runtime is being subscribed, +/// and conflating them would match every rule at the wrong anchor. +/// +/// `start` itself is always included — callers are responsible for not asking +/// about a directory that is ignored. +/// +/// Symlinked directories are not descended into, matching the walker (the +/// `ignore` crate does not follow links by default). That also keeps a +/// symlink cycle from turning this into an infinite walk. +fn watchable_dirs( + root: &Path, + start: &Path, + exclude_dirs: &[String], + gitignore: Option<&tgrep_core::gitignore::IgnoreMatcher>, +) -> std::collections::HashSet { + let mut found = std::collections::HashSet::new(); + found.insert(start.to_path_buf()); + + let mut stack = vec![start.to_path_buf()]; + while let Some(dir) = stack.pop() { + let Ok(entries) = std::fs::read_dir(&dir) else { + // An unreadable directory is not a reason to abandon the rest of + // the tree; the periodic reconcile is what catches what we miss. + continue; + }; + for entry in entries.flatten() { + if !entry.file_type().is_ok_and(|t| t.is_dir()) { + continue; + } + let path = entry.path(); + let Ok(rel) = path.strip_prefix(root) else { + continue; + }; + let rel = rel.to_string_lossy().replace('\\', "/"); + if should_skip_watcher_dir(&rel, exclude_dirs, gitignore) { + continue; + } + stack.push(path.clone()); + found.insert(path); + } + } + found +} + +/// Recompute the watcher's subscriptions against the ignore rules in force. +/// +/// Called when the watcher starts and every time the ignore matcher is +/// published, so relaxing a rule subscribes to the tree it used to hide and +/// tightening one drops it. +fn sync_watch_registrations(state: &ServerState, root: &Path) { + if !PER_DIRECTORY_WATCHES { + return; + } + let mut registry = state.watch_registry.lock().unwrap(); + let Some(registry) = registry.as_mut() else { + // The watcher has not started yet. It syncs once as it comes up, so + // there is nothing to do and nothing to remember. + return; + }; + + let start = Instant::now(); + let desired = { + let gitignore = state.gitignore.read().unwrap(); + watchable_dirs(root, root, &state.exclude_dirs, gitignore.as_ref()) + }; + let total = desired.len(); + let (added, removed) = registry.sync(&desired); + if added > 0 || removed > 0 { + eprintln!( + "[trace] watcher subscriptions: {total} directories \ + (+{added}, -{removed}) in {:.1}ms", + start.elapsed().as_secs_f64() * 1000.0 + ); + } +} + +/// Subscribe to a directory that has just appeared, and to anything already +/// inside it. +/// +/// Non-recursive subscriptions are not extended by notify — it only auto-adds +/// watches beneath a watch that was registered as recursive — so a new +/// directory has to be picked up here or its contents are invisible. +/// +/// Files that landed between the directory's creation and its subscription +/// would be missed by definition, so the same pass indexes what it finds. +/// The caller must already hold `snapshot_gate`. +fn watch_new_subtree(state: &ServerState, root: &Path, dir: &Path) { + let Ok(rel_dir) = dir.strip_prefix(root) else { + return; + }; + let rel_dir = rel_dir.to_string_lossy().replace('\\', "/"); + + let desired = { + let gitignore = state.gitignore.read().unwrap(); + // The event that brought us here was filtered with file semantics, so + // a `build/`-style rule that only ever matches directories has not + // been applied to this path yet. Re-check before subscribing to it. + if !rel_dir.is_empty() + && should_skip_watcher_dir(&rel_dir, &state.exclude_dirs, gitignore.as_ref()) + { + return; + } + watchable_dirs(root, dir, &state.exclude_dirs, gitignore.as_ref()) + }; + + { + let mut registry = state.watch_registry.lock().unwrap(); + let Some(registry) = registry.as_mut() else { + return; + }; + // A union, not a replacement: `desired` covers only this subtree, and + // `sync` would read anything outside it as newly stale and unsubscribe + // from the entire rest of the repository. + let union: std::collections::HashSet = + registry.watched.union(&desired).cloned().collect(); + registry.sync(&union); + } + + for subdir in &desired { + let Ok(entries) = std::fs::read_dir(subdir) else { + continue; + }; + for entry in entries.flatten() { + if !entry.file_type().is_ok_and(|t| t.is_file()) { + continue; + } + let path = entry.path(); + let Ok(rel) = path.strip_prefix(root) else { + continue; + }; + let rel = rel.to_string_lossy().replace('\\', "/"); + let skip = { + let gitignore = state.gitignore.read().unwrap(); + should_skip_watcher_path(&rel, &state.exclude_dirs, gitignore.as_ref()) + }; + if !skip { + reindex_file(state, &path, &rel); + } + } + } +} + fn schedule_ignore_rules_refresh(state: Arc, root: PathBuf) { if state .ignore_refresh_scheduled @@ -1713,8 +2010,6 @@ fn schedule_ignore_rules_refresh(state: Arc, root: PathBuf) { } fn handle_fs_event(state: &Arc, root: &Path, event: &Event) { - use tgrep_core::meta::FileStamp; - let dominated_kinds = matches!( event.kind, EventKind::Create(_) | EventKind::Modify(_) | EventKind::Remove(_) @@ -1810,66 +2105,83 @@ fn handle_fs_event(state: &Arc, root: &Path, event: &Event) { } if !path.is_file() { + if PER_DIRECTORY_WATCHES && path.is_dir() { + // With non-recursive subscriptions notify will not extend the + // watch set for us, so a directory that just appeared — and + // anything already inside it — has to be picked up here. + watch_new_subtree(state, root, path); + } continue; } - // Compute the file's current stamp and skip if it matches what we - // last indexed. notify on Windows in particular fires Modify events - // for atime/attribute updates, opens, etc. — re-indexing on those - // would re-read large files, churn the live overlay, and produce a - // misleading "modified" trace for files that didn't actually change. - let current = match std::fs::metadata(path) { - Ok(m) => FileStamp { - mtime: m - .modified() - .ok() - .and_then(|t| t.duration_since(std::time::SystemTime::UNIX_EPOCH).ok()) - .map(|d| d.as_secs()) - .unwrap_or(0), - size: m.len(), - }, - Err(_) => continue, - }; - if state.file_stamps.read().unwrap().get(&rel_path) == Some(¤t) { - continue; - } + reindex_file(state, path, &rel_path); + } +} - // Read contents and extract trigrams OUTSIDE the index write lock - // so a concurrent search (which needs a read lock) is not blocked - // on our file I/O and trigram parsing. Windows' SRWLock is - // writer-preferring: a single waiting writer here would otherwise - // stall every subsequent search request. - let data = match std::fs::read(path) { - Ok(d) => d, - Err(_) => continue, - }; - let text = tgrep_core::encoding::decode_for_index(&data); - let is_binary = tgrep_core::trigram::is_binary(&text); - let per_tri = if is_binary { - None - } else { - Some(tgrep_core::live::LiveIndex::compute_trigram_masks(&text)) - }; +/// Read a file and merge it into the live index, unless its stamp says the +/// content we already indexed is current. +/// +/// The caller must hold `snapshot_gate`: the read, the commit, and the stamp +/// update have to be atomic with respect to a flush or auto-save. +fn reindex_file(state: &ServerState, path: &Path, rel_path: &str) { + use tgrep_core::meta::FileStamp; - eprintln!("[trace] reindex: modified {rel_path}"); - // gate acquired at the function level — the commit + stamp - // update is processed atomically with respect to flush/auto-save. - { - let mut index = state.index.write().unwrap(); - match per_tri { - Some(per_tri) => index.live.commit_upsert(&rel_path, per_tri), - None => index.live.delete_file(&rel_path), - } - } - state - .file_stamps - .write() - .unwrap() - .insert(rel_path.clone(), current); - if let Ok(mut cache) = state.cache.write() { - cache.pop(&rel_path); + // Compute the file's current stamp and skip if it matches what we + // last indexed. notify on Windows in particular fires Modify events + // for atime/attribute updates, opens, etc. — re-indexing on those + // would re-read large files, churn the live overlay, and produce a + // misleading "modified" trace for files that didn't actually change. + let current = match std::fs::metadata(path) { + Ok(m) => FileStamp { + mtime: m + .modified() + .ok() + .and_then(|t| t.duration_since(std::time::SystemTime::UNIX_EPOCH).ok()) + .map(|d| d.as_secs()) + .unwrap_or(0), + size: m.len(), + }, + Err(_) => return, + }; + if state.file_stamps.read().unwrap().get(rel_path) == Some(¤t) { + return; + } + + // Read contents and extract trigrams OUTSIDE the index write lock + // so a concurrent search (which needs a read lock) is not blocked + // on our file I/O and trigram parsing. Windows' SRWLock is + // writer-preferring: a single waiting writer here would otherwise + // stall every subsequent search request. + let data = match std::fs::read(path) { + Ok(d) => d, + Err(_) => return, + }; + let text = tgrep_core::encoding::decode_for_index(&data); + let is_binary = tgrep_core::trigram::is_binary(&text); + let per_tri = if is_binary { + None + } else { + Some(tgrep_core::live::LiveIndex::compute_trigram_masks(&text)) + }; + + eprintln!("[trace] reindex: modified {rel_path}"); + // Gate held by the caller — the commit + stamp update is processed + // atomically with respect to flush/auto-save. + { + let mut index = state.index.write().unwrap(); + match per_tri { + Some(per_tri) => index.live.commit_upsert(rel_path, per_tri), + None => index.live.delete_file(rel_path), } } + state + .file_stamps + .write() + .unwrap() + .insert(rel_path.to_string(), current); + if let Ok(mut cache) = state.cache.write() { + cache.pop(rel_path); + } } /// Whether a scheduled reconcile should run now. @@ -2369,7 +2681,7 @@ fn background_refresh_stale( // this function holds `snapshot_gate` for write across its whole body, and // the only reader of `state.gitignore` takes the read side first, so no // event can observe the matcher before this function returns either way. - commit_stale_matcher(state, build_stale_matcher(state, root, &walk)); + publish_ignore_matcher(state, root, build_stale_matcher(state, root, &walk)); if walk.skipped_error > 0 { eprintln!( @@ -2607,8 +2919,7 @@ fn bootstrap_index_build(state: &Arc, root: &Path, index_dir: &Path state.no_require_git, ); let found = matcher.is_some(); - *state.gitignore.write().unwrap() = matcher; - state.gitignore_pending.store(false, Ordering::SeqCst); + publish_ignore_matcher(state, root, matcher); eprintln!( "[trace] gitignore matcher built from {} file(s) in {:.1}ms{}", outcome.gitignore_files.len(), @@ -2701,8 +3012,7 @@ fn background_index_build(state: &Arc, root: &Path, index_dir: &Pat state.no_require_git, ); let has_matcher = matcher.is_some(); - *state.gitignore.write().unwrap() = matcher; - state.gitignore_pending.store(false, Ordering::SeqCst); + publish_ignore_matcher(state, root, matcher); eprintln!( "[trace] gitignore matcher built from index walk in {:.1}ms \ ({} .gitignore + {} .ignore files{})", @@ -3455,6 +3765,7 @@ mod tests { index_progress: std::sync::atomic::AtomicU64::new(0), index_total: std::sync::atomic::AtomicU64::new(0), watch_enabled: true, + watch_registry: Mutex::new(None), exclude_dirs: Vec::new(), no_ignore: false, no_require_git: false, @@ -3676,6 +3987,160 @@ mod tests { )); } + #[test] + fn watchable_dirs_prunes_ignored_and_hidden_subtrees() { + // The point of the subscription set: an ignored directory costs one + // inotify watch descriptor per directory inside it, so pruning has to + // happen before the subtree is walked, not after its events arrive. + let tmp = TempDir::new().unwrap(); + let root = tmp.path(); + std::fs::create_dir(root.join(".git")).unwrap(); + std::fs::write(root.join(".gitignore"), "build/\n").unwrap(); + + for dir in [ + "src", + "src/nested", + "build", + "build/a", + "build/a/deep", + ".git/objects", + "vendor", + "vendor/pkg", + ] { + std::fs::create_dir_all(root.join(dir)).unwrap(); + } + + let gi = tgrep_core::gitignore::build_matcher(root).expect("matcher should build"); + let exclude = vec!["vendor".to_string()]; + let dirs = watchable_dirs(root, root, &exclude, Some(&gi)); + + let rel: std::collections::HashSet = dirs + .iter() + .map(|p| { + p.strip_prefix(root) + .unwrap() + .to_string_lossy() + .replace('\\', "/") + }) + .collect(); + + // The root itself is always watched, plus the directories the indexer + // would descend into. + assert!(rel.contains(""), "root must always be watched: {rel:?}"); + assert!(rel.contains("src")); + assert!(rel.contains("src/nested")); + + // A gitignored directory and everything beneath it. + assert!( + !rel.contains("build"), + "gitignored dir was watched: {rel:?}" + ); + assert!(!rel.contains("build/a")); + assert!(!rel.contains("build/a/deep")); + + // Hidden directories, which the walker skips too. + assert!(!rel.contains(".git")); + assert!(!rel.contains(".git/objects")); + + // `--exclude` names prune the directory itself, not just its children. + assert!(!rel.contains("vendor"), "excluded dir was watched: {rel:?}"); + assert!(!rel.contains("vendor/pkg")); + } + + #[test] + fn watchable_dirs_without_a_matcher_keeps_everything_visible() { + // `--no-ignore` publishes no matcher. The subscription set must then + // be the whole tree minus hidden paths, matching what the walk indexes; + // silently narrowing it would drop events for files that ARE indexed. + let tmp = TempDir::new().unwrap(); + let root = tmp.path(); + std::fs::create_dir_all(root.join("build/a")).unwrap(); + std::fs::create_dir_all(root.join(".hidden")).unwrap(); + + let dirs = watchable_dirs(root, root, &[], None); + let rel: std::collections::HashSet = dirs + .iter() + .map(|p| { + p.strip_prefix(root) + .unwrap() + .to_string_lossy() + .replace('\\', "/") + }) + .collect(); + + assert!(rel.contains("build")); + assert!(rel.contains("build/a")); + assert!(!rel.contains(".hidden")); + } + + #[test] + fn watchable_dirs_anchors_rules_at_the_root_not_the_start_directory() { + // A subtree that appears at runtime is walked from itself, but the + // ignore rules are written against paths relative to the repository + // root. Walking with the subtree as the anchor would test "nested" + // against a rule meant for "src/nested" and prune the wrong things. + let tmp = TempDir::new().unwrap(); + let root = tmp.path(); + std::fs::create_dir(root.join(".git")).unwrap(); + std::fs::write(root.join(".gitignore"), "src/fresh/skipped/\n/keep/\n").unwrap(); + + for dir in ["src/fresh/skipped", "src/fresh/keep", "src/fresh/kept"] { + std::fs::create_dir_all(root.join(dir)).unwrap(); + } + + let gi = tgrep_core::gitignore::build_matcher(root).expect("matcher should build"); + let dirs = watchable_dirs(root, &root.join("src/fresh"), &[], Some(&gi)); + let rel: std::collections::HashSet = dirs + .iter() + .map(|p| { + p.strip_prefix(root) + .unwrap() + .to_string_lossy() + .replace('\\', "/") + }) + .collect(); + + assert!( + rel.contains("src/fresh"), + "start dir must be watched: {rel:?}" + ); + assert!(rel.contains("src/fresh/kept")); + assert!( + !rel.contains("src/fresh/skipped"), + "a root-anchored rule was not applied: {rel:?}" + ); + // `keep/` is anchored at the root, so it must NOT prune + // `src/fresh/keep` just because the walk started at `src/fresh`. + assert!( + rel.contains("src/fresh/keep"), + "a root-anchored rule was applied at the wrong depth: {rel:?}" + ); + } + + #[test] + fn skip_watcher_dir_applies_directory_semantics() { + // A directory-only gitignore rule (`build/`) does not match the path + // `build` when it is tested as a file, which is why the subscription + // set needs its own dir-aware entry point. + let tmp = TempDir::new().unwrap(); + std::fs::create_dir(tmp.path().join(".git")).unwrap(); + std::fs::write(tmp.path().join(".gitignore"), "build/\n").unwrap(); + let gi = tgrep_core::gitignore::build_matcher(tmp.path()).expect("matcher should build"); + + assert!(should_skip_watcher_dir("build", &[], Some(&gi))); + assert!(!should_skip_watcher_path("build", &[], Some(&gi))); + + // `--exclude` prunes the named directory itself... + let exclude = vec!["target".to_string()]; + assert!(should_skip_watcher_dir("target", &exclude, None)); + assert!(should_skip_watcher_dir("src/target", &exclude, None)); + // ...but a *file* of that name is still indexed, so it must not be + // skipped. This is the invariant `should_skip_watcher_path` already + // held, and sharing an implementation must not have changed it. + assert!(!should_skip_watcher_path("target", &exclude, None)); + assert!(!should_skip_watcher_path("src/target", &exclude, None)); + } + #[test] fn identifies_live_ignore_rule_changes() { let root = Path::new("workspace"); diff --git a/tgrep-cli/tests/watcher_watch_registration.rs b/tgrep-cli/tests/watcher_watch_registration.rs new file mode 100644 index 0000000..47376da --- /dev/null +++ b/tgrep-cli/tests/watcher_watch_registration.rs @@ -0,0 +1,308 @@ +//! The watcher must not take OS subscriptions for trees it is going to ignore. +//! +//! On Linux `notify` has no recursive inotify mode: `RecursiveMode::Recursive` +//! walks the tree and spends one watch descriptor per directory. Ignored build +//! output is usually most of the directories in a repository, so subscribing to +//! it burns the per-user `fs.inotify.max_user_watches` budget on events that +//! are then discarded — and because notify propagates the first registration +//! failure, a repo large enough to exhaust that budget loses its watcher +//! entirely. +//! +//! This is observable: the kernel reports a process's inotify watches in +//! `/proc//fdinfo/`, one `inotify wd:` line per watch. The test below +//! counts them for a live server and pins that the ignored subtree is absent, +//! while asserting the watcher still works — a watcher that registered nothing +//! at all would trivially satisfy the count. +//! +//! Windows (ReadDirectoryChangesW) and macOS (FSEvents) subscribe once for the +//! whole subtree, so there is no per-directory registration to withhold and +//! nothing here applies; the count assertion is Linux-only for that reason. + +use std::fs; +use std::io::{BufRead, BufReader, Write}; +use std::net::TcpStream; +use std::path::Path; +use std::process::{Child, Command}; +use std::thread; +use std::time::{Duration, Instant}; + +use tempfile::TempDir; + +/// Directories inside the ignored tree. Large enough that a recursive +/// subscription is unmistakable in the watch count, small enough to create +/// quickly. +#[cfg(target_os = "linux")] +const IGNORED_DIRS: usize = 60; +/// Directories inside the indexed tree. +#[cfg(target_os = "linux")] +const SOURCE_DIRS: usize = 4; + +fn tgrep_bin() -> std::path::PathBuf { + assert_cmd::cargo::cargo_bin("tgrep") +} + +struct ServerGuard { + child: Child, +} + +impl Drop for ServerGuard { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} + +fn send_request(port: u16, request: &str) -> std::io::Result { + let mut stream = TcpStream::connect(format!("127.0.0.1:{port}"))?; + stream.set_read_timeout(Some(Duration::from_secs(30)))?; + writeln!(stream, "{request}")?; + stream.flush()?; + let mut reader = BufReader::new(stream); + let mut response = String::new(); + reader.read_line(&mut response)?; + Ok(response) +} + +fn search_matches(port: u16, pattern: &str) -> u64 { + let request = serde_json::json!({ + "jsonrpc": "2.0", + "method": "search", + "id": 1, + "params": { "pattern": pattern } + }) + .to_string(); + let response = send_request(port, &request).expect("search request failed"); + let value: serde_json::Value = serde_json::from_str(&response).expect("invalid JSON response"); + value + .pointer("/result/num_matches") + .and_then(|v| v.as_u64()) + .unwrap_or_else(|| panic!("missing num_matches in response: {response}")) +} + +fn wait_for_match(port: u16, pattern: &str, timeout: Duration) -> bool { + let start = Instant::now(); + loop { + if search_matches(port, pattern) > 0 { + return true; + } + if start.elapsed() > timeout { + return false; + } + thread::sleep(Duration::from_millis(100)); + } +} + +/// Read `(pid, port)` once the server is accepting connections. +fn wait_for_server(index_dir: &Path) -> (u32, u16) { + let serve_json = index_dir.join("serve.json"); + let start = Instant::now(); + loop { + assert!( + start.elapsed() <= Duration::from_secs(60), + "tgrep serve did not start within 60 seconds" + ); + if let Ok(data) = fs::read_to_string(&serve_json) + && let Ok(info) = serde_json::from_str::(&data) + && let Some(p) = info.get("port").and_then(|v| v.as_u64()) + && let Some(pid) = info.get("pid").and_then(|v| v.as_u64()) + && TcpStream::connect(format!("127.0.0.1:{p}")).is_ok() + { + return (pid as u32, p as u16); + } + thread::sleep(Duration::from_millis(20)); + } +} + +/// Total inotify watch descriptors held by `pid`. +/// +/// Each inotify file descriptor's `fdinfo` lists one `inotify wd:` line per +/// watch, so summing them across the process's descriptors gives the number of +/// directories it is subscribed to. +#[cfg(target_os = "linux")] +fn inotify_watch_count(pid: u32) -> usize { + let dir = format!("/proc/{pid}/fdinfo"); + let Ok(entries) = fs::read_dir(&dir) else { + panic!("could not read {dir}; /proc must be mounted for this test"); + }; + let mut total = 0; + for entry in entries.flatten() { + // Descriptors come and go while we read; a vanished one is not a + // failure, it simply holds no watches we can count. + if let Ok(contents) = fs::read_to_string(entry.path()) { + total += contents + .lines() + .filter(|line| line.starts_with("inotify wd:")) + .count(); + } + } + total +} + +#[cfg(target_os = "linux")] +#[test] +fn watcher_does_not_subscribe_to_gitignored_directories() { + let dir = TempDir::new().unwrap(); + let root = dir.path(); + let index_dir = root.join(".tgrep_test_index"); + + // A real (if empty) `.git` entry: `.gitignore` rules only apply inside a + // repository, so without it the fixture would measure the wrong thing. + fs::create_dir_all(root.join(".git")).unwrap(); + fs::write(root.join(".gitignore"), "build/\n").unwrap(); + + for i in 0..IGNORED_DIRS { + let sub = root.join("build").join(format!("out{i}")); + fs::create_dir_all(&sub).unwrap(); + fs::write(sub.join("artifact.txt"), "ignored_build_output\n").unwrap(); + } + for i in 0..SOURCE_DIRS { + let sub = root.join("src").join(format!("pkg{i}")); + fs::create_dir_all(&sub).unwrap(); + fs::write( + sub.join("lib.rs"), + format!("fn seeded{i}() {{ let normal_source_marker = {i}; }}\n"), + ) + .unwrap(); + } + + let status = Command::new(tgrep_bin()) + .args([ + "index", + root.to_str().unwrap(), + "--index-path", + index_dir.to_str().unwrap(), + ]) + .status() + .expect("failed to run tgrep index"); + assert!(status.success(), "initial index build failed"); + + let child = Command::new(tgrep_bin()) + .args([ + "serve", + "--index-path", + index_dir.to_str().unwrap(), + root.to_str().unwrap(), + ]) + .stderr(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .spawn() + .expect("failed to start tgrep serve"); + let _server = ServerGuard { child }; + + let (pid, port) = wait_for_server(&index_dir); + + // Positive control, and a wait for the watcher to be live: subscriptions + // are deferred until the ignore matcher is published, so counting before + // that would pass for the wrong reason. + assert!( + wait_for_match(port, "normal_source_marker", Duration::from_secs(30)), + "expected the seeded source files to be searchable" + ); + fs::write( + root.join("src").join("added.rs"), + "fn added() { let watcher_added_marker = 1; }\n", + ) + .unwrap(); + assert!( + wait_for_match(port, "watcher_added_marker", Duration::from_secs(30)), + "watcher never indexed a newly created ordinary file, so the watch \ + count below would be meaningless" + ); + + let watches = inotify_watch_count(pid); + + // The tree the watcher legitimately needs is the root, `src`, `src/pkg*`, + // and `.git` is hidden so it is not watched either. Allow generous slack + // for anything else in the process holding an inotify fd, but stay far + // below the ~66 a recursive subscription over this fixture would take. + let allowed = SOURCE_DIRS + 8; + assert!( + watches <= allowed, + "watcher holds {watches} inotify watches for a tree whose only \ + non-ignored directories are the root plus {SOURCE_DIRS} under src/; \ + it is subscribing to the {IGNORED_DIRS} gitignored directories under \ + build/ (expected at most {allowed})" + ); +} + +/// New directories still have to be picked up. +/// +/// Non-recursive subscriptions are not extended by notify, so a directory +/// created after startup is invisible unless the watcher subscribes to it as +/// it appears. This runs everywhere: on a whole-subtree backend it simply +/// re-confirms the existing behaviour, which is the point — the two paths must +/// agree. +#[test] +fn watcher_indexes_files_in_directories_created_after_startup() { + let dir = TempDir::new().unwrap(); + let root = dir.path(); + let index_dir = root.join(".tgrep_test_index"); + + fs::create_dir_all(root.join(".git")).unwrap(); + fs::write(root.join(".gitignore"), "build/\n").unwrap(); + fs::create_dir_all(root.join("src")).unwrap(); + fs::write( + root.join("src").join("lib.rs"), + "fn seeded() { let normal_source_marker = 1; }\n", + ) + .unwrap(); + + let status = Command::new(tgrep_bin()) + .args([ + "index", + root.to_str().unwrap(), + "--index-path", + index_dir.to_str().unwrap(), + ]) + .status() + .expect("failed to run tgrep index"); + assert!(status.success(), "initial index build failed"); + + let child = Command::new(tgrep_bin()) + .args([ + "serve", + "--index-path", + index_dir.to_str().unwrap(), + root.to_str().unwrap(), + ]) + .stderr(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .spawn() + .expect("failed to start tgrep serve"); + let _server = ServerGuard { child }; + + let (_pid, port) = wait_for_server(&index_dir); + assert!( + wait_for_match(port, "normal_source_marker", Duration::from_secs(30)), + "expected the seeded source file to be searchable" + ); + thread::sleep(Duration::from_secs(2)); + + // A whole new subtree, several levels deep, written in one go. The files + // land immediately after their directories, which is exactly the race the + // subscription pass has to close. + let nested = root.join("src").join("fresh").join("deeper"); + fs::create_dir_all(&nested).unwrap(); + fs::write( + nested.join("new.rs"), + "fn fresh() { let nested_new_dir_marker = 2; }\n", + ) + .unwrap(); + + assert!( + wait_for_match(port, "nested_new_dir_marker", Duration::from_secs(30)), + "watcher never indexed a file created in a directory that did not \ + exist when the server started" + ); + + // ...and a directory created inside the ignored tree stays ignored. + let ignored = root.join("build").join("fresh"); + fs::create_dir_all(&ignored).unwrap(); + fs::write(ignored.join("out.txt"), "new_ignored_dir_marker\n").unwrap(); + thread::sleep(Duration::from_secs(3)); + assert_eq!( + search_matches(port, "new_ignored_dir_marker"), + 0, + "watcher indexed a file in a directory created under a gitignored path" + ); +} From 31825bb518f77ebf11c699976cac112c6843e341 Mon Sep 17 00:00:00 2001 From: Shengyu Fu Date: Wed, 26 Aug 2026 21:58:17 -0700 Subject: [PATCH 03/28] Subscribe to new subtrees additively instead of rebuilding the set `watch_new_subtree` merged the new subtree into the live subscription set by cloning both into a union and handing that to `sync`. That is correct but proportional to the whole watched set, and it runs once per directory created at runtime -- so on a repository holding tens of thousands of watched directories, a checkout or a build that creates many directories does quadratic work copying `PathBuf`s. Split the additive half of `sync` into `add_all` and call that instead. It iterates only the new subtree and tests membership, so the cost is proportional to what actually appeared. `sync` keeps its prune-and-add behaviour for the whole-tree case and now shares the same code. The distinction matters beyond performance: passing a subtree to `sync` would treat the entire rest of the repository as stale and unsubscribe from it, so a new folder would silently disable file watching. Added a test that pins both behaviours. Measured against the layout reported by the Office monorepo team -- 40,210 directories, ~200 searchable files, `.git` both hidden and `--exclude`d -- the subscription set is 202 directories, computed in 13.7ms. The walk cost is set by the tree that survives pruning, not the physical tree: the same measurement over 8,458 directories takes 12.6ms. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: bcc85b55-e821-447e-b9a9-144b78829ae4 --- tgrep-cli/src/serve.rs | 117 ++++++++++++++++++++++++++++++----------- 1 file changed, 87 insertions(+), 30 deletions(-) diff --git a/tgrep-cli/src/serve.rs b/tgrep-cli/src/serve.rs index 93fd52f..4cca305 100644 --- a/tgrep-cli/src/serve.rs +++ b/tgrep-cli/src/serve.rs @@ -1771,36 +1771,31 @@ struct WatchRegistry { } impl WatchRegistry { - /// Bring the subscription set in line with `desired`, subscribing to - /// directories that are newly relevant and dropping ones that are not. + /// Subscribe to every directory in `desired` that is not already + /// subscribed, leaving existing subscriptions alone. /// - /// Returns `(added, removed)`. A single directory that cannot be - /// subscribed is reported and skipped rather than failing the sync: the - /// watcher is still useful for everything else, and giving up on the whole - /// tree is exactly the failure mode this registration exists to avoid. - fn sync(&mut self, desired: &std::collections::HashSet) -> (usize, usize) { - let stale: Vec = self.watched.difference(desired).cloned().collect(); - let mut removed = 0; - for dir in stale { - // Best effort. inotify drops a descriptor by itself when the - // directory is deleted, so "not found" is an expected outcome - // here, not an error worth reporting. - let _ = self.watcher.unwatch(&dir); - self.watched.remove(&dir); - removed += 1; - } - - let missing: Vec = desired.difference(&self.watched).cloned().collect(); + /// Returns the number added. A single directory that cannot be subscribed + /// is reported and skipped rather than failing the whole call: the watcher + /// is still useful for everything else, and giving up on the entire tree is + /// exactly the failure mode this registration exists to avoid. + fn add_all(&mut self, desired: &std::collections::HashSet) -> usize { let mut added = 0; let mut failures = 0; - for dir in missing { - match self.watcher.watch(&dir, RecursiveMode::NonRecursive) { + // Iterating `desired` and testing membership is deliberate: a + // `difference` would be proportional to the whole watched set, and + // this runs per newly created directory on repositories where that set + // is tens of thousands of entries. + for dir in desired { + if self.watched.contains(dir) { + continue; + } + match self.watcher.watch(dir, RecursiveMode::NonRecursive) { Ok(()) => { - self.watched.insert(dir); + self.watched.insert(dir.clone()); added += 1; } Err(e) => { - // One line per sync, not per directory: exhausting the + // One line per call, not per directory: exhausting the // inotify budget fails thousands of these at once. if failures == 0 { eprintln!( @@ -1816,7 +1811,28 @@ impl WatchRegistry { if failures > 1 { eprintln!("[trace] warning: {failures} directories could not be watched"); } - (added, removed) + added + } + + /// Bring the subscription set in line with `desired`, subscribing to + /// directories that are newly relevant and dropping ones that are not. + /// + /// Returns `(added, removed)`. Only for a set that describes the whole + /// tree — anything absent from `desired` is unsubscribed. To subscribe to + /// a subtree without disturbing the rest, use [`Self::add_all`]. + fn sync(&mut self, desired: &std::collections::HashSet) -> (usize, usize) { + let stale: Vec = self.watched.difference(desired).cloned().collect(); + let mut removed = 0; + for dir in stale { + // Best effort. inotify drops a descriptor by itself when the + // directory is deleted, so "not found" is an expected outcome + // here, not an error worth reporting. + let _ = self.watcher.unwatch(&dir); + self.watched.remove(&dir); + removed += 1; + } + + (self.add_all(desired), removed) } } @@ -1941,12 +1957,13 @@ fn watch_new_subtree(state: &ServerState, root: &Path, dir: &Path) { let Some(registry) = registry.as_mut() else { return; }; - // A union, not a replacement: `desired` covers only this subtree, and - // `sync` would read anything outside it as newly stale and unsubscribe - // from the entire rest of the repository. - let union: std::collections::HashSet = - registry.watched.union(&desired).cloned().collect(); - registry.sync(&union); + // Additive, not a sync: `desired` covers only this subtree, and `sync` + // would read everything outside it as stale and unsubscribe from the + // entire rest of the repository. Doing this without materialising a + // union also matters at scale — a monorepo can hold tens of thousands + // of watched directories, and copying that set for every newly created + // directory would be quadratic over a checkout or a build. + registry.add_all(&desired); } for subdir in &desired { @@ -3987,6 +4004,46 @@ mod tests { )); } + #[test] + fn watch_registry_add_all_is_additive_but_sync_prunes() { + // These two must not be confused. `watch_new_subtree` learns only + // about the subtree that just appeared, so if it went through `sync` + // every directory outside that subtree would look stale and the server + // would unsubscribe from the entire rest of the repository — turning a + // new folder into a silent, total loss of file watching. + let tmp = TempDir::new().unwrap(); + let a = tmp.path().join("a"); + let b = tmp.path().join("b"); + let c = tmp.path().join("c"); + for dir in [&a, &b, &c] { + std::fs::create_dir(dir).unwrap(); + } + + let watcher = notify::recommended_watcher(|_: notify::Result| {}).unwrap(); + let mut registry = WatchRegistry { + watcher, + watched: std::collections::HashSet::new(), + }; + + let added = registry.add_all(&[a.clone(), b.clone()].into_iter().collect()); + assert_eq!(added, 2); + + // Adding a subtree leaves existing subscriptions untouched, and + // re-adding one already present is a no-op rather than a duplicate. + let added = registry.add_all(&[b.clone(), c.clone()].into_iter().collect()); + assert_eq!(added, 1, "b was already watched and must not be re-added"); + assert_eq!( + registry.watched, + [a.clone(), b.clone(), c.clone()].into_iter().collect(), + "add_all dropped a subscription outside the set it was given" + ); + + // `sync`, by contrast, is authoritative over the whole tree. + let (added, removed) = registry.sync(&[c.clone()].into_iter().collect()); + assert_eq!((added, removed), (0, 2)); + assert_eq!(registry.watched, [c].into_iter().collect()); + } + #[test] fn watchable_dirs_prunes_ignored_and_hidden_subtrees() { // The point of the subscription set: an ignored directory costs one From ecec6e83fd06d401948fc2bf3462ffa01c73edc0 Mon Sep 17 00:00:00 2001 From: Shengyu Fu Date: Wed, 26 Aug 2026 22:21:50 -0700 Subject: [PATCH 04/28] Bump the workspace version to 1.0.3 The watcher fixes on this branch change observable behaviour on Linux -- `tgrep serve` no longer takes an inotify watch per directory -- and the Office monorepo report that corroborated the bug was filed against the bundled 1.0.2. A distinct version is what lets that team tell whether a build contains the fix. Both crates inherit `version.workspace`, so the manifest change is one line; `Cargo.lock` is regenerated with `cargo update --workspace` rather than hand-edited. Note that the third-party `equivalent` crate is also at 1.0.2 and is deliberately untouched. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: bcc85b55-e821-447e-b9a9-144b78829ae4 --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 575f327..e6ef78a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1046,7 +1046,7 @@ checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" [[package]] name = "tgrep-cli" -version = "1.0.2" +version = "1.0.3" dependencies = [ "anyhow", "assert_cmd", @@ -1072,7 +1072,7 @@ dependencies = [ [[package]] name = "tgrep-core" -version = "1.0.2" +version = "1.0.3" dependencies = [ "anyhow", "criterion", diff --git a/Cargo.toml b/Cargo.toml index 8e27680..426b619 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ members = ["tgrep-core", "tgrep-cli"] resolver = "2" [workspace.package] -version = "1.0.2" +version = "1.0.3" edition = "2024" license = "MIT" repository = "https://github.com/microsoft/tgrep" From 6ddba29556746f44db570e800d0a026c625f5326 Mon Sep 17 00:00:00 2001 From: Shengyu Fu Date: Wed, 26 Aug 2026 22:41:08 -0700 Subject: [PATCH 05/28] Close three watcher races found in review Subscriptions were established only after the walk that produced the ignore matcher, so a file written to a directory in that window was in neither place: not in the walk's results, and not able to report itself yet. It stayed invisible until the hourly reconcile. sync_watch_registrations now returns the directories it newly subscribed to, and the stale check rechecks them once the merge has settled. It has to be after the merge: stream_merge_stale_changes replaces file_stamps wholesale, so an earlier scan would be discarded and would re-read every changed file on the way. reindex_file compares stamps first, so on a tree that did not move under us this costs one metadata call per file. The two index-build publishes deliberately skip the scan. There "newly watched" is the whole repository, and the stale check that follows startup already does a full walk-versus-index diff, which is a superset. Scanning there would stat the entire tree while holding the gate, on the path a warm start exists to keep fast. watch_new_subtree had two more problems of its own. It read each directory before subscribing to it, leaving the same race one level down for anything created in between; it now subscribes to a level before enumerating it. And a subtree that arrives already populated -- a clone, a mv, a branch switch -- can carry its own .gitignore. Those files are dot-prefixed, so the recovery scan dropped them silently and indexed the rest of the subtree against rules that had never heard of it. It now looks for ignore rules first and defers to a refresh rather than indexing under stale ones. Finally, Path::is_dir follows symlinks. A link to a directory was therefore subscribed to and walked through, indexing a target the walker never descends into and that may sit outside the root entirely. is_real_dir asks the question we actually mean. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: bcc85b55-e821-447e-b9a9-144b78829ae4 --- tgrep-cli/src/serve.rs | 354 ++++++++++++++---- tgrep-cli/tests/watcher_watch_registration.rs | 81 ++++ 2 files changed, 366 insertions(+), 69 deletions(-) diff --git a/tgrep-cli/src/serve.rs b/tgrep-cli/src/serve.rs index 4cca305..4cf55fb 100644 --- a/tgrep-cli/src/serve.rs +++ b/tgrep-cli/src/serve.rs @@ -756,17 +756,24 @@ fn build_stale_matcher( /// Callers on the stale path hold `snapshot_gate` for write, which is what /// makes the matcher swap and the index decisions around it atomic from the /// watcher's point of view. +/// +/// Returns the directories that were newly subscribed to as a result. Those +/// were unwatched while the caller's walk ran, so anything written to them in +/// that window produced no event and appears in no walk result. Callers pass +/// the list to [`reindex_files_in`] once `state.file_stamps` describes the +/// index they just published. +#[must_use = "newly watched directories need a recovery scan or writes race the subscription"] fn publish_ignore_matcher( state: &ServerState, root: &Path, matcher: Option, -) { +) -> Vec { *state.gitignore.write().unwrap() = matcher; state.gitignore_pending.store(false, Ordering::SeqCst); // New rules mean a different set of directories worth hearing about: // a tightened rule releases the subscriptions under it, and a relaxed // one takes subscriptions for the tree it used to hide. - sync_watch_registrations(state, root); + sync_watch_registrations(state, root) } fn handle_connection(stream: TcpStream, state: &ServerState) -> Result<()> { @@ -1569,7 +1576,13 @@ fn start_file_watcher(state: Arc, root: &Path, queue_cap: usize) -> if state.gitignore_pending.load(Ordering::SeqCst) { eprintln!("[trace] watcher subscriptions deferred until the ignore matcher is ready"); } else { - sync_watch_registrations(&state, root); + // The newly watched directories are deliberately not rechecked here. + // This is every directory in the repository, the index was built or + // opened moments ago, and the stale check that follows startup + // reconciles the same drift while holding `snapshot_gate` — which this + // path does not hold and must not take, since the watcher has to be + // receiving events before that check runs. + let _ = sync_watch_registrations(&state, root); } let worker_state = Arc::clone(&state); @@ -1761,6 +1774,17 @@ fn is_ignore_rules_file(root: &Path, path: &Path) -> bool { /// `PollWatcher` are per-path too, but nothing here builds or tests them. const PER_DIRECTORY_WATCHES: bool = cfg!(any(target_os = "linux", target_os = "android")); +/// Whether `path` is a directory in its own right rather than a symlink to one. +/// +/// [`Path::is_dir`] follows links, so it answers "does this lead to a +/// directory", which is the wrong question here: the walker does not follow +/// symlinks, so a symlinked directory is not part of the indexed tree. Treating +/// one as a directory would subscribe to and index its target — possibly a tree +/// outside `root` entirely, and possibly a cycle. +fn is_real_dir(path: &Path) -> bool { + std::fs::symlink_metadata(path).is_ok_and(|meta| meta.file_type().is_dir()) +} + /// The watcher plus the set of directories it is currently subscribed to. /// /// Only meaningful when [`PER_DIRECTORY_WATCHES`] is true; elsewhere `watched` @@ -1774,12 +1798,13 @@ impl WatchRegistry { /// Subscribe to every directory in `desired` that is not already /// subscribed, leaving existing subscriptions alone. /// - /// Returns the number added. A single directory that cannot be subscribed - /// is reported and skipped rather than failing the whole call: the watcher - /// is still useful for everything else, and giving up on the entire tree is - /// exactly the failure mode this registration exists to avoid. - fn add_all(&mut self, desired: &std::collections::HashSet) -> usize { - let mut added = 0; + /// Returns the directories that were newly subscribed to. A single + /// directory that cannot be subscribed is reported and skipped rather than + /// failing the whole call: the watcher is still useful for everything + /// else, and giving up on the entire tree is exactly the failure mode this + /// registration exists to avoid. + fn add_all<'a>(&mut self, desired: impl IntoIterator) -> Vec { + let mut added = Vec::new(); let mut failures = 0; // Iterating `desired` and testing membership is deliberate: a // `difference` would be proportional to the whole watched set, and @@ -1792,7 +1817,7 @@ impl WatchRegistry { match self.watcher.watch(dir, RecursiveMode::NonRecursive) { Ok(()) => { self.watched.insert(dir.clone()); - added += 1; + added.push(dir.clone()); } Err(e) => { // One line per call, not per directory: exhausting the @@ -1820,7 +1845,7 @@ impl WatchRegistry { /// Returns `(added, removed)`. Only for a set that describes the whole /// tree — anything absent from `desired` is unsubscribed. To subscribe to /// a subtree without disturbing the rest, use [`Self::add_all`]. - fn sync(&mut self, desired: &std::collections::HashSet) -> (usize, usize) { + fn sync(&mut self, desired: &std::collections::HashSet) -> (Vec, usize) { let stale: Vec = self.watched.difference(desired).cloned().collect(); let mut removed = 0; for dir in stale { @@ -1896,15 +1921,21 @@ fn watchable_dirs( /// Called when the watcher starts and every time the ignore matcher is /// published, so relaxing a rule subscribes to the tree it used to hide and /// tightening one drops it. -fn sync_watch_registrations(state: &ServerState, root: &Path) { +/// +/// Returns the directories that were newly subscribed to. Until a directory is +/// subscribed it cannot report anything, so a file written to one of these +/// between the caller's walk and this call is in neither the walk's results nor +/// any event. The caller is expected to hand the list to +/// [`reindex_files_in`] once its own bookkeeping is settled. +fn sync_watch_registrations(state: &ServerState, root: &Path) -> Vec { if !PER_DIRECTORY_WATCHES { - return; + return Vec::new(); } let mut registry = state.watch_registry.lock().unwrap(); let Some(registry) = registry.as_mut() else { // The watcher has not started yet. It syncs once as it comes up, so // there is nothing to do and nothing to remember. - return; + return Vec::new(); }; let start = Instant::now(); @@ -1914,13 +1945,59 @@ fn sync_watch_registrations(state: &ServerState, root: &Path) { }; let total = desired.len(); let (added, removed) = registry.sync(&desired); - if added > 0 || removed > 0 { + if !added.is_empty() || removed > 0 { eprintln!( "[trace] watcher subscriptions: {total} directories \ - (+{added}, -{removed}) in {:.1}ms", + (+{}, -{removed}) in {:.1}ms", + added.len(), start.elapsed().as_secs_f64() * 1000.0 ); } + added +} + +/// Re-check the files directly inside `dirs`, indexing the ones that changed. +/// +/// Used to close the gap between a walk and the subscriptions that follow it: +/// [`reindex_file`] compares stamps first, so for a tree that did not change +/// under us this costs one `metadata` call per file and indexes nothing. +/// +/// Callers must already hold `snapshot_gate`, and `state.file_stamps` must +/// already describe the index as published — a merge that replaces the stamps +/// afterwards would both discard what this records and make every file here +/// look changed. +fn reindex_files_in(state: &ServerState, root: &Path, dirs: &[PathBuf]) { + if dirs.is_empty() { + return; + } + let start = Instant::now(); + for dir in dirs { + let Ok(entries) = std::fs::read_dir(dir) else { + continue; + }; + for entry in entries.flatten() { + if !entry.file_type().is_ok_and(|t| t.is_file()) { + continue; + } + let path = entry.path(); + let Ok(rel) = path.strip_prefix(root) else { + continue; + }; + let rel = rel.to_string_lossy().replace('\\', "/"); + let skip = { + let gitignore = state.gitignore.read().unwrap(); + should_skip_watcher_path(&rel, &state.exclude_dirs, gitignore.as_ref()) + }; + if !skip { + reindex_file(state, &path, &rel); + } + } + } + eprintln!( + "[trace] watcher: rechecked {} newly watched directories in {:.1}ms", + dirs.len(), + start.elapsed().as_secs_f64() * 1000.0 + ); } /// Subscribe to a directory that has just appeared, and to anything already @@ -1933,60 +2010,118 @@ fn sync_watch_registrations(state: &ServerState, root: &Path) { /// Files that landed between the directory's creation and its subscription /// would be missed by definition, so the same pass indexes what it finds. /// The caller must already hold `snapshot_gate`. -fn watch_new_subtree(state: &ServerState, root: &Path, dir: &Path) { +/// +/// The descent subscribes to each level *before* reading it. Reading first +/// leaves a window in which a child created in between is in neither place: +/// not in what this pass enumerates, and not yet able to report itself. That +/// window is small but it is exactly the one a checkout or a build fills, and +/// anything lost in it stays invisible until the hourly reconcile. +fn watch_new_subtree(state: &Arc, root: &Path, dir: &Path) { + // `is_dir` follows symlinks; the walker does not. Refuse a symlinked + // directory here so we never subscribe to, or index, a tree the indexer + // would not have walked into. + if !is_real_dir(dir) { + return; + } let Ok(rel_dir) = dir.strip_prefix(root) else { return; }; let rel_dir = rel_dir.to_string_lossy().replace('\\', "/"); - - let desired = { + // The event that brought us here was filtered with file semantics, so a + // `build/`-style rule that only ever matches directories has not been + // applied to this path yet. Re-check before subscribing to it. + if !rel_dir.is_empty() { let gitignore = state.gitignore.read().unwrap(); - // The event that brought us here was filtered with file semantics, so - // a `build/`-style rule that only ever matches directories has not - // been applied to this path yet. Re-check before subscribing to it. - if !rel_dir.is_empty() - && should_skip_watcher_dir(&rel_dir, &state.exclude_dirs, gitignore.as_ref()) - { + if should_skip_watcher_dir(&rel_dir, &state.exclude_dirs, gitignore.as_ref()) { return; } - watchable_dirs(root, dir, &state.exclude_dirs, gitignore.as_ref()) - }; - - { - let mut registry = state.watch_registry.lock().unwrap(); - let Some(registry) = registry.as_mut() else { - return; - }; - // Additive, not a sync: `desired` covers only this subtree, and `sync` - // would read everything outside it as stale and unsubscribe from the - // entire rest of the repository. Doing this without materialising a - // union also matters at scale — a monorepo can hold tens of thousands - // of watched directories, and copying that set for every newly created - // directory would be quadratic over a checkout or a build. - registry.add_all(&desired); } - for subdir in &desired { - let Ok(entries) = std::fs::read_dir(subdir) else { - continue; - }; - for entry in entries.flatten() { - if !entry.file_type().is_ok_and(|t| t.is_file()) { + let mut level = vec![dir.to_path_buf()]; + let mut seen: std::collections::HashSet = std::collections::HashSet::new(); + let mut files: Vec<(PathBuf, String)> = Vec::new(); + let mut found_ignore_rules = false; + + while !level.is_empty() { + { + let mut registry = state.watch_registry.lock().unwrap(); + let Some(registry) = registry.as_mut() else { + return; + }; + // Additive, not a sync: this covers only the new subtree, and + // `sync` would read everything outside it as stale and unsubscribe + // from the entire rest of the repository. Staying additive also + // matters at scale — a monorepo can hold tens of thousands of + // watched directories, and materialising a union for every newly + // created directory would be quadratic over a checkout. + registry.add_all(level.iter()); + } + + // The registry lock is released before any `read_dir`, so a large + // subtree does not hold it across the I/O for a whole level. + let mut next = Vec::new(); + for subdir in level.drain(..) { + if !seen.insert(subdir.clone()) { continue; } - let path = entry.path(); - let Ok(rel) = path.strip_prefix(root) else { + let Ok(entries) = std::fs::read_dir(&subdir) else { continue; }; - let rel = rel.to_string_lossy().replace('\\', "/"); - let skip = { - let gitignore = state.gitignore.read().unwrap(); - should_skip_watcher_path(&rel, &state.exclude_dirs, gitignore.as_ref()) - }; - if !skip { - reindex_file(state, &path, &rel); + for entry in entries.flatten() { + let Ok(file_type) = entry.file_type() else { + continue; + }; + let path = entry.path(); + let Ok(rel) = path.strip_prefix(root) else { + continue; + }; + let rel = rel.to_string_lossy().replace('\\', "/"); + // `DirEntry::file_type` does not follow symlinks, so a + // symlinked directory is neither descended into nor indexed. + if file_type.is_dir() { + let skip = { + let gitignore = state.gitignore.read().unwrap(); + should_skip_watcher_dir(&rel, &state.exclude_dirs, gitignore.as_ref()) + }; + if !skip { + next.push(path); + } + } else if file_type.is_file() { + // A subtree that arrives whole — a clone, a `mv`, a branch + // switch — can carry its own ignore rules. Those files are + // dot-prefixed, so the scan below would silently drop them + // and index the rest of the subtree against rules that do + // not know about them. + if !state.no_ignore && is_ignore_rules_file(root, &path) { + found_ignore_rules = true; + continue; + } + let skip = { + let gitignore = state.gitignore.read().unwrap(); + should_skip_watcher_path(&rel, &state.exclude_dirs, gitignore.as_ref()) + }; + if !skip { + files.push((path, rel)); + } + } } } + level = next; + } + + if found_ignore_rules { + // Indexing now would apply the wrong rules to the whole subtree, and + // anything wrongly indexed would stay until something touched it + // again. The refresh rewalks and republishes, which covers these files + // correctly; it runs on its own thread and takes `snapshot_gate` + // there, so scheduling it while we hold the gate is safe. + state.ignore_rules_dirty.store(true, Ordering::SeqCst); + schedule_ignore_rules_refresh(Arc::clone(state), root.to_path_buf()); + return; + } + + for (path, rel) in &files { + reindex_file(state, path, rel); } } @@ -2122,7 +2257,11 @@ fn handle_fs_event(state: &Arc, root: &Path, event: &Event) { } if !path.is_file() { - if PER_DIRECTORY_WATCHES && path.is_dir() { + // `is_real_dir` rather than `is_dir`: the latter follows symlinks, + // and a link to a directory is not something the walker descends + // into, so subscribing to and indexing its target would pull in a + // tree the index never contained — possibly outside `root`. + if PER_DIRECTORY_WATCHES && is_real_dir(path) { // With non-recursive subscriptions notify will not extend the // watch set for us, so a directory that just appeared — and // anything already inside it — has to be picked up here. @@ -2655,19 +2794,57 @@ fn stamps_for_indexed( .collect() } +/// Rebuild the ignore matcher and reconcile the index against the filesystem. +/// +/// Returns whether the index can be trusted afterwards. A `false` means the +/// walk or the merge failed and the stamps are not describing the index. fn background_refresh_stale( state: &Arc, root: &Path, index_dir: &Path, compare_index_membership: bool, ) -> bool { - use tgrep_core::meta; - use tgrep_core::walker; - let _refresh = state.stale_refresh_lock.lock().unwrap(); // Keep watcher/auto-save mutations out for the complete walk → matcher → - // merge cycle. Search queries do not take this gate and remain available. + // merge → recovery cycle. Search queries do not take this gate and remain + // available. Held here rather than inside so the recovery scan below is + // still covered by it. let _gate = state.snapshot_gate.write().unwrap(); + + let mut newly_watched = Vec::new(); + let ok = refresh_stale_locked( + state, + root, + index_dir, + compare_index_membership, + &mut newly_watched, + ); + + // Directories that were not subscribed while the walk ran could not report + // a write, and the walk may have passed them before it happened, so a file + // created in that window is in neither place. Recheck them now that the + // subscriptions exist. + // + // Only on success, and only here at the end: a failed walk or merge leaves + // `file_stamps` describing something other than the published index, and + // `stream_merge_stale_changes` replaces the stamps wholesale, so scanning + // any earlier would both be discarded and re-read every changed file. + if ok { + reindex_files_in(state, root, &newly_watched); + } + ok +} + +fn refresh_stale_locked( + state: &Arc, + root: &Path, + index_dir: &Path, + compare_index_membership: bool, + newly_watched: &mut Vec, +) -> bool { + use tgrep_core::meta; + use tgrep_core::walker; + let start = Instant::now(); eprintln!("[trace] stale check: comparing index against filesystem..."); @@ -2695,10 +2872,10 @@ fn background_refresh_stale( // with a delete, would be enough. // // Committing here rather than at each exit is invisible to the watcher: - // this function holds `snapshot_gate` for write across its whole body, and + // the caller holds `snapshot_gate` for write across this whole body, and // the only reader of `state.gitignore` takes the read side first, so no // event can observe the matcher before this function returns either way. - publish_ignore_matcher(state, root, build_stale_matcher(state, root, &walk)); + *newly_watched = publish_ignore_matcher(state, root, build_stale_matcher(state, root, &walk)); if walk.skipped_error > 0 { eprintln!( @@ -2936,7 +3113,13 @@ fn bootstrap_index_build(state: &Arc, root: &Path, index_dir: &Path state.no_require_git, ); let found = matcher.is_some(); - publish_ignore_matcher(state, root, matcher); + // These subscriptions are the repository's first, so "newly watched" + // is every directory in it. No recovery scan: the startup stale check + // runs right after this and does a full walk-versus-index diff, which + // is a strict superset of what a recovery scan would find — and doing + // it here would stat the entire tree while holding the gate, on the + // path a warm start exists to keep fast. + let _ = publish_ignore_matcher(state, root, matcher); eprintln!( "[trace] gitignore matcher built from {} file(s) in {:.1}ms{}", outcome.gitignore_files.len(), @@ -3029,7 +3212,11 @@ fn background_index_build(state: &Arc, root: &Path, index_dir: &Pat state.no_require_git, ); let has_matcher = matcher.is_some(); - publish_ignore_matcher(state, root, matcher); + // Same reasoning as the bootstrap publish: this is the cold-start + // build, so "newly watched" is the whole tree, `indexing` keeps the + // watcher off the index for the rest of the build, and the stale check + // that follows reconciles anything written during it. + let _ = publish_ignore_matcher(state, root, matcher); eprintln!( "[trace] gitignore matcher built from index walk in {:.1}ms \ ({} .gitignore + {} .ignore files{})", @@ -4025,13 +4212,17 @@ mod tests { watched: std::collections::HashSet::new(), }; - let added = registry.add_all(&[a.clone(), b.clone()].into_iter().collect()); - assert_eq!(added, 2); + let added = registry.add_all(&[a.clone(), b.clone()]); + assert_eq!(added.len(), 2); // Adding a subtree leaves existing subscriptions untouched, and // re-adding one already present is a no-op rather than a duplicate. - let added = registry.add_all(&[b.clone(), c.clone()].into_iter().collect()); - assert_eq!(added, 1, "b was already watched and must not be re-added"); + let added = registry.add_all(&[b.clone(), c.clone()]); + assert_eq!( + added, + vec![c.clone()], + "b was already watched and must not be re-added" + ); assert_eq!( registry.watched, [a.clone(), b.clone(), c.clone()].into_iter().collect(), @@ -4040,10 +4231,35 @@ mod tests { // `sync`, by contrast, is authoritative over the whole tree. let (added, removed) = registry.sync(&[c.clone()].into_iter().collect()); - assert_eq!((added, removed), (0, 2)); + assert_eq!((added.len(), removed), (0, 2)); assert_eq!(registry.watched, [c].into_iter().collect()); } + #[cfg(unix)] + #[test] + fn is_real_dir_rejects_a_symlink_to_a_directory() { + // `Path::is_dir` follows links, so it would report a symlinked + // directory as a directory and the watcher would subscribe to and + // index the link's target — a tree the walker never descends into, and + // one that can sit entirely outside the repository root. + let tmp = TempDir::new().unwrap(); + let real = tmp.path().join("real"); + std::fs::create_dir(&real).unwrap(); + let link = tmp.path().join("link"); + std::os::unix::fs::symlink(&real, &link).unwrap(); + + assert!(is_real_dir(&real)); + assert!( + link.is_dir(), + "precondition: is_dir follows the link, which is the trap" + ); + assert!(!is_real_dir(&link)); + assert!(!is_real_dir(&tmp.path().join("missing"))); + let file = tmp.path().join("file.txt"); + std::fs::write(&file, "x").unwrap(); + assert!(!is_real_dir(&file)); + } + #[test] fn watchable_dirs_prunes_ignored_and_hidden_subtrees() { // The point of the subscription set: an ignored directory costs one diff --git a/tgrep-cli/tests/watcher_watch_registration.rs b/tgrep-cli/tests/watcher_watch_registration.rs index 47376da..d9fc215 100644 --- a/tgrep-cli/tests/watcher_watch_registration.rs +++ b/tgrep-cli/tests/watcher_watch_registration.rs @@ -306,3 +306,84 @@ fn watcher_indexes_files_in_directories_created_after_startup() { "watcher indexed a file in a directory created under a gitignored path" ); } + +/// A subtree that arrives already populated can carry its own ignore rules. +/// +/// A clone, a `git mv`, a branch switch or an unpacked archive all land this +/// way: the directory and everything under it appear in one step, so there is +/// no moment at which the `.gitignore` inside it is seen on its own. The +/// recovery pass skips dot-prefixed files, so without explicitly looking for +/// ignore rules it would index the subtree against rules that never mentioned +/// it — and the wrongly indexed files would stay until something touched them +/// again or the hourly reconcile came round. +#[test] +fn watcher_honors_ignore_rules_inside_a_subtree_that_arrives_whole() { + let dir = TempDir::new().unwrap(); + let root = dir.path(); + let index_dir = root.join(".tgrep_test_index"); + + fs::create_dir_all(root.join(".git")).unwrap(); + fs::write(root.join(".gitignore"), "build/\n").unwrap(); + fs::create_dir_all(root.join("src")).unwrap(); + fs::write( + root.join("src").join("lib.rs"), + "fn seeded() { let normal_source_marker = 1; }\n", + ) + .unwrap(); + + let status = Command::new(tgrep_bin()) + .args([ + "index", + root.to_str().unwrap(), + "--index-path", + index_dir.to_str().unwrap(), + ]) + .status() + .expect("failed to run tgrep index"); + assert!(status.success(), "initial index build failed"); + + let child = Command::new(tgrep_bin()) + .args([ + "serve", + "--index-path", + index_dir.to_str().unwrap(), + root.to_str().unwrap(), + ]) + .stderr(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .spawn() + .expect("failed to start tgrep serve"); + let _server = ServerGuard { child }; + + let (_pid, port) = wait_for_server(&index_dir); + assert!( + wait_for_match(port, "normal_source_marker", Duration::from_secs(30)), + "expected the seeded source file to be searchable" + ); + thread::sleep(Duration::from_secs(2)); + + // Staged under a dot-prefixed name so the watcher ignores it while it is + // being built, and on the same filesystem so the move below is atomic. + let staging = root.join(".staging"); + fs::create_dir_all(&staging).unwrap(); + fs::write(staging.join(".gitignore"), "secret.txt\n").unwrap(); + fs::write(staging.join("secret.txt"), "moved_subtree_secret_marker\n").unwrap(); + fs::write( + staging.join("keep.rs"), + "fn keep() { let moved_subtree_keep_marker = 3; }\n", + ) + .unwrap(); + + fs::rename(&staging, root.join("vendor")).unwrap(); + + assert!( + wait_for_match(port, "moved_subtree_keep_marker", Duration::from_secs(60)), + "watcher never indexed the non-ignored file in a subtree that arrived whole" + ); + assert_eq!( + search_matches(port, "moved_subtree_secret_marker"), + 0, + "watcher indexed a file excluded by a .gitignore that arrived inside \ + the same subtree, so the subtree was indexed against stale rules" + ); +} From 2907e1ba1b3deb3e257045eef0bde4da08a293cc Mon Sep 17 00:00:00 2001 From: Shengyu Fu Date: Wed, 26 Aug 2026 22:55:23 -0700 Subject: [PATCH 06/28] Re-subscribe to a directory that is removed and recreated MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The kernel releases an inotify watch by itself when its directory is deleted or moved away, and nothing reports that the descriptor is gone. The path stayed in `watched`, so a directory recreated at the same location looked subscribed while receiving no events at all. Nothing downstream could recover it either. `add_all` skipped it as already watched, and so did every later `sync`: the path is in `desired` *and* in `watched`, which is indistinguishable from a live subscription. The entry stayed poisoned for the life of the process, so `rm -rf build && mkdir build`, a branch switch or a `git clean` silently stopped the directory being watched until the server restarted. Two cheap halves. `forget` clears the entry when a removal event arrives, which is a single hash lookup — deleting a tree delivers one event per directory in it, so anything proportional to the whole watched set would make that quadratic. And `watch_new_subtree` now re-issues subscriptions rather than trusting `watched`, since a directory that has just appeared is precisely the case where that belief is worthless; `inotify_add_watch` is idempotent, so re-adding costs a syscall and returns the existing descriptor. The forced path still reports only genuinely new directories, so the recovery scan does not treat a whole subtree as freshly watched. Descendants carried off by a move deliver no events of their own, but the next sync no longer finds them under the root and unsubscribes them there. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: bcc85b55-e821-447e-b9a9-144b78829ae4 --- tgrep-cli/src/serve.rs | 126 +++++++++++++++++- tgrep-cli/tests/watcher_watch_registration.rs | 83 ++++++++++++ 2 files changed, 205 insertions(+), 4 deletions(-) diff --git a/tgrep-cli/src/serve.rs b/tgrep-cli/src/serve.rs index 4cf55fb..76c9bfd 100644 --- a/tgrep-cli/src/serve.rs +++ b/tgrep-cli/src/serve.rs @@ -1804,6 +1804,33 @@ impl WatchRegistry { /// else, and giving up on the entire tree is exactly the failure mode this /// registration exists to avoid. fn add_all<'a>(&mut self, desired: impl IntoIterator) -> Vec { + self.subscribe(desired, false) + } + + /// Subscribe to every directory in `dirs`, re-issuing the subscription even + /// for ones already recorded as watched. + /// + /// For directories that have just appeared, where membership in `watched` + /// proves nothing. The kernel drops an inotify watch by itself when its + /// directory is deleted or moved away, and nothing tells us the descriptor + /// is gone — so a path recreated at the same location would look + /// subscribed while receiving no events at all. [`Self::add_all`] would + /// skip it, and so would every later [`Self::sync`], since the path is in + /// `desired` *and* in `watched`: the entry stays poisoned until the server + /// restarts. Re-adding is cheap and idempotent (`inotify_add_watch` + /// returns the existing descriptor), so the doubt is worth paying for. + /// + /// Returns only the directories that were not previously recorded, so a + /// caller's notion of "newly watched" keeps its meaning. + fn resubscribe_all<'a>(&mut self, dirs: impl IntoIterator) -> Vec { + self.subscribe(dirs, true) + } + + fn subscribe<'a>( + &mut self, + desired: impl IntoIterator, + force: bool, + ) -> Vec { let mut added = Vec::new(); let mut failures = 0; // Iterating `desired` and testing membership is deliberate: a @@ -1811,15 +1838,24 @@ impl WatchRegistry { // this runs per newly created directory on repositories where that set // is tens of thousands of entries. for dir in desired { - if self.watched.contains(dir) { + let known = self.watched.contains(dir); + if known && !force { continue; } match self.watcher.watch(dir, RecursiveMode::NonRecursive) { Ok(()) => { - self.watched.insert(dir.clone()); - added.push(dir.clone()); + if !known { + self.watched.insert(dir.clone()); + added.push(dir.clone()); + } } Err(e) => { + // A forced re-add that fails means the directory is gone + // again; drop the entry so a later attempt can retry it + // rather than trusting a descriptor that does not exist. + if known { + self.watched.remove(dir); + } // One line per call, not per directory: exhausting the // inotify budget fails thousands of these at once. if failures == 0 { @@ -1839,6 +1875,25 @@ impl WatchRegistry { added } + /// Drop a path that no longer exists from the subscription set. + /// + /// The kernel has already released the descriptor if the directory was + /// deleted; the `unwatch` is for the moved-away case, where it is still + /// live and now pointing outside the tree. What matters either way is + /// clearing `watched`, so that if the path comes back, it is treated as + /// the new directory it is instead of an already-subscribed one. + /// + /// Cheap by design: one hash lookup per removal event, because deleting a + /// tree delivers one event per directory in it and anything proportional + /// to the whole watched set would turn that into quadratic work. + /// Descendants left behind by a move are pruned by the next + /// [`Self::sync`], which no longer finds them under the root. + fn forget(&mut self, path: &Path) { + if self.watched.remove(path) { + let _ = self.watcher.unwatch(path); + } + } + /// Bring the subscription set in line with `desired`, subscribing to /// directories that are newly relevant and dropping ones that are not. /// @@ -2054,7 +2109,12 @@ fn watch_new_subtree(state: &Arc, root: &Path, dir: &Path) { // matters at scale — a monorepo can hold tens of thousands of // watched directories, and materialising a union for every newly // created directory would be quadratic over a checkout. - registry.add_all(level.iter()); + // + // Forced, because these directories have just appeared: a path + // recreated where a watched one used to be is still recorded as + // watched, but the kernel dropped its descriptor when the original + // went away. + registry.resubscribe_all(level.iter()); } // The registry lock is released before any `read_dir`, so a large @@ -2236,6 +2296,18 @@ fn handle_fs_event(state: &Arc, root: &Path, event: &Event) { let is_remove = matches!(event.kind, EventKind::Remove(_)) || !path.exists(); if is_remove { + // A watched directory that disappears takes its descriptor with + // it, but not its entry in the registry. Clearing that entry is + // what lets the path be subscribed again if it comes back — and + // keeps `watched` from accumulating dead paths between syncs. + // Done for every removed path rather than only known directories, + // since by now there is nothing left to ask what it was; a path + // that was never watched is a single failed hash lookup. + if PER_DIRECTORY_WATCHES + && let Some(registry) = state.watch_registry.lock().unwrap().as_mut() + { + registry.forget(path); + } // notify can deliver Remove events for transient/unknown paths // (e.g. a build tool's temp file). Suppress the noisy log line // for those, but still apply the delete unconditionally — if @@ -4235,6 +4307,52 @@ mod tests { assert_eq!(registry.watched, [c].into_iter().collect()); } + #[test] + fn a_recreated_directory_is_subscribed_again_rather_than_assumed_watched() { + // The kernel releases an inotify watch by itself when its directory is + // deleted or moved away, and says nothing about it. A path recreated at + // the same location therefore *looks* subscribed while receiving no + // events — and because it is in `desired` as well as in `watched`, no + // later `sync` can tell the difference either. The entry stays poisoned + // for the life of the process, so the directory silently stops being + // watched forever. + let tmp = TempDir::new().unwrap(); + let a = tmp.path().join("a"); + std::fs::create_dir(&a).unwrap(); + + let watcher = notify::recommended_watcher(|_: notify::Result| {}).unwrap(); + let mut registry = WatchRegistry { + watcher, + watched: std::collections::HashSet::new(), + }; + assert_eq!(registry.add_all(std::slice::from_ref(&a)).len(), 1); + + // What the removal event does. Without it the entry below survives. + registry.forget(&a); + assert!( + !registry.watched.contains(&a), + "a removed directory must not be left recorded as watched" + ); + assert_eq!( + registry.add_all(std::slice::from_ref(&a)).len(), + 1, + "a directory recreated after removal must be subscribed again" + ); + + // And the belt-and-braces half: even with the entry still present — + // a move away delivers no event for the descendants it takes with it — + // a directory that has just appeared gets its subscription re-issued. + // Already-known paths are not reported as new, so the recovery scan + // does not treat the whole subtree as freshly watched. + assert!( + registry + .resubscribe_all(std::slice::from_ref(&a)) + .is_empty(), + "re-issuing a subscription must not report an existing path as new" + ); + assert!(registry.watched.contains(&a)); + } + #[cfg(unix)] #[test] fn is_real_dir_rejects_a_symlink_to_a_directory() { diff --git a/tgrep-cli/tests/watcher_watch_registration.rs b/tgrep-cli/tests/watcher_watch_registration.rs index d9fc215..87d370d 100644 --- a/tgrep-cli/tests/watcher_watch_registration.rs +++ b/tgrep-cli/tests/watcher_watch_registration.rs @@ -387,3 +387,86 @@ fn watcher_honors_ignore_rules_inside_a_subtree_that_arrives_whole() { the same subtree, so the subtree was indexed against stale rules" ); } + +/// A directory removed and recreated at the same path must still be watched. +/// +/// The kernel drops an inotify watch when its directory goes away and does not +/// say so, leaving the path recorded as watched with no descriptor behind it. +/// Nothing later can tell that entry from a live one — it is in the desired set +/// *and* in the watched set — so without explicitly clearing it on removal the +/// directory stops being watched for the life of the process. +/// +/// `rm -rf build && mkdir build`, a branch switch, and a `git clean` all do +/// exactly this. +#[test] +fn watcher_rewatches_a_directory_that_is_removed_and_recreated() { + let dir = TempDir::new().unwrap(); + let root = dir.path(); + let index_dir = root.join(".tgrep_test_index"); + + fs::create_dir_all(root.join(".git")).unwrap(); + fs::write(root.join(".gitignore"), "build/\n").unwrap(); + fs::create_dir_all(root.join("src").join("gen")).unwrap(); + fs::write( + root.join("src").join("lib.rs"), + "fn seeded() { let normal_source_marker = 1; }\n", + ) + .unwrap(); + fs::write( + root.join("src").join("gen").join("old.rs"), + "fn old() { let pre_delete_marker = 2; }\n", + ) + .unwrap(); + + let status = Command::new(tgrep_bin()) + .args([ + "index", + root.to_str().unwrap(), + "--index-path", + index_dir.to_str().unwrap(), + ]) + .status() + .expect("failed to run tgrep index"); + assert!(status.success(), "initial index build failed"); + + let child = Command::new(tgrep_bin()) + .args([ + "serve", + "--index-path", + index_dir.to_str().unwrap(), + root.to_str().unwrap(), + ]) + .stderr(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .spawn() + .expect("failed to start tgrep serve"); + let _server = ServerGuard { child }; + + let (_pid, port) = wait_for_server(&index_dir); + assert!( + wait_for_match(port, "pre_delete_marker", Duration::from_secs(30)), + "expected the seeded file in src/gen to be searchable" + ); + thread::sleep(Duration::from_secs(2)); + + let gen_dir = root.join("src").join("gen"); + fs::remove_dir_all(&gen_dir).unwrap(); + thread::sleep(Duration::from_secs(2)); + fs::create_dir_all(&gen_dir).unwrap(); + + // Deliberately after the recreation has been processed. A file written in + // the same breath would be picked up by the subscription pass's own scan + // and would say nothing about whether the watch itself was re-established. + thread::sleep(Duration::from_secs(3)); + fs::write( + gen_dir.join("new.rs"), + "fn regenerated() { let post_recreate_marker = 3; }\n", + ) + .unwrap(); + + assert!( + wait_for_match(port, "post_recreate_marker", Duration::from_secs(30)), + "watcher never saw a file written to a directory that was removed and \ + recreated, so its subscription was not re-established" + ); +} From 329ec6663dba1b0178a45ab236802c0fbcf811f0 Mon Sep 17 00:00:00 2001 From: Shengyu Fu Date: Wed, 26 Aug 2026 23:17:46 -0700 Subject: [PATCH 07/28] Address review round 3: recovery scans, event gating, eligibility Three follow-ups from review, all cases where the watcher and the walk could disagree about the index. 1. The directories returned by the three non-stale subscription passes were discarded. The justification comments were wrong: on a warm start the stale check runs on a thread spawned before the watcher, so it can publish while the registry is still empty, making the startup sync the first descendant pass with nothing following it. Files written to a directory between the walk reaching it and the subscription being taken were then invisible until the hourly reconcile. All three sites now hand their directories to spawn_recovery_scan, which waits out `indexing` before looking -- during a cold build the stamps are not yet written and every file would read as changed. 2. watch_new_subtree ran for every directory event, including Modify(Metadata). A recursive chmod or a branch switch re-walked and re-subscribed each subtree once per directory in it. Gated to the kinds that can actually introduce a directory: Create and Modify(Name). 3. reindex_file ignored the walker's binary-extension and max-filesize rules, so a file arriving through the watcher was indexed even when a walk of the same tree would reject it -- and the next reconcile deleted it again. The check now mirrors walk_file_metadata, and drops any entry it already holds when a file stops being eligible. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: bcc85b55-e821-447e-b9a9-144b78829ae4 --- tgrep-cli/src/serve.rs | 128 +++++++++++++++--- tgrep-cli/tests/watcher_watch_registration.rs | 96 +++++++++++++ tgrep-core/src/walker.rs | 6 +- 3 files changed, 209 insertions(+), 21 deletions(-) diff --git a/tgrep-cli/src/serve.rs b/tgrep-cli/src/serve.rs index 76c9bfd..fad18ec 100644 --- a/tgrep-cli/src/serve.rs +++ b/tgrep-cli/src/serve.rs @@ -1576,13 +1576,14 @@ fn start_file_watcher(state: Arc, root: &Path, queue_cap: usize) -> if state.gitignore_pending.load(Ordering::SeqCst) { eprintln!("[trace] watcher subscriptions deferred until the ignore matcher is ready"); } else { - // The newly watched directories are deliberately not rechecked here. - // This is every directory in the repository, the index was built or - // opened moments ago, and the stale check that follows startup - // reconciles the same drift while holding `snapshot_gate` — which this - // path does not hold and must not take, since the watcher has to be - // receiving events before that check runs. - let _ = sync_watch_registrations(&state, root); + // This can be the first subscription pass the repository ever gets: + // the stale check runs on a thread spawned before this function, so it + // can publish while `watch_registry` is still `None` and take no + // subscriptions at all. Its walk is then already over by the time we + // subscribe here, and nothing else revisits the tree until the hourly + // reconcile — so the recovery scan is not optional on this path. + let newly_watched = sync_watch_registrations(&state, root); + spawn_recovery_scan(&state, root, newly_watched); } let worker_state = Arc::clone(&state); @@ -2221,6 +2222,49 @@ fn schedule_ignore_rules_refresh(state: Arc, root: PathBuf) { }); } +/// Recheck newly watched directories on a background thread. +/// +/// For the publish sites that cannot scan inline. Until a directory is +/// subscribed it cannot report a write, and the walk that decided to subscribe +/// to it may have passed it before that write happened, so a file landing in +/// that window is in neither place and would wait for the hourly reconcile. +/// +/// Spawned because at startup this is every directory in the repository and +/// the callers are on paths that must not block: `start_file_watcher` has to +/// get its worker running, and an index build must not stop to stat the tree +/// it is already reading. +/// +/// Waits out `indexing` first. During a build the stamps do not describe the +/// index yet, so every file would look changed and the scan would re-read the +/// whole repository alongside the build that is already doing it. +fn spawn_recovery_scan(state: &Arc, root: &Path, dirs: Vec) { + if dirs.is_empty() || !PER_DIRECTORY_WATCHES { + return; + } + let state = Arc::clone(state); + let root = root.to_path_buf(); + let spawned = thread::Builder::new() + .name("tgrep-watch-recovery".into()) + .spawn(move || { + while state.indexing.load(Ordering::SeqCst) { + thread::sleep(Duration::from_millis(200)); + } + // Read, not write: this does exactly what `handle_fs_event` does, + // and that runs under the read side. Taking it at all is what + // keeps the stamp check and the index update from interleaving + // with a flush. + let _gate = state.snapshot_gate.read().unwrap(); + reindex_files_in(&state, &root, &dirs); + }); + if spawned.is_err() { + eprintln!( + "[trace] warning: could not start the watcher recovery scan; \ + files written while subscriptions were being established will \ + wait for the next reconcile" + ); + } +} + fn handle_fs_event(state: &Arc, root: &Path, event: &Event) { let dominated_kinds = matches!( event.kind, @@ -2329,11 +2373,22 @@ fn handle_fs_event(state: &Arc, root: &Path, event: &Event) { } if !path.is_file() { + // Only for events that can actually introduce a directory. Any + // `Modify` would include `Modify(Metadata)`, which a recursive + // chmod or a checkout fires once per directory — and each one + // would re-walk and re-subscribe that directory's whole subtree + // on the single watcher worker, turning a linear operation into + // quadratic work. inotify announces a new directory as `Create` + // and one moved in as `Modify(Name)`; nothing else can. + let introduces_dir = matches!( + event.kind, + EventKind::Create(_) | EventKind::Modify(notify::event::ModifyKind::Name(_)) + ); // `is_real_dir` rather than `is_dir`: the latter follows symlinks, // and a link to a directory is not something the walker descends // into, so subscribing to and indexing its target would pull in a // tree the index never contained — possibly outside `root`. - if PER_DIRECTORY_WATCHES && is_real_dir(path) { + if PER_DIRECTORY_WATCHES && introduces_dir && is_real_dir(path) { // With non-recursive subscriptions notify will not extend the // watch set for us, so a directory that just appeared — and // anything already inside it — has to be picked up here. @@ -2371,6 +2426,37 @@ fn reindex_file(state: &ServerState, path: &Path, rel_path: &str) { }, Err(_) => return, }; + + // The same two rules `walk_file_metadata` applies, and for the same + // reason: the walk is authoritative about what belongs in the index, so + // anything it rejects must not be added here. Without this a file that + // grew past the cap — or an ineligible extension in a directory a relaxed + // ignore rule just exposed — would be read whole and indexed, and the next + // reconcile would silently delete it again. + let eligible = !tgrep_core::walker::is_binary_extension(path) + && !state + .max_file_size + .is_some_and(|limit| current.size > limit); + if !eligible { + // It may have been eligible when it was last indexed — a file can grow + // past the cap. Drop what we hold so the index matches the walk rather + // than keeping a stale copy of the smaller version until the reconcile. + if state + .file_stamps + .write() + .unwrap() + .remove(rel_path) + .is_some() + { + eprintln!("[trace] reindex: dropped {rel_path} (no longer eligible)"); + state.index.write().unwrap().live.delete_file(rel_path); + if let Ok(mut cache) = state.cache.write() { + cache.pop(rel_path); + } + } + return; + } + if state.file_stamps.read().unwrap().get(rel_path) == Some(¤t) { return; } @@ -3185,13 +3271,13 @@ fn bootstrap_index_build(state: &Arc, root: &Path, index_dir: &Path state.no_require_git, ); let found = matcher.is_some(); - // These subscriptions are the repository's first, so "newly watched" - // is every directory in it. No recovery scan: the startup stale check - // runs right after this and does a full walk-versus-index diff, which - // is a strict superset of what a recovery scan would find — and doing - // it here would stat the entire tree while holding the gate, on the - // path a warm start exists to keep fast. - let _ = publish_ignore_matcher(state, root, matcher); + // "Newly watched" here is every directory in the repository, and the + // build's walk ran before any of them were subscribed. Deferred rather + // than skipped: the scan waits out `indexing` and then costs one + // `metadata` call per file, since the stamps this build just wrote + // describe the index exactly. + let newly_watched = publish_ignore_matcher(state, root, matcher); + spawn_recovery_scan(state, root, newly_watched); eprintln!( "[trace] gitignore matcher built from {} file(s) in {:.1}ms{}", outcome.gitignore_files.len(), @@ -3284,11 +3370,13 @@ fn background_index_build(state: &Arc, root: &Path, index_dir: &Pat state.no_require_git, ); let has_matcher = matcher.is_some(); - // Same reasoning as the bootstrap publish: this is the cold-start - // build, so "newly watched" is the whole tree, `indexing` keeps the - // watcher off the index for the rest of the build, and the stale check - // that follows reconciles anything written during it. - let _ = publish_ignore_matcher(state, root, matcher); + // Subscriptions are taken here, partway through the build, so files + // written to a directory the walk has already passed are in neither + // the build's results nor any event. The scan waits for the build to + // finish before looking, because until then the stamps describe + // nothing and every file would read as changed. + let newly_watched = publish_ignore_matcher(state, root, matcher); + spawn_recovery_scan(state, root, newly_watched); eprintln!( "[trace] gitignore matcher built from index walk in {:.1}ms \ ({} .gitignore + {} .ignore files{})", diff --git a/tgrep-cli/tests/watcher_watch_registration.rs b/tgrep-cli/tests/watcher_watch_registration.rs index 87d370d..0ee4474 100644 --- a/tgrep-cli/tests/watcher_watch_registration.rs +++ b/tgrep-cli/tests/watcher_watch_registration.rs @@ -470,3 +470,99 @@ fn watcher_rewatches_a_directory_that_is_removed_and_recreated() { recreated, so its subscription was not re-established" ); } + +/// The walk decides what belongs in the index; the watcher must agree with it. +/// +/// `should_skip_watcher_path` only filters by location — excludes, ignore +/// rules, hidden paths. It says nothing about the two rules the walker applies +/// per file: binary extensions are skipped, and so is anything over +/// `--max-filesize`. A file arriving through the watcher therefore used to be +/// indexed even when a walk of the very same tree would have rejected it, so +/// the index disagreed with itself depending on whether a file was present at +/// startup or written afterwards — and the next reconcile silently deleted it. +/// +/// Both rejections are asserted alongside an eligible file written at the same +/// moment. Without that control the test would pass just as happily against a +/// watcher that had stopped indexing anything at all. +#[test] +fn watcher_applies_the_same_file_eligibility_rules_as_the_walker() { + let dir = TempDir::new().unwrap(); + let root = dir.path(); + let index_dir = root.join(".tgrep_test_index"); + + fs::create_dir_all(root.join(".git")).unwrap(); + fs::create_dir_all(root.join("src")).unwrap(); + fs::write( + root.join("src").join("lib.rs"), + "fn seeded() { let seeded_source_marker = 1; }\n", + ) + .unwrap(); + + let status = Command::new(tgrep_bin()) + .args([ + "index", + root.to_str().unwrap(), + "--index-path", + index_dir.to_str().unwrap(), + "--max-filesize", + "2K", + ]) + .status() + .expect("failed to run tgrep index"); + assert!(status.success(), "initial index build failed"); + + let child = Command::new(tgrep_bin()) + .args([ + "serve", + "--index-path", + index_dir.to_str().unwrap(), + "--max-filesize", + "2K", + root.to_str().unwrap(), + ]) + .stderr(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .spawn() + .expect("failed to start tgrep serve"); + let _server = ServerGuard { child }; + + let (_pid, port) = wait_for_server(&index_dir); + assert!( + wait_for_match(port, "seeded_source_marker", Duration::from_secs(30)), + "expected the seeded file to be searchable" + ); + + let src = root.join("src"); + // Text content, so nothing but the extension can keep it out. + fs::write( + src.join("asset.png"), + "fn decoy() { let binary_extension_marker = 2; }\n", + ) + .unwrap(); + // Same marker, pushed past the 2K cap by padding. + let mut oversized = String::from("fn big() { let oversized_file_marker = 3; }\n"); + oversized.push_str(&"// padding\n".repeat(400)); + fs::write(src.join("huge.rs"), oversized).unwrap(); + fs::write( + src.join("extra.rs"), + "fn extra() { let eligible_file_marker = 4; }\n", + ) + .unwrap(); + + assert!( + wait_for_match(port, "eligible_file_marker", Duration::from_secs(30)), + "the eligible file written next to the rejected ones was never indexed, \ + so this test cannot say anything about the rejections" + ); + + assert_eq!( + search_matches(port, "binary_extension_marker"), + 0, + "the watcher indexed a file whose extension the walker rejects" + ); + assert_eq!( + search_matches(port, "oversized_file_marker"), + 0, + "the watcher indexed a file larger than --max-filesize" + ); +} diff --git a/tgrep-core/src/walker.rs b/tgrep-core/src/walker.rs index 2af0a04..5cc3a1b 100644 --- a/tgrep-core/src/walker.rs +++ b/tgrep-core/src/walker.rs @@ -158,7 +158,11 @@ impl Default for WalkOptions { } /// Check if a file extension indicates a binary format. -fn is_binary_extension(path: &Path) -> bool { +/// +/// Public so the watcher can apply the same rule the walk does. A file the +/// walk rejected here must not be inserted into the index by an incremental +/// update, or the two disagree about what the index contains. +pub fn is_binary_extension(path: &Path) -> bool { path.extension() .and_then(|e| e.to_str()) .is_some_and(|ext| { From 371bc77073ecc9021c66d8e9d478b07efb5aa705 Mon Sep 17 00:00:00 2001 From: Shengyu Fu Date: Thu, 27 Aug 2026 16:53:20 -0700 Subject: [PATCH 08/28] Close five watcher correctness gaps found in review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Symlink escape (high). `reindex_file` stat'd and read through `std::fs::metadata`, which follows links. The indexer walks with `follow_links(false)`, where a symlink is neither file nor dir and is skipped outright, so a link inside the repository had its target's bytes indexed under the link's own path — and the target need not be under the served root. Switched to `symlink_metadata` and made `is_file` on the link's own metadata an eligibility rule, so a link falls into the branch that drops whatever was indexed there before. Conditional drop. That branch only deleted when a stamp entry existed, so a stamp map that no longer describes the overlay left ineligible content searchable. It now also consults the live overlay. Deliberately not unconditional the way the removal branch is: removals are rare, this runs for every ineligible file a recovery scan passes, and `delete_file` records a tombstone even for a path that was never indexed. Stamp publication race. `background_index_build` assigned `state.file_stamps` after dropping the publish gate, but cleared `indexing` before it. The recovery scan waits on `indexing` and then on the gate, so it ran in that window against empty stamps: every file read as changed and the whole repository was re-read — exactly what the wait exists to prevent — and the assignment then discarded the stamps the scan had just recorded. The stamps are now published inside the gate, before the flag flips, and the flush is handed a read guard rather than a clone. Recovery scan gaps. `reindex_files_in` looked only at files directly inside each newly watched directory. It now also picks up subdirectories created in the same window (filtered by subscription membership, so the startup case stays a hash lookup apiece rather than a re-walk per level), drops entries for files removed in it, and defers to an ignore-rules refresh when it finds an ignore file that landed inside the window. That last test is bounded at both ends by mtime so a clock-skewed network mount cannot arm a refresh that arms the next one. Over-subscription. `watch_new_subtree` recorded an ignore-rules file and kept descending, taking a watch descriptor per level of a tree the rules it was about to publish would exclude — in the moved-in `node_modules` case this change exists to fix. It now abandons the descent at the point of discovery. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: bcc85b55-e821-447e-b9a9-144b78829ae4 --- tgrep-cli/src/serve.rs | 385 +++++++++++++++--- tgrep-cli/tests/watcher_watch_registration.rs | 123 ++++++ 2 files changed, 446 insertions(+), 62 deletions(-) diff --git a/tgrep-cli/src/serve.rs b/tgrep-cli/src/serve.rs index fad18ec..0cb8c9f 100644 --- a/tgrep-cli/src/serve.rs +++ b/tgrep-cli/src/serve.rs @@ -10,7 +10,7 @@ use std::path::{Path, PathBuf}; use std::sync::atomic::Ordering; use std::sync::{Arc, Mutex, RwLock}; use std::thread; -use std::time::{Duration, Instant}; +use std::time::{Duration, Instant, SystemTime}; use fs2::FileExt; use lru::LruCache; @@ -757,17 +757,18 @@ fn build_stale_matcher( /// makes the matcher swap and the index decisions around it atomic from the /// watcher's point of view. /// -/// Returns the directories that were newly subscribed to as a result. Those -/// were unwatched while the caller's walk ran, so anything written to them in -/// that window produced no event and appears in no walk result. Callers pass -/// the list to [`reindex_files_in`] once `state.file_stamps` describes the -/// index they just published. +/// Returns the directories that were newly subscribed to as a result, and the +/// moment the walk behind that decision began. Those were unwatched while the +/// caller's walk ran, so anything written to them in that window produced no +/// event and appears in no walk result. Callers pass both to +/// [`reindex_files_in`] once `state.file_stamps` describes the index they just +/// published. #[must_use = "newly watched directories need a recovery scan or writes race the subscription"] fn publish_ignore_matcher( state: &ServerState, root: &Path, matcher: Option, -) -> Vec { +) -> (Vec, SystemTime) { *state.gitignore.write().unwrap() = matcher; state.gitignore_pending.store(false, Ordering::SeqCst); // New rules mean a different set of directories worth hearing about: @@ -1582,8 +1583,8 @@ fn start_file_watcher(state: Arc, root: &Path, queue_cap: usize) -> // subscriptions at all. Its walk is then already over by the time we // subscribe here, and nothing else revisits the tree until the hourly // reconcile — so the recovery scan is not optional on this path. - let newly_watched = sync_watch_registrations(&state, root); - spawn_recovery_scan(&state, root, newly_watched); + let (newly_watched, since) = sync_watch_registrations(&state, root); + spawn_recovery_scan(&state, root, newly_watched, since); } let worker_state = Arc::clone(&state); @@ -1876,6 +1877,16 @@ impl WatchRegistry { added } + /// Whether `path` is already subscribed. + /// + /// For deciding whether a directory found during a recovery scan is one the + /// sync already knew about or one that appeared after it — the latter has + /// to be picked up explicitly, since a non-recursive subscription on its + /// parent says nothing about it. + fn is_watched(&self, path: &Path) -> bool { + self.watched.contains(path) + } + /// Drop a path that no longer exists from the subscription set. /// /// The kernel has already released the descriptor if the directory was @@ -1978,20 +1989,26 @@ fn watchable_dirs( /// published, so relaxing a rule subscribes to the tree it used to hide and /// tightening one drops it. /// -/// Returns the directories that were newly subscribed to. Until a directory is -/// subscribed it cannot report anything, so a file written to one of these -/// between the caller's walk and this call is in neither the walk's results nor -/// any event. The caller is expected to hand the list to -/// [`reindex_files_in`] once its own bookkeeping is settled. -fn sync_watch_registrations(state: &ServerState, root: &Path) -> Vec { +/// Returns the directories that were newly subscribed to, and the moment the +/// walk behind that decision began. Until a directory is subscribed it cannot +/// report anything, so a file written to one of these between the walk and the +/// subscription is in neither the walk's results nor any event. The caller is +/// expected to hand both to [`reindex_files_in`] once its own bookkeeping is +/// settled; the timestamp bounds that window, which is what lets the scan tell +/// an ignore-rules file that landed inside it from the thousands that were +/// already there and are already reflected in the matcher. +fn sync_watch_registrations(state: &ServerState, root: &Path) -> (Vec, SystemTime) { + // Before the early returns as well as the walk: a caller that gets no + // directories back still gets a usable bound. + let since = SystemTime::now(); if !PER_DIRECTORY_WATCHES { - return Vec::new(); + return (Vec::new(), since); } let mut registry = state.watch_registry.lock().unwrap(); let Some(registry) = registry.as_mut() else { // The watcher has not started yet. It syncs once as it comes up, so // there is nothing to do and nothing to remember. - return Vec::new(); + return (Vec::new(), since); }; let start = Instant::now(); @@ -2009,37 +2026,113 @@ fn sync_watch_registrations(state: &ServerState, root: &Path) -> Vec { start.elapsed().as_secs_f64() * 1000.0 ); } - added + (added, since) } -/// Re-check the files directly inside `dirs`, indexing the ones that changed. +/// Re-check the files directly inside `dirs`, indexing the ones that changed, +/// dropping the ones that are gone, and subscribing to subdirectories that +/// appeared while the subscriptions were being established. /// /// Used to close the gap between a walk and the subscriptions that follow it: /// [`reindex_file`] compares stamps first, so for a tree that did not change /// under us this costs one `metadata` call per file and indexes nothing. /// +/// `since` is when the walk behind `dirs` began — the start of the window this +/// is closing. It is only consulted for ignore-rules files, where "did this +/// arrive after the matcher was decided" cannot be answered from the stamps: +/// the dot-prefixed ones are hidden, so they are never indexed and never have +/// one. +/// /// Callers must already hold `snapshot_gate`, and `state.file_stamps` must /// already describe the index as published — a merge that replaces the stamps /// afterwards would both discard what this records and make every file here /// look changed. -fn reindex_files_in(state: &ServerState, root: &Path, dirs: &[PathBuf]) { +fn reindex_files_in(state: &Arc, root: &Path, dirs: &[PathBuf], since: SystemTime) { if dirs.is_empty() { return; } let start = Instant::now(); + // Directories whose listing succeeded, and the files those listings + // contained, for the removal sweep at the end. Only files are recorded: + // stamps describe files, so directory names would just be dead weight on a + // set that at startup spans the whole repository. + let mut swept: std::collections::HashSet = std::collections::HashSet::new(); + let mut present: std::collections::HashSet = std::collections::HashSet::new(); + for dir in dirs { + let Ok(rel_dir) = dir.strip_prefix(root) else { + continue; + }; + let rel_dir = rel_dir.to_string_lossy().replace('\\', "/"); let Ok(entries) = std::fs::read_dir(dir) else { + // No listing means no evidence, and the sweep below must not treat + // silence as absence. continue; }; + let mut subdirs: Vec = Vec::new(); for entry in entries.flatten() { - if !entry.file_type().is_ok_and(|t| t.is_file()) { - continue; - } let path = entry.path(); let Ok(rel) = path.strip_prefix(root) else { continue; }; let rel = rel.to_string_lossy().replace('\\', "/"); + let Ok(file_type) = entry.file_type() else { + // Unclassifiable, so nothing can be concluded about it — + // least of all that it is gone. + present.insert(rel); + continue; + }; + // `DirEntry::file_type` does not follow symlinks, so a symlinked + // file or directory is neither indexed nor descended into, which + // is what the walker does with `follow_links(false)`. + if file_type.is_dir() { + subdirs.push(path); + continue; + } + if !file_type.is_file() { + continue; + } + present.insert(rel.clone()); + + // An ignore-rules file that landed in this window was not seen by + // the walk that built the matcher in force, so every other file in + // this scan is being judged by rules that do not know about it. + // Indexing them now would apply the wrong rules and leave whatever + // was wrongly indexed until something touched it again. + // + // The mtime test is what keeps this quiet: a repository has an + // ignore file in every other directory and the startup scan walks + // past all of them, but they predate the walk and are already + // accounted for. Only one written inside the window can have been + // missed — and a spurious match (a `touch` in the same + // millisecond) costs an idempotent refresh, not correctness. + // + // Bounded at both ends, not just the near one. On a network mount + // whose server clock runs ahead of ours, every recently touched + // file carries a future mtime and would pass a one-sided test — on + // every scan, including the one at the end of the refresh this + // schedules, which walks the whole repository and then arms the + // next. Treating a future mtime as skew rather than as an arrival + // gives up the fix on such a mount and keeps the loop closed. + if !state.no_ignore + && is_ignore_rules_file(root, &path) + && entry + .metadata() + .and_then(|m| m.modified()) + .is_ok_and(|m| m >= since && m <= SystemTime::now()) + { + // Abandon the scan: the refresh rewalks and republishes, which + // covers these directories properly, and anything indexed + // between here and there would be judged by the stale rules. + state.ignore_rules_dirty.store(true, Ordering::SeqCst); + schedule_ignore_rules_refresh(Arc::clone(state), root.to_path_buf()); + eprintln!( + "[trace] watcher: ignore rules changed during recovery ({rel}); \ + deferring to a refresh" + ); + return; + } + let skip = { let gitignore = state.gitignore.read().unwrap(); should_skip_watcher_path(&rel, &state.exclude_dirs, gitignore.as_ref()) @@ -2048,7 +2141,39 @@ fn reindex_files_in(state: &ServerState, root: &Path, dirs: &[PathBuf]) { reindex_file(state, &path, &rel); } } + swept.insert(rel_dir); + + // A directory created in the same window is in neither `dirs` (the + // walk did not see it) nor any event (its parent's subscription is + // non-recursive, so notify does not extend to it), and would stay + // invisible until the hourly reconcile. + // + // Only the ones not already subscribed: at startup `dirs` is every + // directory in the repository and each one is a subdirectory of + // another, so descending into all of them would re-walk the tree once + // per level. The membership test reduces that to a hash lookup apiece. + if !subdirs.is_empty() { + let unwatched: Vec = { + let mut registry = state.watch_registry.lock().unwrap(); + match registry.as_mut() { + // Filtered under the lock but subscribed outside it: + // `watch_new_subtree` takes the same lock, and it is not + // reentrant. + Some(registry) => subdirs + .into_iter() + .filter(|p| !registry.is_watched(p)) + .collect(), + None => Vec::new(), + } + }; + for subdir in &unwatched { + watch_new_subtree(state, root, subdir); + } + } } + + sweep_removed_files(state, &swept, &present); + eprintln!( "[trace] watcher: rechecked {} newly watched directories in {:.1}ms", dirs.len(), @@ -2056,6 +2181,70 @@ fn reindex_files_in(state: &ServerState, root: &Path, dirs: &[PathBuf]) { ); } +/// Drop index entries for files that were deleted while subscriptions were +/// being established. +/// +/// The counterpart to the indexing pass in [`reindex_files_in`]: a file removed +/// in that window produced no event either, and unlike a modified one nothing +/// later brings it back to the watcher's attention, so it keeps answering +/// searches until the hourly reconcile. +/// +/// `swept` holds the relative directories whose listing succeeded — a failed +/// `read_dir` proves nothing and must not be read as an empty directory — and +/// `present` every file those listings contained, regardless of ignore rules or +/// eligibility. Filtering `present` would delete entries for files that are +/// still on disk and were indexed under a laxer configuration. +/// +/// The caller must already hold `snapshot_gate`. +fn sweep_removed_files( + state: &ServerState, + swept: &std::collections::HashSet, + present: &std::collections::HashSet, +) { + if swept.is_empty() { + return; + } + // One pass over the stamps rather than a lookup per swept directory: at + // startup both sides of this span the whole repository, and anything + // proportional to their product would not finish. + let gone: Vec = { + let stamps = state.file_stamps.read().unwrap(); + stamps + .keys() + .filter(|rel| { + let parent = rel.rsplit_once('/').map_or("", |(dir, _)| dir); + swept.contains(parent) && !present.contains(rel.as_str()) + }) + .cloned() + .collect() + }; + if gone.is_empty() { + return; + } + eprintln!( + "[trace] watcher: dropped {} file(s) removed while subscriptions were \ + being established", + gone.len() + ); + { + let mut index = state.index.write().unwrap(); + for rel in &gone { + index.live.delete_file(rel); + } + } + { + let mut stamps = state.file_stamps.write().unwrap(); + for rel in &gone { + stamps.remove(rel); + } + } + if let Ok(mut cache) = state.cache.write() { + for rel in &gone { + cache.pop(rel); + } + } +} + /// Subscribe to a directory that has just appeared, and to anything already /// inside it. /// @@ -2098,7 +2287,7 @@ fn watch_new_subtree(state: &Arc, root: &Path, dir: &Path) { let mut files: Vec<(PathBuf, String)> = Vec::new(); let mut found_ignore_rules = false; - while !level.is_empty() { + 'descend: while !level.is_empty() { { let mut registry = state.watch_registry.lock().unwrap(); let Some(registry) = registry.as_mut() else { @@ -2153,9 +2342,20 @@ fn watch_new_subtree(state: &Arc, root: &Path, dir: &Path) { // dot-prefixed, so the scan below would silently drop them // and index the rest of the subtree against rules that do // not know about them. + // + // Abandon the descent immediately rather than finishing it. + // Everything gathered from here on is discarded by the + // refresh anyway, and the rules that are about to be + // published are the ones that decide whether these + // directories should be watched at all — continuing would + // subscribe to every level of, say, a `node_modules/` that + // was just moved into place, which on Linux is a watch + // descriptor apiece and the exhaustion this pass exists to + // avoid. The refresh's `sync` would prune them, but only + // after they had already been taken. if !state.no_ignore && is_ignore_rules_file(root, &path) { found_ignore_rules = true; - continue; + break 'descend; } let skip = { let gitignore = state.gitignore.read().unwrap(); @@ -2237,7 +2437,12 @@ fn schedule_ignore_rules_refresh(state: Arc, root: PathBuf) { /// Waits out `indexing` first. During a build the stamps do not describe the /// index yet, so every file would look changed and the scan would re-read the /// whole repository alongside the build that is already doing it. -fn spawn_recovery_scan(state: &Arc, root: &Path, dirs: Vec) { +fn spawn_recovery_scan( + state: &Arc, + root: &Path, + dirs: Vec, + since: SystemTime, +) { if dirs.is_empty() || !PER_DIRECTORY_WATCHES { return; } @@ -2254,7 +2459,7 @@ fn spawn_recovery_scan(state: &Arc, root: &Path, dirs: Vec // keeps the stamp check and the index update from interleaving // with a flush. let _gate = state.snapshot_gate.read().unwrap(); - reindex_files_in(&state, &root, &dirs); + reindex_files_in(&state, &root, &dirs, since); }); if spawned.is_err() { eprintln!( @@ -2372,6 +2577,10 @@ fn handle_fs_event(state: &Arc, root: &Path, event: &Event) { continue; } + // `is_file` follows symlinks, so a link to a file lands in + // `reindex_file` below rather than here — deliberately: that is where + // it is recognised as ineligible and any content indexed under that + // path before it became a link is dropped. if !path.is_file() { // Only for events that can actually introduce a directory. Any // `Modify` would include `Modify(Metadata)`, which a recursive @@ -2414,40 +2623,72 @@ fn reindex_file(state: &ServerState, path: &Path, rel_path: &str) { // for atime/attribute updates, opens, etc. — re-indexing on those // would re-read large files, churn the live overlay, and produce a // misleading "modified" trace for files that didn't actually change. - let current = match std::fs::metadata(path) { - Ok(m) => FileStamp { - mtime: m - .modified() - .ok() - .and_then(|t| t.duration_since(std::time::SystemTime::UNIX_EPOCH).ok()) - .map(|d| d.as_secs()) - .unwrap_or(0), - size: m.len(), - }, + // + // `symlink_metadata` describes the link itself; `metadata` would describe + // its target. See the eligibility check below for why that distinction + // matters here. + let meta = match std::fs::symlink_metadata(path) { + Ok(m) => m, Err(_) => return, }; + let current = FileStamp { + mtime: meta + .modified() + .ok() + .and_then(|t| t.duration_since(std::time::SystemTime::UNIX_EPOCH).ok()) + .map(|d| d.as_secs()) + .unwrap_or(0), + size: meta.len(), + }; - // The same two rules `walk_file_metadata` applies, and for the same - // reason: the walk is authoritative about what belongs in the index, so - // anything it rejects must not be added here. Without this a file that - // grew past the cap — or an ineligible extension in a directory a relaxed - // ignore rule just exposed — would be read whole and indexed, and the next - // reconcile would silently delete it again. - let eligible = !tgrep_core::walker::is_binary_extension(path) + // The rules `walk_file_metadata` applies, and for the same reason: the walk + // is authoritative about what belongs in the index, so anything it rejects + // must not be added here. Without this a file that grew past the cap — or + // an ineligible extension in a directory a relaxed ignore rule just exposed + // — would be read whole and indexed, and the next reconcile would silently + // delete it again. + // + // `is_file` on the link's own metadata is the third rule, and the one with + // teeth: the walker runs with `follow_links(false)`, where a symlink is + // neither file nor dir and is skipped outright. Following one here would + // read the target and index its bytes under the link's path — and the + // target need not be under `root` at all, so a link committed to a branch + // (or dropped in by a build) is enough to pull `~/.ssh/id_rsa` into an + // index whose whole contract is that it covers the served tree. + let eligible = meta.is_file() + && !tgrep_core::walker::is_binary_extension(path) && !state .max_file_size .is_some_and(|limit| current.size > limit); if !eligible { // It may have been eligible when it was last indexed — a file can grow - // past the cap. Drop what we hold so the index matches the walk rather - // than keeping a stale copy of the smaller version until the reconcile. - if state + // past the cap, and a real file can be replaced by a link to one. Drop + // what we hold so the index matches the walk rather than keeping a + // stale copy of the smaller version until the reconcile. + // + // The stamp is not the only evidence that something is indexed. A file + // the watcher added since the last flush lives in the live overlay, + // and a stamp map that was replaced wholesale — by a stale merge, or + // by a load that failed and left it empty — no longer mentions it, so + // testing the stamp alone would skip the drop and keep serving content + // the walk rejects. `has_path` covers the overlay; an entry that is + // only in the persisted reader with no stamp to match is beyond what + // can be checked cheaply here (the reader has no path index) and is + // left to the reconcile's membership diff. + // + // Both are checked before deleting rather than deleting unconditionally + // the way the removal branch does: removals are rare, but this runs for + // every ineligible file a recovery scan walks past, and `delete_file` + // records a tombstone and dirties the overlay even when the path was + // never indexed. + let had_stamp = state .file_stamps .write() .unwrap() .remove(rel_path) - .is_some() - { + .is_some(); + let in_overlay = state.index.read().unwrap().live.has_path(rel_path); + if had_stamp || in_overlay { eprintln!("[trace] reindex: dropped {rel_path} (no longer eligible)"); state.index.write().unwrap().live.delete_file(rel_path); if let Ok(mut cache) = state.cache.write() { @@ -2970,6 +3211,10 @@ fn background_refresh_stale( let _gate = state.snapshot_gate.write().unwrap(); let mut newly_watched = Vec::new(); + // Before the walk, not after the subscriptions: this bounds the window the + // recovery scan is closing, and the window opens the moment the traversal + // that decided what to subscribe to begins. + let since = SystemTime::now(); let ok = refresh_stale_locked( state, root, @@ -2988,7 +3233,7 @@ fn background_refresh_stale( // `stream_merge_stale_changes` replaces the stamps wholesale, so scanning // any earlier would both be discarded and re-read every changed file. if ok { - reindex_files_in(state, root, &newly_watched); + reindex_files_in(state, root, &newly_watched, since); } ok } @@ -3033,7 +3278,10 @@ fn refresh_stale_locked( // the caller holds `snapshot_gate` for write across this whole body, and // the only reader of `state.gitignore` takes the read side first, so no // event can observe the matcher before this function returns either way. - *newly_watched = publish_ignore_matcher(state, root, build_stale_matcher(state, root, &walk)); + // The caller anchored the recovery window at its own walk, which starts + // earlier than the subscription sync inside this call, so the timestamp + // that comes back here is the looser of the two and is dropped. + *newly_watched = publish_ignore_matcher(state, root, build_stale_matcher(state, root, &walk)).0; if walk.skipped_error > 0 { eprintln!( @@ -3276,8 +3524,8 @@ fn bootstrap_index_build(state: &Arc, root: &Path, index_dir: &Path // than skipped: the scan waits out `indexing` and then costs one // `metadata` call per file, since the stamps this build just wrote // describe the index exactly. - let newly_watched = publish_ignore_matcher(state, root, matcher); - spawn_recovery_scan(state, root, newly_watched); + let (newly_watched, since) = publish_ignore_matcher(state, root, matcher); + spawn_recovery_scan(state, root, newly_watched, since); eprintln!( "[trace] gitignore matcher built from {} file(s) in {:.1}ms{}", outcome.gitignore_files.len(), @@ -3375,8 +3623,8 @@ fn background_index_build(state: &Arc, root: &Path, index_dir: &Pat // the build's results nor any event. The scan waits for the build to // finish before looking, because until then the stamps describe // nothing and every file would read as changed. - let newly_watched = publish_ignore_matcher(state, root, matcher); - spawn_recovery_scan(state, root, newly_watched); + let (newly_watched, since) = publish_ignore_matcher(state, root, matcher); + spawn_recovery_scan(state, root, newly_watched, since); eprintln!( "[trace] gitignore matcher built from index walk in {:.1}ms \ ({} .gitignore + {} .ignore files{})", @@ -3583,6 +3831,19 @@ fn background_index_build(state: &Arc, root: &Path, index_dir: &Pat // the newly published reader; no event is lost. let gate = state.snapshot_gate.write().unwrap(); state.flushing.store(true, Ordering::SeqCst); + + // Publish the stamps *before* clearing `indexing`, not after the flush. + // The recovery scan started at publish time waits for `indexing` to clear + // and then blocks on this gate, so it runs the instant the gate drops. If + // the stamps were still unpublished at that point every file it walked + // would compare as changed and it would re-read the entire repository — + // the exact work the wait exists to avoid — and the assignment would then + // overwrite the stamps it had just recorded for anything that really did + // change during the build, losing them until the next reconcile. + // + // Done even if the flush below fails: the live overlay already reflects + // what was just indexed, and the stamps describe that. + *state.file_stamps.write().unwrap() = stamps; state.indexing.store(false, Ordering::SeqCst); // Final flush to disk for the bulk build. Use the same streaming @@ -3592,17 +3853,17 @@ fn background_index_build(state: &Arc, root: &Path, index_dir: &Pat // intermediate incremental flushes published `complete = false` so a // mid-build kill would resume rather than be treated as finished. eprintln!("[trace] persisting final index to disk..."); - let pruned = flush_append_only_overlay_locked(state, index_dir, true, Some(&stamps)); + let pruned = { + // A read guard rather than a clone: these maps hold an entry per file + // in the repo. Nothing reachable from the flush takes this lock, and + // every other writer is behind the publish gate we hold. + let stamps = state.file_stamps.read().unwrap(); + flush_append_only_overlay_locked(state, index_dir, true, Some(&stamps)) + }; drop(gate); state.flushing.store(false, Ordering::SeqCst); - // Refresh the in-memory file_stamps so the file watcher can recognize - // unchanged files and skip spurious notify events (e.g. atime/attribute - // updates on Windows). Done even if the flush failed — the live overlay - // already reflects what we just indexed, and the stamps describe that. - *state.file_stamps.write().unwrap() = stamps; - // Reclaim memory held by the indexing-time live overlay — but only when // the flush actually completed and `prune_persisted_entries` ran. If the // flush failed, the overlay is still the source of truth and shrinking diff --git a/tgrep-cli/tests/watcher_watch_registration.rs b/tgrep-cli/tests/watcher_watch_registration.rs index 0ee4474..e8e765e 100644 --- a/tgrep-cli/tests/watcher_watch_registration.rs +++ b/tgrep-cli/tests/watcher_watch_registration.rs @@ -92,6 +92,25 @@ fn wait_for_match(port: u16, pattern: &str, timeout: Duration) -> bool { } } +/// Wait until `pattern` stops matching. +/// +/// For content that has to be *dropped* from the index. The barrier the other +/// assertions use — a second file appearing — only proves that some later event +/// was processed, which on a backend that coalesces or reorders events (macOS) +/// says nothing about the drop. +fn wait_for_no_match(port: u16, pattern: &str, timeout: Duration) -> bool { + let start = Instant::now(); + loop { + if search_matches(port, pattern) == 0 { + return true; + } + if start.elapsed() > timeout { + return false; + } + thread::sleep(Duration::from_millis(100)); + } +} + /// Read `(pid, port)` once the server is accepting connections. fn wait_for_server(index_dir: &Path) -> (u32, u16) { let serve_json = index_dir.join("serve.json"); @@ -497,6 +516,12 @@ fn watcher_applies_the_same_file_eligibility_rules_as_the_walker() { "fn seeded() { let seeded_source_marker = 1; }\n", ) .unwrap(); + // Small enough to be indexed now, and grown past the cap below. + fs::write( + root.join("src").join("grows.rs"), + "fn grows() { let outgrew_the_cap_marker = 5; }\n", + ) + .unwrap(); let status = Command::new(tgrep_bin()) .args([ @@ -543,6 +568,11 @@ fn watcher_applies_the_same_file_eligibility_rules_as_the_walker() { let mut oversized = String::from("fn big() { let oversized_file_marker = 3; }\n"); oversized.push_str(&"// padding\n".repeat(400)); fs::write(src.join("huge.rs"), oversized).unwrap(); + // Already in the index, and now over the cap: what was indexed has to go, + // not just stop being updated. + let mut grown = String::from("fn grows() { let outgrew_the_cap_marker = 5; }\n"); + grown.push_str(&"// padding\n".repeat(400)); + fs::write(src.join("grows.rs"), grown).unwrap(); fs::write( src.join("extra.rs"), "fn extra() { let eligible_file_marker = 4; }\n", @@ -565,4 +595,97 @@ fn watcher_applies_the_same_file_eligibility_rules_as_the_walker() { 0, "the watcher indexed a file larger than --max-filesize" ); + assert!( + wait_for_no_match(port, "outgrew_the_cap_marker", Duration::from_secs(30)), + "a file that grew past --max-filesize kept its indexed content" + ); +} + +#[cfg(unix)] +#[test] +fn watcher_does_not_index_through_symlinks() { + let dir = TempDir::new().unwrap(); + let root = dir.path(); + let index_dir = root.join(".tgrep_test_index"); + + // Deliberately outside the served root: this stands in for anything a + // symlink committed to a branch could point at. + let outside = TempDir::new().unwrap(); + let secret = outside.path().join("secret.txt"); + fs::write(&secret, "fn leak() { let outside_root_marker = 1; }\n").unwrap(); + + fs::create_dir_all(root.join(".git")).unwrap(); + fs::create_dir_all(root.join("src")).unwrap(); + fs::write( + root.join("src").join("lib.rs"), + "fn seeded() { let seeded_source_marker = 1; }\n", + ) + .unwrap(); + // Indexed as a real file first, then replaced by a link below. + fs::write( + root.join("src").join("swapped.rs"), + "fn swapped() { let replaced_by_symlink_marker = 2; }\n", + ) + .unwrap(); + + let status = Command::new(tgrep_bin()) + .args([ + "index", + root.to_str().unwrap(), + "--index-path", + index_dir.to_str().unwrap(), + ]) + .status() + .expect("failed to run tgrep index"); + assert!(status.success(), "initial index build failed"); + + let child = Command::new(tgrep_bin()) + .args([ + "serve", + "--index-path", + index_dir.to_str().unwrap(), + root.to_str().unwrap(), + ]) + .stderr(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .spawn() + .expect("failed to start tgrep serve"); + let _server = ServerGuard { child }; + + let (_pid, port) = wait_for_server(&index_dir); + assert!( + wait_for_match(port, "seeded_source_marker", Duration::from_secs(30)), + "expected the seeded file to be searchable" + ); + assert_eq!( + search_matches(port, "replaced_by_symlink_marker"), + 1, + "expected the file that is about to be replaced to start out indexed" + ); + + let src = root.join("src"); + std::os::unix::fs::symlink(&secret, src.join("link.rs")).unwrap(); + fs::remove_file(src.join("swapped.rs")).unwrap(); + std::os::unix::fs::symlink(&secret, src.join("swapped.rs")).unwrap(); + fs::write( + src.join("extra.rs"), + "fn extra() { let eligible_file_marker = 3; }\n", + ) + .unwrap(); + + assert!( + wait_for_match(port, "eligible_file_marker", Duration::from_secs(30)), + "the eligible file written alongside the symlinks was never indexed, \ + so this test cannot say anything about the symlinks" + ); + + assert!( + wait_for_no_match(port, "replaced_by_symlink_marker", Duration::from_secs(30)), + "a real file replaced by a symlink kept its old content in the index" + ); + assert_eq!( + search_matches(port, "outside_root_marker"), + 0, + "the watcher followed a symlink and indexed content from outside the served root" + ); } From 700e64eeae3a16182b1863b9b21422a88315906a Mon Sep 17 00:00:00 2001 From: Shengyu Fu Date: Thu, 27 Aug 2026 17:19:18 -0700 Subject: [PATCH 09/28] Close the remaining watcher recovery and symlink gaps Review round 5. Read the file through one no-follow handle. `symlink_metadata` and a later `std::fs::read` are two lookups of the same name, and a tree being rewritten underneath us can swap a regular file for a link in between, so the bytes indexed were not necessarily the ones judged eligible. `open_no_follow` opens the path itself -- O_NOFOLLOW on unix, the reparse point on Windows -- and the type, size, mtime and contents all come off that handle. Stop keying the ineligible-file drop on a stamp entry. `ServerState` accepts an empty stamp map when filestamps.json is missing or unreadable, and the reader can still hold the path in that state, so the delete now always happens. An existing tombstone is taken as proof there is nothing left to do, which keeps a startup scan over a repository full of binary assets from dirtying the overlay once per file. Anchor the recovery window at the walk that produced the matcher, not at the subscription sync that follows it. A nested `.ignore` written between the two carries an mtime predating a timestamp taken later and would be read as already accounted for, leaving its subtree indexed under rules that never saw it. `publish_ignore_matcher` now takes `since` from the caller. Replay the events discarded during the initial build. They could not be applied then -- the stamps do not describe the index yet -- but they are the only record that those paths moved, and the build's own walk misses anything written to a directory it has already passed. Buffered, capped, and replayed as synthetic create events once the build publishes, so they go through the same filtering an ordinary event gets; an overflowing burst falls back to a full reconcile. This is also the first recovery any of this offers on a whole-subtree backend, where there are no per-directory subscriptions to have raced. Include the root in the recovery scan. It is subscribed as the watcher starts, so every later sync sees it as already watched and it never appears in the newly-watched list -- nothing covered a file written to the top level while a build walk was deeper in the tree. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: bcc85b55-e821-447e-b9a9-144b78829ae4 --- tgrep-cli/src/serve.rs | 512 +++++++++++++++++++++++++++++++++++------ 1 file changed, 438 insertions(+), 74 deletions(-) diff --git a/tgrep-cli/src/serve.rs b/tgrep-cli/src/serve.rs index 0cb8c9f..da9938d 100644 --- a/tgrep-cli/src/serve.rs +++ b/tgrep-cli/src/serve.rs @@ -290,6 +290,23 @@ struct ServerState { ignore_rules_dirty: std::sync::atomic::AtomicBool, /// Ensures a burst of ignore-file events uses at most one refresh worker. ignore_refresh_scheduled: std::sync::atomic::AtomicBool, + /// Paths the watcher saw while `indexing` was set, kept so they can be + /// replayed once the build publishes. + /// + /// Events that arrive during a build cannot be applied — the stamps do not + /// describe the index yet, so every path would read as changed — but they + /// are the only record that those paths moved. The build's own walk misses + /// anything written to a directory it has already passed, and on a + /// whole-subtree backend there is no per-directory recovery scan to fall + /// back on, so discarding them leaves the change invisible until the hourly + /// reconcile. + /// + /// `None` means the buffer overflowed and the paths were dropped; the + /// replay then falls back to a full stale refresh, which is slower but + /// complete. Bounded because a build can run for minutes on a large + /// repository and a checkout or a build tree churning underneath it is + /// unbounded. + deferred_events: Mutex>>, /// Progress: number of files indexed so far. index_progress: std::sync::atomic::AtomicU64, /// Total files discovered for indexing. @@ -573,6 +590,7 @@ pub fn run(root: &Path, index_path: Option<&Path>, options: ServeOptions<'_>) -> gitignore_pending: std::sync::atomic::AtomicBool::new(!no_watch && !no_ignore), ignore_rules_dirty: std::sync::atomic::AtomicBool::new(false), ignore_refresh_scheduled: std::sync::atomic::AtomicBool::new(false), + deferred_events: Mutex::new(Some(std::collections::HashSet::new())), index_progress: std::sync::atomic::AtomicU64::new(0), index_total: std::sync::atomic::AtomicU64::new(0), watch_enabled: !no_watch, @@ -757,24 +775,29 @@ fn build_stale_matcher( /// makes the matcher swap and the index decisions around it atomic from the /// watcher's point of view. /// -/// Returns the directories that were newly subscribed to as a result, and the -/// moment the walk behind that decision began. Those were unwatched while the -/// caller's walk ran, so anything written to them in that window produced no -/// event and appears in no walk result. Callers pass both to -/// [`reindex_files_in`] once `state.file_stamps` describes the index they just -/// published. +/// Returns the directories that were newly subscribed to as a result. Those +/// were unwatched while the caller's walk ran, so anything written to them in +/// that window produced no event and appears in no walk result. Callers pass +/// them to [`reindex_files_in`] once `state.file_stamps` describes the index +/// they just published. +/// +/// `since` is when the walk that produced `matcher` began, and is only passed +/// through so the recovery scan can use it. It has to come from the caller: the +/// subscription walk inside this function starts later, and an ignore file +/// written between the two would predate a timestamp taken here and be read as +/// already accounted for by rules that never saw it. #[must_use = "newly watched directories need a recovery scan or writes race the subscription"] fn publish_ignore_matcher( state: &ServerState, root: &Path, matcher: Option, -) -> (Vec, SystemTime) { +) -> Vec { *state.gitignore.write().unwrap() = matcher; state.gitignore_pending.store(false, Ordering::SeqCst); // New rules mean a different set of directories worth hearing about: // a tightened rule releases the subscriptions under it, and a relaxed // one takes subscriptions for the tree it used to hide. - sync_watch_registrations(state, root) + sync_watch_registrations(state, root).0 } fn handle_connection(stream: TcpStream, state: &ServerState) -> Result<()> { @@ -2424,6 +2447,103 @@ fn schedule_ignore_rules_refresh(state: Arc, root: PathBuf) { /// Recheck newly watched directories on a background thread. /// +/// Remember the paths in an event that arrived mid-build, for replay once the +/// build publishes. +/// +/// The event cannot be applied now: until `indexing` clears, `file_stamps` does +/// not describe the index, so every path would compare as changed and the +/// watcher would re-read the repository alongside the build already reading it. +/// It cannot simply be dropped either — see [`ServerState::deferred_events`]. +/// +/// Capped, and the cap discards the whole set rather than truncating it: a +/// partial set is indistinguishable from a complete one at replay time, and +/// silently recovering nine tenths of a checkout is worse than knowing to fall +/// back on a full refresh. +fn defer_events_during_build(state: &ServerState, event: &Event) { + /// A checkout of the Linux kernel is ~90k files. Above this the fallback + /// refresh is cheaper than the replay would be anyway. + const MAX_DEFERRED: usize = 100_000; + + let Ok(mut deferred) = state.deferred_events.lock() else { + return; + }; + let Some(paths) = deferred.as_mut() else { + return; + }; + if paths.len().saturating_add(event.paths.len()) > MAX_DEFERRED { + *deferred = None; + eprintln!( + "[trace] warning: too many file changes during the initial index build to replay \ + individually; a full reconcile will run instead" + ); + return; + } + paths.extend(event.paths.iter().cloned()); +} + +/// Apply the events that arrived during the build, now that it has published. +/// +/// Replayed as synthetic create events rather than handled directly so they go +/// through exactly the filtering an ordinary event gets — ignore rules, the +/// exclude list, the index directory, new-subtree subscription. `Create(Any)` +/// is the right kind for all of them: `handle_fs_event` decides removal from +/// whether the path still exists, and modifications and creations take the same +/// route, so one kind covers arrivals, edits and deletions alike. +/// +/// The caller must *not* hold `snapshot_gate`; `handle_fs_event` takes it. +fn replay_deferred_events(state: &Arc, root: &Path) { + let deferred = match state.deferred_events.lock() { + // Leaves `Some(empty)` behind, so anything deferred by a later build + // (a reconcile sets `indexing` again) is collected from scratch. + Ok(mut guard) => guard.replace(std::collections::HashSet::new()), + Err(_) => return, + }; + let Some(paths) = deferred else { + // Overflowed. A stale refresh rewalks the tree and diffs it against the + // index, which covers every path the replay would have, and it is what + // already runs when ignore rules change mid-build. + eprintln!("[trace] watcher: reconciling after too many changes during the initial build"); + let state = Arc::clone(state); + let root = root.to_path_buf(); + if thread::Builder::new() + .name("tgrep-deferred-reconcile".into()) + .spawn(move || { + if !background_refresh_stale(&state, &root, &state.index_dir, true) { + eprintln!( + "[trace] warning: the post-build reconcile did not complete; changes made \ + during the build wait for the next one" + ); + } + }) + .is_err() + { + eprintln!("[trace] warning: could not start the post-build reconcile"); + } + return; + }; + if paths.is_empty() { + return; + } + + let start = Instant::now(); + let count = paths.len(); + for path in paths { + handle_fs_event( + state, + root, + &Event { + kind: EventKind::Create(notify::event::CreateKind::Any), + paths: vec![path], + attrs: Default::default(), + }, + ); + } + eprintln!( + "[trace] watcher: replayed {count} change(s) deferred during the initial build in {:.1}ms", + start.elapsed().as_secs_f64() * 1000.0 + ); +} + /// For the publish sites that cannot scan inline. Until a directory is /// subscribed it cannot report a write, and the walk that decided to subscribe /// to it may have passed it before that write happened, so a file landing in @@ -2436,16 +2556,16 @@ fn schedule_ignore_rules_refresh(state: Arc, root: PathBuf) { /// /// Waits out `indexing` first. During a build the stamps do not describe the /// index yet, so every file would look changed and the scan would re-read the -/// whole repository alongside the build that is already doing it. +/// whole repository alongside the build that is already doing it. That wait is +/// also what makes this the right place to replay the events the build made the +/// watcher discard, which is why it runs even on a backend that has nothing +/// per-directory to recover. fn spawn_recovery_scan( state: &Arc, root: &Path, dirs: Vec, since: SystemTime, ) { - if dirs.is_empty() || !PER_DIRECTORY_WATCHES { - return; - } let state = Arc::clone(state); let root = root.to_path_buf(); let spawned = thread::Builder::new() @@ -2454,6 +2574,15 @@ fn spawn_recovery_scan( while state.indexing.load(Ordering::SeqCst) { thread::sleep(Duration::from_millis(200)); } + // Before the gate, not under it: this takes `snapshot_gate` for + // read itself, and std's `RwLock` may deadlock on a recursive read + // if a writer queues up in between. + replay_deferred_events(&state, &root); + + let dirs = recovery_scan_dirs(&state, &root, dirs); + if dirs.is_empty() { + return; + } // Read, not write: this does exactly what `handle_fs_event` does, // and that runs under the read side. Taking it at all is what // keeps the stamp check and the index update from interleaving @@ -2470,6 +2599,30 @@ fn spawn_recovery_scan( } } +/// The directories a recovery scan should recheck, given the ones a +/// subscription sync reported as newly watched. +/// +/// The root is added because it is never in that list: it is subscribed as the +/// watcher starts, before any matcher exists, so every later sync sees it as +/// already watched. Nothing else covers it — a file written to the top level +/// while a build walk was deeper in the tree produced no event the build could +/// use and no event the watcher would keep — and it costs one directory +/// listing. +/// +/// Empty on a whole-subtree backend, where there are no per-directory +/// subscriptions to have raced and `reindex_files_in`'s pickup of unwatched +/// subdirectories would take exactly the per-directory watches that backend +/// exists to avoid. +fn recovery_scan_dirs(state: &ServerState, root: &Path, mut dirs: Vec) -> Vec { + if !PER_DIRECTORY_WATCHES || !state.watch_enabled { + return Vec::new(); + } + if !dirs.iter().any(|d| d == root) { + dirs.push(root.to_path_buf()); + } + dirs +} + fn handle_fs_event(state: &Arc, root: &Path, event: &Event) { let dominated_kinds = matches!( event.kind, @@ -2494,8 +2647,11 @@ fn handle_fs_event(state: &Arc, root: &Path, event: &Event) { } // Skip ordinary file events while the initial background index build is in - // progress. The indexer will pick up those files itself. + // progress. The indexer will pick up those files itself — but only for the + // parts of the tree it has not reached yet, so remember these and replay + // them once it publishes. if state.indexing.load(Ordering::SeqCst) { + defer_events_during_build(state, event); return; } @@ -2610,26 +2766,110 @@ fn handle_fs_event(state: &Arc, root: &Path, event: &Event) { } } +/// Drop everything the index holds for a path. +/// +/// The delete is not conditional on a stamp entry. `ServerState` accepts an +/// empty stamp map when `filestamps.json` is missing or unreadable, and the +/// reader can still hold the path in that state, so keying the delete on the +/// stamp alone would leave content the walk now rejects searchable. The stamp +/// removal is best-effort; the index and cache deletes always happen. +/// +/// The cost of that is a tombstone in the overlay for paths that were never +/// indexed — `live::delete_file` records one either way — and this runs for +/// every ineligible file a recovery scan walks past, which at startup is every +/// binary asset in the repository. An existing tombstone is therefore taken as +/// proof there is nothing left to do, which bounds that to one per distinct +/// path. The trace line, which is the part that would be actively misleading, +/// stays conditional on there having been something to drop. +/// +/// The caller must already hold `snapshot_gate`. +fn drop_indexed_file(state: &ServerState, rel_path: &str, reason: &str) { + let had_stamp = state + .file_stamps + .write() + .unwrap() + .remove(rel_path) + .is_some(); + { + let index = state.index.read().unwrap(); + if index.live.has_path(rel_path) || had_stamp { + eprintln!("[trace] reindex: dropped {rel_path} ({reason})"); + } else if index.live.is_deleted(rel_path) { + // Already tombstoned, so there is nothing to record and no reason + // to dirty the overlay again. This is the repeat case: a recovery + // scan or a chatty editor can bring the same rejected path back + // here any number of times. + return; + } + } + state.index.write().unwrap().live.delete_file(rel_path); + if let Ok(mut cache) = state.cache.write() { + cache.pop(rel_path); + } +} + +/// Open a file without following a final symlink, so the handle is the path +/// itself rather than wherever it points. +/// +/// Every later question — is this a regular file, how big is it, what is in it +/// — is then answered from the one handle, which is what makes the eligibility +/// decision and the read describe the same object. Checking the path and then +/// reading it separately leaves a window in which a tree being rewritten under +/// us (a checkout, a build, `git mv`) can swap a regular file for a link, and +/// the read would follow it out of the served root. +fn open_no_follow(path: &Path) -> std::io::Result { + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + // Fails outright on a symlink, which is the answer we want. + std::fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW) + .open(path) + } + #[cfg(windows)] + { + use std::os::windows::fs::OpenOptionsExt; + // Opens the reparse point rather than its target. Unlike O_NOFOLLOW + // this succeeds, so the caller's `is_file` check on the handle's + // metadata is what rejects it — a reparse point is not a regular file. + const FILE_FLAG_OPEN_REPARSE_POINT: u32 = 0x0020_0000; + std::fs::OpenOptions::new() + .read(true) + .custom_flags(FILE_FLAG_OPEN_REPARSE_POINT) + .open(path) + } + #[cfg(not(any(unix, windows)))] + { + std::fs::File::open(path) + } +} + /// Read a file and merge it into the live index, unless its stamp says the /// content we already indexed is current. /// /// The caller must hold `snapshot_gate`: the read, the commit, and the stamp /// update have to be atomic with respect to a flush or auto-save. fn reindex_file(state: &ServerState, path: &Path, rel_path: &str) { + use std::io::Read; use tgrep_core::meta::FileStamp; - // Compute the file's current stamp and skip if it matches what we - // last indexed. notify on Windows in particular fires Modify events - // for atime/attribute updates, opens, etc. — re-indexing on those - // would re-read large files, churn the live overlay, and produce a - // misleading "modified" trace for files that didn't actually change. - // - // `symlink_metadata` describes the link itself; `metadata` would describe - // its target. See the eligibility check below for why that distinction - // matters here. - let meta = match std::fs::symlink_metadata(path) { - Ok(m) => m, - Err(_) => return, + // One handle for the whole decision. Opened without following a final + // symlink, and every fact below — type, size, mtime, bytes — read back off + // it, so nothing that happens to the path in the meantime can make the + // content we index disagree with the metadata we judged it by. + let file = match open_no_follow(path) { + Ok(f) => f, + Err(_) => { + // Includes the ELOOP that a symlink gets on unix. It may still be + // a path we indexed before it became one, so fall through to the + // drop rather than returning. + drop_indexed_file(state, rel_path, "no longer eligible"); + return; + } + }; + let Ok(meta) = file.metadata() else { + return; }; let current = FileStamp { mtime: meta @@ -2648,13 +2888,14 @@ fn reindex_file(state: &ServerState, path: &Path, rel_path: &str) { // — would be read whole and indexed, and the next reconcile would silently // delete it again. // - // `is_file` on the link's own metadata is the third rule, and the one with - // teeth: the walker runs with `follow_links(false)`, where a symlink is - // neither file nor dir and is skipped outright. Following one here would - // read the target and index its bytes under the link's path — and the - // target need not be under `root` at all, so a link committed to a branch - // (or dropped in by a build) is enough to pull `~/.ssh/id_rsa` into an - // index whose whole contract is that it covers the served tree. + // `is_file` is the third rule, and the one with teeth: the walker runs with + // `follow_links(false)`, where a symlink is neither file nor dir and is + // skipped outright. Indexing through one would put the target's bytes under + // the link's path — and the target need not be under `root` at all, so a + // link committed to a branch (or dropped in by a build) is enough to pull + // `~/.ssh/id_rsa` into an index whose whole contract is that it covers the + // served tree. On unix the open above has already failed for a link; this + // is what rejects one on Windows, where the reparse point opens fine. let eligible = meta.is_file() && !tgrep_core::walker::is_binary_extension(path) && !state @@ -2665,36 +2906,7 @@ fn reindex_file(state: &ServerState, path: &Path, rel_path: &str) { // past the cap, and a real file can be replaced by a link to one. Drop // what we hold so the index matches the walk rather than keeping a // stale copy of the smaller version until the reconcile. - // - // The stamp is not the only evidence that something is indexed. A file - // the watcher added since the last flush lives in the live overlay, - // and a stamp map that was replaced wholesale — by a stale merge, or - // by a load that failed and left it empty — no longer mentions it, so - // testing the stamp alone would skip the drop and keep serving content - // the walk rejects. `has_path` covers the overlay; an entry that is - // only in the persisted reader with no stamp to match is beyond what - // can be checked cheaply here (the reader has no path index) and is - // left to the reconcile's membership diff. - // - // Both are checked before deleting rather than deleting unconditionally - // the way the removal branch does: removals are rare, but this runs for - // every ineligible file a recovery scan walks past, and `delete_file` - // records a tombstone and dirties the overlay even when the path was - // never indexed. - let had_stamp = state - .file_stamps - .write() - .unwrap() - .remove(rel_path) - .is_some(); - let in_overlay = state.index.read().unwrap().live.has_path(rel_path); - if had_stamp || in_overlay { - eprintln!("[trace] reindex: dropped {rel_path} (no longer eligible)"); - state.index.write().unwrap().live.delete_file(rel_path); - if let Ok(mut cache) = state.cache.write() { - cache.pop(rel_path); - } - } + drop_indexed_file(state, rel_path, "no longer eligible"); return; } @@ -2707,10 +2919,14 @@ fn reindex_file(state: &ServerState, path: &Path, rel_path: &str) { // on our file I/O and trigram parsing. Windows' SRWLock is // writer-preferring: a single waiting writer here would otherwise // stall every subsequent search request. - let data = match std::fs::read(path) { - Ok(d) => d, - Err(_) => return, - }; + // + // From the handle, not the path: re-opening here is what would let a + // symlink take the place of the file we just approved. + let mut file = file; + let mut data = Vec::with_capacity(current.size.min(1 << 20) as usize); + if file.read_to_end(&mut data).is_err() { + return; + } let text = tgrep_core::encoding::decode_for_index(&data); let is_binary = tgrep_core::trigram::is_binary(&text); let per_tri = if is_binary { @@ -3278,10 +3494,9 @@ fn refresh_stale_locked( // the caller holds `snapshot_gate` for write across this whole body, and // the only reader of `state.gitignore` takes the read side first, so no // event can observe the matcher before this function returns either way. - // The caller anchored the recovery window at its own walk, which starts - // earlier than the subscription sync inside this call, so the timestamp - // that comes back here is the looser of the two and is dropped. - *newly_watched = publish_ignore_matcher(state, root, build_stale_matcher(state, root, &walk)).0; + // The caller's walk started before this publish; its timestamp is what the + // recovery scan needs, so the one the subscription sync derives is dropped. + *newly_watched = publish_ignore_matcher(state, root, build_stale_matcher(state, root, &walk)); if walk.skipped_error > 0 { eprintln!( @@ -3435,6 +3650,13 @@ fn reset_to_empty_index(state: &ServerState, root: &Path, index_dir: &Path) { /// caller to fall back. fn bootstrap_index_build(state: &Arc, root: &Path, index_dir: &Path) -> bool { let start = Instant::now(); + // Anchors the recovery window at the start of the build's traversal, which + // is the point from which writes could be missed: nothing under `root` is + // subscribed yet, and the walk below has not reached most of it. Taking it + // after the build — or letting the later subscription sync derive its own — + // would exclude everything written while the build ran, which is precisely + // the window that needs recovering. + let since = SystemTime::now(); eprintln!("[trace] bootstrapping index with the external merge sort (memory-bounded)..."); // Dropped once the build is done so the sampled peak (on platforms without @@ -3510,6 +3732,7 @@ fn bootstrap_index_build(state: &Arc, root: &Path, index_dir: &Path the watcher may reindex on spurious events" ), } + let mut newly_watched = Vec::new(); if state.watch_enabled && !state.no_ignore { let t_gi = Instant::now(); let matcher = tgrep_core::walker::build_gitignore_matcher_from_files( @@ -3524,8 +3747,7 @@ fn bootstrap_index_build(state: &Arc, root: &Path, index_dir: &Path // than skipped: the scan waits out `indexing` and then costs one // `metadata` call per file, since the stamps this build just wrote // describe the index exactly. - let (newly_watched, since) = publish_ignore_matcher(state, root, matcher); - spawn_recovery_scan(state, root, newly_watched, since); + newly_watched = publish_ignore_matcher(state, root, matcher); eprintln!( "[trace] gitignore matcher built from {} file(s) in {:.1}ms{}", outcome.gitignore_files.len(), @@ -3533,6 +3755,12 @@ fn bootstrap_index_build(state: &Arc, root: &Path, index_dir: &Path if found { "" } else { " (no rules found)" } ); } + // Outside that block, because it also drains the events the watcher had to + // discard while this build ran, and those pile up whether or not there are + // ignore rules to publish. + if state.watch_enabled { + spawn_recovery_scan(state, root, newly_watched, since); + } state.indexing.store(false, Ordering::SeqCst); drop(gate); @@ -3596,6 +3824,11 @@ fn background_index_build(state: &Arc, root: &Path, index_dir: &Pat // Phase 1: Walk file paths (no content reads) let t_walk = Instant::now(); + // The recovery window opens with this traversal, not with the subscriptions + // it later feeds: a nested `.ignore` written after the walk read its parent + // directory but before the matcher is published is invisible to both, and a + // timestamp taken any later would date it as already accounted for. + let since = SystemTime::now(); let walk = walker::walk_dir( root, &WalkOptions { @@ -3609,6 +3842,7 @@ fn background_index_build(state: &Arc, root: &Path, index_dir: &Pat }, ); + let mut newly_watched = Vec::new(); if state.watch_enabled && !state.no_ignore { let start = Instant::now(); let matcher = walker::build_gitignore_matcher_from_files( @@ -3623,8 +3857,7 @@ fn background_index_build(state: &Arc, root: &Path, index_dir: &Pat // the build's results nor any event. The scan waits for the build to // finish before looking, because until then the stamps describe // nothing and every file would read as changed. - let (newly_watched, since) = publish_ignore_matcher(state, root, matcher); - spawn_recovery_scan(state, root, newly_watched, since); + newly_watched = publish_ignore_matcher(state, root, matcher); eprintln!( "[trace] gitignore matcher built from index walk in {:.1}ms \ ({} .gitignore + {} .ignore files{})", @@ -3634,6 +3867,11 @@ fn background_index_build(state: &Arc, root: &Path, index_dir: &Pat if has_matcher { "" } else { ", no rules found" } ); } + // Outside that block: the scan also drains the events discarded while this + // build ran, which accumulate with or without ignore rules. + if state.watch_enabled { + spawn_recovery_scan(state, root, newly_watched, since); + } // Filter out already-indexed files let new_files: Vec<_> = if skip_paths.is_empty() { @@ -4387,6 +4625,7 @@ mod tests { gitignore_pending: std::sync::atomic::AtomicBool::new(true), ignore_rules_dirty: std::sync::atomic::AtomicBool::new(false), ignore_refresh_scheduled: std::sync::atomic::AtomicBool::new(false), + deferred_events: Mutex::new(Some(std::collections::HashSet::new())), index_progress: std::sync::atomic::AtomicU64::new(0), index_total: std::sync::atomic::AtomicU64::new(0), watch_enabled: true, @@ -5154,6 +5393,131 @@ mod tests { ); } + /// The metadata the eligibility check uses and the bytes that get indexed + /// have to describe the same object, which means one handle. + #[test] + fn open_no_follow_reads_a_regular_file() { + use std::io::Read; + + let tmp = TempDir::new().unwrap(); + let path = tmp.path().join("plain.rs"); + std::fs::write(&path, "fn main() {}\n").unwrap(); + + let file = open_no_follow(&path).expect("a regular file opens"); + let meta = file.metadata().expect("metadata off the handle"); + assert!(meta.is_file()); + assert_eq!(meta.len(), 13); + + let mut data = String::new(); + (&file).read_to_string(&mut data).unwrap(); + assert_eq!(data, "fn main() {}\n"); + } + + /// A symlink must not open as its target, or a link committed to a branch + /// would pull a file from outside the served root into the index. + #[cfg(unix)] + #[test] + fn open_no_follow_refuses_a_symlink() { + let tmp = TempDir::new().unwrap(); + let target = tmp.path().join("secret.txt"); + std::fs::write(&target, "sensitive\n").unwrap(); + let link = tmp.path().join("link.txt"); + std::os::unix::fs::symlink(&target, &link).unwrap(); + + assert!( + open_no_follow(&link).is_err(), + "O_NOFOLLOW must reject the link rather than open its target" + ); + } + + /// A burst big enough to be worth replaying individually is not worth + /// replaying individually. The buffer gives up as a whole, because a + /// truncated set looks exactly like a complete one at replay time. + #[cfg(unix)] + #[test] + fn deferring_more_changes_than_the_cap_gives_up_on_the_whole_set() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + + let small = Event { + kind: EventKind::Create(notify::event::CreateKind::Any), + paths: vec![root.join("a.rs")], + attrs: Default::default(), + }; + defer_events_during_build(&state, &small); + assert_eq!( + state.deferred_events.lock().unwrap().as_ref().unwrap().len(), + 1 + ); + + let flood = Event { + kind: EventKind::Create(notify::event::CreateKind::Any), + paths: (0..200_001).map(|i| root.join(format!("f{i}.rs"))).collect(), + attrs: Default::default(), + }; + defer_events_during_build(&state, &flood); + assert!( + state.deferred_events.lock().unwrap().is_none(), + "an overflowing burst must mark the buffer unusable, not truncate it" + ); + + // And stays given up on, rather than resuming a partial record. + defer_events_during_build(&state, &small); + assert!(state.deferred_events.lock().unwrap().is_none()); + } + + /// Events seen while a build ran are not applied then, but they are the + /// only record that those paths moved: the build's walk misses anything + /// written to a directory it already passed. + #[cfg(unix)] + #[test] + fn changes_deferred_during_a_build_are_applied_once_it_publishes() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + // As it is once a build publishes: no rules to wait for, nothing + // indexing. + state.gitignore_pending.store(false, Ordering::SeqCst); + + let path = root.join("late.rs"); + std::fs::write(&path, "fn written_during_the_build() {}\n").unwrap(); + + state.indexing.store(true, Ordering::SeqCst); + handle_fs_event( + &state, + &root, + &Event { + kind: EventKind::Create(notify::event::CreateKind::Any), + paths: vec![path.clone()], + attrs: Default::default(), + }, + ); + assert!( + !state.index.read().unwrap().live.has_path("late.rs"), + "an event during a build must not touch the index" + ); + + state.indexing.store(false, Ordering::SeqCst); + replay_deferred_events(&state, &root); + + assert!( + state.index.read().unwrap().live.has_path("late.rs"), + "the deferred change must be applied once the build is done" + ); + assert!( + state + .deferred_events + .lock() + .unwrap() + .as_ref() + .is_some_and(|p| p.is_empty()), + "the buffer must be drained, and left usable for the next build" + ); + } + #[test] fn glob_filter_unix_patterns() { use crate::glob_filter::GlobFilter; From d564fe51eafd5d14effb3b1e0e7ca4348b6e334f Mon Sep 17 00:00:00 2001 From: Shengyu Fu Date: Thu, 27 Aug 2026 17:20:32 -0700 Subject: [PATCH 10/28] Format the round 5 tests Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: bcc85b55-e821-447e-b9a9-144b78829ae4 --- tgrep-cli/src/serve.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tgrep-cli/src/serve.rs b/tgrep-cli/src/serve.rs index da9938d..5c020fd 100644 --- a/tgrep-cli/src/serve.rs +++ b/tgrep-cli/src/serve.rs @@ -5448,13 +5448,21 @@ mod tests { }; defer_events_during_build(&state, &small); assert_eq!( - state.deferred_events.lock().unwrap().as_ref().unwrap().len(), + state + .deferred_events + .lock() + .unwrap() + .as_ref() + .unwrap() + .len(), 1 ); let flood = Event { kind: EventKind::Create(notify::event::CreateKind::Any), - paths: (0..200_001).map(|i| root.join(format!("f{i}.rs"))).collect(), + paths: (0..100_000) + .map(|i| root.join(format!("f{i}.rs"))) + .collect(), attrs: Default::default(), }; defer_events_during_build(&state, &flood); From f9b9579c40fb09bb3857d15905959651d3c84778 Mon Sep 17 00:00:00 2001 From: Shengyu Fu Date: Thu, 27 Aug 2026 17:33:02 -0700 Subject: [PATCH 11/28] Contain the read to the served root, and serialize reindexing Review round 6. Guard every path component, not just the last one. `O_NOFOLLOW` and `FILE_FLAG_OPEN_REPARSE_POINT` protect the file being opened and nothing above it, so `root/a/file` still resolves through `a` if `a` is a link -- and the file at the end of that is a perfectly ordinary file outside the tree we serve, which is exactly the containment the walker's `follow_links(false)` promises. `open_within_root` resolves the path a component at a time from the root: with `openat` on unix, where the name is never re-resolved and there is no window to swap a directory for a link, and by checking each ancestor on Windows, where there is no `openat` and creating a symlink needs a privilege that is not granted by default. Non-literal components are refused rather than interpreted. Serialize the check-read-commit cycle in `reindex_file`. The snapshot gate is held for read by everything that indexes a file, so a recovery scan and the watcher worker can both be inside it for the same path, both see the same old stamp, and the one that read the older content can commit last -- the newer event already consumed, the stale version surviving until the next reconcile. The new lock is taken per file, so the two interleave rather than one waiting out the other, and searches do not take it at all. Also open with `O_NONBLOCK`, so a fifo in the tree answers immediately instead of blocking the watcher until someone opens the write end. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: bcc85b55-e821-447e-b9a9-144b78829ae4 --- tgrep-cli/src/serve.rs | 233 +++++++++++++++++++++++++++++++++++------ 1 file changed, 202 insertions(+), 31 deletions(-) diff --git a/tgrep-cli/src/serve.rs b/tgrep-cli/src/serve.rs index 5c020fd..7e503be 100644 --- a/tgrep-cli/src/serve.rs +++ b/tgrep-cli/src/serve.rs @@ -290,6 +290,19 @@ struct ServerState { ignore_rules_dirty: std::sync::atomic::AtomicBool, /// Ensures a burst of ignore-file events uses at most one refresh worker. ignore_refresh_scheduled: std::sync::atomic::AtomicBool, + /// Serializes the whole check-read-commit cycle in [`reindex_file`]. + /// + /// `snapshot_gate` is held for *read* by everything that indexes a file, so + /// the watcher worker and a recovery scan can be inside `reindex_file` for + /// the same path at once. Both then see the same old stamp, both read, and + /// whichever commits last wins — which is not necessarily the one that read + /// the newer content. The losing write is already consumed, so the stale + /// version survives until the next reconcile. + /// + /// Taken per file rather than per scan, so a recovery pass and the watcher + /// interleave instead of one waiting out the other. It is never held across + /// anything but one file's read, and searches do not take it at all. + reindex_lock: Mutex<()>, /// Paths the watcher saw while `indexing` was set, kept so they can be /// replayed once the build publishes. /// @@ -590,6 +603,7 @@ pub fn run(root: &Path, index_path: Option<&Path>, options: ServeOptions<'_>) -> gitignore_pending: std::sync::atomic::AtomicBool::new(!no_watch && !no_ignore), ignore_rules_dirty: std::sync::atomic::AtomicBool::new(false), ignore_refresh_scheduled: std::sync::atomic::AtomicBool::new(false), + reindex_lock: Mutex::new(()), deferred_events: Mutex::new(Some(std::collections::HashSet::new())), index_progress: std::sync::atomic::AtomicU64::new(0), index_total: std::sync::atomic::AtomicU64::new(0), @@ -2811,22 +2825,11 @@ fn drop_indexed_file(state: &ServerState, rel_path: &str, reason: &str) { /// Open a file without following a final symlink, so the handle is the path /// itself rather than wherever it points. /// -/// Every later question — is this a regular file, how big is it, what is in it -/// — is then answered from the one handle, which is what makes the eligibility -/// decision and the read describe the same object. Checking the path and then -/// reading it separately leaves a window in which a tree being rewritten under -/// us (a checkout, a build, `git mv`) can swap a regular file for a link, and -/// the read would follow it out of the served root. +/// Only the last component. For a path whose ancestors are not already trusted, +/// use [`open_within_root`] — which on unix has no use for this, since `openat` +/// resolves the final component the same way as every other one. +#[cfg(not(unix))] fn open_no_follow(path: &Path) -> std::io::Result { - #[cfg(unix)] - { - use std::os::unix::fs::OpenOptionsExt; - // Fails outright on a symlink, which is the answer we want. - std::fs::OpenOptions::new() - .read(true) - .custom_flags(libc::O_NOFOLLOW) - .open(path) - } #[cfg(windows)] { use std::os::windows::fs::OpenOptionsExt; @@ -2845,6 +2848,115 @@ fn open_no_follow(path: &Path) -> std::io::Result { } } +/// Open a file under `root` without traversing a symlink at *any* level. +/// +/// Refusing to follow the final component is not enough. A path arrives here +/// from an event or a replay as a name, and `root/a/file` reads the same +/// whether `a` is a directory or a link to one — so an intermediate link is +/// enough to hand back a file outside the served tree, which is exactly the +/// containment the walker's `follow_links(false)` promises and the index's +/// contract depends on. +/// +/// `root` itself is the trust anchor and is opened normally: it is the +/// directory the user asked us to serve, so a link there is theirs to have. +/// +/// On unix this is race-free. Each component is resolved with `openat` against +/// the handle for its parent, so the name is never re-resolved and there is no +/// window in which a directory can be swapped for a link between the check and +/// the use. +#[cfg(unix)] +fn open_within_root(root: &Path, path: &Path) -> std::io::Result { + use std::io::{Error, ErrorKind}; + use std::os::fd::{AsRawFd, FromRawFd, OwnedFd}; + use std::os::unix::ffi::OsStrExt; + + let components = relative_components(root, path)?; + let mut dir: OwnedFd = std::fs::File::open(root)?.into(); + let last = components.len() - 1; + for (i, component) in components.iter().enumerate() { + let name = std::ffi::CString::new(component.as_bytes()) + .map_err(|_| Error::new(ErrorKind::InvalidInput, "path component contains a NUL"))?; + // `O_DIRECTORY` on the intermediates so a *file* in the middle of the + // path fails here rather than at the next `openat`, and `O_NOFOLLOW` on + // every one of them, including the last. `O_NONBLOCK`: see + // `open_no_follow`. + let mut flags = libc::O_RDONLY | libc::O_NOFOLLOW | libc::O_CLOEXEC | libc::O_NONBLOCK; + if i != last { + flags |= libc::O_DIRECTORY; + } + // SAFETY: `dir` is a live directory descriptor for the parent, and + // `name` is a NUL-terminated single path component that outlives the + // call. + let fd = unsafe { libc::openat(dir.as_raw_fd(), name.as_ptr(), flags) }; + if fd < 0 { + return Err(Error::last_os_error()); + } + // SAFETY: `openat` returned a fresh owned descriptor. Assigning it + // drops the previous one, closing the parent we no longer need. + dir = unsafe { OwnedFd::from_raw_fd(fd) }; + } + Ok(std::fs::File::from(dir)) +} + +/// As above. Windows has no `openat`, so the ancestors are checked by path +/// instead of by handle: this rejects a symlink or junction that is actually +/// there, but not one substituted between the check and the open. Closing that +/// window needs handle-relative opens (`NtCreateFile` with a `RootDirectory`), +/// which is a great deal of unsafe code for a platform where creating a symlink +/// needs a privilege the machine does not grant by default. +#[cfg(not(unix))] +fn open_within_root(root: &Path, path: &Path) -> std::io::Result { + use std::io::{Error, ErrorKind}; + + let components = relative_components(root, path)?; + let mut walked = root.to_path_buf(); + for component in &components[..components.len() - 1] { + walked.push(component); + let meta = std::fs::symlink_metadata(&walked)?; + // `is_symlink` covers junctions too: both are reparse points tagged as + // name surrogates, which is what std tests for. + if meta.file_type().is_symlink() { + return Err(Error::new( + ErrorKind::InvalidInput, + "path traverses a symlink", + )); + } + if !meta.is_dir() { + return Err(Error::new(ErrorKind::NotADirectory, "not a directory")); + } + } + open_no_follow(path) +} + +/// `path` split into the literal components below `root`. +/// +/// Anything that is not a plain name — `..`, a root, a prefix — is refused +/// rather than interpreted, since resolving those is the whole business +/// [`open_within_root`] is avoiding. +fn relative_components(root: &Path, path: &Path) -> std::io::Result> { + use std::io::{Error, ErrorKind}; + + let rel = path + .strip_prefix(root) + .map_err(|_| Error::new(ErrorKind::InvalidInput, "path is outside the served root"))?; + let mut components = Vec::new(); + for component in rel.components() { + match component { + std::path::Component::Normal(name) => components.push(name.to_os_string()), + _ => { + return Err(Error::new( + ErrorKind::InvalidInput, + "path has a non-literal component", + )); + } + } + } + if components.is_empty() { + return Err(Error::new(ErrorKind::InvalidInput, "path is the root")); + } + Ok(components) +} + /// Read a file and merge it into the live index, unless its stamp says the /// content we already indexed is current. /// @@ -2854,11 +2966,21 @@ fn reindex_file(state: &ServerState, path: &Path, rel_path: &str) { use std::io::Read; use tgrep_core::meta::FileStamp; - // One handle for the whole decision. Opened without following a final - // symlink, and every fact below — type, size, mtime, bytes — read back off - // it, so nothing that happens to the path in the meantime can make the - // content we index disagree with the metadata we judged it by. - let file = match open_no_follow(path) { + // Against other indexers, not against searches. The gate above is held for + // read, so without this a recovery scan and the watcher worker can both be + // here for the same path, both read, and the one that read the *older* + // content can commit last. See `ServerState::reindex_lock`. + let _reindex = match state.reindex_lock.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + }; + + // One handle for the whole decision, resolved a component at a time from + // the root so no part of the path can be a symlink, and every fact below — + // type, size, mtime, bytes — read back off it. Nothing that happens to the + // path in the meantime can then make the content we index disagree with the + // metadata we judged it by, or put it outside the tree we serve. + let file = match open_within_root(&state.root, path) { Ok(f) => f, Err(_) => { // Includes the ELOOP that a symlink gets on unix. It may still be @@ -2894,8 +3016,9 @@ fn reindex_file(state: &ServerState, path: &Path, rel_path: &str) { // the link's path — and the target need not be under `root` at all, so a // link committed to a branch (or dropped in by a build) is enough to pull // `~/.ssh/id_rsa` into an index whose whole contract is that it covers the - // served tree. On unix the open above has already failed for a link; this - // is what rejects one on Windows, where the reparse point opens fine. + // served tree. On unix the open above has already failed for a link at any + // level; this is what rejects a final one on Windows, where the reparse + // point opens fine. let eligible = meta.is_file() && !tgrep_core::walker::is_binary_extension(path) && !state @@ -4625,6 +4748,7 @@ mod tests { gitignore_pending: std::sync::atomic::AtomicBool::new(true), ignore_rules_dirty: std::sync::atomic::AtomicBool::new(false), ignore_refresh_scheduled: std::sync::atomic::AtomicBool::new(false), + reindex_lock: Mutex::new(()), deferred_events: Mutex::new(Some(std::collections::HashSet::new())), index_progress: std::sync::atomic::AtomicU64::new(0), index_total: std::sync::atomic::AtomicU64::new(0), @@ -5396,14 +5520,15 @@ mod tests { /// The metadata the eligibility check uses and the bytes that get indexed /// have to describe the same object, which means one handle. #[test] - fn open_no_follow_reads_a_regular_file() { + fn open_within_root_reads_a_regular_file() { use std::io::Read; let tmp = TempDir::new().unwrap(); - let path = tmp.path().join("plain.rs"); + std::fs::create_dir(tmp.path().join("src")).unwrap(); + let path = tmp.path().join("src").join("plain.rs"); std::fs::write(&path, "fn main() {}\n").unwrap(); - let file = open_no_follow(&path).expect("a regular file opens"); + let file = open_within_root(tmp.path(), &path).expect("a regular file opens"); let meta = file.metadata().expect("metadata off the handle"); assert!(meta.is_file()); assert_eq!(meta.len(), 13); @@ -5417,16 +5542,62 @@ mod tests { /// would pull a file from outside the served root into the index. #[cfg(unix)] #[test] - fn open_no_follow_refuses_a_symlink() { - let tmp = TempDir::new().unwrap(); - let target = tmp.path().join("secret.txt"); + fn open_within_root_refuses_a_symlinked_file() { + let outside = TempDir::new().unwrap(); + let target = outside.path().join("secret.txt"); std::fs::write(&target, "sensitive\n").unwrap(); - let link = tmp.path().join("link.txt"); + + let root = TempDir::new().unwrap(); + let link = root.path().join("link.txt"); std::os::unix::fs::symlink(&target, &link).unwrap(); assert!( - open_no_follow(&link).is_err(), - "O_NOFOLLOW must reject the link rather than open its target" + open_within_root(root.path(), &link).is_err(), + "the link must not open as its target" + ); + } + + /// And neither must a symlink anywhere *above* the file: `root/a/file` + /// reads the same whether `a` is a directory or a link to one, so guarding + /// only the last component still lets a whole tree in from outside. + #[cfg(unix)] + #[test] + fn open_within_root_refuses_a_symlinked_ancestor() { + let outside = TempDir::new().unwrap(); + std::fs::write(outside.path().join("secret.txt"), "sensitive\n").unwrap(); + + let root = TempDir::new().unwrap(); + let link = root.path().join("a"); + std::os::unix::fs::symlink(outside.path(), &link).unwrap(); + let through_link = link.join("secret.txt"); + + // The file at the end of that path is a perfectly ordinary file, and + // opening it by name works — which is the point. + assert!(std::fs::File::open(&through_link).is_ok()); + assert!( + open_within_root(root.path(), &through_link).is_err(), + "an intermediate symlink must not be traversed" + ); + } + + /// Nothing may be resolved that could climb back out of the root. + #[test] + fn open_within_root_refuses_paths_that_escape_or_are_not_literal() { + let tmp = TempDir::new().unwrap(); + std::fs::create_dir(tmp.path().join("src")).unwrap(); + std::fs::write(tmp.path().join("src").join("a.rs"), "x\n").unwrap(); + + assert!( + open_within_root(tmp.path(), tmp.path()).is_err(), + "the root" + ); + assert!( + open_within_root(tmp.path(), &tmp.path().join("..").join("a.rs")).is_err(), + "a parent component" + ); + assert!( + open_within_root(&tmp.path().join("src"), &tmp.path().join("src")).is_err(), + "outside the given root" ); } From bf94c684eced6e1605d358c807e7c231e8f2cb53 Mon Sep 17 00:00:00 2001 From: Shengyu Fu Date: Thu, 27 Aug 2026 17:55:40 -0700 Subject: [PATCH 12/28] Keep the watcher's recovery honest about what it can and cannot see MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five follow-ups from review, all about a decision made on evidence that does not support it. Replaying a deferred event reconstructed it as a creation regardless of what actually happened, so a recursive chmod during a build put every directory through watch_new_subtree. The flag that distinguishes them is now carried with the path. The deferral itself re-checked `indexing` outside the buffer lock, so an event could land in a set that had already been replayed and swapped out. It is now read under the lock, which is what makes the handoff provable: the replay cannot swap without that lock. reindex_file treated any failure to open as proof of ineligibility, so a permission error or a Windows sharing violation evicted content that was still perfectly valid — while the stale path, deliberately, keeps unreadable files and retries them. Only errors that establish something structural drop the entry now. A recovery scan could see an ignore file that arrived, by its mtime, but not one that was deleted, since a deleted file leaves nothing to stat. The published matcher's sources are recorded and checked directly. And containment on Windows was still check-then-open: an ancestor could become a junction between the check and the open. The handle is now asked where it ended up, which cannot be raced, and the per-ancestor stat walk goes away with it. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: bcc85b55-e821-447e-b9a9-144b78829ae4 --- tgrep-cli/Cargo.toml | 2 +- tgrep-cli/src/serve.rs | 442 ++++++++++++++++++++++++++++++++++++----- 2 files changed, 398 insertions(+), 46 deletions(-) diff --git a/tgrep-cli/Cargo.toml b/tgrep-cli/Cargo.toml index 3dfd3ae..eeaa75f 100644 --- a/tgrep-cli/Cargo.toml +++ b/tgrep-cli/Cargo.toml @@ -31,7 +31,7 @@ memmap2 = "0.9" memchr = "2" [target.'cfg(windows)'.dependencies] -windows-sys = { version = "0.61", features = ["Win32_System_ProcessStatus", "Win32_System_SystemInformation", "Win32_System_Threading"] } +windows-sys = { version = "0.61", features = ["Win32_Storage_FileSystem", "Win32_System_ProcessStatus", "Win32_System_SystemInformation", "Win32_System_Threading"] } [target.'cfg(unix)'.dependencies] libc = "0.2" diff --git a/tgrep-cli/src/serve.rs b/tgrep-cli/src/serve.rs index 7e503be..ad902ca 100644 --- a/tgrep-cli/src/serve.rs +++ b/tgrep-cli/src/serve.rs @@ -290,6 +290,15 @@ struct ServerState { ignore_rules_dirty: std::sync::atomic::AtomicBool, /// Ensures a burst of ignore-file events uses at most one refresh worker. ignore_refresh_scheduled: std::sync::atomic::AtomicBool, + /// The ignore files the published matcher was built from. + /// + /// A recovery scan can spot an ignore file that *arrived* during its window + /// by its mtime, but a deleted one leaves nothing behind to notice. That is + /// the more damaging direction: the matcher keeps enforcing rules whose + /// source is gone, so an entire subtree stays unsubscribed and unindexed + /// until something else forces a rebuild. Keeping the source list lets the + /// scan test for it directly, at one stat per ignore file per scan. + ignore_sources: RwLock>, /// Serializes the whole check-read-commit cycle in [`reindex_file`]. /// /// `snapshot_gate` is held for *read* by everything that indexes a file, so @@ -304,7 +313,8 @@ struct ServerState { /// anything but one file's read, and searches do not take it at all. reindex_lock: Mutex<()>, /// Paths the watcher saw while `indexing` was set, kept so they can be - /// replayed once the build publishes. + /// replayed once the build publishes, each with whether its original event + /// could have introduced a directory. /// /// Events that arrive during a build cannot be applied — the stamps do not /// describe the index yet, so every path would read as changed — but they @@ -314,12 +324,19 @@ struct ServerState { /// back on, so discarding them leaves the change invisible until the hourly /// reconcile. /// + /// The flag has to be carried rather than reconstructed. Replaying + /// everything as a create would put every recorded path through + /// `watch_new_subtree`, and a recursive `chmod` or a checkout fires a + /// metadata-only modify per directory — so a tree that produced no new + /// directories at all would be walked and force-resubscribed once per + /// recorded path, which is quadratic over a deep checkout. + /// /// `None` means the buffer overflowed and the paths were dropped; the /// replay then falls back to a full stale refresh, which is slower but /// complete. Bounded because a build can run for minutes on a large /// repository and a checkout or a build tree churning underneath it is /// unbounded. - deferred_events: Mutex>>, + deferred_events: Mutex>>, /// Progress: number of files indexed so far. index_progress: std::sync::atomic::AtomicU64, /// Total files discovered for indexing. @@ -603,8 +620,9 @@ pub fn run(root: &Path, index_path: Option<&Path>, options: ServeOptions<'_>) -> gitignore_pending: std::sync::atomic::AtomicBool::new(!no_watch && !no_ignore), ignore_rules_dirty: std::sync::atomic::AtomicBool::new(false), ignore_refresh_scheduled: std::sync::atomic::AtomicBool::new(false), + ignore_sources: RwLock::new(Vec::new()), reindex_lock: Mutex::new(()), - deferred_events: Mutex::new(Some(std::collections::HashSet::new())), + deferred_events: Mutex::new(Some(std::collections::HashMap::new())), index_progress: std::sync::atomic::AtomicU64::new(0), index_total: std::sync::atomic::AtomicU64::new(0), watch_enabled: !no_watch, @@ -782,6 +800,27 @@ fn build_stale_matcher( matcher } +/// The ignore files a matcher was built from, as one list. +/// +/// Root-level `p4ignore.ini` is a separate source from the walker's point of +/// view — it is applied as its own filter rather than collected with the +/// gitignore files — but deleting it invalidates the published rules exactly +/// the same way, so it belongs in the list. +fn ignore_sources_of( + root: &Path, + gitignore_files: &[PathBuf], + ignore_files: &[PathBuf], +) -> Vec { + let mut sources = Vec::with_capacity(gitignore_files.len() + ignore_files.len() + 1); + sources.extend_from_slice(gitignore_files); + sources.extend_from_slice(ignore_files); + let p4 = root.join(tgrep_core::gitignore::P4IGNORE_FILENAME); + if p4.is_file() { + sources.push(p4); + } + sources +} + /// Publish a new ignore matcher and bring everything that depends on it up to /// date. `None` is a legitimate matcher when no rules exist. /// @@ -800,12 +839,18 @@ fn build_stale_matcher( /// subscription walk inside this function starts later, and an ignore file /// written between the two would predate a timestamp taken here and be read as /// already accounted for by rules that never saw it. +/// +/// `sources` are the ignore files `matcher` was built from, recorded so a +/// recovery scan can notice one of them being deleted — an event the mtime +/// heuristic cannot see, since a deleted file leaves nothing to stat. #[must_use = "newly watched directories need a recovery scan or writes race the subscription"] fn publish_ignore_matcher( state: &ServerState, root: &Path, matcher: Option, + sources: Vec, ) -> Vec { + *state.ignore_sources.write().unwrap() = sources; *state.gitignore.write().unwrap() = matcher; state.gitignore_pending.store(false, Ordering::SeqCst); // New rules mean a different set of directories worth hearing about: @@ -2078,7 +2123,9 @@ fn sync_watch_registrations(state: &ServerState, root: &Path) -> (Vec, /// is closing. It is only consulted for ignore-rules files, where "did this /// arrive after the matcher was decided" cannot be answered from the stamps: /// the dot-prefixed ones are hidden, so they are never indexed and never have -/// one. +/// one. The opposite case — one that was *deleted* in the window — is handled +/// separately, from `state.ignore_sources`, since a deleted file leaves nothing +/// to stat. /// /// Callers must already hold `snapshot_gate`, and `state.file_stamps` must /// already describe the index as published — a merge that replaces the stamps @@ -2089,6 +2136,30 @@ fn reindex_files_in(state: &Arc, root: &Path, dirs: &[PathBuf], sin return; } let start = Instant::now(); + + // An ignore file that was deleted during the window is invisible to the + // per-entry mtime test below — there is no entry left to stat. It is also + // the more damaging direction: rules that no longer have a source keep + // being enforced, so the subtree they hide stays unsubscribed and + // unindexed until an unrelated rebuild happens along. Checking the sources + // the published matcher was built from catches it at one stat apiece, once + // per scan rather than once per file. + if !state.no_ignore { + let vanished = { + let sources = state.ignore_sources.read().unwrap(); + sources.iter().find(|p| !p.exists()).cloned() + }; + if let Some(gone) = vanished { + state.ignore_rules_dirty.store(true, Ordering::SeqCst); + schedule_ignore_rules_refresh(Arc::clone(state), root.to_path_buf()); + eprintln!( + "[trace] watcher: ignore rules source {} disappeared; deferring to a refresh", + gone.display() + ); + return; + } + } + // Directories whose listing succeeded, and the files those listings // contained, for the removal sweep at the end. Only files are recorded: // stamps describe files, so directory names would just be dead weight on a @@ -2459,30 +2530,41 @@ fn schedule_ignore_rules_refresh(state: Arc, root: PathBuf) { }); } -/// Recheck newly watched directories on a background thread. -/// /// Remember the paths in an event that arrived mid-build, for replay once the -/// build publishes. +/// build publishes. Returns whether it did — a `false` means the build finished +/// underneath us and the caller should handle the event normally. /// /// The event cannot be applied now: until `indexing` clears, `file_stamps` does /// not describe the index, so every path would compare as changed and the /// watcher would re-read the repository alongside the build already reading it. /// It cannot simply be dropped either — see [`ServerState::deferred_events`]. /// +/// `indexing` is re-read *under the buffer lock*, and that is what makes the +/// handoff safe. The caller's own check is only a hint: between it and this +/// call the build can finish and [`replay_deferred_events`] can swap the buffer +/// out, and an insert landing after that swap is in a set nothing will ever +/// look at again. Because the replay cannot swap until `indexing` is false, and +/// cannot swap without this lock, seeing `indexing` set while holding it proves +/// the swap has not happened yet. +/// /// Capped, and the cap discards the whole set rather than truncating it: a /// partial set is indistinguishable from a complete one at replay time, and /// silently recovering nine tenths of a checkout is worse than knowing to fall /// back on a full refresh. -fn defer_events_during_build(state: &ServerState, event: &Event) { +fn defer_events_during_build(state: &ServerState, event: &Event) -> bool { /// A checkout of the Linux kernel is ~90k files. Above this the fallback /// refresh is cheaper than the replay would be anyway. const MAX_DEFERRED: usize = 100_000; let Ok(mut deferred) = state.deferred_events.lock() else { - return; + return false; }; + if !state.indexing.load(Ordering::SeqCst) { + return false; + } let Some(paths) = deferred.as_mut() else { - return; + // Already overflowed, so the fallback refresh will cover this path too. + return true; }; if paths.len().saturating_add(event.paths.len()) > MAX_DEFERRED { *deferred = None; @@ -2490,26 +2572,43 @@ fn defer_events_during_build(state: &ServerState, event: &Event) { "[trace] warning: too many file changes during the initial index build to replay \ individually; a full reconcile will run instead" ); - return; + return true; + } + // Only these kinds can put a directory somewhere, and only they should + // trigger a subtree walk on replay. See `ServerState::deferred_events`. + let introduces_dir = matches!( + event.kind, + EventKind::Create(_) | EventKind::Modify(notify::event::ModifyKind::Name(_)) + ); + for path in &event.paths { + // A path seen both ways keeps the stronger claim: a directory that was + // created and then chmod'd still needs its subtree picked up. + let entry = paths.entry(path.clone()).or_insert(false); + *entry |= introduces_dir; } - paths.extend(event.paths.iter().cloned()); + true } /// Apply the events that arrived during the build, now that it has published. /// -/// Replayed as synthetic create events rather than handled directly so they go -/// through exactly the filtering an ordinary event gets — ignore rules, the -/// exclude list, the index directory, new-subtree subscription. `Create(Any)` -/// is the right kind for all of them: `handle_fs_event` decides removal from -/// whether the path still exists, and modifications and creations take the same -/// route, so one kind covers arrivals, edits and deletions alike. +/// Replayed as synthetic events rather than handled directly so they go through +/// exactly the filtering an ordinary event gets — ignore rules, the exclude +/// list, the index directory, new-subtree subscription. The kind is +/// reconstructed from the flag recorded with each path so the directory gate +/// still holds; beyond that gate, creations and modifications take the same +/// route, and `handle_fs_event` decides removal from whether the path still +/// exists, so one kind of each class covers arrivals, edits and deletions +/// alike. /// /// The caller must *not* hold `snapshot_gate`; `handle_fs_event` takes it. fn replay_deferred_events(state: &Arc, root: &Path) { let deferred = match state.deferred_events.lock() { - // Leaves `Some(empty)` behind, so anything deferred by a later build - // (a reconcile sets `indexing` again) is collected from scratch. - Ok(mut guard) => guard.replace(std::collections::HashSet::new()), + // Leaves an empty map behind, so anything deferred by a later build + // (a reconcile sets `indexing` again) is collected from scratch. The + // caller has already waited out `indexing`, and `defer_events_during_ + // build` re-reads it under this same lock, so nothing can be inserted + // into the old map after this point. + Ok(mut guard) => guard.replace(std::collections::HashMap::new()), Err(_) => return, }; let Some(paths) = deferred else { @@ -2541,12 +2640,19 @@ fn replay_deferred_events(state: &Arc, root: &Path) { let start = Instant::now(); let count = paths.len(); - for path in paths { + for (path, introduces_dir) in paths { + let kind = if introduces_dir { + EventKind::Create(notify::event::CreateKind::Any) + } else { + EventKind::Modify(notify::event::ModifyKind::Data( + notify::event::DataChange::Any, + )) + }; handle_fs_event( state, root, &Event { - kind: EventKind::Create(notify::event::CreateKind::Any), + kind, paths: vec![path], attrs: Default::default(), }, @@ -2664,8 +2770,11 @@ fn handle_fs_event(state: &Arc, root: &Path, event: &Event) { // progress. The indexer will pick up those files itself — but only for the // parts of the tree it has not reached yet, so remember these and replay // them once it publishes. - if state.indexing.load(Ordering::SeqCst) { - defer_events_during_build(state, event); + // + // The load is a fast path that keeps the mutex out of the common case; the + // decision is made under the lock, since the build can finish between the + // two and an event deferred after that is never replayed. + if state.indexing.load(Ordering::SeqCst) && defer_events_during_build(state, event) { return; } @@ -2822,6 +2931,37 @@ fn drop_indexed_file(state: &ServerState, rel_path: &str, reason: &str) { } } +/// Whether a failure to open a path establishes that it does not belong in the +/// index, as opposed to merely being unreadable right now. +/// +/// The distinction decides whether the watcher drops what it has indexed. A +/// path that is gone, or that cannot be reached without traversing a symlink, +/// is genuinely ineligible and the entry has to go. A path that is locked, +/// unreadable, or lost to a descriptor limit is none of those things — dropping +/// on that would evict live content because a build held the file open for a +/// moment, and the stale path deliberately keeps unreadable files and retries +/// them later. +fn proves_ineligible(error: &std::io::Error) -> bool { + use std::io::ErrorKind; + + // `InvalidInput` and `NotADirectory` are what `open_within_root` itself + // returns for a path that escapes the root, has a non-literal component, or + // runs through something that is not a directory. + if matches!( + error.kind(), + ErrorKind::NotFound | ErrorKind::NotADirectory | ErrorKind::InvalidInput + ) { + return true; + } + // A symlink met under `O_NOFOLLOW`, at any level. Not yet a stable + // `ErrorKind`, so it has to be read from the raw code. + #[cfg(unix)] + if error.raw_os_error() == Some(libc::ELOOP) { + return true; + } + false +} + /// Open a file without following a final symlink, so the handle is the path /// itself rather than wherever it points. /// @@ -2898,13 +3038,80 @@ fn open_within_root(root: &Path, path: &Path) -> std::io::Result Ok(std::fs::File::from(dir)) } -/// As above. Windows has no `openat`, so the ancestors are checked by path -/// instead of by handle: this rejects a symlink or junction that is actually -/// there, but not one substituted between the check and the open. Closing that -/// window needs handle-relative opens (`NtCreateFile` with a `RootDirectory`), -/// which is a great deal of unsafe code for a platform where creating a symlink -/// needs a privilege the machine does not grant by default. -#[cfg(not(unix))] +/// As above. Windows has no `openat`, so containment is established after the +/// fact instead of during resolution: the file is opened without following a +/// final reparse point, and the *handle* is then asked where it actually ended +/// up. Anything that is not under the root's own resolved path is refused. +/// +/// This is race-free in the way that matters. Checking each ancestor by path +/// first would only reject a junction that happened to be there at the time of +/// the check — one substituted between the check and the open would still be +/// followed. Asking the handle removes the second lookup entirely: whatever the +/// open resolved through, the answer describes the object we are actually +/// holding. +#[cfg(windows)] +fn open_within_root(root: &Path, path: &Path) -> std::io::Result { + use std::io::{Error, ErrorKind}; + + // Rejects escapes and non-literal components before anything is opened. + relative_components(root, path)?; + let file = open_no_follow(path)?; + + // The root's own resolved path, since it may itself sit under a junction or + // a substituted drive — comparing against the path as given would then + // reject every file in the tree. `canonicalize` is the same + // `GetFinalPathNameByHandleW` query underneath, so the two agree on + // verbatim prefix and casing. + let anchor = std::fs::canonicalize(root)?; + let opened = final_path_of(&file)?; + if !opened.starts_with(&anchor) { + return Err(Error::new( + ErrorKind::InvalidInput, + "path resolves outside the served root", + )); + } + Ok(file) +} + +/// Where an open handle actually is, with every reparse point on the way +/// resolved. +#[cfg(windows)] +fn final_path_of(file: &std::fs::File) -> std::io::Result { + use std::os::windows::ffi::OsStringExt; + use std::os::windows::io::AsRawHandle; + use windows_sys::Win32::Storage::FileSystem::{ + FILE_NAME_NORMALIZED, GetFinalPathNameByHandleW, VOLUME_NAME_DOS, + }; + + let handle = file.as_raw_handle() as isize; + let mut buf = vec![0u16; 512]; + loop { + // SAFETY: `handle` is a live file handle borrowed from `file`, and the + // buffer's length is passed as its capacity in `u16`s. + let needed = unsafe { + GetFinalPathNameByHandleW( + handle as _, + buf.as_mut_ptr(), + buf.len() as u32, + FILE_NAME_NORMALIZED | VOLUME_NAME_DOS, + ) + }; + if needed == 0 { + return Err(std::io::Error::last_os_error()); + } + // The return value excludes the NUL when it fits and includes it when + // it does not, so a value at or past the capacity means "too small". + if (needed as usize) < buf.len() { + buf.truncate(needed as usize); + return Ok(PathBuf::from(std::ffi::OsString::from_wide(&buf))); + } + buf.resize(needed as usize + 1, 0); + } +} + +/// A fallback for platforms that are neither unix nor Windows, where there is +/// no way to do better than refusing a link that is actually there. +#[cfg(not(any(unix, windows)))] fn open_within_root(root: &Path, path: &Path) -> std::io::Result { use std::io::{Error, ErrorKind}; @@ -2913,8 +3120,6 @@ fn open_within_root(root: &Path, path: &Path) -> std::io::Result for component in &components[..components.len() - 1] { walked.push(component); let meta = std::fs::symlink_metadata(&walked)?; - // `is_symlink` covers junctions too: both are reparse points tagged as - // name surrogates, which is what std tests for. if meta.file_type().is_symlink() { return Err(Error::new( ErrorKind::InvalidInput, @@ -2982,13 +3187,22 @@ fn reindex_file(state: &ServerState, path: &Path, rel_path: &str) { // metadata we judged it by, or put it outside the tree we serve. let file = match open_within_root(&state.root, path) { Ok(f) => f, - Err(_) => { - // Includes the ELOOP that a symlink gets on unix. It may still be - // a path we indexed before it became one, so fall through to the - // drop rather than returning. + Err(e) if proves_ineligible(&e) => { + // Gone, or not a regular file reachable without traversing a link. + // It may still be a path we indexed before it became one, so fall + // through to the drop rather than returning. drop_indexed_file(state, rel_path, "no longer eligible"); return; } + Err(_) => { + // A permission error, a Windows sharing violation, a descriptor + // limit — none of which say anything about whether the file + // belongs in the index. Dropping on those would evict live content + // because a build held the file open for a moment. The stale path + // already treats unreadable files this way, keeping what it has and + // retrying later, and the watcher should not disagree with it. + return; + } }; let Ok(meta) = file.metadata() else { return; @@ -3619,7 +3833,12 @@ fn refresh_stale_locked( // event can observe the matcher before this function returns either way. // The caller's walk started before this publish; its timestamp is what the // recovery scan needs, so the one the subscription sync derives is dropped. - *newly_watched = publish_ignore_matcher(state, root, build_stale_matcher(state, root, &walk)); + *newly_watched = publish_ignore_matcher( + state, + root, + build_stale_matcher(state, root, &walk), + ignore_sources_of(root, &walk.gitignore_files, &walk.ignore_files), + ); if walk.skipped_error > 0 { eprintln!( @@ -3870,7 +4089,12 @@ fn bootstrap_index_build(state: &Arc, root: &Path, index_dir: &Path // than skipped: the scan waits out `indexing` and then costs one // `metadata` call per file, since the stamps this build just wrote // describe the index exactly. - newly_watched = publish_ignore_matcher(state, root, matcher); + newly_watched = publish_ignore_matcher( + state, + root, + matcher, + ignore_sources_of(root, &outcome.gitignore_files, &outcome.ignore_files), + ); eprintln!( "[trace] gitignore matcher built from {} file(s) in {:.1}ms{}", outcome.gitignore_files.len(), @@ -3980,7 +4204,12 @@ fn background_index_build(state: &Arc, root: &Path, index_dir: &Pat // the build's results nor any event. The scan waits for the build to // finish before looking, because until then the stamps describe // nothing and every file would read as changed. - newly_watched = publish_ignore_matcher(state, root, matcher); + newly_watched = publish_ignore_matcher( + state, + root, + matcher, + ignore_sources_of(root, &walk.gitignore_files, &walk.ignore_files), + ); eprintln!( "[trace] gitignore matcher built from index walk in {:.1}ms \ ({} .gitignore + {} .ignore files{})", @@ -4748,8 +4977,9 @@ mod tests { gitignore_pending: std::sync::atomic::AtomicBool::new(true), ignore_rules_dirty: std::sync::atomic::AtomicBool::new(false), ignore_refresh_scheduled: std::sync::atomic::AtomicBool::new(false), + ignore_sources: RwLock::new(Vec::new()), reindex_lock: Mutex::new(()), - deferred_events: Mutex::new(Some(std::collections::HashSet::new())), + deferred_events: Mutex::new(Some(std::collections::HashMap::new())), index_progress: std::sync::atomic::AtomicU64::new(0), index_total: std::sync::atomic::AtomicU64::new(0), watch_enabled: true, @@ -5611,13 +5841,16 @@ mod tests { let root = tmp.path().to_path_buf(); let index_dir = root.join(".tgrep"); let state = test_server_state(&root, &index_dir); + // The function only defers while a build is running; it now reports + // back so the caller can handle an event that arrived after one ended. + state.indexing.store(true, Ordering::SeqCst); let small = Event { kind: EventKind::Create(notify::event::CreateKind::Any), paths: vec![root.join("a.rs")], attrs: Default::default(), }; - defer_events_during_build(&state, &small); + assert!(defer_events_during_build(&state, &small)); assert_eq!( state .deferred_events @@ -5636,14 +5869,14 @@ mod tests { .collect(), attrs: Default::default(), }; - defer_events_during_build(&state, &flood); + assert!(defer_events_during_build(&state, &flood)); assert!( state.deferred_events.lock().unwrap().is_none(), "an overflowing burst must mark the buffer unusable, not truncate it" ); // And stays given up on, rather than resuming a partial record. - defer_events_during_build(&state, &small); + assert!(defer_events_during_build(&state, &small)); assert!(state.deferred_events.lock().unwrap().is_none()); } @@ -5697,6 +5930,125 @@ mod tests { ); } + /// A metadata-only change to a directory is not a claim that anything new + /// is under it. Replaying every deferred path as a creation would turn a + /// recursive `chmod` during a build into one subtree walk per directory. + #[cfg(unix)] + #[test] + fn a_deferred_metadata_change_does_not_replay_as_a_subtree_arrival() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + state.gitignore_pending.store(false, Ordering::SeqCst); + + let dir = root.join("vendor"); + std::fs::create_dir(&dir).unwrap(); + std::fs::write(dir.join("deep.rs"), "fn deep() {}\n").unwrap(); + + state.indexing.store(true, Ordering::SeqCst); + handle_fs_event( + &state, + &root, + &Event { + kind: EventKind::Modify(notify::event::ModifyKind::Metadata( + notify::event::MetadataKind::Permissions, + )), + paths: vec![dir.clone()], + attrs: Default::default(), + }, + ); + assert_eq!( + state + .deferred_events + .lock() + .unwrap() + .as_ref() + .unwrap() + .get(&dir), + Some(&false), + "a metadata modify must not be recorded as introducing a directory" + ); + + state.indexing.store(false, Ordering::SeqCst); + replay_deferred_events(&state, &root); + + // The replay reconstructs a modify, which stops at the directory gate. + // Had it reconstructed a create, `watch_new_subtree` would have walked + // in and indexed the file below. + assert!( + !state.index.read().unwrap().live.has_path("vendor/deep.rs"), + "a metadata modify must not trigger a subtree walk on replay" + ); + } + + /// A file that cannot be opened right now is not a file that stopped + /// belonging in the index. Evicting on a transient error would drop live + /// content because something else held the file open for a moment. + #[cfg(unix)] + #[test] + fn an_unreadable_file_keeps_its_indexed_content() { + use std::os::unix::fs::PermissionsExt; + + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + state.gitignore_pending.store(false, Ordering::SeqCst); + + let path = root.join("locked.rs"); + std::fs::write(&path, "fn readable() {}\n").unwrap(); + reindex_file(&state, &path, "locked.rs"); + assert!(state.index.read().unwrap().live.has_path("locked.rs")); + + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o000)).unwrap(); + if std::fs::File::open(&path).is_ok() { + // Running as root, where the mode is advisory. Nothing to test. + return; + } + reindex_file(&state, &path, "locked.rs"); + + assert!( + !state.index.read().unwrap().live.is_deleted("locked.rs"), + "an unreadable file must keep what was already indexed for it" + ); + } + + /// Deleting an ignore file is invisible to a scan that looks for *arrivals* + /// by mtime, and leaves rules in force whose source is gone — so the + /// published sources are checked directly. + #[cfg(unix)] + #[test] + fn a_recovery_scan_notices_an_ignore_file_that_was_deleted() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + state.gitignore_pending.store(false, Ordering::SeqCst); + + // Published as a source, then removed behind the matcher's back. + let rules = root.join(".gitignore"); + std::fs::write(&rules, "build/\n").unwrap(); + *state.ignore_sources.write().unwrap() = vec![rules.clone()]; + std::fs::remove_file(&rules).unwrap(); + + std::fs::write(root.join("kept.rs"), "fn kept() {}\n").unwrap(); + // Pretend a refresh worker is already running, so the scan's request + // for one is coalesced into it instead of spawning a real rewalk that + // would race the assertions below. + state.ignore_refresh_scheduled.store(true, Ordering::SeqCst); + reindex_files_in(&state, &root, &[root.clone()], SystemTime::UNIX_EPOCH); + + assert!( + state.ignore_rules_dirty.load(Ordering::SeqCst), + "a vanished ignore source must schedule a refresh" + ); + assert!( + !state.index.read().unwrap().live.has_path("kept.rs"), + "the scan must abandon rather than index under rules it knows are stale" + ); + } + #[test] fn glob_filter_unix_patterns() { use crate::glob_filter::GlobFilter; From f8bedb0150f888e3337b9009fb456686fd6c130c Mon Sep 17 00:00:00 2001 From: Shengyu Fu Date: Thu, 27 Aug 2026 18:00:30 -0700 Subject: [PATCH 13/28] Take the slice by reference in the new recovery test Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: bcc85b55-e821-447e-b9a9-144b78829ae4 --- tgrep-cli/src/serve.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tgrep-cli/src/serve.rs b/tgrep-cli/src/serve.rs index ad902ca..2b09ed8 100644 --- a/tgrep-cli/src/serve.rs +++ b/tgrep-cli/src/serve.rs @@ -6037,7 +6037,12 @@ mod tests { // for one is coalesced into it instead of spawning a real rewalk that // would race the assertions below. state.ignore_refresh_scheduled.store(true, Ordering::SeqCst); - reindex_files_in(&state, &root, &[root.clone()], SystemTime::UNIX_EPOCH); + reindex_files_in( + &state, + &root, + std::slice::from_ref(&root), + SystemTime::UNIX_EPOCH, + ); assert!( state.ignore_rules_dirty.load(Ordering::SeqCst), From e284cabde511f8a0398663352a7ddacee1a470b7 Mon Sep 17 00:00:00 2001 From: Shengyu Fu Date: Thu, 27 Aug 2026 18:16:16 -0700 Subject: [PATCH 14/28] Stop concluding absence from an incomplete listing, and from a path that still exists MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three more from review, all about the watcher acting on evidence it does not have. read_dir's per-entry errors were being flattened away, and the directory was then recorded as fully enumerated. A name that failed to yield is simply missing from `present`, so the sweep read it as a deletion and tombstoned a file it had no reason to believe was gone. The directory now has to have enumerated cleanly before it can claim to have been swept, which is what a failed listing already did. An indexed file atomically replaced by a fifo, a socket, or a symlink to a directory is not a removal — the path still exists — and it is not a directory either, so the non-file branch returned without touching the index and the old contents stayed searchable. Real directories keep the subtree handling; everything else drops what was indexed. And notify registers inotify watches without IN_DONT_FOLLOW, so the descriptor lands on whatever the name resolves to at that instant, not on the directory the walk validated earlier. A swap in between left a descriptor watching outside the root while the registry recorded the name as covered. The registration is now re-checked no-follow and undone on a mismatch. That narrows the window rather than closing it, since notify takes a path and not a handle; what remains is missed events on a real directory, which the reconcile picks up, and never misplaced content, which open_within_root settles from the handle it reads. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: bcc85b55-e821-447e-b9a9-144b78829ae4 --- tgrep-cli/src/serve.rs | 165 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 147 insertions(+), 18 deletions(-) diff --git a/tgrep-cli/src/serve.rs b/tgrep-cli/src/serve.rs index 2b09ed8..94a5428 100644 --- a/tgrep-cli/src/serve.rs +++ b/tgrep-cli/src/serve.rs @@ -1928,6 +1928,34 @@ impl WatchRegistry { } match self.watcher.watch(dir, RecursiveMode::NonRecursive) { Ok(()) => { + // notify's inotify backend registers without + // `IN_DONT_FOLLOW`, so the descriptor lands on whatever the + // name resolves to at that instant — and the no-follow + // check that qualified this directory happened earlier, in + // the walk. A checkout or a rename can replace it with a + // symlink in between, leaving the descriptor watching an + // inode outside the root while `watched` records an + // in-root name as covered. + // + // Re-checking after the fact catches that: if the name is + // no longer a real directory, the registration is undone + // and the entry is not recorded, so a later `sync` retries + // it rather than treating a poisoned subscription as live. + // + // This narrows the window rather than closing it — notify + // takes a path, not a handle, so a swap that is reverted + // before this check is undetectable through its API. What + // that costs is bounded: it is missed *events* on a real + // directory, which the periodic reconcile picks up, and + // never misplaced content, since `open_within_root` + // establishes containment from the handle it reads. + if !is_real_dir(dir) { + let _ = self.watcher.unwatch(dir); + if known { + self.watched.remove(dir); + } + continue; + } if !known { self.watched.insert(dir.clone()); added.push(dir.clone()); @@ -2178,7 +2206,17 @@ fn reindex_files_in(state: &Arc, root: &Path, dirs: &[PathBuf], sin continue; }; let mut subdirs: Vec = Vec::new(); - for entry in entries.flatten() { + // A per-entry error is as much a gap in the evidence as a failed + // listing: the name it would have yielded is simply absent from + // `present`, and the sweep below would read that as a deletion. The + // entry is skipped either way, but the directory then does not get to + // claim it was enumerated. + let mut listing_complete = true; + for entry in entries { + let Ok(entry) = entry else { + listing_complete = false; + continue; + }; let path = entry.path(); let Ok(rel) = path.strip_prefix(root) else { continue; @@ -2249,7 +2287,9 @@ fn reindex_files_in(state: &Arc, root: &Path, dirs: &[PathBuf], sin reindex_file(state, &path, &rel); } } - swept.insert(rel_dir); + if listing_complete { + swept.insert(rel_dir); + } // A directory created in the same window is in neither `dirs` (the // walk did not see it) nor any event (its parent's subscription is @@ -2861,27 +2901,37 @@ fn handle_fs_event(state: &Arc, root: &Path, event: &Event) { // it is recognised as ineligible and any content indexed under that // path before it became a link is dropped. if !path.is_file() { - // Only for events that can actually introduce a directory. Any - // `Modify` would include `Modify(Metadata)`, which a recursive - // chmod or a checkout fires once per directory — and each one - // would re-walk and re-subscribe that directory's whole subtree - // on the single watcher worker, turning a linear operation into - // quadratic work. inotify announces a new directory as `Create` - // and one moved in as `Modify(Name)`; nothing else can. - let introduces_dir = matches!( - event.kind, - EventKind::Create(_) | EventKind::Modify(notify::event::ModifyKind::Name(_)) - ); // `is_real_dir` rather than `is_dir`: the latter follows symlinks, // and a link to a directory is not something the walker descends // into, so subscribing to and indexing its target would pull in a // tree the index never contained — possibly outside `root`. - if PER_DIRECTORY_WATCHES && introduces_dir && is_real_dir(path) { - // With non-recursive subscriptions notify will not extend the - // watch set for us, so a directory that just appeared — and - // anything already inside it — has to be picked up here. - watch_new_subtree(state, root, path); + if is_real_dir(path) { + // Only for events that can actually introduce a directory. Any + // `Modify` would include `Modify(Metadata)`, which a recursive + // chmod or a checkout fires once per directory — and each one + // would re-walk and re-subscribe that directory's whole subtree + // on the single watcher worker, turning a linear operation into + // quadratic work. inotify announces a new directory as `Create` + // and one moved in as `Modify(Name)`; nothing else can. + let introduces_dir = matches!( + event.kind, + EventKind::Create(_) | EventKind::Modify(notify::event::ModifyKind::Name(_)) + ); + if PER_DIRECTORY_WATCHES && introduces_dir { + // With non-recursive subscriptions notify will not extend + // the watch set for us, so a directory that just appeared — + // and anything already inside it — has to be picked up here. + watch_new_subtree(state, root, path); + } + continue; } + // Neither a regular file nor a directory: a fifo, a socket, a + // device, or a symlink of any kind. An indexed `x.rs` atomically + // replaced by one of those is not a removal — `path.exists()` is + // still true and inotify may report only the rename destination — + // so nothing above catches it, and without this the old contents + // stay searchable indefinitely. + drop_indexed_file(state, &rel_path, "no longer a regular file"); continue; } @@ -6054,6 +6104,85 @@ mod tests { ); } + /// A directory whose listing was only partly enumerated has not proved + /// anything about the names it did not yield, so the sweep must not treat + /// them as deleted. + #[cfg(unix)] + #[test] + fn a_directory_that_was_not_fully_enumerated_is_not_swept() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + + let path = root.join("d").join("a.rs"); + std::fs::create_dir(root.join("d")).unwrap(); + std::fs::write(&path, "fn a() {}\n").unwrap(); + reindex_file(&state, &path, "d/a.rs"); + assert!(state.index.read().unwrap().live.has_path("d/a.rs")); + + // What `reindex_files_in` produces when an entry in `d` failed to + // enumerate: the file is missing from `present`, and `d` is therefore + // withheld from `swept`. + let swept = std::collections::HashSet::new(); + let present = std::collections::HashSet::new(); + sweep_removed_files(&state, &swept, &present); + assert!( + !state.index.read().unwrap().live.is_deleted("d/a.rs"), + "an unenumerated directory must not tombstone the files under it" + ); + + // And with the listing complete, the same absence is a deletion. + let swept = std::collections::HashSet::from(["d".to_string()]); + sweep_removed_files(&state, &swept, &present); + assert!( + state.index.read().unwrap().live.is_deleted("d/a.rs"), + "a fully enumerated directory must sweep what it no longer contains" + ); + } + + /// An indexed file replaced in place by something that is not a regular + /// file is not a removal — the path still exists — but its contents are no + /// longer there to be found. + #[cfg(unix)] + #[test] + fn a_file_replaced_by_a_fifo_loses_its_indexed_content() { + use std::os::unix::ffi::OsStrExt; + + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + state.gitignore_pending.store(false, Ordering::SeqCst); + + let path = root.join("x.rs"); + std::fs::write(&path, "fn was_a_real_file() {}\n").unwrap(); + reindex_file(&state, &path, "x.rs"); + assert!(state.index.read().unwrap().live.has_path("x.rs")); + + std::fs::remove_file(&path).unwrap(); + let name = std::ffi::CString::new(path.as_os_str().as_bytes()).unwrap(); + // SAFETY: `name` is a NUL-terminated path that outlives the call. + assert_eq!(unsafe { libc::mkfifo(name.as_ptr(), 0o644) }, 0); + + handle_fs_event( + &state, + &root, + &Event { + kind: EventKind::Modify(notify::event::ModifyKind::Name( + notify::event::RenameMode::To, + )), + paths: vec![path.clone()], + attrs: Default::default(), + }, + ); + + assert!( + state.index.read().unwrap().live.is_deleted("x.rs"), + "content indexed before the path became a fifo must not stay searchable" + ); + } + #[test] fn glob_filter_unix_patterns() { use crate::glob_filter::GlobFilter; From 03adea4e7fdd4552ae3584e2803d588e0c55fc53 Mon Sep 17 00:00:00 2001 From: Shengyu Fu Date: Thu, 27 Aug 2026 18:25:34 -0700 Subject: [PATCH 15/28] Name the sweep test after the invariant it actually pins A control run showed it passing with the incomplete-listing fix reverted, which is correct: it exercises sweep_removed_files directly, so it guards the contract that withholding a directory is sufficient, not the wiring that withholds it. A per-entry readdir failure cannot be induced portably, so that half stays untested and the doc comment now says so. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: bcc85b55-e821-447e-b9a9-144b78829ae4 --- tgrep-cli/src/serve.rs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tgrep-cli/src/serve.rs b/tgrep-cli/src/serve.rs index 94a5428..6df454d 100644 --- a/tgrep-cli/src/serve.rs +++ b/tgrep-cli/src/serve.rs @@ -6104,12 +6104,17 @@ mod tests { ); } - /// A directory whose listing was only partly enumerated has not proved - /// anything about the names it did not yield, so the sweep must not treat - /// them as deleted. + /// The invariant the incomplete-listing fix relies on: a directory that is + /// not in `swept` has proved nothing about the names under it, so the + /// sweep must leave them alone. + /// + /// The wiring above it — withholding a directory whose `read_dir` iterator + /// yielded an error — has no test of its own, because a per-entry + /// `readdir` failure cannot be induced portably. This pins the half that + /// makes withholding sufficient. #[cfg(unix)] #[test] - fn a_directory_that_was_not_fully_enumerated_is_not_swept() { + fn sweep_removed_files_only_deletes_from_directories_it_enumerated() { let tmp = TempDir::new().unwrap(); let root = tmp.path().to_path_buf(); let index_dir = root.join(".tgrep"); From c88e345e23c9b1c58c6f3f3fe50e55b157c9ac27 Mon Sep 17 00:00:00 2001 From: Shengyu Fu Date: Thu, 27 Aug 2026 18:36:10 -0700 Subject: [PATCH 16/28] Recheck sweep candidates under the reindex lock, and bound the read at the cap Two races the reviewer found in the reconcile path. The sweep deleted files on the strength of a directory listing taken earlier in the scan, without the lock `reindex_file` holds. A file recreated in between has already had its create event consumed by the watcher, so deleting it here lost it until the next reconcile with nothing left to replay. Each candidate is now rechecked against the filesystem while holding `reindex_lock`, so the observation the delete acts on is its own and cannot be overtaken. The content read was unbounded even though the size that qualified the file was stat'd before it. A file appended to in between -- a log, a build artifact -- was pulled into memory whole and indexed past `--max-filesize`. `read_within_limit` reads at most one byte past the cap and drops what the index holds when that byte is there. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: bcc85b55-e821-447e-b9a9-144b78829ae4 --- tgrep-cli/src/serve.rs | 203 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 179 insertions(+), 24 deletions(-) diff --git a/tgrep-cli/src/serve.rs b/tgrep-cli/src/serve.rs index 6df454d..4c6360f 100644 --- a/tgrep-cli/src/serve.rs +++ b/tgrep-cli/src/serve.rs @@ -2369,27 +2369,39 @@ fn sweep_removed_files( if gone.is_empty() { return; } - eprintln!( - "[trace] watcher: dropped {} file(s) removed while subscriptions were \ - being established", - gone.len() - ); - { - let mut index = state.index.write().unwrap(); - for rel in &gone { - index.live.delete_file(rel); - } - } - { - let mut stamps = state.file_stamps.write().unwrap(); - for rel in &gone { - stamps.remove(rel); + // Per candidate, under the same lock `reindex_file` takes, and re-checked + // against the filesystem rather than against the listing that produced + // `gone`. That listing is from earlier in the scan; a file recreated since + // then has already had its create event consumed by the watcher, so + // deleting it here on the strength of a stale observation would lose it + // until the next reconcile — and there is nothing left to replay. + // + // The lock is what makes the recheck mean anything: without it the file + // could be reindexed between the check and the delete, which is the same + // bug one instruction later. + let mut dropped = 0usize; + for rel in &gone { + let _reindex = match state.reindex_lock.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + }; + // No-follow: a path that came back as a symlink is still not something + // the index should hold, so it stays swept. + if std::fs::symlink_metadata(state.root.join(rel)).is_ok_and(|m| m.file_type().is_file()) { + continue; } - } - if let Ok(mut cache) = state.cache.write() { - for rel in &gone { + state.index.write().unwrap().live.delete_file(rel); + state.file_stamps.write().unwrap().remove(rel); + if let Ok(mut cache) = state.cache.write() { cache.pop(rel); } + dropped += 1; + } + if dropped > 0 { + eprintln!( + "[trace] watcher: dropped {dropped} file(s) removed while subscriptions were \ + being established" + ); } } @@ -3212,13 +3224,53 @@ fn relative_components(root: &Path, path: &Path) -> std::io::Result), + /// The file yielded more bytes than the cap allows, whatever its size said. + TooLarge, + Failed, +} + +/// Reads a file's contents, never pulling in more than one byte past the cap. +/// +/// The size that qualified this file was stat'd before the read, and appending +/// between the two is exactly what a log or a build artifact does. An +/// unbounded read would then hold the whole of it in memory and index it past +/// the limit the user set. One byte over is enough to prove it no longer +/// qualifies, and is all that is ever read beyond the limit. +fn read_within_limit(file: &mut std::fs::File, limit: Option, capacity: usize) -> CappedRead { + use std::io::Read; + + let mut data = Vec::with_capacity(capacity); + match limit { + Some(limit) => { + if file + .take(limit.saturating_add(1)) + .read_to_end(&mut data) + .is_err() + { + return CappedRead::Failed; + } + if data.len() as u64 > limit { + return CappedRead::TooLarge; + } + } + None => { + if file.read_to_end(&mut data).is_err() { + return CappedRead::Failed; + } + } + } + CappedRead::Data(data) +} + /// Read a file and merge it into the live index, unless its stamp says the /// content we already indexed is current. /// /// The caller must hold `snapshot_gate`: the read, the commit, and the stamp /// update have to be atomic with respect to a flush or auto-save. fn reindex_file(state: &ServerState, path: &Path, rel_path: &str) { - use std::io::Read; use tgrep_core::meta::FileStamp; // Against other indexers, not against searches. The gate above is held for @@ -3310,10 +3362,18 @@ fn reindex_file(state: &ServerState, path: &Path, rel_path: &str) { // From the handle, not the path: re-opening here is what would let a // symlink take the place of the file we just approved. let mut file = file; - let mut data = Vec::with_capacity(current.size.min(1 << 20) as usize); - if file.read_to_end(&mut data).is_err() { - return; - } + let data = match read_within_limit( + &mut file, + state.max_file_size, + current.size.min(1 << 20) as usize, + ) { + CappedRead::Data(data) => data, + CappedRead::TooLarge => { + drop_indexed_file(state, rel_path, "grew past the size limit while being read"); + return; + } + CappedRead::Failed => return, + }; let text = tgrep_core::encoding::decode_for_index(&data); let is_binary = tgrep_core::trigram::is_binary(&text); let per_tri = if is_binary { @@ -6137,7 +6197,9 @@ mod tests { "an unenumerated directory must not tombstone the files under it" ); - // And with the listing complete, the same absence is a deletion. + // And with the listing complete, the same absence is a deletion — once + // the file is actually gone, which the sweep now confirms itself. + std::fs::remove_file(&path).unwrap(); let swept = std::collections::HashSet::from(["d".to_string()]); sweep_removed_files(&state, &swept, &present); assert!( @@ -6146,6 +6208,35 @@ mod tests { ); } + /// The listing that decides what to sweep is from earlier in the scan. A + /// file recreated since then has already had its event consumed, so + /// deleting it on that stale evidence loses it until the next reconcile. + #[cfg(unix)] + #[test] + fn the_sweep_does_not_delete_a_file_that_came_back() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + + let path = root.join("d").join("a.rs"); + std::fs::create_dir(root.join("d")).unwrap(); + std::fs::write(&path, "fn a() {}\n").unwrap(); + reindex_file(&state, &path, "d/a.rs"); + assert!(state.index.read().unwrap().live.has_path("d/a.rs")); + + // `d` enumerated cleanly and did not contain `a.rs` at the time — but + // it is back on disk by the time the sweep runs. + let swept = std::collections::HashSet::from(["d".to_string()]); + let present = std::collections::HashSet::new(); + sweep_removed_files(&state, &swept, &present); + + assert!( + !state.index.read().unwrap().live.is_deleted("d/a.rs"), + "a file that exists again must not be swept on a stale listing" + ); + } + /// An indexed file replaced in place by something that is not a regular /// file is not a removal — the path still exists — but its contents are no /// longer there to be found. @@ -6188,6 +6279,70 @@ mod tests { ); } + /// The size that qualifies a file is stat'd before its contents are read. + /// A file that grows in between must not be read into memory without + /// bound, nor indexed past the cap. + #[test] + fn a_read_stops_one_byte_past_the_cap() { + let tmp = TempDir::new().unwrap(); + let path = tmp.path().join("grew.txt"); + std::fs::write(&path, "x".repeat(4096)).unwrap(); + + // Stat said 16 bytes; the file is 4096 by the time it is read. + let mut file = std::fs::File::open(&path).unwrap(); + assert!(matches!( + read_within_limit(&mut file, Some(64), 16), + CappedRead::TooLarge + )); + + let mut file = std::fs::File::open(&path).unwrap(); + assert!( + matches!(read_within_limit(&mut file, None, 16), CappedRead::Data(d) if d.len() == 4096), + "no cap means no bound" + ); + + std::fs::write(&path, "small").unwrap(); + let mut file = std::fs::File::open(&path).unwrap(); + assert!( + matches!(read_within_limit(&mut file, Some(64), 5), CappedRead::Data(d) if d == b"small"), + "a file within the cap reads whole" + ); + + // Exactly at the cap is still within it. + std::fs::write(&path, "x".repeat(64)).unwrap(); + let mut file = std::fs::File::open(&path).unwrap(); + assert!( + matches!(read_within_limit(&mut file, Some(64), 64), CappedRead::Data(d) if d.len() == 64) + ); + } + + /// The whole point of the bound: a file past the cap loses what the index + /// held for it rather than keeping a stale, smaller copy. + #[cfg(unix)] + #[test] + fn a_file_that_outgrows_the_cap_loses_its_indexed_content() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let index_dir = root.join(".tgrep"); + let mut state = test_server_state(&root, &index_dir); + Arc::get_mut(&mut state).unwrap().max_file_size = Some(64); + + let path = root.join("grows.rs"); + std::fs::write(&path, "fn small_enough() {}\n").unwrap(); + reindex_file(&state, &path, "grows.rs"); + assert!( + state.index.read().unwrap().live.has_path("grows.rs"), + "a file under the cap should index normally" + ); + + std::fs::write(&path, "x".repeat(4096)).unwrap(); + reindex_file(&state, &path, "grows.rs"); + assert!( + state.index.read().unwrap().live.is_deleted("grows.rs"), + "content past the cap must not stay searchable" + ); + } + #[test] fn glob_filter_unix_patterns() { use crate::glob_filter::GlobFilter; From 04a9af2cf3375404f47062840f8999c30584b1c1 Mon Sep 17 00:00:00 2001 From: Shengyu Fu Date: Thu, 27 Aug 2026 18:41:59 -0700 Subject: [PATCH 17/28] Name the size-gate test for the growth it actually pins A control run with the read bound reverted left this test passing: the eligibility check catches a file that outgrew the cap between visits long before the read does. It pins that gate, not the new bound, and the name now says so. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: bcc85b55-e821-447e-b9a9-144b78829ae4 --- tgrep-cli/src/serve.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tgrep-cli/src/serve.rs b/tgrep-cli/src/serve.rs index 4c6360f..e1eb03b 100644 --- a/tgrep-cli/src/serve.rs +++ b/tgrep-cli/src/serve.rs @@ -6316,11 +6316,14 @@ mod tests { ); } - /// The whole point of the bound: a file past the cap loses what the index - /// held for it rather than keeping a stale, smaller copy. + /// The size gate is checked against a fresh stat on every visit, so a file + /// that outgrew the cap since it was indexed loses what the index holds + /// rather than keeping the smaller version until the reconcile. (The + /// growth this pins is between visits — growth *during* a read is what + /// `read_within_limit` above covers.) #[cfg(unix)] #[test] - fn a_file_that_outgrows_the_cap_loses_its_indexed_content() { + fn a_file_that_outgrew_the_cap_between_visits_loses_its_indexed_content() { let tmp = TempDir::new().unwrap(); let root = tmp.path().to_path_buf(); let index_dir = root.join(".tgrep"); From 1ab05044f528ce640d065bf8a71d29e088eec722 Mon Sep 17 00:00:00 2001 From: Shengyu Fu Date: Thu, 27 Aug 2026 20:42:47 -0700 Subject: [PATCH 18/28] Serialise removals with reindex, and stop missing ignore files Three defects the reviewer found, all in the reconcile path. A removal handled by the watcher mutated the index without `reindex_lock`, so a `reindex_file` already holding a file's bytes could commit them after the delete. That resurrects a file that is gone, with a fresh stamp, so nothing afterwards disagrees and no further event is coming to correct it. Both removal branches now take the lock; it is the caller's rather than `drop_indexed_file`'s because `reindex_file` calls in while already holding it. Arrival of an ignore file was detected by mtime alone, and `git checkout`, `tar -x` and `rsync -a` all restore mtimes from what they unpack -- so a nested `.gitignore` could arrive dated months ago and sail past the window. Absence from `ignore_sources` is the exact question instead: this file did not feed the published matcher. The mtime window stays for the case the source list cannot answer, an existing source that has just been edited. Ignore files reached through a symlink were dropped before either test ran. `DirEntry::file_type` does not follow links, but the walker collects rule files with `Path::is_file`, which does -- so those files carried rules that the recovery scan and the new-subtree descent were both blind to. The check now runs ahead of the type dispatch and follows links to decide. They are still never indexed. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: bcc85b55-e821-447e-b9a9-144b78829ae4 --- tgrep-cli/src/serve.rs | 338 +++++++++++++++++++++++++++++++++-------- 1 file changed, 274 insertions(+), 64 deletions(-) diff --git a/tgrep-cli/src/serve.rs b/tgrep-cli/src/serve.rs index e1eb03b..1ca4176 100644 --- a/tgrep-cli/src/serve.rs +++ b/tgrep-cli/src/serve.rs @@ -2166,16 +2166,31 @@ fn reindex_files_in(state: &Arc, root: &Path, dirs: &[PathBuf], sin let start = Instant::now(); // An ignore file that was deleted during the window is invisible to the - // per-entry mtime test below — there is no entry left to stat. It is also - // the more damaging direction: rules that no longer have a source keep - // being enforced, so the subtree they hide stays unsubscribed and - // unindexed until an unrelated rebuild happens along. Checking the sources - // the published matcher was built from catches it at one stat apiece, once - // per scan rather than once per file. - if !state.no_ignore { - let vanished = { + // per-entry test below — there is no entry left to stat. It is also the + // more damaging direction: rules that no longer have a source keep being + // enforced, so the subtree they hide stays unsubscribed and unindexed until + // an unrelated rebuild happens along. Checking the sources the published + // matcher was built from catches it at one stat apiece, once per scan + // rather than once per file. + // + // The same list answers the opposite question for free: an ignore file that + // is *not* among the sources is one the published matcher never read. + let known_sources: std::collections::HashSet = if state.no_ignore { + std::collections::HashSet::new() + } else { + let (vanished, known) = { let sources = state.ignore_sources.read().unwrap(); - sources.iter().find(|p| !p.exists()).cloned() + ( + // `exists` follows links, matching how the walker collected + // these — a symlinked source whose target is gone has stopped + // contributing rules just as surely as a deleted one. + sources.iter().find(|p| !p.exists()).cloned(), + sources + .iter() + .filter_map(|p| p.strip_prefix(root).ok()) + .map(|rel| rel.to_string_lossy().replace('\\', "/")) + .collect::>(), + ) }; if let Some(gone) = vanished { state.ignore_rules_dirty.store(true, Ordering::SeqCst); @@ -2186,7 +2201,8 @@ fn reindex_files_in(state: &Arc, root: &Path, dirs: &[PathBuf], sin ); return; } - } + known + }; // Directories whose listing succeeded, and the files those listings // contained, for the removal sweep at the end. Only files are recorded: @@ -2235,23 +2251,36 @@ fn reindex_files_in(state: &Arc, root: &Path, dirs: &[PathBuf], sin subdirs.push(path); continue; } - if !file_type.is_file() { - continue; - } - present.insert(rel.clone()); + // Ahead of the regular-file test, and following links to decide. + // The walker collects rule files with `Path::is_file`, which + // resolves symlinks, so a symlinked `.gitignore` contributes rules + // exactly like a real one — while `DirEntry::file_type` above does + // not resolve them, which left those files falling through the + // `!is_file` bail and never triggering a refresh. They are still + // never indexed; they are only allowed to announce themselves. + // // An ignore-rules file that landed in this window was not seen by // the walk that built the matcher in force, so every other file in // this scan is being judged by rules that do not know about it. // Indexing them now would apply the wrong rules and leave whatever // was wrongly indexed until something touched it again. // - // The mtime test is what keeps this quiet: a repository has an - // ignore file in every other directory and the startup scan walks - // past all of them, but they predate the walk and are already + // Two tests, because neither alone is enough. Absence from the + // published sources is the exact question — this file did not feed + // the matcher — and it catches the arrival however old the file + // says it is, which matters because `git checkout`, `tar -x` and + // `rsync -a` all restore mtimes from the archive and would sail + // past a recency test. The mtime window then covers the case the + // source list cannot: a file that was already a source and has just + // been *edited*. + // + // The mtime test is what keeps the second one quiet: a repository + // has an ignore file in every other directory and the startup scan + // walks past all of them, but they predate the walk and are already // accounted for. Only one written inside the window can have been - // missed — and a spurious match (a `touch` in the same - // millisecond) costs an idempotent refresh, not correctness. + // missed — and a spurious match (a `touch` in the same millisecond) + // costs an idempotent refresh, not correctness. // // Bounded at both ends, not just the near one. On a network mount // whose server clock runs ahead of ours, every recently touched @@ -2260,25 +2289,36 @@ fn reindex_files_in(state: &Arc, root: &Path, dirs: &[PathBuf], sin // schedules, which walks the whole repository and then arms the // next. Treating a future mtime as skew rather than as an arrival // gives up the fix on such a mount and keeps the loop closed. - if !state.no_ignore - && is_ignore_rules_file(root, &path) - && entry - .metadata() + if !state.no_ignore && is_ignore_rules_file(root, &path) && path.is_file() { + let unknown = !known_sources.contains(&rel); + let touched = std::fs::metadata(&path) .and_then(|m| m.modified()) - .is_ok_and(|m| m >= since && m <= SystemTime::now()) - { - // Abandon the scan: the refresh rewalks and republishes, which - // covers these directories properly, and anything indexed - // between here and there would be judged by the stale rules. - state.ignore_rules_dirty.store(true, Ordering::SeqCst); - schedule_ignore_rules_refresh(Arc::clone(state), root.to_path_buf()); - eprintln!( - "[trace] watcher: ignore rules changed during recovery ({rel}); \ - deferring to a refresh" - ); - return; + .is_ok_and(|m| m >= since && m <= SystemTime::now()); + if unknown || touched { + // Abandon the scan: the refresh rewalks and republishes, + // which covers these directories properly, and anything + // indexed between here and there would be judged by the + // stale rules. + state.ignore_rules_dirty.store(true, Ordering::SeqCst); + schedule_ignore_rules_refresh(Arc::clone(state), root.to_path_buf()); + let why = if unknown { + "not a known source" + } else { + "modified" + }; + eprintln!( + "[trace] watcher: ignore rules changed during recovery ({rel}, {why}); \ + deferring to a refresh" + ); + return; + } } + if !file_type.is_file() { + continue; + } + present.insert(rel.clone()); + let skip = { let gitignore = state.gitignore.read().unwrap(); should_skip_watcher_path(&rel, &state.exclude_dirs, gitignore.as_ref()) @@ -2381,10 +2421,7 @@ fn sweep_removed_files( // bug one instruction later. let mut dropped = 0usize; for rel in &gone { - let _reindex = match state.reindex_lock.lock() { - Ok(guard) => guard, - Err(poisoned) => poisoned.into_inner(), - }; + let _reindex = lock_reindex(state); // No-follow: a path that came back as a symlink is still not something // the index should hold, so it stays swept. if std::fs::symlink_metadata(state.root.join(rel)).is_ok_and(|m| m.file_type().is_file()) { @@ -2486,6 +2523,32 @@ fn watch_new_subtree(state: &Arc, root: &Path, dir: &Path) { continue; }; let rel = rel.to_string_lossy().replace('\\', "/"); + // A subtree that arrives whole — a clone, a `mv`, a branch + // switch — can carry its own ignore rules. Those files are + // dot-prefixed, so the scan below would silently drop them and + // index the rest of the subtree against rules that do not know + // about them. + // + // Ahead of the type dispatch, and following links: the walker + // collects rule files with `Path::is_file`, which resolves + // symlinks, whereas `DirEntry::file_type` does not — so a + // symlinked `.gitignore` fell between the two branches below + // and was never noticed, despite carrying rules the walker + // would read. + // + // Abandon the descent immediately rather than finishing it. + // Everything gathered from here on is discarded by the refresh + // anyway, and the rules that are about to be published are the + // ones that decide whether these directories should be watched + // at all — continuing would subscribe to every level of, say, a + // `node_modules/` that was just moved into place, which on + // Linux is a watch descriptor apiece and the exhaustion this + // pass exists to avoid. The refresh's `sync` would prune them, + // but only after they had already been taken. + if !state.no_ignore && is_ignore_rules_file(root, &path) && path.is_file() { + found_ignore_rules = true; + break 'descend; + } // `DirEntry::file_type` does not follow symlinks, so a // symlinked directory is neither descended into nor indexed. if file_type.is_dir() { @@ -2497,26 +2560,6 @@ fn watch_new_subtree(state: &Arc, root: &Path, dir: &Path) { next.push(path); } } else if file_type.is_file() { - // A subtree that arrives whole — a clone, a `mv`, a branch - // switch — can carry its own ignore rules. Those files are - // dot-prefixed, so the scan below would silently drop them - // and index the rest of the subtree against rules that do - // not know about them. - // - // Abandon the descent immediately rather than finishing it. - // Everything gathered from here on is discarded by the - // refresh anyway, and the rules that are about to be - // published are the ones that decide whether these - // directories should be watched at all — continuing would - // subscribe to every level of, say, a `node_modules/` that - // was just moved into place, which on Linux is a watch - // descriptor apiece and the exhaustion this pass exists to - // avoid. The refresh's `sync` would prune them, but only - // after they had already been taken. - if !state.no_ignore && is_ignore_rules_file(root, &path) { - found_ignore_rules = true; - break 'descend; - } let skip = { let gitignore = state.gitignore.read().unwrap(); should_skip_watcher_path(&rel, &state.exclude_dirs, gitignore.as_ref()) @@ -2894,6 +2937,15 @@ fn handle_fs_event(state: &Arc, root: &Path, event: &Event) { // `file_stamps` is missing/out-of-date (e.g. first run after // an older index), skipping the delete entirely would leave // stale entries for files that no longer exist. + // + // Under `reindex_lock`, or a concurrent `reindex_file` that has + // already read the file's bytes commits them *after* this delete + // and resurrects a file that is gone — with a fresh stamp, so + // nothing afterwards disagrees and no further event is coming to + // correct it. The lock makes the two orderings the only two: the + // delete lands on content that was committed, or the reindex opens + // a path that is already gone and drops it. + let _reindex = lock_reindex(state); let known_path = state.file_stamps.read().unwrap().contains_key(&rel_path); if known_path { eprintln!("[trace] reindex: removed {rel_path}"); @@ -2943,6 +2995,10 @@ fn handle_fs_event(state: &Arc, root: &Path, event: &Event) { // still true and inotify may report only the rename destination — // so nothing above catches it, and without this the old contents // stay searchable indefinitely. + // + // Same lock as the removal above, for the same reason: a reindex + // already holding the old bytes must not commit them after this. + let _reindex = lock_reindex(state); drop_indexed_file(state, &rel_path, "no longer a regular file"); continue; } @@ -2951,6 +3007,18 @@ fn handle_fs_event(state: &Arc, root: &Path, event: &Event) { } } +/// Take the mutation lock, tolerating a previous holder's panic. +/// +/// Poisoning here means some other indexer panicked partway through an update, +/// not that the index is unusable. Refusing to serialise from then on would +/// turn one failure into the resurrection race this lock exists to prevent. +fn lock_reindex(state: &ServerState) -> std::sync::MutexGuard<'_, ()> { + match state.reindex_lock.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + } +} + /// Drop everything the index holds for a path. /// /// The delete is not conditional on a stamp entry. `ServerState` accepts an @@ -2967,7 +3035,9 @@ fn handle_fs_event(state: &Arc, root: &Path, event: &Event) { /// path. The trace line, which is the part that would be actively misleading, /// stays conditional on there having been something to drop. /// -/// The caller must already hold `snapshot_gate`. +/// The caller must already hold `snapshot_gate` and `reindex_lock`. The lock is +/// the caller's rather than this function's because `reindex_file` calls in +/// while holding it, and a `Mutex` is not reentrant. fn drop_indexed_file(state: &ServerState, rel_path: &str, reason: &str) { let had_stamp = state .file_stamps @@ -3277,10 +3347,7 @@ fn reindex_file(state: &ServerState, path: &Path, rel_path: &str) { // read, so without this a recovery scan and the watcher worker can both be // here for the same path, both read, and the one that read the *older* // content can commit last. See `ServerState::reindex_lock`. - let _reindex = match state.reindex_lock.lock() { - Ok(guard) => guard, - Err(poisoned) => poisoned.into_inner(), - }; + let _reindex = lock_reindex(state); // One handle for the whole decision, resolved a component at a time from // the root so no part of the path can be a symlink, and every fact below — @@ -6279,6 +6346,149 @@ mod tests { ); } + /// A removal must not land while another thread is midway through indexing + /// the same path, or the reindex commits bytes it read earlier and + /// resurrects a file that is gone — with a fresh stamp, so nothing + /// afterwards disagrees and no further event is coming to correct it. + #[cfg(unix)] + #[test] + fn a_removal_waits_for_an_in_flight_reindex() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + state.gitignore_pending.store(false, Ordering::SeqCst); + + let path = root.join("x.rs"); + std::fs::write(&path, "fn indexed() {}\n").unwrap(); + reindex_file(&state, &path, "x.rs"); + assert!(state.index.read().unwrap().live.has_path("x.rs")); + + std::fs::remove_file(&path).unwrap(); + + // Stands in for a `reindex_file` that has read the old bytes and not + // yet committed them: it holds exactly this lock across that window. + let held = state.reindex_lock.lock().unwrap(); + + let worker = { + let state = Arc::clone(&state); + let root = root.clone(); + let path = path.clone(); + std::thread::spawn(move || { + handle_fs_event( + &state, + &root, + &Event { + kind: EventKind::Remove(notify::event::RemoveKind::File), + paths: vec![path], + attrs: Default::default(), + }, + ); + }) + }; + + // The delete has to wait its turn. Without the lock it lands + // immediately, which is the ordering that loses the file. + std::thread::sleep(std::time::Duration::from_millis(300)); + assert!( + !state.index.read().unwrap().live.is_deleted("x.rs"), + "a removal must not mutate the index while an indexer holds the lock" + ); + + drop(held); + worker.join().unwrap(); + assert!( + state.index.read().unwrap().live.is_deleted("x.rs"), + "and it must still apply once the lock is free" + ); + } + + /// `git checkout`, `tar -x` and `rsync -a` all restore mtimes from what + /// they unpack, so a nested ignore file can arrive carrying a timestamp + /// from months ago. A recency test cannot see that; absence from the + /// published sources can. + #[cfg(unix)] + #[test] + fn an_arriving_ignore_file_with_a_preserved_mtime_still_refreshes_the_matcher() { + use std::os::unix::ffi::OsStrExt; + + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + state.gitignore_pending.store(false, Ordering::SeqCst); + + // The matcher in force was built without it. + *state.ignore_sources.write().unwrap() = Vec::new(); + + let sub = root.join("sub"); + std::fs::create_dir(&sub).unwrap(); + let rules = sub.join(".gitignore"); + std::fs::write(&rules, "kept.rs\n").unwrap(); + std::fs::write(sub.join("kept.rs"), "fn kept() {}\n").unwrap(); + + // Backdated well outside any plausible scan window. + let name = std::ffi::CString::new(rules.as_os_str().as_bytes()).unwrap(); + let stamp = libc::timeval { + tv_sec: 1_000_000, + tv_usec: 0, + }; + let times = [stamp, stamp]; + // SAFETY: `name` is NUL-terminated and `times` is a two-element array, + // both outliving the call. + assert_eq!(unsafe { libc::utimes(name.as_ptr(), times.as_ptr()) }, 0); + + state.ignore_refresh_scheduled.store(true, Ordering::SeqCst); + let since = SystemTime::now() - std::time::Duration::from_secs(60); + reindex_files_in(&state, &root, std::slice::from_ref(&sub), since); + + assert!( + state.ignore_rules_dirty.load(Ordering::SeqCst), + "an ignore file the matcher never read must schedule a refresh, \ + however old its mtime is" + ); + assert!( + !state.index.read().unwrap().live.has_path("sub/kept.rs"), + "the scan must abandon rather than index under rules it knows are stale" + ); + } + + /// The walker collects rule files with `Path::is_file`, which follows + /// links, so a symlinked `.gitignore` contributes rules like any other — + /// but `DirEntry::file_type` does not follow links, and the scan used to + /// drop those entries before ever asking what they were. + #[cfg(unix)] + #[test] + fn a_symlinked_ignore_file_is_still_seen_by_a_recovery_scan() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + state.gitignore_pending.store(false, Ordering::SeqCst); + + let sub = root.join("sub"); + std::fs::create_dir(&sub).unwrap(); + let target = root.join("shared-rules"); + std::fs::write(&target, "kept.rs\n").unwrap(); + std::os::unix::fs::symlink(&target, sub.join(".gitignore")).unwrap(); + std::fs::write(sub.join("kept.rs"), "fn kept() {}\n").unwrap(); + + // Recent enough that the mtime window alone would catch a *regular* + // file here: what this pins is that a symlink gets that far at all. + state.ignore_refresh_scheduled.store(true, Ordering::SeqCst); + let since = SystemTime::now() - std::time::Duration::from_secs(60); + reindex_files_in(&state, &root, std::slice::from_ref(&sub), since); + + assert!( + state.ignore_rules_dirty.load(Ordering::SeqCst), + "a symlinked ignore file carries rules and must schedule a refresh" + ); + assert!( + !state.index.read().unwrap().live.has_path("sub/kept.rs"), + "the scan must abandon rather than index under rules it knows are stale" + ); + } + /// The size that qualifies a file is stat'd before its contents are read. /// A file that grows in between must not be read into memory without /// bound, nor indexed past the cap. From 8eebd99f5a9c3c58f9b9c36af7a66edf5932f634 Mon Sep 17 00:00:00 2001 From: Shengyu Fu Date: Thu, 27 Aug 2026 20:49:06 -0700 Subject: [PATCH 19/28] Ask every directory for ignore rules before indexing any file The macOS runner caught this: `read_dir` promises no ordering, and there `.gitignore` routinely comes back after its siblings -- so a per-entry check indexes part of a directory under the stale rules before it ever reaches the file that changes them. Across the scan it is worse: rules in a directory later in `dirs` arrive after earlier ones were indexed. `changed_ignore_rules_in` answers for the whole scan up front, probing by name the way the walker discovers these files. That agrees with the walker by construction, is ordering-independent, and follows symlinks through `Path::is_file` -- which is also what makes a symlinked `.gitignore` visible, so the per-entry check it replaces is gone. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: bcc85b55-e821-447e-b9a9-144b78829ae4 --- tgrep-cli/src/serve.rs | 183 +++++++++++++++++++++++++++-------------- 1 file changed, 119 insertions(+), 64 deletions(-) diff --git a/tgrep-cli/src/serve.rs b/tgrep-cli/src/serve.rs index 1ca4176..a693b89 100644 --- a/tgrep-cli/src/serve.rs +++ b/tgrep-cli/src/serve.rs @@ -2139,10 +2139,75 @@ fn sync_watch_registrations(state: &ServerState, root: &Path) -> (Vec, (added, since) } +/// The first ignore-rules file in `dirs` that the published matcher did not +/// read, or that has been edited since `since`. +/// +/// Probing by name rather than inspecting listings, for three reasons. It is +/// how the walker itself discovers these files, so the two agree by +/// construction. `Path::is_file` follows symlinks, so a symlinked `.gitignore` +/// — which carries rules exactly like a real one — is seen, where +/// `DirEntry::file_type` does not resolve it and the entry gets dropped as "not +/// a regular file". And it is ordering-independent: `read_dir` offers no +/// ordering, and on macOS `.gitignore` routinely comes back *after* its +/// siblings, so a per-entry check indexes part of a directory under the stale +/// rules before it ever reaches the file that changes them. Answering for the +/// whole scan up front closes that window across directories as well. +/// +/// Two tests, because neither alone is enough. Absence from the published +/// sources is the exact question — this file did not feed the matcher in force +/// — and it catches an arrival however old the file says it is, which matters +/// because `git checkout`, `tar -x` and `rsync -a` all restore mtimes from what +/// they unpack and would sail past a recency test. The mtime window then covers +/// what the source list cannot: a file that was already a source and has just +/// been edited. +/// +/// That window is bounded at both ends, not just the near one. On a network +/// mount whose server clock runs ahead of ours, every recently touched file +/// carries a future mtime and would pass a one-sided test — on every scan, +/// including the one at the end of the refresh this schedules, which walks the +/// whole repository and then arms the next. Treating a future mtime as skew +/// rather than as an edit keeps that loop closed. +fn changed_ignore_rules_in( + root: &Path, + dirs: &[PathBuf], + known_sources: &std::collections::HashSet, + since: SystemTime, +) -> Option<(String, &'static str)> { + let mut probed: std::collections::HashSet = std::collections::HashSet::new(); + for dir in dirs { + let mut candidates = vec![ + dir.join(tgrep_core::gitignore::GITIGNORE_FILENAME), + dir.join(tgrep_core::gitignore::DOT_IGNORE_FILENAME), + ]; + // Root-scoped, mirroring the walker, which only reads the root file. + if dir == root { + candidates.push(root.join(tgrep_core::gitignore::P4IGNORE_FILENAME)); + } + for candidate in candidates { + if !probed.insert(candidate.clone()) || !candidate.is_file() { + continue; + } + let Ok(rel) = candidate.strip_prefix(root) else { + continue; + }; + let rel = rel.to_string_lossy().replace('\\', "/"); + if !known_sources.contains(&rel) { + return Some((rel, "not a known source")); + } + if std::fs::metadata(&candidate) + .and_then(|m| m.modified()) + .is_ok_and(|m| m >= since && m <= SystemTime::now()) + { + return Some((rel, "modified")); + } + } + } + None +} + /// Re-check the files directly inside `dirs`, indexing the ones that changed, /// dropping the ones that are gone, and subscribing to subdirectories that /// appeared while the subscriptions were being established. -/// /// Used to close the gap between a walk and the subscriptions that follow it: /// [`reindex_file`] compares stamps first, so for a tree that did not change /// under us this costs one `metadata` call per file and indexes nothing. @@ -2204,6 +2269,24 @@ fn reindex_files_in(state: &Arc, root: &Path, dirs: &[PathBuf], sin known }; + // Before a single file is indexed: an ignore-rules file that landed in this + // window was not seen by the walk that built the matcher in force, so every + // file in this scan would be judged by rules that do not know about it, and + // whatever was wrongly indexed would stay until something touched it again. + if !state.no_ignore + && let Some((rel, why)) = changed_ignore_rules_in(root, dirs, &known_sources, since) + { + // Abandon the scan: the refresh rewalks and republishes, which covers + // these directories properly. + state.ignore_rules_dirty.store(true, Ordering::SeqCst); + schedule_ignore_rules_refresh(Arc::clone(state), root.to_path_buf()); + eprintln!( + "[trace] watcher: ignore rules changed during recovery ({rel}, {why}); \ + deferring to a refresh" + ); + return; + } + // Directories whose listing succeeded, and the files those listings // contained, for the removal sweep at the end. Only files are recorded: // stamps describe files, so directory names would just be dead weight on a @@ -2251,69 +2334,6 @@ fn reindex_files_in(state: &Arc, root: &Path, dirs: &[PathBuf], sin subdirs.push(path); continue; } - - // Ahead of the regular-file test, and following links to decide. - // The walker collects rule files with `Path::is_file`, which - // resolves symlinks, so a symlinked `.gitignore` contributes rules - // exactly like a real one — while `DirEntry::file_type` above does - // not resolve them, which left those files falling through the - // `!is_file` bail and never triggering a refresh. They are still - // never indexed; they are only allowed to announce themselves. - // - // An ignore-rules file that landed in this window was not seen by - // the walk that built the matcher in force, so every other file in - // this scan is being judged by rules that do not know about it. - // Indexing them now would apply the wrong rules and leave whatever - // was wrongly indexed until something touched it again. - // - // Two tests, because neither alone is enough. Absence from the - // published sources is the exact question — this file did not feed - // the matcher — and it catches the arrival however old the file - // says it is, which matters because `git checkout`, `tar -x` and - // `rsync -a` all restore mtimes from the archive and would sail - // past a recency test. The mtime window then covers the case the - // source list cannot: a file that was already a source and has just - // been *edited*. - // - // The mtime test is what keeps the second one quiet: a repository - // has an ignore file in every other directory and the startup scan - // walks past all of them, but they predate the walk and are already - // accounted for. Only one written inside the window can have been - // missed — and a spurious match (a `touch` in the same millisecond) - // costs an idempotent refresh, not correctness. - // - // Bounded at both ends, not just the near one. On a network mount - // whose server clock runs ahead of ours, every recently touched - // file carries a future mtime and would pass a one-sided test — on - // every scan, including the one at the end of the refresh this - // schedules, which walks the whole repository and then arms the - // next. Treating a future mtime as skew rather than as an arrival - // gives up the fix on such a mount and keeps the loop closed. - if !state.no_ignore && is_ignore_rules_file(root, &path) && path.is_file() { - let unknown = !known_sources.contains(&rel); - let touched = std::fs::metadata(&path) - .and_then(|m| m.modified()) - .is_ok_and(|m| m >= since && m <= SystemTime::now()); - if unknown || touched { - // Abandon the scan: the refresh rewalks and republishes, - // which covers these directories properly, and anything - // indexed between here and there would be judged by the - // stale rules. - state.ignore_rules_dirty.store(true, Ordering::SeqCst); - schedule_ignore_rules_refresh(Arc::clone(state), root.to_path_buf()); - let why = if unknown { - "not a known source" - } else { - "modified" - }; - eprintln!( - "[trace] watcher: ignore rules changed during recovery ({rel}, {why}); \ - deferring to a refresh" - ); - return; - } - } - if !file_type.is_file() { continue; } @@ -6489,6 +6509,41 @@ mod tests { ); } + /// `read_dir` promises no ordering, and the rules a scan must respect can + /// live in a directory it has not reached yet — so every directory in the + /// scan is asked before any file in it is indexed. + #[cfg(unix)] + #[test] + fn a_scan_checks_every_directory_for_rules_before_indexing_any_file() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + state.gitignore_pending.store(false, Ordering::SeqCst); + *state.ignore_sources.write().unwrap() = Vec::new(); + + // The rules are in `b`, the file is in `a`, and `a` is scanned first. + let a = root.join("a"); + let b = root.join("b"); + std::fs::create_dir(&a).unwrap(); + std::fs::create_dir(&b).unwrap(); + std::fs::write(a.join("keep.rs"), "fn keep() {}\n").unwrap(); + std::fs::write(b.join(".gitignore"), "keep.rs\n").unwrap(); + + state.ignore_refresh_scheduled.store(true, Ordering::SeqCst); + let since = SystemTime::now() - std::time::Duration::from_secs(60); + reindex_files_in(&state, &root, &[a, b], since); + + assert!( + state.ignore_rules_dirty.load(Ordering::SeqCst), + "the scan must notice rules that live later in its own list" + ); + assert!( + !state.index.read().unwrap().live.has_path("a/keep.rs"), + "nothing may be indexed before every directory has been asked for rules" + ); + } + /// The size that qualifies a file is stat'd before its contents are read. /// A file that grows in between must not be read into memory without /// bound, nor indexed past the cap. From d9bbd728f573cf367a3ff3b402089173de3829be Mon Sep 17 00:00:00 2001 From: Shengyu Fu Date: Thu, 27 Aug 2026 21:04:14 -0700 Subject: [PATCH 20/28] Recognise a rules file by what was read, not by its name Three gaps left by the previous round, all of the same shape: a pathname was being treated as evidence about contents. A `.gitignore` symlinked to `shared-rules` contributes the target's rules, because the walker resolves links. Editing the target produces an event naming `shared-rules`, whose basename means nothing to the live check, so no refresh was scheduled and the matcher stayed stale until the hourly reconcile. A source that is replaced in place -- `git checkout`, `tar -x`, a restore from an archive -- keeps its path and can carry an mtime that predates the scan window. Known name, untouched clock: neither test in the recovery scan fired, and the subtree was indexed under rules that were never read. `publish_ignore_matcher` still documented a `since` parameter it no longer takes. Records what the published matcher actually read -- size and mtime per source, plus an entry for the target of any symlinked source that is itself under the root -- and asks against that. The mtime window stays: it covers writes landing between the walk and the stat. Targets outside the root cannot be watched at all, so for those the reconcile remains the backstop, now stated rather than implied. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: bcc85b55-e821-447e-b9a9-144b78829ae4 --- tgrep-cli/src/serve.rs | 279 +++++++++++++++++++++++++++++++++-------- 1 file changed, 228 insertions(+), 51 deletions(-) diff --git a/tgrep-cli/src/serve.rs b/tgrep-cli/src/serve.rs index a693b89..b6196b6 100644 --- a/tgrep-cli/src/serve.rs +++ b/tgrep-cli/src/serve.rs @@ -299,6 +299,24 @@ struct ServerState { /// until something else forces a rebuild. Keeping the source list lets the /// scan test for it directly, at one stat per ignore file per scan. ignore_sources: RwLock>, + /// What the published matcher actually read, per ignore source: the size + /// and mtime each file had, keyed by its path relative to `root`. + /// + /// A pathname is not evidence about contents. An existing `.gitignore` can + /// be replaced after the matcher walk by one restored from an archive, + /// carrying an mtime older than the scan window — the path is still a known + /// source and the mtime still predates the window, so neither test in + /// [`changed_ignore_rules_in`] fires and the subtree is indexed under rules + /// that were never read. Comparing against what was read closes that. + /// + /// Also holds an entry for the target of any source reached through a + /// symlink, when that target is itself under `root`. Following links is + /// what the walker does, so the matcher's contents come from the target — + /// but an edit to the target does not touch the link, and no event names a + /// path whose basename is `.gitignore`. The target's own entry is what + /// lets that event be recognised. A target outside `root` is not watched at + /// all, so for those the reconcile stays the backstop. + ignore_source_stamps: RwLock, /// Serializes the whole check-read-commit cycle in [`reindex_file`]. /// /// `snapshot_gate` is held for *read* by everything that indexes a file, so @@ -621,6 +639,7 @@ pub fn run(root: &Path, index_path: Option<&Path>, options: ServeOptions<'_>) -> ignore_rules_dirty: std::sync::atomic::AtomicBool::new(false), ignore_refresh_scheduled: std::sync::atomic::AtomicBool::new(false), ignore_sources: RwLock::new(Vec::new()), + ignore_source_stamps: RwLock::new(IgnoreStamps::new()), reindex_lock: Mutex::new(()), deferred_events: Mutex::new(Some(std::collections::HashMap::new())), index_progress: std::sync::atomic::AtomicU64::new(0), @@ -800,6 +819,10 @@ fn build_stale_matcher( matcher } +/// The size and mtime an ignore source had when the matcher read it, keyed by +/// its path relative to the served root. +type IgnoreStamps = std::collections::HashMap)>; + /// The ignore files a matcher was built from, as one list. /// /// Root-level `p4ignore.ini` is a separate source from the walker's point of @@ -821,6 +844,42 @@ fn ignore_sources_of( sources } +/// Stat every source so a later scan can ask whether the file it finds is the +/// one the matcher read, rather than merely whether something of that name is +/// there. +/// +/// A source reached through a symlink gets a second entry under its target's +/// own relative path, when the target is under `root`. The stat follows links +/// either way, so both entries describe the contents that were read. +fn ignore_stamps_of(root: &Path, sources: &[PathBuf]) -> IgnoreStamps { + let canonical_root = std::fs::canonicalize(root).ok(); + let mut stamps = IgnoreStamps::with_capacity(sources.len()); + let mut record = |path: &Path, base: &Path| { + let Ok(rel) = path.strip_prefix(base) else { + return; + }; + let rel = rel.to_string_lossy().replace('\\', "/"); + // Follows links: what matters is the content behind the name. + if let Ok(meta) = std::fs::metadata(path) { + stamps.insert(rel, (meta.len(), meta.modified().ok())); + } + }; + for source in sources { + record(source, root); + let is_link = std::fs::symlink_metadata(source).is_ok_and(|m| m.file_type().is_symlink()); + if !is_link { + continue; + } + if let Some(canonical_root) = canonical_root.as_ref() + && let Ok(target) = std::fs::canonicalize(source) + && target.starts_with(canonical_root) + { + record(&target, canonical_root); + } + } + stamps +} + /// Publish a new ignore matcher and bring everything that depends on it up to /// date. `None` is a legitimate matcher when no rules exist. /// @@ -834,15 +893,17 @@ fn ignore_sources_of( /// them to [`reindex_files_in`] once `state.file_stamps` describes the index /// they just published. /// -/// `since` is when the walk that produced `matcher` began, and is only passed -/// through so the recovery scan can use it. It has to come from the caller: the -/// subscription walk inside this function starts later, and an ignore file -/// written between the two would predate a timestamp taken here and be read as -/// already accounted for by rules that never saw it. +/// The returned directories must be paired with a timestamp the *caller* +/// captured before the walk that produced `matcher`, and handed to +/// [`reindex_files_in`] as its `since`. This function cannot supply it: the +/// subscription walk below starts later, so an ignore file written between the +/// caller's walk and this point would predate any timestamp taken here and be +/// read as already accounted for by rules that never saw it. /// -/// `sources` are the ignore files `matcher` was built from, recorded so a -/// recovery scan can notice one of them being deleted — an event the mtime -/// heuristic cannot see, since a deleted file leaves nothing to stat. +/// `sources` are the ignore files `matcher` was built from. They are recorded +/// so a recovery scan can notice one being deleted — which no mtime test can +/// see, a deleted file leaving nothing to stat — and stat'd so it can also +/// notice one being replaced, which a pathname cannot show. #[must_use = "newly watched directories need a recovery scan or writes race the subscription"] fn publish_ignore_matcher( state: &ServerState, @@ -850,6 +911,7 @@ fn publish_ignore_matcher( matcher: Option, sources: Vec, ) -> Vec { + *state.ignore_source_stamps.write().unwrap() = ignore_stamps_of(root, &sources); *state.ignore_sources.write().unwrap() = sources; *state.gitignore.write().unwrap() = matcher; state.gitignore_pending.store(false, Ordering::SeqCst); @@ -2140,7 +2202,8 @@ fn sync_watch_registrations(state: &ServerState, root: &Path) -> (Vec, } /// The first ignore-rules file in `dirs` that the published matcher did not -/// read, or that has been edited since `since`. +/// read, that has been replaced since it did, or that has been edited since +/// `since`. /// /// Probing by name rather than inspecting listings, for three reasons. It is /// how the walker itself discovers these files, so the two agree by @@ -2153,13 +2216,22 @@ fn sync_watch_registrations(state: &ServerState, root: &Path) -> (Vec, /// rules before it ever reaches the file that changes them. Answering for the /// whole scan up front closes that window across directories as well. /// -/// Two tests, because neither alone is enough. Absence from the published -/// sources is the exact question — this file did not feed the matcher in force -/// — and it catches an arrival however old the file says it is, which matters -/// because `git checkout`, `tar -x` and `rsync -a` all restore mtimes from what -/// they unpack and would sail past a recency test. The mtime window then covers -/// what the source list cannot: a file that was already a source and has just -/// been edited. +/// Three tests, because none alone is enough. +/// +/// Absence from the published sources is the exact question for an arrival — +/// this file did not feed the matcher in force — and it catches one however old +/// the file says it is, which matters because `git checkout`, `tar -x` and +/// `rsync -a` all restore mtimes from what they unpack and would sail past a +/// recency test. +/// +/// A stamp mismatch answers the same question for a file that was *already* a +/// source: a pathname proves nothing about contents, and the same archive +/// restore can swap a source for a different file bearing an older mtime, which +/// is invisible to both of the other tests. +/// +/// The mtime window then covers the gap the stamps cannot: they are taken when +/// the matcher is published, which is after the walk that read these files, so +/// a write landing between the two is recorded as if it had been read. /// /// That window is bounded at both ends, not just the near one. On a network /// mount whose server clock runs ahead of ours, every recently touched file @@ -2170,7 +2242,7 @@ fn sync_watch_registrations(state: &ServerState, root: &Path) -> (Vec, fn changed_ignore_rules_in( root: &Path, dirs: &[PathBuf], - known_sources: &std::collections::HashSet, + known_sources: &IgnoreStamps, since: SystemTime, ) -> Option<(String, &'static str)> { let mut probed: std::collections::HashSet = std::collections::HashSet::new(); @@ -2191,11 +2263,19 @@ fn changed_ignore_rules_in( continue; }; let rel = rel.to_string_lossy().replace('\\', "/"); - if !known_sources.contains(&rel) { + let Some(read_as) = known_sources.get(&rel) else { return Some((rel, "not a known source")); + }; + // Follows links, matching how the stamp was taken. + let Ok(meta) = std::fs::metadata(&candidate) else { + continue; + }; + let current = (meta.len(), meta.modified().ok()); + if current != *read_as { + return Some((rel, "not the file the matcher read")); } - if std::fs::metadata(&candidate) - .and_then(|m| m.modified()) + if meta + .modified() .is_ok_and(|m| m >= since && m <= SystemTime::now()) { return Some((rel, "modified")); @@ -2230,32 +2310,21 @@ fn reindex_files_in(state: &Arc, root: &Path, dirs: &[PathBuf], sin } let start = Instant::now(); - // An ignore file that was deleted during the window is invisible to the - // per-entry test below — there is no entry left to stat. It is also the - // more damaging direction: rules that no longer have a source keep being - // enforced, so the subtree they hide stays unsubscribed and unindexed until - // an unrelated rebuild happens along. Checking the sources the published - // matcher was built from catches it at one stat apiece, once per scan - // rather than once per file. - // - // The same list answers the opposite question for free: an ignore file that - // is *not* among the sources is one the published matcher never read. - let known_sources: std::collections::HashSet = if state.no_ignore { - std::collections::HashSet::new() + // An ignore file that was deleted during the window leaves nothing to stat, + // so no test over what is on disk can see it. It is also the more damaging + // direction: rules that no longer have a source keep being enforced, so the + // subtree they hide stays unsubscribed and unindexed until an unrelated + // rebuild happens along. Checking the sources the published matcher was + // built from catches it at one stat apiece, once per scan. + let known_sources: IgnoreStamps = if state.no_ignore { + IgnoreStamps::new() } else { - let (vanished, known) = { + let vanished = { let sources = state.ignore_sources.read().unwrap(); - ( - // `exists` follows links, matching how the walker collected - // these — a symlinked source whose target is gone has stopped - // contributing rules just as surely as a deleted one. - sources.iter().find(|p| !p.exists()).cloned(), - sources - .iter() - .filter_map(|p| p.strip_prefix(root).ok()) - .map(|rel| rel.to_string_lossy().replace('\\', "/")) - .collect::>(), - ) + // `exists` follows links, matching how the walker collected these — + // a symlinked source whose target is gone has stopped contributing + // rules just as surely as a deleted one. + sources.iter().find(|p| !p.exists()).cloned() }; if let Some(gone) = vanished { state.ignore_rules_dirty.store(true, Ordering::SeqCst); @@ -2266,7 +2335,7 @@ fn reindex_files_in(state: &Arc, root: &Path, dirs: &[PathBuf], sin ); return; } - known + state.ignore_source_stamps.read().unwrap().clone() }; // Before a single file is indexed: an ignore-rules file that landed in this @@ -2867,11 +2936,29 @@ fn handle_fs_event(state: &Arc, root: &Path, event: &Event) { return; } - let ignore_rules_changed = !state.no_ignore - && event - .paths - .iter() - .any(|path| is_ignore_rules_file(root, path)); + // Two ways an event can carry a rules change. The obvious one is a path the + // walker would read as a rules file, recognised by name. + // + // The other is a path the published matcher actually read through a + // symlink. `ignore_files_in` uses `Path::is_file`, which follows links, so + // a `.gitignore` symlinked to `shared-rules` contributes the *target's* + // contents — but editing the target produces an event naming `shared-rules`, + // whose basename means nothing to `is_ignore_rules_file`, and touches + // nothing whose name does. Recognising the paths that were read, and not + // just the names rules usually go by, is what closes that. + // + // Only targets inside `root` can appear here, because only those are + // watched; for one outside, no event arrives at all and the periodic + // reconcile remains the backstop. + let ignore_rules_changed = !state.no_ignore && { + let stamps = state.ignore_source_stamps.read().unwrap(); + event.paths.iter().any(|path| { + is_ignore_rules_file(root, path) + || path + .strip_prefix(root) + .is_ok_and(|rel| stamps.contains_key(&rel.to_string_lossy().replace('\\', "/"))) + }) + }; if ignore_rules_changed { state.ignore_rules_dirty.store(true, Ordering::SeqCst); if state.indexing.load(Ordering::SeqCst) { @@ -5156,7 +5243,6 @@ mod tests { /// A `ServerState` over an empty index, for exercising the stale path /// directly. Mirrors the defaults `run` uses with a watcher and ignore /// rules enabled, which is the configuration `gitignore_pending` gates. - #[cfg(unix)] fn test_server_state(root: &Path, index_dir: &Path) -> Arc { create_empty_index(index_dir).expect("create empty index"); let hybrid = HybridIndex::open(index_dir, root).expect("open empty index"); @@ -5175,6 +5261,7 @@ mod tests { ignore_rules_dirty: std::sync::atomic::AtomicBool::new(false), ignore_refresh_scheduled: std::sync::atomic::AtomicBool::new(false), ignore_sources: RwLock::new(Vec::new()), + ignore_source_stamps: RwLock::new(IgnoreStamps::new()), reindex_lock: Mutex::new(()), deferred_events: Mutex::new(Some(std::collections::HashMap::new())), index_progress: std::sync::atomic::AtomicU64::new(0), @@ -6544,6 +6631,96 @@ mod tests { ); } + /// A path is not evidence about contents. An archive restore can put a + /// different `.gitignore` at a path the matcher already read, carrying an + /// mtime older than the scan window — known name, untouched by the clock, + /// and yet not the file whose rules are being enforced. + #[test] + fn a_rule_file_swapped_for_an_older_one_is_not_taken_on_faith() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + state.gitignore_pending.store(false, Ordering::SeqCst); + + let rules = root.join(".gitignore"); + std::fs::write(&rules, "nothing-at-all.rs\n").unwrap(); + std::fs::write(root.join("keep.rs"), "fn keep() {}\n").unwrap(); + *state.ignore_source_stamps.write().unwrap() = + ignore_stamps_of(&root, std::slice::from_ref(&rules)); + *state.ignore_sources.write().unwrap() = vec![rules.clone()]; + + // Far enough ahead that the mtime window cannot fire for anything on + // disk: what is left is the question of whether the file is the one + // that was read. + let since = SystemTime::now() + std::time::Duration::from_secs(3600); + state.ignore_refresh_scheduled.store(true, Ordering::SeqCst); + reindex_files_in(&state, &root, std::slice::from_ref(&root), since); + assert!( + !state.ignore_rules_dirty.load(Ordering::SeqCst), + "the file the matcher read must not be reported as changed" + ); + assert!( + state.index.read().unwrap().live.has_path("keep.rs"), + "and the scan must get on with its work" + ); + + // Same path, same age as far as the window is concerned, different + // rules. + std::fs::write(&rules, "keep.rs\n").unwrap(); + reindex_files_in(&state, &root, std::slice::from_ref(&root), since); + assert!( + state.ignore_rules_dirty.load(Ordering::SeqCst), + "a source that is no longer the file that was read must schedule a refresh" + ); + } + + /// A `.gitignore` symlinked to `shared-rules` contributes the target's + /// contents, because the walker follows links. Editing the target is + /// therefore a rules change — but it touches nothing named like one. + #[cfg(unix)] + #[test] + fn an_edit_to_a_symlinked_rule_files_target_schedules_a_refresh() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + state.gitignore_pending.store(false, Ordering::SeqCst); + + let target = root.join("shared-rules"); + std::fs::write(&target, "keep.rs\n").unwrap(); + let link = root.join(".gitignore"); + std::os::unix::fs::symlink(&target, &link).unwrap(); + + let stamps = ignore_stamps_of(&root, std::slice::from_ref(&link)); + assert!( + stamps.contains_key("shared-rules"), + "the file the rules were actually read from has to be recorded too" + ); + *state.ignore_source_stamps.write().unwrap() = stamps; + + // Stop at the flag: what is being pinned is that the event is + // recognised, not what the refresh then does. + state.indexing.store(true, Ordering::SeqCst); + handle_fs_event( + &state, + &root, + &Event { + kind: EventKind::Modify(notify::event::ModifyKind::Data( + notify::event::DataChange::Any, + )), + paths: vec![target], + attrs: Default::default(), + }, + ); + + assert!( + state.ignore_rules_dirty.load(Ordering::SeqCst), + "an edit to the file a rules symlink resolves to is a rules change, \ + whatever the path is called" + ); + } + /// The size that qualifies a file is stat'd before its contents are read. /// A file that grows in between must not be read into memory without /// bound, nor indexed past the cap. From 6b345311c70135b4dee785b37e775cfd8c041f39 Mon Sep 17 00:00:00 2001 From: Shengyu Fu Date: Thu, 27 Aug 2026 21:14:59 -0700 Subject: [PATCH 21/28] Sweep what can answer a search, and allow for a coarse clock Two more of the same kind: a proxy standing in for the thing itself. `sweep_removed_files` took its candidates from `file_stamps`, but a stamp is not what makes a file searchable -- the index is. `filestamps.json` is optional by design, and missing or unreadable is tolerated everywhere else, so a seeded index with no stamps had no candidates at all and files deleted during the unwatched window kept answering searches until the hourly reconcile. Candidates now come from the reader and the overlay as well, skipping reader paths already hidden by a tombstone so a repeated scan does not count the same deletion as fresh churn. `reader_paths` would have allocated a copy of every path in the index to answer this, so `reader_paths_matching` filters in place. The recovery scan's mtime window compared a rounded timestamp against a wall-clock instant. HFS+ and ext3 store whole seconds and FAT two, so a write that followed the walk can be dated before it, and for a source edited between the walk and the publication the stamp matches -- leaving the window as the only test, failing by up to two seconds. Widened by that granularity at the near end. The far end stays where it was, since a future mtime is clock skew and treating it as an edit is what loops. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: bcc85b55-e821-447e-b9a9-144b78829ae4 --- tgrep-cli/src/serve.rs | 147 ++++++++++++++++++++++++++++++++++----- tgrep-core/src/hybrid.rs | 15 ++++ 2 files changed, 144 insertions(+), 18 deletions(-) diff --git a/tgrep-cli/src/serve.rs b/tgrep-cli/src/serve.rs index b6196b6..ef21d1e 100644 --- a/tgrep-cli/src/serve.rs +++ b/tgrep-cli/src/serve.rs @@ -823,6 +823,15 @@ fn build_stale_matcher( /// its path relative to the served root. type IgnoreStamps = std::collections::HashMap)>; +/// How far an mtime may lag the write it records. +/// +/// Two seconds, which covers the coarsest granularity still in use: FAT and +/// its descendants store modification times in two-second units, HFS+ and +/// ext3 in whole seconds. Used to widen comparisons against a wall-clock +/// instant, which has no such rounding, so a write cannot be dated before a +/// moment it actually followed. +const MTIME_GRANULARITY: Duration = Duration::from_secs(2); + /// The ignore files a matcher was built from, as one list. /// /// Root-level `p4ignore.ini` is a separate source from the walker's point of @@ -2233,18 +2242,27 @@ fn sync_watch_registrations(state: &ServerState, root: &Path) -> (Vec, /// the matcher is published, which is after the walk that read these files, so /// a write landing between the two is recorded as if it had been read. /// -/// That window is bounded at both ends, not just the near one. On a network -/// mount whose server clock runs ahead of ours, every recently touched file -/// carries a future mtime and would pass a one-sided test — on every scan, -/// including the one at the end of the refresh this schedules, which walks the -/// whole repository and then arms the next. Treating a future mtime as skew -/// rather than as an edit keeps that loop closed. +/// That window is widened by [`MTIME_GRANULARITY`] at the near end, because +/// `since` is a wall-clock instant with nanosecond precision and an mtime is +/// not. HFS+ and ext3 store whole seconds, FAT-derived filesystems two, so a +/// write that happens after `since` can be stamped before it and read as +/// historical. Over-triggering costs one rewalk that finds nothing; the slack +/// is bounded, so a file whose mtime keeps qualifying stops doing so as later +/// scans take later timestamps. +/// +/// The far end is bounded too. On a network mount whose server clock runs +/// ahead of ours, every recently touched file carries a future mtime and would +/// pass a one-sided test — on every scan, including the one at the end of the +/// refresh this schedules, which walks the whole repository and then arms the +/// next. Treating a future mtime as skew rather than as an edit keeps that +/// loop closed. fn changed_ignore_rules_in( root: &Path, dirs: &[PathBuf], known_sources: &IgnoreStamps, since: SystemTime, ) -> Option<(String, &'static str)> { + let since = since.checked_sub(MTIME_GRANULARITY).unwrap_or(since); let mut probed: std::collections::HashSet = std::collections::HashSet::new(); for dir in dirs { let mut candidates = vec![ @@ -2472,6 +2490,19 @@ fn reindex_files_in(state: &Arc, root: &Path, dirs: &[PathBuf], sin /// eligibility. Filtering `present` would delete entries for files that are /// still on disk and were indexed under a laxer configuration. /// +/// Candidates come from everything that can answer a search, not from +/// `file_stamps` alone. A stamp is not a precondition for being searchable: +/// `filestamps.json` is optional by design — missing or unreadable leaves the +/// map empty, and a build that predates a given file's stamp leaves it partial +/// — while the reader still holds that file's content. Sweeping only what has +/// a stamp would then delete nothing at all, and the deleted files would keep +/// answering searches until the hourly reconcile. +/// +/// Reader paths already hidden by a tombstone are skipped. `delete_file` +/// tombstones unconditionally and counts a mutation for it, so re-deleting +/// them would make every scan over a directory with deletions in it look like +/// fresh churn and pull flushes forward for no reason. +/// /// The caller must already hold `snapshot_gate`. fn sweep_removed_files( state: &ServerState, @@ -2481,20 +2512,28 @@ fn sweep_removed_files( if swept.is_empty() { return; } - // One pass over the stamps rather than a lookup per swept directory: at - // startup both sides of this span the whole repository, and anything - // proportional to their product would not finish. - let gone: Vec = { + // One pass per source rather than a lookup per swept directory: at startup + // both sides of this span the whole repository, and anything proportional + // to their product would not finish. + let missing = |rel: &str| { + let parent = rel.rsplit_once('/').map_or("", |(dir, _)| dir); + swept.contains(parent) && !present.contains(rel) + }; + let mut gone: std::collections::HashSet = { let stamps = state.file_stamps.read().unwrap(); - stamps - .keys() - .filter(|rel| { - let parent = rel.rsplit_once('/').map_or("", |(dir, _)| dir); - swept.contains(parent) && !present.contains(rel.as_str()) - }) - .cloned() - .collect() + stamps.keys().filter(|rel| missing(rel)).cloned().collect() }; + { + let index = state.index.read().unwrap(); + gone.extend(index.reader_paths_matching(|rel| missing(rel) && !index.live.is_deleted(rel))); + gone.extend( + index + .live + .overlay_paths() + .into_iter() + .filter(|rel| missing(rel)), + ); + } if gone.is_empty() { return; } @@ -6721,6 +6760,78 @@ mod tests { ); } + /// An mtime is not a wall-clock instant. Whole-second (HFS+, ext3) or + /// two-second (FAT) granularity can date a write before the moment it + /// actually followed, and a source edited between the walk and the + /// publication carries a stamp that matches it — so the window is the only + /// test left, and it has to allow for the rounding. + #[test] + fn a_rule_file_stamped_a_second_early_is_still_inside_the_window() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + state.gitignore_pending.store(false, Ordering::SeqCst); + + let rules = root.join(".gitignore"); + std::fs::write(&rules, "nothing-at-all.rs\n").unwrap(); + *state.ignore_source_stamps.write().unwrap() = + ignore_stamps_of(&root, std::slice::from_ref(&rules)); + *state.ignore_sources.write().unwrap() = vec![rules.clone()]; + let stamped = std::fs::metadata(&rules).unwrap().modified().unwrap(); + state.ignore_refresh_scheduled.store(true, Ordering::SeqCst); + + // Far outside any rounding: this one really is history. + let old = stamped + std::time::Duration::from_secs(3600); + reindex_files_in(&state, &root, std::slice::from_ref(&root), old); + assert!( + !state.ignore_rules_dirty.load(Ordering::SeqCst), + "a source last written an hour before the walk is not a change" + ); + + // Within the granularity of a coarse filesystem's clock: the file may + // well have been written after the walk began and been rounded down. + let rounded = stamped + std::time::Duration::from_secs(1); + reindex_files_in(&state, &root, std::slice::from_ref(&root), rounded); + assert!( + state.ignore_rules_dirty.load(Ordering::SeqCst), + "an mtime that sits just under the window has to be treated as recent" + ); + } + + /// A stamp is not what makes a file searchable — the index is. + /// `filestamps.json` is optional, and a partial or absent map used to mean + /// the sweep had no candidates and deleted files kept answering searches. + #[test] + fn the_sweep_drops_a_deleted_file_that_never_had_a_stamp() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + state.gitignore_pending.store(false, Ordering::SeqCst); + + let path = root.join("seeded.rs"); + std::fs::write(&path, "fn seeded() {}\n").unwrap(); + let gate = state.snapshot_gate.read().unwrap(); + reindex_file(&state, &path, "seeded.rs"); + assert!(state.index.read().unwrap().live.has_path("seeded.rs")); + + // Indexed, and searchable, but with nothing in the stamp map to say so + // — as after a seed whose stamps could not be read. + state.file_stamps.write().unwrap().remove("seeded.rs"); + std::fs::remove_file(&path).unwrap(); + + let swept: std::collections::HashSet = [String::new()].into_iter().collect(); + sweep_removed_files(&state, &swept, &std::collections::HashSet::new()); + drop(gate); + + assert!( + state.index.read().unwrap().live.is_deleted("seeded.rs"), + "a file that is gone from disk must stop answering searches whether or \ + not it had a stamp" + ); + } + /// The size that qualifies a file is stat'd before its contents are read. /// A file that grows in between must not be read into memory without /// bound, nor indexed past the cap. diff --git a/tgrep-core/src/hybrid.rs b/tgrep-core/src/hybrid.rs index 24197ff..bf96237 100644 --- a/tgrep-core/src/hybrid.rs +++ b/tgrep-core/src/hybrid.rs @@ -293,6 +293,21 @@ impl HybridIndex { self.reader().all_paths().iter().cloned().collect() } + /// The reader paths `keep` accepts. + /// + /// For callers that want a few of them — everything under one directory, + /// say. [`Self::reader_paths`] allocates a copy of every path in the index + /// to answer that, which at repository scale is the bulk of the cost and + /// all of it wasted. + pub fn reader_paths_matching(&self, mut keep: impl FnMut(&str) -> bool) -> Vec { + self.reader() + .all_paths() + .iter() + .filter(|path| keep(path)) + .cloned() + .collect() + } + /// Number of files in the on-disk reader. pub fn reader_file_count(&self) -> usize { self.reader().num_files() From 80a8d6e4c4928775a434fad12e4461bd254a748a Mon Sep 17 00:00:00 2001 From: Shengyu Fu Date: Thu, 27 Aug 2026 21:23:54 -0700 Subject: [PATCH 22/28] Compare ignore sources by their bytes, not their metadata Size and mtime do not establish that a rule file is the one the matcher read. `rsync -a`, `tar -x` and a restore from an archive all preserve mtime, and two different sets of rules are easily the same length -- so the pair is identical across the replacement, and the recovery scan accepts a matcher built from rules that are gone. That is precisely the case this path exists for: a change made while no event was observable. `ignore_source_stamps` now records a hash of each source's bytes, and `changed_ignore_rules_in` compares against it. Never persisted, so the hash only has to be stable within a run. These are rule files -- a few hundred bytes each, already in the page cache from the walk that found them -- so this is not the same proposition as hashing indexed content, where the cost is the whole repository and a wrong stamp costs one stale file rather than a whole subtree indexed under the wrong rules. The digest is taken at publication rather than inside the matcher builder because the `ignore` crate opens these files itself and does not hand back what it read. The mtime window stays, and is what covers that residual gap. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: bcc85b55-e821-447e-b9a9-144b78829ae4 --- tgrep-cli/src/serve.rs | 144 ++++++++++++++++++++++++++++++++--------- 1 file changed, 114 insertions(+), 30 deletions(-) diff --git a/tgrep-cli/src/serve.rs b/tgrep-cli/src/serve.rs index ef21d1e..551bd3f 100644 --- a/tgrep-cli/src/serve.rs +++ b/tgrep-cli/src/serve.rs @@ -299,15 +299,16 @@ struct ServerState { /// until something else forces a rebuild. Keeping the source list lets the /// scan test for it directly, at one stat per ignore file per scan. ignore_sources: RwLock>, - /// What the published matcher actually read, per ignore source: the size - /// and mtime each file had, keyed by its path relative to `root`. + /// What the published matcher actually read, per ignore source: a hash of + /// the bytes, keyed by the file's path relative to `root`. /// - /// A pathname is not evidence about contents. An existing `.gitignore` can - /// be replaced after the matcher walk by one restored from an archive, - /// carrying an mtime older than the scan window — the path is still a known - /// source and the mtime still predates the window, so neither test in - /// [`changed_ignore_rules_in`] fires and the subtree is indexed under rules - /// that were never read. Comparing against what was read closes that. + /// A pathname is not evidence about contents, and neither is metadata. An + /// existing `.gitignore` can be replaced after the matcher was built by one + /// restored from an archive — same length, mtime preserved by the restore, + /// so the path is still a known source and nothing about its metadata has + /// moved. Neither test in [`changed_ignore_rules_in`] would fire, and the + /// subtree would be indexed under rules that were never read. Comparing + /// against what was read closes that. /// /// Also holds an entry for the target of any source reached through a /// symlink, when that target is itself under `root`. Following links is @@ -819,9 +820,34 @@ fn build_stale_matcher( matcher } -/// The size and mtime an ignore source had when the matcher read it, keyed by -/// its path relative to the served root. -type IgnoreStamps = std::collections::HashMap)>; +/// What an ignore source contained when the matcher read it: its length and a +/// hash of its bytes, keyed by its path relative to the served root. +/// +/// A digest rather than metadata, because metadata does not answer the +/// question. `rsync -a`, `tar -x` and a restore from an archive all preserve +/// mtime, and two different sets of rules can easily be the same number of +/// bytes — at which point a size-and-mtime pair is identical across a +/// replacement and the scan accepts a matcher built from rules that are gone. +/// +/// Never persisted, so the hash only has to be stable within a run. +type IgnoreStamps = std::collections::HashMap; + +/// The length and content hash of `path`, following links. +/// +/// `None` when it cannot be read, which is treated as "not what was read": +/// a source that has become unreadable has stopped contributing the rules the +/// matcher is enforcing, and that is a change. +fn ignore_digest_of(path: &Path) -> Option<(u64, u64)> { + use std::hash::{Hash, Hasher}; + + // The whole file, as the matcher builder reads it. These are rule files: + // a few hundred bytes each in practice, and already in the page cache from + // the walk that found them. + let bytes = std::fs::read(path).ok()?; + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + bytes.hash(&mut hasher); + Some((bytes.len() as u64, hasher.finish())) +} /// How far an mtime may lag the write it records. /// @@ -853,13 +879,18 @@ fn ignore_sources_of( sources } -/// Stat every source so a later scan can ask whether the file it finds is the +/// Read every source so a later scan can ask whether the file it finds is the /// one the matcher read, rather than merely whether something of that name is /// there. /// /// A source reached through a symlink gets a second entry under its target's -/// own relative path, when the target is under `root`. The stat follows links +/// own relative path, when the target is under `root`. The read follows links /// either way, so both entries describe the contents that were read. +/// +/// Taken here rather than inside the matcher builder because the `ignore` +/// crate opens these files itself and does not hand back what it read. That +/// leaves a window between its read and this one, which is what the mtime test +/// in [`changed_ignore_rules_in`] is for. fn ignore_stamps_of(root: &Path, sources: &[PathBuf]) -> IgnoreStamps { let canonical_root = std::fs::canonicalize(root).ok(); let mut stamps = IgnoreStamps::with_capacity(sources.len()); @@ -868,9 +899,8 @@ fn ignore_stamps_of(root: &Path, sources: &[PathBuf]) -> IgnoreStamps { return; }; let rel = rel.to_string_lossy().replace('\\', "/"); - // Follows links: what matters is the content behind the name. - if let Ok(meta) = std::fs::metadata(path) { - stamps.insert(rel, (meta.len(), meta.modified().ok())); + if let Some(digest) = ignore_digest_of(path) { + stamps.insert(rel, digest); } }; for source in sources { @@ -910,9 +940,10 @@ fn ignore_stamps_of(root: &Path, sources: &[PathBuf]) -> IgnoreStamps { /// read as already accounted for by rules that never saw it. /// /// `sources` are the ignore files `matcher` was built from. They are recorded -/// so a recovery scan can notice one being deleted — which no mtime test can -/// see, a deleted file leaving nothing to stat — and stat'd so it can also -/// notice one being replaced, which a pathname cannot show. +/// so a recovery scan can notice one being deleted — which no test against +/// what is on disk can see, a deleted file leaving nothing to read — and read +/// so it can also notice one being replaced, which neither a pathname nor a +/// timestamp can show. #[must_use = "newly watched directories need a recovery scan or writes race the subscription"] fn publish_ignore_matcher( state: &ServerState, @@ -2234,13 +2265,14 @@ fn sync_watch_registrations(state: &ServerState, root: &Path) -> (Vec, /// recency test. /// /// A stamp mismatch answers the same question for a file that was *already* a -/// source: a pathname proves nothing about contents, and the same archive -/// restore can swap a source for a different file bearing an older mtime, which -/// is invisible to both of the other tests. +/// source: a pathname proves nothing about contents, and neither does +/// metadata. `rsync -a` and `tar -x` preserve mtime, and two different sets of +/// rules are easily the same length, so the comparison is against a hash of +/// what was actually read. /// -/// The mtime window then covers the gap the stamps cannot: they are taken when -/// the matcher is published, which is after the walk that read these files, so -/// a write landing between the two is recorded as if it had been read. +/// The mtime window then covers the gap the digests cannot: they are taken +/// when the matcher is published, which is after the builder read these files, +/// so a write landing between the two is recorded as if it had been read. /// /// That window is widened by [`MTIME_GRANULARITY`] at the near end, because /// `since` is a wall-clock instant with nanosecond precision and an mtime is @@ -2284,14 +2316,13 @@ fn changed_ignore_rules_in( let Some(read_as) = known_sources.get(&rel) else { return Some((rel, "not a known source")); }; - // Follows links, matching how the stamp was taken. + if ignore_digest_of(&candidate).as_ref() != Some(read_as) { + return Some((rel, "not the file the matcher read")); + } + // Follows links, matching how the digest was taken. let Ok(meta) = std::fs::metadata(&candidate) else { continue; }; - let current = (meta.len(), meta.modified().ok()); - if current != *read_as { - return Some((rel, "not the file the matcher read")); - } if meta .modified() .is_ok_and(|m| m >= since && m <= SystemTime::now()) @@ -6832,6 +6863,59 @@ mod tests { ); } + /// Metadata is not content either. `rsync -a` and `tar -x` preserve mtime, + /// and two different sets of rules are easily the same length — so a + /// size-and-mtime pair is identical across the replacement and only the + /// bytes tell them apart. + #[cfg(unix)] + #[test] + fn a_rule_file_swapped_for_one_of_the_same_size_and_age_is_still_caught() { + use std::os::unix::ffi::OsStrExt; + + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + state.gitignore_pending.store(false, Ordering::SeqCst); + + let rules = root.join(".gitignore"); + std::fs::write(&rules, "aaa.rs\n").unwrap(); + let before = std::fs::metadata(&rules).unwrap(); + *state.ignore_source_stamps.write().unwrap() = + ignore_stamps_of(&root, std::slice::from_ref(&rules)); + *state.ignore_sources.write().unwrap() = vec![rules.clone()]; + + // Different rules, same seven bytes, and the restore puts the old + // timestamp back. + std::fs::write(&rules, "bbb.rs\n").unwrap(); + let secs = before + .modified() + .unwrap() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap(); + let name = std::ffi::CString::new(rules.as_os_str().as_bytes()).unwrap(); + let stamp = libc::timeval { + tv_sec: secs.as_secs() as libc::time_t, + tv_usec: secs.subsec_micros() as libc::suseconds_t, + }; + let times = [stamp, stamp]; + // SAFETY: `name` is NUL-terminated and `times` is a two-element array, + // both outliving the call. + assert_eq!(unsafe { libc::utimes(name.as_ptr(), times.as_ptr()) }, 0); + let after = std::fs::metadata(&rules).unwrap(); + assert_eq!(before.len(), after.len()); + assert_eq!(before.modified().unwrap(), after.modified().unwrap()); + + state.ignore_refresh_scheduled.store(true, Ordering::SeqCst); + let since = SystemTime::now() + std::time::Duration::from_secs(3600); + reindex_files_in(&state, &root, std::slice::from_ref(&root), since); + assert!( + state.ignore_rules_dirty.load(Ordering::SeqCst), + "rules that were swapped for different ones of the same size and age \ + must still be noticed" + ); + } + /// The size that qualifies a file is stat'd before its contents are read. /// A file that grows in between must not be read into memory without /// bound, nor indexed past the cap. From de056e4c9b6855b27e23d7da1cd46d1828afc32c Mon Sep 17 00:00:00 2001 From: Shengyu Fu Date: Thu, 27 Aug 2026 21:26:16 -0700 Subject: [PATCH 23/28] Backdate both writes so the restored mtime is exactly reproducible utimes takes microseconds and ext4 stores nanoseconds, so restoring a timestamp read back from the filesystem loses precision and the fixture's own precondition fails. Pin both writes to a whole second instead. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: bcc85b55-e821-447e-b9a9-144b78829ae4 --- tgrep-cli/src/serve.rs | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/tgrep-cli/src/serve.rs b/tgrep-cli/src/serve.rs index 551bd3f..ce0f5df 100644 --- a/tgrep-cli/src/serve.rs +++ b/tgrep-cli/src/serve.rs @@ -6879,7 +6879,20 @@ mod tests { state.gitignore_pending.store(false, Ordering::SeqCst); let rules = root.join(".gitignore"); + let name = std::ffi::CString::new(rules.as_os_str().as_bytes()).unwrap(); + // Whole seconds, so it survives a round trip through `utimes`, which + // takes microseconds while ext4 stores nanoseconds. + let stamp = libc::timeval { + tv_sec: 1_000_000, + tv_usec: 0, + }; + let times = [stamp, stamp]; + // SAFETY: `name` is NUL-terminated and `times` is a two-element array, + // both outliving each call. + let backdate = || assert_eq!(unsafe { libc::utimes(name.as_ptr(), times.as_ptr()) }, 0); + std::fs::write(&rules, "aaa.rs\n").unwrap(); + backdate(); let before = std::fs::metadata(&rules).unwrap(); *state.ignore_source_stamps.write().unwrap() = ignore_stamps_of(&root, std::slice::from_ref(&rules)); @@ -6888,20 +6901,7 @@ mod tests { // Different rules, same seven bytes, and the restore puts the old // timestamp back. std::fs::write(&rules, "bbb.rs\n").unwrap(); - let secs = before - .modified() - .unwrap() - .duration_since(SystemTime::UNIX_EPOCH) - .unwrap(); - let name = std::ffi::CString::new(rules.as_os_str().as_bytes()).unwrap(); - let stamp = libc::timeval { - tv_sec: secs.as_secs() as libc::time_t, - tv_usec: secs.subsec_micros() as libc::suseconds_t, - }; - let times = [stamp, stamp]; - // SAFETY: `name` is NUL-terminated and `times` is a two-element array, - // both outliving the call. - assert_eq!(unsafe { libc::utimes(name.as_ptr(), times.as_ptr()) }, 0); + backdate(); let after = std::fs::metadata(&rules).unwrap(); assert_eq!(before.len(), after.len()); assert_eq!(before.modified().unwrap(), after.modified().unwrap()); From 16d0081779a930b4ca173bda740ee7e8b5302e22 Mon Sep 17 00:00:00 2001 From: Shengyu Fu Date: Thu, 27 Aug 2026 21:45:55 -0700 Subject: [PATCH 24/28] Round 14: containment, stamp truth, and rules read under a race MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four defects the fourteenth review wave found, all in the watcher's trust boundaries. `is_real_dir` only ever inspected the last component, so `root/a/b` qualified while `a` was a symlink to anywhere on the machine. Add `is_contained_dir`, which walks from the served root and requires a real directory at every level, and use it where a path from an event becomes a subscription. `WatchRegistry` now carries the root and short-circuits through its own watched set, so the startup sync still costs one stat per directory rather than one per level. The vanished-source test used `exists`, which a rule file replaced by a directory, a FIFO or a socket passes — while the walker would no longer collect it and the digest check skips it for not being a file. `is_file` is the question that was meant. The background build stamped from a metadata walk taken after the walk that fed the index, so a file created between the two got a stamp with no index entry: `reindex_file` returns early on a matching stamp and the periodic reconcile compares stamps alone, so it would never be searchable. Stamp only what the index holds. The matcher reads its own sources inside the build, so digests taken afterwards describe whatever the replace left behind. Build the matcher inside `publish_ignore_matcher` and digest its sources on both sides; a mismatch publishes the matcher but marks it stale and schedules a refresh. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: bcc85b55-e821-447e-b9a9-144b78829ae4 --- tgrep-cli/src/serve.rs | 368 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 327 insertions(+), 41 deletions(-) diff --git a/tgrep-cli/src/serve.rs b/tgrep-cli/src/serve.rs index ce0f5df..7f36ff2 100644 --- a/tgrep-cli/src/serve.rs +++ b/tgrep-cli/src/serve.rs @@ -944,21 +944,55 @@ fn ignore_stamps_of(root: &Path, sources: &[PathBuf]) -> IgnoreStamps { /// what is on disk can see, a deleted file leaving nothing to read — and read /// so it can also notice one being replaced, which neither a pathname nor a /// timestamp can show. +/// +/// The matcher is built *here*, from `build`, rather than being handed in +/// already made. The stamps have to describe the bytes the matcher actually +/// read, and the read happens inside the build — `GitignoreBuilder::add` opens +/// each source itself. Stamping afterwards alone would record whatever is on +/// disk when the build finishes, so an mtime-preserving atomic replace during +/// the build would leave the matcher enforcing the old rules while the stamps +/// swore they were current, and every later check — pathname, timestamp, +/// digest — would agree that nothing needed rereading. Taking the digests on +/// both sides of the build turns that into something visible: if a source +/// moved underneath it, the published matcher is marked stale and a refresh is +/// scheduled. It is still published, because the alternative is no matcher at +/// all, which means indexing ignored paths until the refresh lands. #[must_use = "newly watched directories need a recovery scan or writes race the subscription"] fn publish_ignore_matcher( - state: &ServerState, + state: &Arc, root: &Path, - matcher: Option, sources: Vec, + build: impl FnOnce() -> Option, ) -> Vec { - *state.ignore_source_stamps.write().unwrap() = ignore_stamps_of(root, &sources); + let before = ignore_stamps_of(root, &sources); + let matcher = build(); + let stamps = ignore_stamps_of(root, &sources); + let raced = stamps != before; + + *state.ignore_source_stamps.write().unwrap() = stamps; *state.ignore_sources.write().unwrap() = sources; *state.gitignore.write().unwrap() = matcher; state.gitignore_pending.store(false, Ordering::SeqCst); // New rules mean a different set of directories worth hearing about: // a tightened rule releases the subscriptions under it, and a relaxed // one takes subscriptions for the tree it used to hide. - sync_watch_registrations(state, root).0 + let newly_watched = sync_watch_registrations(state, root).0; + + if raced { + // After the publish, so the refresh runs against the matcher and the + // subscriptions this call just established rather than racing them. + // The refresh rewalks and republishes, and a filesystem that has + // stopped moving produces matching digests next time, so this + // converges rather than looping. + eprintln!( + "[trace] warning: an ignore rules file changed while the matcher was \ + being built; scheduling a refresh" + ); + state.ignore_rules_dirty.store(true, Ordering::SeqCst); + schedule_ignore_rules_refresh(Arc::clone(state), root.to_path_buf()); + } + + newly_watched } fn handle_connection(stream: TcpStream, state: &ServerState) -> Result<()> { @@ -1748,6 +1782,7 @@ fn start_file_watcher(state: Arc, root: &Path, queue_cap: usize) -> *state.watch_registry.lock().unwrap() = Some(WatchRegistry { watcher, + root: root.to_path_buf(), watched: std::iter::once(root.to_path_buf()).collect(), }); @@ -1971,12 +2006,56 @@ fn is_real_dir(path: &Path) -> bool { std::fs::symlink_metadata(path).is_ok_and(|meta| meta.file_type().is_dir()) } +/// Whether `path` is a directory reached from `root` without crossing a symlink. +/// +/// [`is_real_dir`] only inspects the last component, which answers the wrong +/// question for a path assembled from an event: `root/a/b` is a perfectly real +/// directory while `a` is a symlink pointing anywhere on the machine. The +/// walker never descends through `a`, so nothing under it belongs to the served +/// tree, yet a `Create` for `root/a/b` would subscribe to it and enumerate and +/// index whatever is inside — a watch descriptor per directory of a tree that +/// is not ours, and file content filed under paths that do not lead to it. +/// +/// `root` itself is the trusted anchor and is not tested. It may legitimately +/// be reached through a link (`tgrep serve /var/tmp/...` on macOS is the common +/// case), and refusing it would leave nothing watchable at all. This is the +/// same contract [`open_within_root`] works to: containment is established +/// relative to the root that was served, not against the real filesystem. +/// +/// Component-by-component with `symlink_metadata`, so the answer is a snapshot +/// rather than a guarantee — a link swapped in afterwards is not visible here. +/// That residual window is what the post-registration re-check in +/// [`WatchRegistry::subscribe`] and `open_within_root`'s no-follow descent +/// exist to bound. +fn is_contained_dir(root: &Path, path: &Path) -> bool { + let Ok(rel) = path.strip_prefix(root) else { + return false; + }; + let mut cursor = root.to_path_buf(); + for component in rel.components() { + // `..` would climb back out of the tree and `.` cannot appear in a path + // built from an event; anything but a plain name is not a descent. + let std::path::Component::Normal(name) = component else { + return false; + }; + cursor.push(name); + if !is_real_dir(&cursor) { + return false; + } + } + true +} + /// The watcher plus the set of directories it is currently subscribed to. /// /// Only meaningful when [`PER_DIRECTORY_WATCHES`] is true; elsewhere `watched` /// holds just the root, which is subscribed recursively. struct WatchRegistry { watcher: RecommendedWatcher, + /// The served root. Subscriptions are only ever taken for directories + /// reachable from it without crossing a symlink; see + /// [`WatchRegistry::contained`]. + root: PathBuf, watched: std::collections::HashSet, } @@ -2012,6 +2091,27 @@ impl WatchRegistry { self.subscribe(dirs, true) } + /// Whether `dir` is still a directory the served tree actually contains. + /// + /// Every entry in `watched` was checked by this method before it went in, + /// so a directory whose parent is already watched (or is the root) inherits + /// that proof and only its own last component needs testing. That short + /// circuit is what keeps the startup sync affordable: the full walk in + /// [`is_contained_dir`] costs one `symlink_metadata` per level, and paying + /// it for each of forty thousand directories at depth ten would be four + /// hundred thousand syscalls to re-derive what the previous entry proved. + /// The sync feeds directories parent-first, so the cheap path is the one + /// nearly every call takes; anything arriving out of order still gets the + /// full walk and the right answer. + fn contained(&self, dir: &Path) -> bool { + match dir.parent() { + Some(parent) if parent == self.root || self.watched.contains(parent) => { + is_real_dir(dir) + } + _ => is_contained_dir(&self.root, dir), + } + } + fn subscribe<'a>( &mut self, desired: impl IntoIterator, @@ -2051,7 +2151,7 @@ impl WatchRegistry { // directory, which the periodic reconcile picks up, and // never misplaced content, since `open_within_root` // establishes containment from the handle it reads. - if !is_real_dir(dir) { + if !self.contained(dir) { let _ = self.watcher.unwatch(dir); if known { self.watched.remove(dir); @@ -2370,16 +2470,26 @@ fn reindex_files_in(state: &Arc, root: &Path, dirs: &[PathBuf], sin } else { let vanished = { let sources = state.ignore_sources.read().unwrap(); - // `exists` follows links, matching how the walker collected these — - // a symlinked source whose target is gone has stopped contributing + // `is_file` follows links, matching how the walker collected these + // (`ignore_files_in` qualifies candidates with `Path::is_file`) — a + // symlinked source whose target is gone has stopped contributing // rules just as surely as a deleted one. - sources.iter().find(|p| !p.exists()).cloned() + // + // `is_file` rather than `exists`: a source replaced by a directory, + // a FIFO or a socket still exists, but the walker would no longer + // collect it and a rebuild would no longer read it. Testing only + // for absence leaves the matcher enforcing rules from a file that + // is not a file any more, with nothing else able to notice — the + // digest check below only runs for candidates the scan walks past, + // and a rule file that has become a directory is not one of them. + sources.iter().find(|p| !p.is_file()).cloned() }; if let Some(gone) = vanished { state.ignore_rules_dirty.store(true, Ordering::SeqCst); schedule_ignore_rules_refresh(Arc::clone(state), root.to_path_buf()); eprintln!( - "[trace] watcher: ignore rules source {} disappeared; deferring to a refresh", + "[trace] watcher: ignore rules source {} is gone or no longer a file; \ + deferring to a refresh", gone.display() ); return; @@ -2620,8 +2730,10 @@ fn sweep_removed_files( fn watch_new_subtree(state: &Arc, root: &Path, dir: &Path) { // `is_dir` follows symlinks; the walker does not. Refuse a symlinked // directory here so we never subscribe to, or index, a tree the indexer - // would not have walked into. - if !is_real_dir(dir) { + // would not have walked into — and check every level, not just the last, + // since a real directory inside a symlinked one is just as far outside the + // served tree as the link itself. + if !is_contained_dir(root, dir) { return; } let Ok(rel_dir) = dir.strip_prefix(root) else { @@ -3793,6 +3905,36 @@ fn create_empty_index(index_dir: &Path) -> Result<()> { Ok(()) } +/// Stamps for the walked files that are actually in the index. +/// +/// The build stamps its work from a *second* traversal, taken after the content +/// walk that fed the index, so a file created between the two appears here and +/// nowhere else. Publishing a stamp for it would be a lie the rest of the +/// server believes: `reindex_file` returns early when the stamp already matches +/// what is on disk, and the periodic reconcile runs with +/// `compare_index_membership` off, so it compares stamps alone too — the file +/// would stay unsearchable until something changed it again. Withholding the +/// stamp instead makes the very next event or scan treat it as new, which is +/// what it is. +fn stamps_for_index_members( + files: Vec, + indexed: &std::collections::HashSet, +) -> std::collections::HashMap { + files + .into_iter() + .filter(|fm| indexed.contains(&fm.relative_path)) + .map(|fm| { + ( + fm.relative_path, + tgrep_core::meta::FileStamp { + mtime: fm.mtime, + size: fm.size, + }, + ) + }) + .collect() +} + /// Detect files that changed while the server was not running. /// Compares stored filestamps against current filesystem metadata, then upserts /// changed/new files and removes deleted files from the LiveIndex. @@ -4190,8 +4332,8 @@ fn refresh_stale_locked( *newly_watched = publish_ignore_matcher( state, root, - build_stale_matcher(state, root, &walk), ignore_sources_of(root, &walk.gitignore_files, &walk.ignore_files), + || build_stale_matcher(state, root, &walk), ); if walk.skipped_error > 0 { @@ -4431,13 +4573,6 @@ fn bootstrap_index_build(state: &Arc, root: &Path, index_dir: &Path let mut newly_watched = Vec::new(); if state.watch_enabled && !state.no_ignore { let t_gi = Instant::now(); - let matcher = tgrep_core::walker::build_gitignore_matcher_from_files( - root, - &outcome.gitignore_files, - &outcome.ignore_files, - state.no_require_git, - ); - let found = matcher.is_some(); // "Newly watched" here is every directory in the repository, and the // build's walk ran before any of them were subscribed. Deferred rather // than skipped: the scan waits out `indexing` and then costs one @@ -4446,9 +4581,17 @@ fn bootstrap_index_build(state: &Arc, root: &Path, index_dir: &Path newly_watched = publish_ignore_matcher( state, root, - matcher, ignore_sources_of(root, &outcome.gitignore_files, &outcome.ignore_files), + || { + tgrep_core::walker::build_gitignore_matcher_from_files( + root, + &outcome.gitignore_files, + &outcome.ignore_files, + state.no_require_git, + ) + }, ); + let found = state.gitignore.read().unwrap().is_some(); eprintln!( "[trace] gitignore matcher built from {} file(s) in {:.1}ms{}", outcome.gitignore_files.len(), @@ -4546,13 +4689,6 @@ fn background_index_build(state: &Arc, root: &Path, index_dir: &Pat let mut newly_watched = Vec::new(); if state.watch_enabled && !state.no_ignore { let start = Instant::now(); - let matcher = walker::build_gitignore_matcher_from_files( - root, - &walk.gitignore_files, - &walk.ignore_files, - state.no_require_git, - ); - let has_matcher = matcher.is_some(); // Subscriptions are taken here, partway through the build, so files // written to a directory the walk has already passed are in neither // the build's results nor any event. The scan waits for the build to @@ -4561,9 +4697,17 @@ fn background_index_build(state: &Arc, root: &Path, index_dir: &Pat newly_watched = publish_ignore_matcher( state, root, - matcher, ignore_sources_of(root, &walk.gitignore_files, &walk.ignore_files), + || { + walker::build_gitignore_matcher_from_files( + root, + &walk.gitignore_files, + &walk.ignore_files, + state.no_require_git, + ) + }, ); + let has_matcher = state.gitignore.read().unwrap().is_some(); eprintln!( "[trace] gitignore matcher built from index walk in {:.1}ms \ ({} .gitignore + {} .ignore files{})", @@ -4744,19 +4888,15 @@ fn background_index_build(state: &Arc, root: &Path, index_dir: &Pat max_file_size: state.max_file_size, }, ); - let stamps: std::collections::HashMap = walk_meta - .files - .into_iter() - .map(|fm| { - ( - fm.relative_path, - tgrep_core::meta::FileStamp { - mtime: fm.mtime, - size: fm.size, - }, - ) - }) - .collect(); + let stamps: std::collections::HashMap = { + let indexed = { + let index = state.index.read().unwrap(); + let mut paths = index.reader_paths(); + paths.extend(index.live.overlay_paths()); + paths + }; + stamps_for_index_members(walk_meta.files, &indexed) + }; // The in-memory build is done — surface "complete" in status now even // though the final disk flush below can take minutes for very large @@ -5577,6 +5717,7 @@ mod tests { let watcher = notify::recommended_watcher(|_: notify::Result| {}).unwrap(); let mut registry = WatchRegistry { watcher, + root: tmp.path().to_path_buf(), watched: std::collections::HashSet::new(), }; @@ -5619,6 +5760,7 @@ mod tests { let watcher = notify::recommended_watcher(|_: notify::Result| {}).unwrap(); let mut registry = WatchRegistry { watcher, + root: tmp.path().to_path_buf(), watched: std::collections::HashSet::new(), }; assert_eq!(registry.add_all(std::slice::from_ref(&a)).len(), 1); @@ -6916,6 +7058,150 @@ mod tests { ); } + /// A rule file that has become a directory, a FIFO or a socket is as gone + /// as a deleted one: the walker would no longer collect it and a rebuild + /// would no longer read it, so the rules it contributed are being enforced + /// by nothing. `exists` says otherwise, and nothing downstream corrects it + /// — the digest check skips candidates that are not files. + #[test] + fn a_rule_file_replaced_by_a_directory_counts_as_gone() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + state.gitignore_pending.store(false, Ordering::SeqCst); + + let rules = root.join(".gitignore"); + std::fs::write(&rules, "aaa.rs\n").unwrap(); + *state.ignore_source_stamps.write().unwrap() = + ignore_stamps_of(&root, std::slice::from_ref(&rules)); + *state.ignore_sources.write().unwrap() = vec![rules.clone()]; + + std::fs::remove_file(&rules).unwrap(); + std::fs::create_dir(&rules).unwrap(); + + // Keeps the refresh this schedules from clearing the flag underneath + // the assertion. + state.ignore_refresh_scheduled.store(true, Ordering::SeqCst); + let since = SystemTime::now() + Duration::from_secs(3600); + reindex_files_in(&state, &root, std::slice::from_ref(&root), since); + assert!( + state.ignore_rules_dirty.load(Ordering::SeqCst), + "a source that is no longer a file must be treated as gone" + ); + } + + /// Containment is a property of the whole path, not of its last component. + /// + /// `root/link/inner` is a perfectly real directory while `link` is a + /// symlink to somewhere else entirely. The walker never descends through + /// the link, so nothing under it is part of the served tree — subscribing + /// to it spends a watch descriptor per directory of a tree that is not + /// ours, which on a large linked-in tree is the inotify exhaustion this + /// registration exists to avoid. + #[cfg(unix)] + #[test] + fn a_directory_below_a_symlinked_one_is_not_subscribed_to() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path().join("root"); + let outside = tmp.path().join("outside"); + std::fs::create_dir(&root).unwrap(); + std::fs::create_dir_all(outside.join("inner")).unwrap(); + std::os::unix::fs::symlink(&outside, root.join("link")).unwrap(); + + let escaped = root.join("link").join("inner"); + assert!( + is_real_dir(&escaped), + "the fixture must be a real directory, or it proves nothing" + ); + assert!(!is_contained_dir(&root, &escaped)); + + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + state.gitignore_pending.store(false, Ordering::SeqCst); + let watcher = notify::recommended_watcher(|_: notify::Result| {}).unwrap(); + *state.watch_registry.lock().unwrap() = Some(WatchRegistry { + watcher, + root: root.clone(), + watched: std::iter::once(root.clone()).collect(), + }); + + let _gate = state.snapshot_gate.read().unwrap(); + watch_new_subtree(&state, &root, &escaped); + + let registry = state.watch_registry.lock().unwrap(); + assert!( + !registry.as_ref().unwrap().watched.contains(&escaped), + "a directory reached through a symlink must not be subscribed to" + ); + } + + /// A stamp is a claim about the index, not about the filesystem. + /// + /// The build stamps from a second traversal that runs after the one that + /// fed the index, so a file created between them is on disk and in no + /// index. Stamping it makes every later check agree it is up to date: + /// `reindex_file` returns early on a matching stamp, and the periodic + /// reconcile compares stamps alone. The file would never be searchable. + #[test] + fn stamps_are_published_only_for_what_the_build_indexed() { + use tgrep_core::walker::FileMeta; + + let files = vec![ + FileMeta { + relative_path: "indexed.rs".to_string(), + mtime: 1, + size: 10, + }, + FileMeta { + relative_path: "arrived_between_the_walks.rs".to_string(), + mtime: 2, + size: 20, + }, + ]; + let indexed = std::iter::once("indexed.rs".to_string()).collect(); + + let stamps = stamps_for_index_members(files, &indexed); + assert!(stamps.contains_key("indexed.rs")); + assert!( + !stamps.contains_key("arrived_between_the_walks.rs"), + "a file the build never indexed must not be stamped as if it had been" + ); + } + + /// The matcher reads its sources itself, inside the build. A replace that + /// lands while it is reading leaves it enforcing the old rules, and stamps + /// taken afterwards describe the new file — so pathname, timestamp and + /// digest all agree there is nothing to reread, and the stale rules stay in + /// force until something unrelated rebuilds them. + #[test] + fn a_rule_file_rewritten_during_the_build_marks_the_matcher_stale() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + let rules = root.join(".gitignore"); + std::fs::write(&rules, "aaa.rs\n").unwrap(); + state.ignore_refresh_scheduled.store(true, Ordering::SeqCst); + + // A build nothing raced publishes without complaint. Asserted first, or + // the test below passes for a matcher that is always stale. + let quiet = publish_ignore_matcher(&state, &root, vec![rules.clone()], || None); + assert!(quiet.is_empty()); + assert!(!state.ignore_rules_dirty.load(Ordering::SeqCst)); + + let newly = publish_ignore_matcher(&state, &root, vec![rules.clone()], || { + // The checkout that lands while the builder is reading. + std::fs::write(&rules, "bbb.rs\n").unwrap(); + None + }); + assert!(newly.is_empty()); + assert!( + state.ignore_rules_dirty.load(Ordering::SeqCst), + "a matcher built over a moving source must be marked stale" + ); + } + /// The size that qualifies a file is stat'd before its contents are read. /// A file that grows in between must not be read into memory without /// bound, nor indexed past the cap. From 6afe2b03a77b6c8c215741243a8f24ee5d856d92 Mon Sep 17 00:00:00 2001 From: Shengyu Fu Date: Thu, 27 Aug 2026 22:07:34 -0700 Subject: [PATCH 25/28] Sweep vanished directories, order subscriptions, distrust raced stamps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review round 15. A whole directory that goes away leaves indexed files whose own parent was never enumerated. The recovery sweep judged a file only from its immediate parent's listing, so everything under a directory that was deleted or moved away stayed searchable — and no event names those files either: a removal delivers one event for the directory, a move away delivers nothing at all. The scan now records which directories it saw present and which it could not list, derives the ones that are gone from a parent listing that did succeed, and the sweep walks ancestors so descendants go with them. `WatchRegistry::sync` fed `add_all` straight from a `HashSet`, so a child could be subscribed before its parent and the containment check had no watched parent to lean on — one `symlink_metadata` per level per directory instead of one per directory. Sorting by depth restores the fast path. Stamps published by a build described a metadata walk taken after the content walk that fed the index, so a file written in between was stamped as indexed while the index held the older bytes. `reindex_file` returns early on a matching stamp, so the replay of the very event that reported the write read nothing and the reconcile behind it agreed. Those paths are already named by the deferred-event buffer, so their stamps are withheld; when the buffer has overflowed nothing can be told apart from what changed and no stamp from that build is published at all. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: bcc85b55-e821-447e-b9a9-144b78829ae4 --- tgrep-cli/src/serve.rs | 312 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 295 insertions(+), 17 deletions(-) diff --git a/tgrep-cli/src/serve.rs b/tgrep-cli/src/serve.rs index 7f36ff2..54010e6 100644 --- a/tgrep-cli/src/serve.rs +++ b/tgrep-cli/src/serve.rs @@ -2100,9 +2100,9 @@ impl WatchRegistry { /// [`is_contained_dir`] costs one `symlink_metadata` per level, and paying /// it for each of forty thousand directories at depth ten would be four /// hundred thousand syscalls to re-derive what the previous entry proved. - /// The sync feeds directories parent-first, so the cheap path is the one - /// nearly every call takes; anything arriving out of order still gets the - /// full walk and the right answer. + /// The sync feeds directories shallowest-first for exactly this reason, so + /// the cheap path is the one nearly every call takes; anything arriving + /// out of order still gets the full walk and the right answer. fn contained(&self, dir: &Path) -> bool { match dir.parent() { Some(parent) if parent == self.root || self.watched.contains(parent) => { @@ -2236,7 +2236,20 @@ impl WatchRegistry { removed += 1; } - (self.add_all(desired), removed) + // Shallowest first, because `desired` is a `HashSet` and hands its + // contents out in whatever order hashing produced. [`Self::contained`] + // establishes containment cheaply by leaning on the parent already + // being watched; a child that arrives before its parent gets no such + // proof and walks every ancestor instead. Unordered, that is the + // common case rather than the exception, and it turns the startup + // sync from one `symlink_metadata` per directory into one per level + // per directory — on a monorepo, hundreds of thousands of syscalls in + // the path that exists to make startup cheap. Sorting by depth is + // enough: a parent is always strictly shallower than its children. + let mut ordered: Vec<&PathBuf> = desired.iter().collect(); + ordered.sort_by_key(|dir| dir.components().count()); + + (self.add_all(ordered), removed) } } @@ -2516,11 +2529,14 @@ fn reindex_files_in(state: &Arc, root: &Path, dirs: &[PathBuf], sin } // Directories whose listing succeeded, and the files those listings - // contained, for the removal sweep at the end. Only files are recorded: - // stamps describe files, so directory names would just be dead weight on a - // set that at startup spans the whole repository. + // contained, for the removal sweep at the end. Directory names are kept + // apart from the files: a directory that vanished with its contents leaves + // indexed paths whose own parent was never enumerated, and its absence + // from its parent's listing is the only evidence there is. let mut swept: std::collections::HashSet = std::collections::HashSet::new(); let mut present: std::collections::HashSet = std::collections::HashSet::new(); + let mut present_dirs: std::collections::HashSet = std::collections::HashSet::new(); + let mut unreadable_dirs: Vec = Vec::new(); for dir in dirs { let Ok(rel_dir) = dir.strip_prefix(root) else { @@ -2529,7 +2545,9 @@ fn reindex_files_in(state: &Arc, root: &Path, dirs: &[PathBuf], sin let rel_dir = rel_dir.to_string_lossy().replace('\\', "/"); let Ok(entries) = std::fs::read_dir(dir) else { // No listing means no evidence, and the sweep below must not treat - // silence as absence. + // silence as absence. Whether the directory is gone or merely + // unreadable is decided later, from its parent's listing. + unreadable_dirs.push(rel_dir); continue; }; let mut subdirs: Vec = Vec::new(); @@ -2559,6 +2577,7 @@ fn reindex_files_in(state: &Arc, root: &Path, dirs: &[PathBuf], sin // file or directory is neither indexed nor descended into, which // is what the walker does with `follow_links(false)`. if file_type.is_dir() { + present_dirs.insert(rel); subdirs.push(path); continue; } @@ -2608,7 +2627,21 @@ fn reindex_files_in(state: &Arc, root: &Path, dirs: &[PathBuf], sin } } - sweep_removed_files(state, &swept, &present); + // A directory that could not be listed is either gone or merely + // unreadable, and only its parent's listing can tell the two apart. The + // ones proven absent stand in for every file under them: those files have + // a parent nothing enumerated, so the per-directory evidence above says + // nothing about them at all, and a subtree deleted or moved away in this + // window would keep answering searches until the hourly reconcile. + let vanished_dirs: std::collections::HashSet = unreadable_dirs + .into_iter() + .filter(|rel| { + let parent = rel.rsplit_once('/').map_or("", |(dir, _)| dir); + swept.contains(parent) && !present_dirs.contains(rel) && !present.contains(rel) + }) + .collect(); + + sweep_removed_files(state, &swept, &present, &vanished_dirs); eprintln!( "[trace] watcher: rechecked {} newly watched directories in {:.1}ms", @@ -2631,6 +2664,15 @@ fn reindex_files_in(state: &Arc, root: &Path, dirs: &[PathBuf], sin /// eligibility. Filtering `present` would delete entries for files that are /// still on disk and were indexed under a laxer configuration. /// +/// `vanished_dirs` are directories that could not be listed *and* were absent +/// from a parent listing that did succeed. Their descendants cannot be judged +/// by `swept` and `present`, which only speak for a file's immediate parent: a +/// directory deleted or moved away whole leaves indexed paths whose parent was +/// never enumerated, and no event names them either — a removal delivers one +/// event for the directory, and a move away delivers nothing at all for what +/// was inside it. Anything under one of these is swept on the strength of the +/// directory's absence. +/// /// Candidates come from everything that can answer a search, not from /// `file_stamps` alone. A stamp is not a precondition for being searchable: /// `filestamps.json` is optional by design — missing or unreadable leaves the @@ -2649,6 +2691,7 @@ fn sweep_removed_files( state: &ServerState, swept: &std::collections::HashSet, present: &std::collections::HashSet, + vanished_dirs: &std::collections::HashSet, ) { if swept.is_empty() { return; @@ -2658,7 +2701,25 @@ fn sweep_removed_files( // to their product would not finish. let missing = |rel: &str| { let parent = rel.rsplit_once('/').map_or("", |(dir, _)| dir); - swept.contains(parent) && !present.contains(rel) + if swept.contains(parent) && !present.contains(rel) { + return true; + } + // Walking ancestors costs one lookup per level, so it is done only + // when something actually vanished — which is rare, while this closure + // runs once per indexed path in the repository. + if vanished_dirs.is_empty() { + return false; + } + let mut ancestor = parent; + loop { + if vanished_dirs.contains(ancestor) { + return true; + } + match ancestor.rsplit_once('/') { + Some((next, _)) => ancestor = next, + None => return false, + } + } }; let mut gone: std::collections::HashSet = { let stamps = state.file_stamps.read().unwrap(); @@ -3935,6 +3996,67 @@ fn stamps_for_index_members( .collect() } +/// Drop the stamps the build has no right to publish, because an event for +/// those paths arrived while it ran. +/// +/// The stamp map describes what the index holds, and the build derives it from +/// a metadata walk taken *after* the content walk. For a file written between +/// the two the index holds the old bytes while the stamp describes the new +/// ones, so the stamp says "current" about content that is stale. That claim is +/// load-bearing in exactly the place that should have repaired it: +/// `reindex_file` returns early on a matching stamp, so the replay of the very +/// event that reported the write reads nothing, and the reconcile behind it +/// compares the same stamps and agrees. The old content stays searchable +/// indefinitely. +/// +/// The deferred buffer already names those paths — it is what replay is about +/// to walk — so withholding their stamps costs one map lookup each and makes +/// the replay do the read it was deferred for. +/// +/// When the buffer overflowed it names nothing, and nothing distinguishes the +/// files that changed from the ones that did not, so no stamp from this build +/// can be trusted and none is published. The reconcile that overflow already +/// schedules then re-reads the tree rather than believing a walk that raced +/// 100k changes. +/// +/// Takes the deferred lock while `snapshot_gate` is held, which is the one +/// order in use: the watcher defers *before* it takes the gate, and replay +/// releases the buffer before handling anything. +fn withhold_stamps_for_deferred( + state: &ServerState, + root: &Path, + mut stamps: std::collections::HashMap, +) -> std::collections::HashMap { + let deferred = match state.deferred_events.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + }; + let Some(paths) = deferred.as_ref() else { + eprintln!( + "[trace] warning: too many changes during the initial build to say which files the \ + walk raced; publishing no stamps so the reconcile re-reads them" + ); + return std::collections::HashMap::new(); + }; + let mut withheld = 0usize; + for path in paths.keys() { + let Ok(rel) = path.strip_prefix(root) else { + continue; + }; + let rel = rel.to_string_lossy().replace('\\', "/"); + if stamps.remove(&rel).is_some() { + withheld += 1; + } + } + if withheld > 0 { + eprintln!( + "[trace] watcher: {withheld} file(s) changed during the initial build; their stamps \ + are withheld so the replay re-reads them" + ); + } + stamps +} + /// Detect files that changed while the server was not running. /// Compares stored filestamps against current filesystem metadata, then upserts /// changed/new files and removes deleted files from the LiveIndex. @@ -4564,7 +4686,13 @@ fn bootstrap_index_build(state: &Arc, root: &Path, index_dir: &Path // matcher from those is what keeps this cheap — `gitignore::build_matcher` // would rewalk the whole tree, which cost 49 s on a 289k-file repo. match tgrep_core::meta::read_filestamps(index_dir) { - Ok(stamps) => *state.file_stamps.write().unwrap() = stamps, + // Minus the paths whose events arrived while the build ran: the builder + // read those files at some point during its walk and stamped what it + // saw, so for anything written afterwards the stamp describes bytes the + // index does not hold. See `withhold_stamps_for_deferred`. + Ok(stamps) => { + *state.file_stamps.write().unwrap() = withhold_stamps_for_deferred(state, root, stamps) + } Err(e) => eprintln!( "[trace] warning: could not load file stamps ({e}); \ the watcher may reindex on spurious events" @@ -4927,7 +5055,10 @@ fn background_index_build(state: &Arc, root: &Path, index_dir: &Pat // // Done even if the flush below fails: the live overlay already reflects // what was just indexed, and the stamps describe that. - *state.file_stamps.write().unwrap() = stamps; + // + // Minus whatever changed underneath the walk, which the stamps would + // otherwise describe as indexed when the index holds the older bytes. + *state.file_stamps.write().unwrap() = withhold_stamps_for_deferred(state, root, stamps); state.indexing.store(false, Ordering::SeqCst); // Final flush to disk for the bulk build. Use the same streaming @@ -6577,7 +6708,8 @@ mod tests { // withheld from `swept`. let swept = std::collections::HashSet::new(); let present = std::collections::HashSet::new(); - sweep_removed_files(&state, &swept, &present); + let no_vanished = std::collections::HashSet::new(); + sweep_removed_files(&state, &swept, &present, &no_vanished); assert!( !state.index.read().unwrap().live.is_deleted("d/a.rs"), "an unenumerated directory must not tombstone the files under it" @@ -6587,7 +6719,7 @@ mod tests { // the file is actually gone, which the sweep now confirms itself. std::fs::remove_file(&path).unwrap(); let swept = std::collections::HashSet::from(["d".to_string()]); - sweep_removed_files(&state, &swept, &present); + sweep_removed_files(&state, &swept, &present, &no_vanished); assert!( state.index.read().unwrap().live.is_deleted("d/a.rs"), "a fully enumerated directory must sweep what it no longer contains" @@ -6615,7 +6747,8 @@ mod tests { // it is back on disk by the time the sweep runs. let swept = std::collections::HashSet::from(["d".to_string()]); let present = std::collections::HashSet::new(); - sweep_removed_files(&state, &swept, &present); + let no_vanished = std::collections::HashSet::new(); + sweep_removed_files(&state, &swept, &present, &no_vanished); assert!( !state.index.read().unwrap().live.is_deleted("d/a.rs"), @@ -6995,7 +7128,13 @@ mod tests { std::fs::remove_file(&path).unwrap(); let swept: std::collections::HashSet = [String::new()].into_iter().collect(); - sweep_removed_files(&state, &swept, &std::collections::HashSet::new()); + let no_vanished = std::collections::HashSet::new(); + sweep_removed_files( + &state, + &swept, + &std::collections::HashSet::new(), + &no_vanished, + ); drop(gate); assert!( @@ -7202,7 +7341,146 @@ mod tests { ); } - /// The size that qualifies a file is stat'd before its contents are read. + /// A directory that vanished whole leaves indexed files whose own parent + /// was never enumerated. `swept`/`present` only speak for a file's + /// immediate parent, so without the vanished-directory evidence every one + /// of those files stays searchable — and no event names them either: a + /// move away delivers nothing at all for what was inside. + #[test] + fn a_directory_that_went_away_whole_takes_its_files_with_it() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + + let dir = root.join("d"); + let nested = dir.join("deep"); + std::fs::create_dir_all(&nested).unwrap(); + std::fs::write(dir.join("a.rs"), "fn a() {}\n").unwrap(); + std::fs::write(nested.join("b.rs"), "fn b() {}\n").unwrap(); + reindex_file(&state, &dir.join("a.rs"), "d/a.rs"); + reindex_file(&state, &nested.join("b.rs"), "d/deep/b.rs"); + assert!(state.index.read().unwrap().live.has_path("d/a.rs")); + + // Still there, merely unlistable from the scan's point of view: the + // directory it was named in enumerated it, so silence proves nothing. + { + let _gate = state.snapshot_gate.read().unwrap(); + let since = SystemTime::now() + Duration::from_secs(3600); + reindex_files_in(&state, &root, std::slice::from_ref(&root), since); + } + assert!( + !state.index.read().unwrap().live.is_deleted("d/a.rs"), + "a directory that is still on disk must not sweep its files" + ); + + // Gone, and the scan is told to recheck it — which is what a recovery + // scan over a newly watched directory does. Its parent lists cleanly + // and does not contain it. + std::fs::remove_dir_all(&dir).unwrap(); + { + let _gate = state.snapshot_gate.read().unwrap(); + let since = SystemTime::now() + Duration::from_secs(3600); + reindex_files_in(&state, &root, &[root.clone(), dir.clone()], since); + } + let index = state.index.read().unwrap(); + assert!( + index.live.is_deleted("d/a.rs"), + "a file directly inside a vanished directory must be swept" + ); + assert!( + index.live.is_deleted("d/deep/b.rs"), + "and so must one further down, whose parent vanished with it" + ); + } + + /// `desired` is a set, so the order it iterates in is not the order the + /// tree is in. Subscribing a child before its parent makes the containment + /// check walk every level from the root for each one, which is the cost + /// the parent-first fast path exists to avoid on a tree with 40k + /// directories in it. + #[test] + fn subscriptions_are_established_from_the_root_down() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let deep = root.join("a").join("b").join("c"); + std::fs::create_dir_all(&deep).unwrap(); + + let watcher = notify::recommended_watcher(|_: notify::Result| {}).unwrap(); + let mut registry = WatchRegistry { + watcher, + root: root.clone(), + watched: std::collections::HashSet::new(), + }; + + let desired: std::collections::HashSet = [ + deep.clone(), + root.join("a").join("b"), + root.join("a"), + root.clone(), + ] + .into_iter() + .collect(); + let (added, _removed) = registry.sync(&desired); + + assert_eq!(added.len(), 4); + let depths: Vec = added.iter().map(|d| d.components().count()).collect(); + assert!( + depths.windows(2).all(|w| w[0] <= w[1]), + "a parent must be subscribed before anything under it, got {added:?}" + ); + assert!(registry.watched.contains(&deep)); + } + + /// Stamps describe what the index holds. A file written while the build + /// ran is read by the metadata walk that produces them but not by the + /// content walk that fed the index, so its stamp says "current" about + /// bytes that are already stale — and `reindex_file` returns early on a + /// matching stamp, so the replay of the very event that reported the write + /// reads nothing. + #[test] + fn a_file_written_during_the_build_is_not_stamped_by_it() { + use tgrep_core::meta::FileStamp; + + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + + let stamps: std::collections::HashMap = [ + ("quiet.rs".to_string(), FileStamp { mtime: 1, size: 10 }), + ("racy.rs".to_string(), FileStamp { mtime: 2, size: 20 }), + ] + .into_iter() + .collect(); + + state + .deferred_events + .lock() + .unwrap() + .as_mut() + .unwrap() + .insert(root.join("racy.rs"), false); + + let published = withhold_stamps_for_deferred(&state, &root, stamps.clone()); + assert!( + published.contains_key("quiet.rs"), + "a file nothing touched keeps its stamp, or the build re-reads the repository" + ); + assert!( + !published.contains_key("racy.rs"), + "a file whose event is waiting to be replayed must not be stamped as indexed" + ); + + // Overflowed: the buffer names nothing, so nothing in the map can be + // told apart from what changed underneath it. + *state.deferred_events.lock().unwrap() = None; + assert!( + withhold_stamps_for_deferred(&state, &root, stamps).is_empty(), + "with the buffer overflowed no stamp from this build can be trusted" + ); + } + /// A file that grows in between must not be read into memory without /// bound, nor indexed past the cap. #[test] From e585f3701b624479b6b94e5bb63fe38262781a23 Mon Sep 17 00:00:00 2001 From: Shengyu Fu Date: Thu, 27 Aug 2026 22:14:57 -0700 Subject: [PATCH 26/28] Recheck a swept path under the same containment contract as indexing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review round 16. The sweep asks whether a candidate is back on disk before dropping it, and asked with `symlink_metadata`, which refuses to follow only the *final* component. A directory that vanished and returned as a link to another tree then makes `root/gone-dir/a.rs` resolve to an ordinary file outside the root, the recheck reads that as "it came back", and the stale in-root entry is kept — permanently, since nothing under a link is walked or watched and no later event names it. Rechecked through `open_within_root` instead, which is the contract `reindex_file` opens under: every ancestor is resolved without following a link. Errors are classified with the existing `proves_ineligible`, so a descriptor limit or a sharing violation preserves the entry for the next reconcile rather than evicting live content. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: bcc85b55-e821-447e-b9a9-144b78829ae4 --- tgrep-cli/src/serve.rs | 69 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 65 insertions(+), 4 deletions(-) diff --git a/tgrep-cli/src/serve.rs b/tgrep-cli/src/serve.rs index 54010e6..46aec7d 100644 --- a/tgrep-cli/src/serve.rs +++ b/tgrep-cli/src/serve.rs @@ -2752,10 +2752,27 @@ fn sweep_removed_files( let mut dropped = 0usize; for rel in &gone { let _reindex = lock_reindex(state); - // No-follow: a path that came back as a symlink is still not something - // the index should hold, so it stays swept. - if std::fs::symlink_metadata(state.root.join(rel)).is_ok_and(|m| m.file_type().is_file()) { - continue; + // Through the same containment contract `reindex_file` opens under, + // not a bare `symlink_metadata`. That call refuses to follow only the + // *final* component: a directory that vanished and came back as a link + // to somewhere else makes `root/gone-dir/a.rs` resolve to a perfectly + // ordinary file outside the tree, and reading that as "it is back" + // keeps the stale in-root entry forever — nothing under a link is + // walked or watched, so no later event corrects it. + // + // Transient failures preserve, as they do in `reindex_file`: a + // descriptor limit or a sharing violation says nothing about whether + // the path belongs in the index, and the next reconcile will ask again. + match open_within_root(&state.root, &state.root.join(rel)) { + Ok(file) => match file.metadata() { + // Back, and reachable without leaving the tree. + Ok(meta) if meta.file_type().is_file() => continue, + // There, but not something the index should hold. + Ok(_) => {} + Err(_) => continue, + }, + Err(e) if proves_ineligible(&e) => {} + Err(_) => continue, } state.index.write().unwrap().live.delete_file(rel); state.file_stamps.write().unwrap().remove(rel); @@ -6756,6 +6773,50 @@ mod tests { ); } + /// A vanished directory that comes back as a link to somewhere else has + /// not brought its files back: nothing under a link is walked, watched or + /// indexed. The recheck that decides "it is here again" has to answer that + /// under the same containment contract indexing does, or the stale in-root + /// entry is kept and no later event ever corrects it. + #[cfg(unix)] + #[test] + fn a_path_that_returns_through_a_symlinked_ancestor_is_still_swept() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path().join("root"); + std::fs::create_dir(&root).unwrap(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + + let dir = root.join("d"); + std::fs::create_dir(&dir).unwrap(); + std::fs::write(dir.join("a.rs"), "fn ours() {}\n").unwrap(); + reindex_file(&state, &dir.join("a.rs"), "d/a.rs"); + assert!(state.index.read().unwrap().live.has_path("d/a.rs")); + + // The scan listed the root, did not find `d`, and is about to sweep + // what was under it. In between, `d` comes back — as a link to a tree + // that is not ours, holding a file by the same name. + let outside = tmp.path().join("elsewhere"); + std::fs::create_dir(&outside).unwrap(); + std::fs::write(outside.join("a.rs"), "fn theirs() {}\n").unwrap(); + std::fs::remove_dir_all(&dir).unwrap(); + std::os::unix::fs::symlink(&outside, &dir).unwrap(); + assert!( + root.join("d").join("a.rs").is_file(), + "the fixture must look like a returning file, or it proves nothing" + ); + + let swept = std::collections::HashSet::from([String::new()]); + let present = std::collections::HashSet::new(); + let vanished = std::collections::HashSet::from(["d".to_string()]); + sweep_removed_files(&state, &swept, &present, &vanished); + + assert!( + state.index.read().unwrap().live.is_deleted("d/a.rs"), + "a file reachable only through a symlink has not come back" + ); + } + /// An indexed file replaced in place by something that is not a regular /// file is not a removal — the path still exists — but its contents are no /// longer there to be found. From c07eb8189ca428e4c555029d0ab8b0b62f527622 Mon Sep 17 00:00:00 2001 From: Shengyu Fu Date: Thu, 27 Aug 2026 23:56:17 -0700 Subject: [PATCH 27/28] Round 17: align the watcher with the walk, and stop trusting lost events MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five defects, all in how the watcher decides what to look at. 1. An ignore-rules refresh could race a resumed build. `background_index_build` publishes its matcher part-way through Phase 2 while holding no gate, so the refresh it schedules took `snapshot_gate` uncontended and replaced `file_stamps` from its own walk, only for the build to overwrite them from a walk that predates the new rules. The refresh worker now waits out `indexing`, mirroring the recovery scan. 2. The point-query matcher omitted git's `core.ignorecase` narrowing that both walks apply as a `filter_entry`, so the watcher subscribed to and indexed exactly the trees the walk excluded — and the next stale check evicted them again. `IgnoreMatcher` now carries the same `CaseInsensitiveIgnore`, applied where the walk applies it. 3. Overflow left dead watches recorded as live. A native drop (`IN_Q_OVERFLOW`, a lost `ReadDirectoryChangesW` buffer) only logged and triggered no reconcile at all, and after any overflow the dropped removal events left the registry recording watches the kernel had released — which every later sync skipped as already present. The error branch now reconciles too, and sets a flag one sync consumes to re-issue subscriptions once. 4. A populated directory moved in from outside the root stayed unindexed on Windows and macOS: the enumeration was gated on per-directory watches, but a recursive backend reports the move as a single event and never describes the contents. The indexing half now runs everywhere; only subscribing stays per-directory. 5. Ignore-source tracking omitted parent rule files and the repository exclude, and resolved the exclude at a literal `.git/info/exclude` — wrong in a linked worktree, where `WalkBuilder` follows the gitdir/commondir chain and this did not. Both are now tracked and resolved the way the walk resolves them. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: bcc85b55-e821-447e-b9a9-144b78829ae4 --- tgrep-cli/src/serve.rs | 351 +++++++++++++++++++++++++++++++++--- tgrep-core/src/git_index.rs | 2 +- tgrep-core/src/gitignore.rs | 220 ++++++++++++++++++---- tgrep-core/src/walker.rs | 55 ++++++ 4 files changed, 569 insertions(+), 59 deletions(-) diff --git a/tgrep-cli/src/serve.rs b/tgrep-cli/src/serve.rs index 46aec7d..ee70f04 100644 --- a/tgrep-cli/src/serve.rs +++ b/tgrep-cli/src/serve.rs @@ -290,6 +290,15 @@ struct ServerState { ignore_rules_dirty: std::sync::atomic::AtomicBool, /// Ensures a burst of ignore-file events uses at most one refresh worker. ignore_refresh_scheduled: std::sync::atomic::AtomicBool, + /// Set when events were lost, cleared by the next subscription sync, which + /// then re-issues every subscription instead of trusting its own records. + /// + /// A dropped directory-removal event leaves the registry recording a watch + /// the kernel has already released. Nothing later contradicts that record: + /// a path recreated at the same location is both wanted and believed + /// watched, so every sync skips it and it never reports again. Only + /// overflow can produce that state, so only overflow pays for the repair. + watch_resubscribe: std::sync::atomic::AtomicBool, /// The ignore files the published matcher was built from. /// /// A recovery scan can spot an ignore file that *arrived* during its window @@ -639,6 +648,7 @@ pub fn run(root: &Path, index_path: Option<&Path>, options: ServeOptions<'_>) -> gitignore_pending: std::sync::atomic::AtomicBool::new(!no_watch && !no_ignore), ignore_rules_dirty: std::sync::atomic::AtomicBool::new(false), ignore_refresh_scheduled: std::sync::atomic::AtomicBool::new(false), + watch_resubscribe: std::sync::atomic::AtomicBool::new(false), ignore_sources: RwLock::new(Vec::new()), ignore_source_stamps: RwLock::new(IgnoreStamps::new()), reindex_lock: Mutex::new(()), @@ -864,10 +874,24 @@ const MTIME_GRANULARITY: Duration = Duration::from_secs(2); /// view — it is applied as its own filter rather than collected with the /// gitignore files — but deleting it invalidates the published rules exactly /// the same way, so it belongs in the list. +/// +/// So do the sources that sit *outside* the served tree: parent-directory +/// `.ignore` / `.gitignore` files and the repository's `info/exclude`. The +/// walk that found everything else never visits them, so without this nothing +/// would notice one being deleted — and a rule with no source left keeps being +/// enforced, holding a subtree unsubscribed and unindexed until an unrelated +/// rebuild happens along. Only files that exist are listed; a path that was +/// never there is not a source that went missing. +/// +/// The user's global ignore file is deliberately absent: the `ignore` crate +/// resolves it through git's config precedence and does not hand back the path +/// it chose, and guessing wrong would report a source as vanished on every +/// scan. fn ignore_sources_of( root: &Path, gitignore_files: &[PathBuf], ignore_files: &[PathBuf], + no_require_git: bool, ) -> Vec { let mut sources = Vec::with_capacity(gitignore_files.len() + ignore_files.len() + 1); sources.extend_from_slice(gitignore_files); @@ -876,6 +900,12 @@ fn ignore_sources_of( if p4.is_file() { sources.push(p4); } + sources.extend( + tgrep_core::gitignore::ancestor_ignore_paths(root, no_require_git) + .into_iter() + .map(|(path, _)| path), + ); + sources.extend(tgrep_core::gitignore::repo_exclude_path(root)); sources } @@ -895,12 +925,19 @@ fn ignore_stamps_of(root: &Path, sources: &[PathBuf]) -> IgnoreStamps { let canonical_root = std::fs::canonicalize(root).ok(); let mut stamps = IgnoreStamps::with_capacity(sources.len()); let mut record = |path: &Path, base: &Path| { - let Ok(rel) = path.strip_prefix(base) else { - return; + // A source above the root — a parent `.gitignore`, the repository's + // `info/exclude` — has no path relative to it. Key it by its own full + // path rather than dropping it: these keys only have to tell one source + // from another within a single run, and `changed_ignore_rules_in` looks + // up by relative path, so an absolute key is simply never hit there. + // Dropped, the digest comparison in `publish_ignore_matcher` would be + // blind to a source it is being asked to watch. + let key = match path.strip_prefix(base) { + Ok(rel) => rel.to_string_lossy().replace('\\', "/"), + Err(_) => path.to_string_lossy().replace('\\', "/"), }; - let rel = rel.to_string_lossy().replace('\\', "/"); if let Some(digest) = ignore_digest_of(path) { - stamps.insert(rel, digest); + stamps.insert(key, digest); } }; for source in sources { @@ -1738,6 +1775,7 @@ fn start_file_watcher(state: Arc, root: &Path, queue_cap: usize) -> let overflowed = Arc::new(std::sync::atomic::AtomicBool::new(false)); let callback_overflow = Arc::clone(&overflowed); + let callback_state = Arc::clone(&state); let mut watcher = match notify::recommended_watcher( move |result: std::result::Result| match result { Ok(event) => match tx.try_send(event) { @@ -1749,14 +1787,29 @@ fn start_file_watcher(state: Arc, root: &Path, queue_cap: usize) -> // trying to replay an unknown number of lost events. Err(TrySendError::Full(_)) => { callback_overflow.store(true, Ordering::SeqCst); + callback_state + .watch_resubscribe + .store(true, Ordering::SeqCst); } Err(TrySendError::Disconnected(_)) => {} }, - // Surface these. A dropped ReadDirectoryChangesW buffer looks - // exactly like "the watcher stopped working" from the outside, - // and silence makes it impossible to tell apart from a bug in - // our own filtering. - Err(e) => eprintln!("[trace] warning: file watcher error: {e}"), + // A native drop is the same loss as a full channel, and the OS + // will not say what it lost — inotify's `IN_Q_OVERFLOW` and a + // dropped `ReadDirectoryChangesW` buffer both arrive here with no + // paths attached. Reconcile on them too: reporting without + // recovering left exactly one of the two overflow paths handled, + // and it was the one the kernel does not use. + // + // Surfaced as well. A dropped buffer looks exactly like "the + // watcher stopped working" from the outside, and silence makes it + // impossible to tell apart from a bug in our own filtering. + Err(e) => { + eprintln!("[trace] warning: file watcher error: {e}"); + callback_overflow.store(true, Ordering::SeqCst); + callback_state + .watch_resubscribe + .store(true, Ordering::SeqCst); + } }, ) { Ok(w) => w, @@ -2221,10 +2274,22 @@ impl WatchRegistry { /// Bring the subscription set in line with `desired`, subscribing to /// directories that are newly relevant and dropping ones that are not. /// + /// `force` re-issues the subscription for directories already recorded as + /// watched. Only needed after events were lost: a directory removal that + /// never arrived leaves the kernel's descriptor gone and this registry's + /// entry intact, and a path recreated there is then in `desired` *and* in + /// `watched`, so an ordinary sync skips it forever. Off by default because + /// re-registering costs a syscall per directory, and a monorepo reconcile + /// would pay forty thousand of them for a doubt only overflow raises. + /// /// Returns `(added, removed)`. Only for a set that describes the whole /// tree — anything absent from `desired` is unsubscribed. To subscribe to /// a subtree without disturbing the rest, use [`Self::add_all`]. - fn sync(&mut self, desired: &std::collections::HashSet) -> (Vec, usize) { + fn sync( + &mut self, + desired: &std::collections::HashSet, + force: bool, + ) -> (Vec, usize) { let stale: Vec = self.watched.difference(desired).cloned().collect(); let mut removed = 0; for dir in stale { @@ -2249,7 +2314,12 @@ impl WatchRegistry { let mut ordered: Vec<&PathBuf> = desired.iter().collect(); ordered.sort_by_key(|dir| dir.components().count()); - (self.add_all(ordered), removed) + let added = if force { + self.resubscribe_all(ordered) + } else { + self.add_all(ordered) + }; + (added, removed) } } @@ -2342,7 +2412,10 @@ fn sync_watch_registrations(state: &ServerState, root: &Path) -> (Vec, watchable_dirs(root, root, &state.exclude_dirs, gitignore.as_ref()) }; let total = desired.len(); - let (added, removed) = registry.sync(&desired); + // Consumed here, so one overflow buys one forced pass rather than making + // every later reconcile re-register the whole tree. + let force = state.watch_resubscribe.swap(false, Ordering::SeqCst); + let (added, removed) = registry.sync(&desired, force); if !added.is_empty() || removed > 0 { eprintln!( "[trace] watcher subscriptions: {total} directories \ @@ -2789,12 +2862,17 @@ fn sweep_removed_files( } } -/// Subscribe to a directory that has just appeared, and to anything already -/// inside it. +/// Index a directory that has just appeared and anything already inside it, +/// subscribing to it as well on a per-directory backend. /// +/// Two separate reasons to be here, and they apply on different platforms. /// Non-recursive subscriptions are not extended by notify — it only auto-adds -/// watches beneath a watch that was registered as recursive — so a new -/// directory has to be picked up here or its contents are invisible. +/// watches beneath a watch that was registered as recursive — so on Linux a new +/// directory has to be subscribed here or its contents are invisible. And on +/// every backend, a directory that arrives already populated (a `mv` from +/// outside the root, a checkout, an unpacked archive) reports itself and +/// nothing else: the kernel does not enumerate what moved in. Both leave files +/// that appear in no walk and no event. /// /// Files that landed between the directory's creation and its subscription /// would be missed by definition, so the same pass indexes what it finds. @@ -2834,7 +2912,12 @@ fn watch_new_subtree(state: &Arc, root: &Path, dir: &Path) { let mut found_ignore_rules = false; 'descend: while !level.is_empty() { - { + // Subscribing is per-directory work. On a recursive backend the root's + // one subscription already covers everything below it, and taking a + // watch per directory there would be the exhaustion this whole pass + // exists to avoid — so only the enumeration below runs on those + // platforms. + if PER_DIRECTORY_WATCHES { let mut registry = state.watch_registry.lock().unwrap(); let Some(registry) = registry.as_mut() else { return; @@ -2950,6 +3033,21 @@ fn schedule_ignore_rules_refresh(state: Arc, root: PathBuf) { thread::spawn(move || { loop { if state.ignore_rules_dirty.swap(false, Ordering::SeqCst) { + // Wait out a build first. `background_index_build` publishes its + // matcher — and so reaches here — while it is still only + // part-way through Phase 2, and it holds `snapshot_gate` for + // none of that. The refresh would take the gate uncontended, + // replace `file_stamps` wholesale from its own walk, and then + // have the build overwrite them again from a walk that predates + // the new rules: an index and a stamp map describing two + // different trees, with no scan left to notice. + // + // A wait rather than a lock, so it cannot deadlock against the + // build; and nothing is lost by waiting, because the build is + // still walking the tree the refresh would walk. + while state.indexing.load(Ordering::SeqCst) { + thread::sleep(Duration::from_millis(200)); + } // The stale refresh walks the tree anyway and republishes the // matcher from that walk, so the reload costs one traversal // rather than a rebuild plus a re-scan. @@ -3348,10 +3446,21 @@ fn handle_fs_event(state: &Arc, root: &Path, event: &Event) { event.kind, EventKind::Create(_) | EventKind::Modify(notify::event::ModifyKind::Name(_)) ); - if PER_DIRECTORY_WATCHES && introduces_dir { - // With non-recursive subscriptions notify will not extend - // the watch set for us, so a directory that just appeared — - // and anything already inside it — has to be picked up here. + if introduces_dir { + // A directory that just appeared can already be full — a + // `mv` of a populated tree from outside the root, a + // checkout, an unpacked archive — and nothing reports the + // contents it arrived with. + // + // On a per-directory backend that is because notify does + // not extend a non-recursive watch set for us. On a + // recursive backend it is the kernel's own doing: both + // `ReadDirectoryChangesW` and FSEvents report a moved-in + // tree as one event for the directory and say nothing about + // what is inside it. Either way those files are in no walk + // and no event, and stay unsearchable until the hourly + // reconcile — so the enumeration runs on every platform and + // only the subscribing part stays per-directory. watch_new_subtree(state, root, path); } continue; @@ -4471,7 +4580,12 @@ fn refresh_stale_locked( *newly_watched = publish_ignore_matcher( state, root, - ignore_sources_of(root, &walk.gitignore_files, &walk.ignore_files), + ignore_sources_of( + root, + &walk.gitignore_files, + &walk.ignore_files, + state.no_require_git, + ), || build_stale_matcher(state, root, &walk), ); @@ -4726,7 +4840,12 @@ fn bootstrap_index_build(state: &Arc, root: &Path, index_dir: &Path newly_watched = publish_ignore_matcher( state, root, - ignore_sources_of(root, &outcome.gitignore_files, &outcome.ignore_files), + ignore_sources_of( + root, + &outcome.gitignore_files, + &outcome.ignore_files, + state.no_require_git, + ), || { tgrep_core::walker::build_gitignore_matcher_from_files( root, @@ -4842,7 +4961,12 @@ fn background_index_build(state: &Arc, root: &Path, index_dir: &Pat newly_watched = publish_ignore_matcher( state, root, - ignore_sources_of(root, &walk.gitignore_files, &walk.ignore_files), + ignore_sources_of( + root, + &walk.gitignore_files, + &walk.ignore_files, + state.no_require_git, + ), || { walker::build_gitignore_matcher_from_files( root, @@ -5618,6 +5742,7 @@ mod tests { gitignore_pending: std::sync::atomic::AtomicBool::new(true), ignore_rules_dirty: std::sync::atomic::AtomicBool::new(false), ignore_refresh_scheduled: std::sync::atomic::AtomicBool::new(false), + watch_resubscribe: std::sync::atomic::AtomicBool::new(false), ignore_sources: RwLock::new(Vec::new()), ignore_source_stamps: RwLock::new(IgnoreStamps::new()), reindex_lock: Mutex::new(()), @@ -5887,7 +6012,7 @@ mod tests { ); // `sync`, by contrast, is authoritative over the whole tree. - let (added, removed) = registry.sync(&[c.clone()].into_iter().collect()); + let (added, removed) = registry.sync(&[c.clone()].into_iter().collect(), false); assert_eq!((added.len(), removed), (0, 2)); assert_eq!(registry.watched, [c].into_iter().collect()); } @@ -7482,7 +7607,7 @@ mod tests { ] .into_iter() .collect(); - let (added, _removed) = registry.sync(&desired); + let (added, _removed) = registry.sync(&desired, false); assert_eq!(added.len(), 4); let depths: Vec = added.iter().map(|d| d.components().count()).collect(); @@ -7493,6 +7618,180 @@ mod tests { assert!(registry.watched.contains(&deep)); } + /// After an overflow the registry's records are not evidence. A directory + /// removal that was dropped leaves the kernel's descriptor released and the + /// entry here intact, and an ordinary sync skips anything it already + /// believes it watches — so the entry never gets corrected and a path + /// recreated there reports nothing for the life of the server. + #[test] + fn a_forced_sync_retires_a_subscription_the_kernel_already_dropped() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let gone = root.join("gone"); + std::fs::create_dir(&gone).unwrap(); + + let watcher = notify::recommended_watcher(|_: notify::Result| {}).unwrap(); + let mut registry = WatchRegistry { + watcher, + root: root.clone(), + watched: std::collections::HashSet::new(), + }; + assert_eq!(registry.add_all(std::slice::from_ref(&gone)).len(), 1); + + // The removal event that would have called `forget` was one of the + // ones the overflow ate. + std::fs::remove_dir(&gone).unwrap(); + let desired: std::collections::HashSet = std::iter::once(gone.clone()).collect(); + + registry.sync(&desired, false); + assert!( + registry.is_watched(&gone), + "the fixture must reproduce the poisoned entry, or it proves nothing" + ); + + registry.sync(&desired, true); + assert!( + !registry.is_watched(&gone), + "a forced sync must re-issue the subscription and drop the entry \ + when it fails, rather than trusting a descriptor that is gone" + ); + } + + /// A directory can arrive already full — a `mv` from outside the root, a + /// checkout, an unpacked archive — and nothing reports what came with it. + /// Linux needs the descent to subscribe; Windows and macOS get the whole + /// move as a single event for the directory and no per-file events at all. + /// The indexing half therefore has to run everywhere, not only where the + /// subscribing half does. + #[test] + fn a_populated_directory_that_arrives_whole_is_indexed_on_every_platform() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + state.gitignore_pending.store(false, Ordering::SeqCst); + + let watcher = notify::recommended_watcher(|_: notify::Result| {}).unwrap(); + *state.watch_registry.lock().unwrap() = Some(WatchRegistry { + watcher, + root: root.clone(), + watched: std::iter::once(root.clone()).collect(), + }); + + // Built somewhere else and moved in, so no event ever described its + // contents. + let staging = tmp.path().join("staging"); + std::fs::create_dir_all(staging.join("deep")).unwrap(); + std::fs::write(staging.join("top.rs"), "fn top() {}\n").unwrap(); + std::fs::write(staging.join("deep").join("low.rs"), "fn low() {}\n").unwrap(); + let moved = root.join("moved"); + std::fs::rename(&staging, &moved).unwrap(); + + handle_fs_event( + &state, + &root, + &Event { + kind: EventKind::Create(notify::event::CreateKind::Any), + paths: vec![moved.clone()], + attrs: Default::default(), + }, + ); + + let index = state.index.read().unwrap(); + assert!( + index.live.has_path("moved/top.rs"), + "a file that moved in with its directory must be indexed" + ); + assert!( + index.live.has_path("moved/deep/low.rs"), + "and so must one further down" + ); + } + + /// Parent-directory rule files and the repository's `info/exclude` are + /// enforced by the published matcher but sit outside the walk that finds + /// everything else, so nothing would notice one being deleted — and rules + /// with no source left keep hiding a subtree from the index. + #[test] + fn ignore_sources_include_the_rules_that_live_outside_the_tree() { + let tmp = TempDir::new().unwrap(); + let repo = tmp.path().join("repo"); + let root = repo.join("sub"); + std::fs::create_dir_all(root.join("nested")).unwrap(); + std::fs::create_dir_all(repo.join(".git").join("info")).unwrap(); + + std::fs::write(repo.join(".git").join("info").join("exclude"), "*.tmp\n").unwrap(); + std::fs::write(repo.join(".gitignore"), "build/\n").unwrap(); + std::fs::write(root.join(".gitignore"), "target/\n").unwrap(); + + let sources = ignore_sources_of(&root, &[root.join(".gitignore")], &[], false); + assert!( + sources.contains(&repo.join(".gitignore")), + "a parent .gitignore the matcher enforces must be tracked: {sources:?}" + ); + assert!( + sources.contains(&repo.join(".git").join("info").join("exclude")), + "so must the repository's own exclude file: {sources:?}" + ); + // Every listed source must exist, or the vanished-source check treats a + // path that was never there as one that just disappeared and schedules + // a refresh on every single scan. + for source in &sources { + assert!( + source.is_file(), + "listed a source that is not there: {source:?}" + ); + } + + // And they are digested, rather than silently dropped for having no + // path relative to the served root. + let stamps = ignore_stamps_of(&root, &sources); + assert_eq!( + stamps.len(), + sources.len(), + "every source must be stamped: {sources:?} -> {stamps:?}" + ); + } + + /// `background_index_build` publishes its matcher while it is still + /// part-way through Phase 2 and holds `snapshot_gate` for none of that. A + /// refresh scheduled from that publish would take the gate uncontended and + /// replace `file_stamps` from its own walk, only for the build to overwrite + /// them from a walk that predates the new rules — an index and a stamp map + /// describing two different trees, with no scan left to notice. + #[test] + fn an_ignore_refresh_waits_for_a_running_build() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let index_dir = root.join(".tgrep"); + std::fs::write(root.join("a.rs"), "fn a() {}\n").unwrap(); + let state = test_server_state(&root, &index_dir); + + state.indexing.store(true, Ordering::SeqCst); + state.gitignore_pending.store(true, Ordering::SeqCst); + state.ignore_rules_dirty.store(true, Ordering::SeqCst); + schedule_ignore_rules_refresh(Arc::clone(&state), root.clone()); + + // Publishing a matcher is the first thing the refresh does that anyone + // outside it can see, so `gitignore_pending` still being set is proof + // that it has not started. + thread::sleep(Duration::from_millis(400)); + assert!( + state.gitignore_pending.load(Ordering::SeqCst), + "a refresh must not run against an index that is still being built" + ); + + state.indexing.store(false, Ordering::SeqCst); + let deadline = Instant::now() + Duration::from_secs(30); + while state.gitignore_pending.load(Ordering::SeqCst) && Instant::now() < deadline { + thread::sleep(Duration::from_millis(20)); + } + assert!( + !state.gitignore_pending.load(Ordering::SeqCst), + "and must run once the build is done, rather than waiting forever" + ); + } + /// Stamps describe what the index holds. A file written while the build /// ran is read by the metadata walk that produces them but not by the /// content walk that fed the index, so its stamp says "current" about diff --git a/tgrep-core/src/git_index.rs b/tgrep-core/src/git_index.rs index f8fbdac..6966de8 100644 --- a/tgrep-core/src/git_index.rs +++ b/tgrep-core/src/git_index.rs @@ -84,7 +84,7 @@ fn normalise(path: &str) -> String { /// /// `.git` is usually a directory, but is a file holding `gitdir: ` in a /// linked worktree or a submodule. -fn git_dir(repo_root: &Path) -> Option { +pub(crate) fn git_dir(repo_root: &Path) -> Option { let dot_git = repo_root.join(".git"); let meta = std::fs::metadata(&dot_git).ok()?; if meta.is_dir() { diff --git a/tgrep-core/src/gitignore.rs b/tgrep-core/src/gitignore.rs index e32bb34..e1d18a9 100644 --- a/tgrep-core/src/gitignore.rs +++ b/tgrep-core/src/gitignore.rs @@ -10,7 +10,7 @@ use crate::walker::walker_thread_count; use ignore::WalkBuilder; -use std::path::Path; +use std::path::{Path, PathBuf}; pub const P4IGNORE_FILENAME: &str = "p4ignore.ini"; @@ -94,6 +94,19 @@ pub struct IgnoreMatcher { /// Repository-local `.git/info/exclude`, below global rules in precedence. repo_exclude: Option<(String, Gitignore)>, global: Gitignore, + /// The directory the relative paths handed to [`Self::is_ignored`] are + /// relative to, needed to rebuild the absolute path + /// [`CaseInsensitiveIgnore`] matches against. + root: std::path::PathBuf, + /// Git's `core.ignorecase` narrowing, when the repository asks for it. + /// + /// The indexing walk applies this as a `filter_entry` alongside the + /// case-sensitive rules, so a matcher without it answers a different + /// question than the walk did. That gap is not academic: on a Windows + /// enlistment it left the watcher subscribing to, and indexing, a 13.4 GiB + /// build artifact the walk had already excluded — and every event under it + /// re-added a file the next stale check then evicted. + ignorecase: Option, } impl IgnoreMatcher { @@ -117,9 +130,10 @@ impl IgnoreMatcher { nested: Vec<(String, IgnoreKind, Gitignore)>, global: Gitignore, ) -> Option { - Self::with_all_sources(local, false, nested, Vec::new(), None, global) + Self::with_all_sources(local, false, nested, Vec::new(), None, global, None) } + #[allow(clippy::too_many_arguments)] fn with_all_sources( local: Gitignore, local_is_filter: bool, @@ -127,6 +141,7 @@ impl IgnoreMatcher { ancestors: Vec<(String, IgnoreKind, Gitignore)>, repo_exclude: Option<(String, Gitignore)>, global: Gitignore, + ignorecase: Option, ) -> Option { let mut nested: Vec = nested .into_iter() @@ -164,14 +179,20 @@ impl IgnoreMatcher { || !nested.is_empty() || !ancestors.is_empty() || repo_exclude.is_some() - || !global.is_empty()) + || !global.is_empty() + || ignorecase.is_some()) .then_some(Self { + // `GitignoreBuilder::new(root)` records `root`, and every caller + // builds `local` from the served root, so this is that root + // without threading it through three more signatures. + root: local.path().to_path_buf(), local, local_is_filter, nested, ancestors, repo_exclude, global, + ignorecase, }) } @@ -271,11 +292,21 @@ impl IgnoreMatcher { if standard_decision == Some(true) { return true; } - self.local_is_filter + if self.local_is_filter && self .local .matched_path_or_any_parents(path, is_dir) .is_ignore() + { + return true; + } + // Applied last, and to whitelisted paths too, because that is where the + // indexing walk applies it: a `filter_entry` rejection is not undone by + // a whitelist rule. Both passes must agree on what the tree contains, + // or the watcher indexes a file the stale check immediately evicts. + self.ignorecase + .as_ref() + .is_some_and(|ignorecase| ignorecase.excludes(&self.root.join(rel), is_dir)) } } @@ -460,8 +491,8 @@ impl CaseInsensitiveIgnore { if use_gitignore { let _ = builder.add(repo_root.join(GITIGNORE_FILENAME)); } - if use_exclude { - let _ = builder.add(repo_root.join(".git").join("info").join("exclude")); + if use_exclude && let Some(exclude) = repo_exclude_path(&repo_root) { + let _ = builder.add(exclude); } let matcher = builder.build().ok()?; if matcher.is_empty() { @@ -531,6 +562,62 @@ fn git_repo_root(root: &Path) -> Option<&Path> { root.ancestors().find(|dir| dir.join(".git").exists()) } +/// The `info/exclude` file whose rules apply to `root`, if there is one. +/// +/// Not `.git/info/exclude`. In a linked worktree or a submodule `.git` is a +/// file holding a `gitdir:` pointer, and that directory in turn holds a +/// `commondir` naming the repository every worktree shares — which is where the +/// one `info/exclude` lives. `WalkBuilder` resolves that chain, so a matcher +/// that stopped at the literal path enforced different rules than the walk in +/// exactly the layouts where the two differ, and the watcher would index a file +/// the next stale check evicts. +pub fn repo_exclude_path(root: &Path) -> Option { + let git_dir = crate::git_index::git_dir(git_repo_root(root)?)?; + let common = match std::fs::read_to_string(git_dir.join("commondir")) { + Ok(target) => { + let target = target.trim(); + let path = Path::new(target); + if target.is_empty() { + git_dir + } else if path.is_absolute() { + path.to_path_buf() + } else { + git_dir.join(path) + } + } + Err(_) => git_dir, + }; + let path = common.join("info").join("exclude"); + path.is_file().then_some(path) +} + +/// The parent-directory `.ignore` / `.gitignore` files that apply to `root`, +/// closest directory first, paired with the directory that anchors them. +/// +/// `WalkBuilder` applies `.ignore` files from every ancestor. Its git boundary +/// is independent: with the default `require_git`, ancestor `.gitignore` files +/// stop after the nearest repository root; with `--no-require-git` they +/// continue to the filesystem root. +pub fn ancestor_ignore_paths(root: &Path, no_require_git: bool) -> Vec<(PathBuf, IgnoreKind)> { + let repo_root = git_repo_root(root); + let mut found = Vec::new(); + for dir in root.ancestors().skip(1) { + for (kind, path, enabled) in [ + (IgnoreKind::DotIgnore, dir.join(DOT_IGNORE_FILENAME), true), + ( + IgnoreKind::GitIgnore, + dir.join(GITIGNORE_FILENAME), + no_require_git || repo_root.is_some_and(|repo| dir.starts_with(repo)), + ), + ] { + if enabled && path.is_file() { + found.push((path, kind)); + } + } + } + found +} + /// `.gitignore` and `.git/info/exclude` are **git-gated** to match the indexing /// walk (`WalkBuilder`'s `require_git` default): they apply only when `root` is /// inside a git repository, detected by scanning `root` and its ancestors for a @@ -602,41 +689,27 @@ pub fn matcher_from_ignore_paths_with_options( } } - // WalkBuilder applies `.ignore` files from every ancestor. Its git boundary - // is independent: with the default `require_git`, ancestor `.gitignore` - // files stop after the nearest repository root; with `--no-require-git` - // they continue to the filesystem root. + // WalkBuilder applies `.ignore` files from every ancestor, with its own + // git boundary — see [`ancestor_ignore_paths`]. let mut ancestors = Vec::new(); - for dir in root.ancestors().skip(1) { + for (path, kind) in ancestor_ignore_paths(root, no_require_git) { + let Some(dir) = path.parent() else { + continue; + }; let prefix = root .strip_prefix(dir) .unwrap_or(root) .to_string_lossy() .replace('\\', "/"); - for (kind, path, enabled) in [ - (IgnoreKind::DotIgnore, dir.join(DOT_IGNORE_FILENAME), true), - ( - IgnoreKind::GitIgnore, - dir.join(GITIGNORE_FILENAME), - no_require_git || repo_root.is_some_and(|repo| dir.starts_with(repo)), - ), - ] { - if !enabled || !path.is_file() { - continue; - } - let mut builder = GitignoreBuilder::new(dir); - let _ = builder.add(&path); - if let Ok(matcher) = builder.build() { - ancestors.push((prefix.clone(), kind, matcher)); - } + let mut builder = GitignoreBuilder::new(dir); + let _ = builder.add(&path); + if let Ok(matcher) = builder.build() { + ancestors.push((prefix, kind, matcher)); } } let repo_exclude = repo_root.and_then(|repo| { - let path = repo.join(".git").join("info").join("exclude"); - if !path.is_file() { - return None; - } + let path = repo_exclude_path(root)?; let mut builder = GitignoreBuilder::new(repo); let _ = builder.add(&path); let matcher = builder.build().ok()?; @@ -654,7 +727,21 @@ pub fn matcher_from_ignore_paths_with_options( } else { GitignoreBuilder::new(root).build().ok()? }; - IgnoreMatcher::with_all_sources(local, true, nested, ancestors, repo_exclude, global) + // The same narrowing `walker::walk_dir` and `walk_file_metadata` apply as a + // `filter_entry`. Serving takes no `--no-ignore-vcs` / `--no-ignore-exclude` + // / `--no-ignore-parent`, so the flags the walk was built with are the + // defaults; `no_ignore` is handled by the caller, which does not build a + // matcher at all in that case. + let ignorecase = CaseInsensitiveIgnore::new(root, true, true, true); + IgnoreMatcher::with_all_sources( + local, + true, + nested, + ancestors, + repo_exclude, + global, + ignorecase, + ) } /// Convenience wrapper for callers that only have `.gitignore` paths. @@ -886,6 +973,75 @@ mod tests { assert!(build_matcher(tmp.path()).is_none()); } + #[test] + fn the_repository_exclude_is_found_through_a_worktree_pointer() { + // In a linked worktree `.git` is a file naming the worktree's own git + // directory, which in turn names the repository every worktree shares + // — and that is where the one `info/exclude` lives. `WalkBuilder` + // resolves the whole chain, so a matcher that stopped at the literal + // `.git/info/exclude` enforced different rules than the walk did, and + // the watcher indexed files the next stale check evicted. + let tmp = tempfile::tempdir().unwrap(); + let common = tmp.path().join("main").join(".git"); + std::fs::create_dir_all(common.join("info")).unwrap(); + std::fs::write(common.join("info").join("exclude"), "*.secret\n").unwrap(); + + let worktree_git = common.join("worktrees").join("wt"); + std::fs::create_dir_all(&worktree_git).unwrap(); + std::fs::write( + worktree_git.join("commondir"), + format!("{}\n", common.display()), + ) + .unwrap(); + + let worktree = tmp.path().join("wt"); + std::fs::create_dir_all(&worktree).unwrap(); + std::fs::write( + worktree.join(".git"), + format!("gitdir: {}\n", worktree_git.display()), + ) + .unwrap(); + + assert_eq!( + repo_exclude_path(&worktree).as_deref(), + Some(common.join("info").join("exclude").as_path()), + "the exclude file has to be reached through the pointer chain" + ); + + let matcher = matcher_from_ignore_paths(&worktree, &[], &[]) + .expect("the exclude file supplies rules"); + assert!(matcher.is_ignored(Path::new("keys.secret"), false)); + assert!(!matcher.is_ignored(Path::new("main.rs"), false)); + } + + #[test] + fn a_relative_commondir_resolves_against_the_worktree_git_dir() { + let tmp = tempfile::tempdir().unwrap(); + let common = tmp.path().join(".git"); + std::fs::create_dir_all(common.join("info")).unwrap(); + std::fs::write(common.join("info").join("exclude"), "*.bin\n").unwrap(); + + // What git actually writes: a path relative to the worktree's git dir. + let worktree_git = common.join("worktrees").join("wt"); + std::fs::create_dir_all(&worktree_git).unwrap(); + std::fs::write(worktree_git.join("commondir"), "../..\n").unwrap(); + + let worktree = tmp.path().join("wt"); + std::fs::create_dir_all(&worktree).unwrap(); + std::fs::write( + worktree.join(".git"), + format!("gitdir: {}\n", worktree_git.display()), + ) + .unwrap(); + + let found = repo_exclude_path(&worktree).expect("resolved through commondir"); + assert!( + std::fs::canonicalize(&found).unwrap() + == std::fs::canonicalize(common.join("info").join("exclude")).unwrap(), + "got {found:?}" + ); + } + #[test] fn local_whitelist_overrides_global_ignore() { use ignore::gitignore::GitignoreBuilder; diff --git a/tgrep-core/src/walker.rs b/tgrep-core/src/walker.rs index 5cc3a1b..98d9554 100644 --- a/tgrep-core/src/walker.rs +++ b/tgrep-core/src/walker.rs @@ -1417,6 +1417,61 @@ mod tests { ); } + /// The watcher cannot walk per event, so it asks the same question as a + /// point query. If the two disagree the watcher subscribes to and indexes a + /// tree the walk excluded, and the next stale check evicts every file it + /// added — on the enlistment this came from, a 13.4 GiB build artifact + /// making up 71% of the corpus, re-read and re-evicted on every pass. + #[test] + fn the_point_query_matcher_hides_exactly_what_the_walk_hides() { + let dir = ignorecase_fixture(true, &["src/main.rs", "src/Kept.TXT"]); + let root = dir.path(); + let matcher = crate::gitignore::matcher_from_ignore_paths( + root, + std::slice::from_ref(&root.join(".gitignore")), + &[], + ) + .expect("the fixture has rules"); + + assert!( + matcher.is_ignored(Path::new("qlogs"), true), + "a directory the walk prunes must not be subscribed to" + ); + assert!( + matcher.is_ignored(Path::new("qlogs/artifact.rs"), false), + "nor may a file inside it be indexed" + ); + assert!( + matcher.is_ignored(Path::new("src/Gone.TXT"), false), + "an untracked file the rule matches once case is ignored" + ); + // The tracked-file exemption comes with it, or the watcher would drop + // events for files git never hides. + assert!( + !matcher.is_ignored(Path::new("src/Kept.TXT"), false), + "a tracked file must stay visible" + ); + assert!(!matcher.is_ignored(Path::new("src/main.rs"), false)); + } + + #[test] + fn the_point_query_matcher_follows_the_case_sensitivity_gate() { + // The other direction, which is the one that loses files: a repository + // that distinguishes case must not have anything hidden from it. + let dir = ignorecase_fixture(false, &["src/main.rs", "src/Kept.TXT"]); + let root = dir.path(); + let matcher = crate::gitignore::matcher_from_ignore_paths( + root, + std::slice::from_ref(&root.join(".gitignore")), + &[], + ) + .expect("the fixture has rules"); + + assert!(!matcher.is_ignored(Path::new("qlogs"), true)); + assert!(!matcher.is_ignored(Path::new("qlogs/artifact.rs"), false)); + assert!(!matcher.is_ignored(Path::new("src/Gone.TXT"), false)); + } + #[test] fn no_ignore_turns_the_whole_thing_off() { let dir = ignorecase_fixture(true, &["src/main.rs"]); From a784a01c763ec3265e69dc11ccd4c64112a0793a Mon Sep 17 00:00:00 2001 From: Shengyu Fu Date: Fri, 28 Aug 2026 00:24:37 -0700 Subject: [PATCH 28/28] Round 18: stop reading I/O failures, and a frozen cache, as facts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. A transient stat failure was read as a deletion. `Path::exists` and `Path::is_file` fold every metadata error into `false`, so one `EACCES` or a Windows sharing violation from a build holding a file open made the watcher tombstone content that was still perfectly valid — and it did so before `reindex_file`, whose whole policy is to preserve entries through exactly those errors, was ever called. The event path now classifies its stat through the same `proves_ineligible` contract the recovery sweep answers to, with an explicit "this stat proves nothing" case that leaves the index alone. 2. The tracked-file exemption froze at the first path it was asked about. That was sound while the matcher was a walk-local object, but round 17 put it inside the watcher's long-lived matcher, and there a `git add -f` or a `git rm --cached` rewrites only `.git/index` — hidden, so no ignore source changes and nothing republishes. The exemption kept answering from the set as it stood at startup and diverged from a fresh walk until the hourly reconcile. It is now reloaded when the index it was read from changes, at the cost of one stat on the paths that reach it, which for most repositories is none. 3. A rule file symlinked into a directory the rules hide was unobservable. `handle_fs_event` already recognises an event naming a symlinked source's target rather than a name rules usually go by, and its comment claimed such targets are watched "because only those are watched". On a per-directory backend they were not: nothing subscribes to `build/` when `build/` is ignored, so editing the file the matcher was built from produced no event at all. Their directory is now subscribed to explicitly — the file, not its subtree, and only for targets inside the root. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: bcc85b55-e821-447e-b9a9-144b78829ae4 --- tgrep-cli/src/serve.rs | 223 +++++++++++++++++++++++++++++++++++- tgrep-core/src/git_index.rs | 8 +- tgrep-core/src/gitignore.rs | 75 ++++++++++-- tgrep-core/src/walker.rs | 34 ++++++ 4 files changed, 321 insertions(+), 19 deletions(-) diff --git a/tgrep-cli/src/serve.rs b/tgrep-cli/src/serve.rs index ee70f04..a2272b0 100644 --- a/tgrep-cli/src/serve.rs +++ b/tgrep-cli/src/serve.rs @@ -2378,6 +2378,51 @@ fn watchable_dirs( found } +/// The directories that must be subscribed to for the ignore sources +/// themselves to be observable, beyond the ones the rules allow. +/// +/// A `.gitignore` symlinked to `build/shared-rules` contributes the *target's* +/// contents, and [`handle_fs_event`] already recognises an event naming that +/// target rather than a name rules usually go by. But only if one arrives: on a +/// per-directory backend nothing subscribes to `build/` when the rules hide it, +/// so the edit that changes what the matcher enforces produces no event at all, +/// and the matcher stays stale until the hourly reconcile — the one case where +/// the source of the rules is invisible to the rules' own watcher. +/// +/// One watch on the target's own directory, not its subtree: this is about +/// seeing a single file that the matcher was built from, not about indexing +/// anything under it. `should_skip_watcher_path` still discards everything else +/// delivered from there, and the target itself is matched by path against the +/// recorded stamps before any of that filtering runs. +/// +/// Targets outside `root` are deliberately not covered. Watching them would +/// mean subscribing outside the tree the server was asked to serve, and the +/// periodic reconcile remains the backstop there. +fn ignore_target_dirs(root: &Path, sources: &[PathBuf]) -> std::collections::HashSet { + let mut dirs = std::collections::HashSet::new(); + let Ok(canonical_root) = std::fs::canonicalize(root) else { + return dirs; + }; + for source in sources { + if !std::fs::symlink_metadata(source).is_ok_and(|m| m.file_type().is_symlink()) { + continue; + } + let Ok(target) = std::fs::canonicalize(source) else { + continue; + }; + let Ok(rel) = target.strip_prefix(&canonical_root) else { + continue; + }; + // Re-anchored on `root` as given rather than kept canonical: the + // registry compares paths literally, and a `\\?\` or `/private` prefix + // would register a second subscription for a directory already watched. + if let Some(parent) = root.join(rel).parent() { + dirs.insert(parent.to_path_buf()); + } + } + dirs +} + /// Recompute the watcher's subscriptions against the ignore rules in force. /// /// Called when the watcher starts and every time the ignore matcher is @@ -2407,10 +2452,14 @@ fn sync_watch_registrations(state: &ServerState, root: &Path) -> (Vec, }; let start = Instant::now(); - let desired = { + let mut desired = { let gitignore = state.gitignore.read().unwrap(); watchable_dirs(root, root, &state.exclude_dirs, gitignore.as_ref()) }; + if !state.no_ignore { + let sources = state.ignore_sources.read().unwrap(); + desired.extend(ignore_target_dirs(root, &sources)); + } let total = desired.len(); // Consumed here, so one overflow buys one forced pass rather than making // every later reconcile re-register the whole tree. @@ -3305,9 +3354,11 @@ fn handle_fs_event(state: &Arc, root: &Path, event: &Event) { // nothing whose name does. Recognising the paths that were read, and not // just the names rules usually go by, is what closes that. // - // Only targets inside `root` can appear here, because only those are - // watched; for one outside, no event arrives at all and the periodic - // reconcile remains the backstop. + // Only targets inside `root` can appear here, and [`ignore_target_dirs`] is + // what makes them observable: their directory is subscribed to even when the + // rules hide it, precisely so this lookup has an event to run against. For a + // target outside the root none arrives and the periodic reconcile remains + // the backstop. let ignore_rules_changed = !state.no_ignore && { let stamps = state.ignore_source_stamps.read().unwrap(); event.paths.iter().any(|path| { @@ -3381,7 +3432,22 @@ fn handle_fs_event(state: &Arc, root: &Path, event: &Event) { continue; } - let is_remove = matches!(event.kind, EventKind::Remove(_)) || !path.exists(); + // Classified through the same contract the sweep uses, from one stat. + // `Path::exists` and `Path::is_file` map every metadata error to + // `false`, so a file held open by a build, a Windows sharing violation, + // or a momentary `EACCES` used to read as "gone" and "not a regular + // file" respectively — and both branches below then evicted content + // that was still perfectly valid. `reindex_file` deliberately preserves + // entries through exactly those failures, but it never got the chance: + // the drop happens here, before it is ever called. + let target = classify_event_target(&std::fs::metadata(path)); + if target == EventTarget::Unknown { + // Unreadable right now is not proof of anything. Leave what is + // indexed alone; the stale path keeps such files and retries them. + continue; + } + + let is_remove = matches!(event.kind, EventKind::Remove(_)) || target == EventTarget::Gone; if is_remove { // A watched directory that disappears takes its descriptor with @@ -3429,7 +3495,7 @@ fn handle_fs_event(state: &Arc, root: &Path, event: &Event) { // `reindex_file` below rather than here — deliberately: that is where // it is recognised as ineligible and any content indexed under that // path before it became a link is dropped. - if !path.is_file() { + if target != EventTarget::Regular { // `is_real_dir` rather than `is_dir`: the latter follows symlinks, // and a link to a directory is not something the walker descends // into, so subscribing to and indexing its target would pull in a @@ -3539,6 +3605,44 @@ fn drop_indexed_file(state: &ServerState, rel_path: &str, reason: &str) { } } +/// What an event's stat result says about the path it named. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum EventTarget { + /// A regular file: index it. + Regular, + /// Something that exists but is not a regular file — a directory, a fifo, a + /// socket, a device. Whatever was indexed under this path has to go. + NotRegular, + /// Proven not to be there: absent, unreachable except through a symlink, or + /// behind a component that is not a directory. + Gone, + /// Exists or not, this stat cannot say. Nothing may be concluded from it. + Unknown, +} + +/// Classify the target of an event from a stat of its path. +/// +/// The `Unknown` case is the point of this. `Path::exists` and `Path::is_file` +/// fold every error into `false`, which turns "a build has this file open" and +/// "the directory was briefly unreadable" into "it is gone" — and the watcher +/// then evicts live content on the strength of it. [`proves_ineligible`] is the +/// contract that separates the two, and it is the same one the recovery sweep +/// and [`reindex_file`] answer to, so all three agree about what an I/O failure +/// is allowed to mean. +/// +/// The stat follows symlinks, which is deliberate: a link to a regular file is +/// classified `Regular` here and refused by `open_within_root` in +/// [`reindex_file`], which is where content indexed under a path that has since +/// become a link is dropped. +fn classify_event_target(meta: &std::io::Result) -> EventTarget { + match meta { + Ok(meta) if meta.is_file() => EventTarget::Regular, + Ok(_) => EventTarget::NotRegular, + Err(error) if proves_ineligible(error) => EventTarget::Gone, + Err(_) => EventTarget::Unknown, + } +} + /// Whether a failure to open a path establishes that it does not belong in the /// index, as opposed to merely being unreadable right now. /// @@ -6219,6 +6323,113 @@ mod tests { ); } + /// A transient stat failure is not a deletion. `Path::exists` said it was, + /// which meant one `EACCES` — or a Windows sharing violation from a build + /// holding the file open — tombstoned content that was still valid, and + /// bypassed the preservation policy `reindex_file` applies to exactly those + /// errors. + #[test] + fn an_unreadable_path_is_not_treated_as_a_deletion() { + use std::io::{Error, ErrorKind}; + + let unreadable: std::io::Result = + Err(Error::from(ErrorKind::PermissionDenied)); + assert_eq!( + classify_event_target(&unreadable), + EventTarget::Unknown, + "a locked or unreadable file must leave the index alone" + ); + + // Windows reports a file opened without FILE_SHARE_* this way, and it + // is the single most common way a stat fails on a live repository. + let sharing: std::io::Result = Err(Error::from_raw_os_error(32)); + assert_eq!(classify_event_target(&sharing), EventTarget::Unknown); + + // The other direction still has to work, or a real deletion is never + // applied. + let absent: std::io::Result = Err(Error::from(ErrorKind::NotFound)); + assert_eq!(classify_event_target(&absent), EventTarget::Gone); + + let tmp = TempDir::new().unwrap(); + let file = tmp.path().join("real.rs"); + std::fs::write(&file, "fn main() {}").unwrap(); + assert_eq!( + classify_event_target(&std::fs::metadata(&file)), + EventTarget::Regular + ); + assert_eq!( + classify_event_target(&std::fs::metadata(tmp.path())), + EventTarget::NotRegular + ); + } + + /// Create a symlink, or return `false` where the platform will not allow + /// one — an unprivileged Windows runner without Developer Mode. + fn try_symlink(target: &Path, link: &Path) -> bool { + #[cfg(unix)] + { + std::os::unix::fs::symlink(target, link).is_ok() + } + #[cfg(windows)] + { + std::os::windows::fs::symlink_file(target, link).is_ok() + } + #[cfg(not(any(unix, windows)))] + { + let _ = (target, link); + false + } + } + + /// The matcher reads a symlinked source through the link, so an edit to the + /// target changes the rules in force. `handle_fs_event` recognises an event + /// naming that target — but on a per-directory backend no event ever + /// arrived, because the directory holding it is one the rules hide and + /// nothing subscribed to it. + #[test] + fn a_rule_file_symlinked_into_a_hidden_directory_is_still_watched() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path(); + std::fs::create_dir(root.join("build")).unwrap(); + std::fs::write(root.join("build").join("shared-rules"), "target/\n").unwrap(); + if !try_symlink( + &root.join("build").join("shared-rules"), + &root.join(".gitignore"), + ) { + return; + } + + let sources = vec![root.join(".gitignore")]; + let dirs = ignore_target_dirs(root, &sources); + + assert!( + dirs.contains(&root.join("build")), + "the directory holding a rule file the matcher read must be watched \ + even when the rules hide it: {dirs:?}" + ); + } + + /// A source that is a plain file needs nothing extra, and one whose target + /// is outside the root must not pull a subscription outside the tree the + /// server was asked to serve. + #[test] + fn ordinary_and_outside_rule_files_add_no_subscriptions() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path(); + std::fs::write(root.join(".gitignore"), "target/\n").unwrap(); + assert!(ignore_target_dirs(root, &[root.join(".gitignore")]).is_empty()); + + let outside = TempDir::new().unwrap(); + std::fs::write(outside.path().join("rules"), "target/\n").unwrap(); + if !try_symlink(&outside.path().join("rules"), &root.join(".ignore")) { + return; + } + assert!( + ignore_target_dirs(root, &[root.join(".ignore")]).is_empty(), + "a target outside the root must not be subscribed to" + ); + } + #[test] fn skip_watcher_dir_applies_directory_semantics() { // A directory-only gitignore rule (`build/`) does not match the path diff --git a/tgrep-core/src/git_index.rs b/tgrep-core/src/git_index.rs index 6966de8..2b01e6e 100644 --- a/tgrep-core/src/git_index.rs +++ b/tgrep-core/src/git_index.rs @@ -144,9 +144,13 @@ pub fn ignores_case(repo_root: &Path) -> bool { /// /// Returns `None` when there is no readable index, which the caller must treat /// as "exempt nothing" only where that is the safe direction. +/// The index file a repository's tracked-file set is read from. +pub(crate) fn index_path(repo_root: &Path) -> Option { + Some(git_dir(repo_root)?.join("index")) +} + pub fn load_tracked(repo_root: &Path) -> Option { - let git_dir = git_dir(repo_root)?; - let bytes = std::fs::read(git_dir.join("index")).ok()?; + let bytes = std::fs::read(index_path(repo_root)?).ok()?; parse_index(&bytes) } diff --git a/tgrep-core/src/gitignore.rs b/tgrep-core/src/gitignore.rs index e1d18a9..173dea4 100644 --- a/tgrep-core/src/gitignore.rs +++ b/tgrep-core/src/gitignore.rs @@ -447,7 +447,36 @@ pub struct CaseInsensitiveIgnore { /// Loaded on the first path this matcher actually claims, which for most /// repositories is never. Reading it costs 163 ms and ~30 MB on a /// 299k-file repository, and nothing at all if no rule ever matches. - tracked: std::sync::OnceLock>, + /// + /// Reloaded when the index it was read from changes. A walk builds this + /// matcher, uses it and drops it, so a snapshot would do; the file watcher + /// holds one for the life of the server, and there a `git add -f` or a + /// `git rm --cached` rewrites only `.git/index` — which is hidden, so no + /// ignore source changes and nothing republishes the matcher. Frozen, the + /// exemption would keep answering from the tracked set as it stood at + /// startup, and the watcher would disagree with a fresh walk about which + /// files exist until the hourly reconcile. + tracked: std::sync::RwLock, +} + +/// The tracked-file set, together with the identity of the index it came from. +#[derive(Default)] +struct TrackedCache { + /// `None` until something is loaded; `Some` even when the load failed, so + /// an unreadable index is not retried on every path. + loaded_from: Option>, + tracked: Option, +} + +/// Modification time and length of the index, which is what identifies it. +/// +/// git replaces the index by renaming `index.lock` over it, so any rewrite +/// lands as a new mtime — the same pair git's own racy-index handling relies +/// on. Two rewrites inside one filesystem mtime tick that leave the length +/// unchanged are the gap, and the periodic reconcile is what closes it. +fn index_identity(repo_root: &Path) -> Option<(std::time::SystemTime, u64)> { + let meta = std::fs::metadata(crate::git_index::index_path(repo_root)?).ok()?; + Some((meta.modified().ok()?, meta.len())) } impl CaseInsensitiveIgnore { @@ -501,7 +530,7 @@ impl CaseInsensitiveIgnore { Some(Self { matcher, repo_root, - tracked: std::sync::OnceLock::new(), + tracked: std::sync::RwLock::new(TrackedCache::default()), }) } @@ -518,21 +547,45 @@ impl CaseInsensitiveIgnore { return false; } // Only now is the index worth reading. - let Some(tracked) = self - .tracked - .get_or_init(|| crate::git_index::load_tracked(&self.repo_root)) - else { - // No readable index means no way to tell tracked from untracked. - // Excluding could hide real source, so decline instead. + let relative = relative.to_string_lossy(); + self.tracked_hides(&relative, is_dir) + } + + /// Whether the tracked-file set leaves `relative` hidden. + /// + /// `false` when the index cannot be read: with no way to tell tracked from + /// untracked, excluding could hide real source, so it declines instead. + fn tracked_hides(&self, relative: &str, is_dir: bool) -> bool { + let identity = index_identity(&self.repo_root); + { + let cache = self.tracked.read().unwrap(); + if cache.loaded_from.as_ref() == Some(&identity) { + return Self::hides(cache.tracked.as_ref(), relative, is_dir); + } + } + let mut cache = self.tracked.write().unwrap(); + // Another thread may have reloaded it while this one waited. + if cache.loaded_from.as_ref() != Some(&identity) { + cache.tracked = crate::git_index::load_tracked(&self.repo_root); + cache.loaded_from = Some(identity); + } + Self::hides(cache.tracked.as_ref(), relative, is_dir) + } + + fn hides( + tracked: Option<&crate::git_index::TrackedFiles>, + relative: &str, + is_dir: bool, + ) -> bool { + let Some(tracked) = tracked else { return false; }; - let relative = relative.to_string_lossy(); if is_dir { // A rule matching a directory does not hide tracked files inside // it, so the walk still has to descend. - !tracked.contains_any_under(&relative) + !tracked.contains_any_under(relative) } else { - !tracked.contains(&relative) + !tracked.contains(relative) } } } diff --git a/tgrep-core/src/walker.rs b/tgrep-core/src/walker.rs index 98d9554..cc5be0b 100644 --- a/tgrep-core/src/walker.rs +++ b/tgrep-core/src/walker.rs @@ -1472,6 +1472,40 @@ mod tests { assert!(!matcher.is_ignored(Path::new("src/Gone.TXT"), false)); } + /// The exemption is answered from a cached read of `.git/index`. A walk + /// builds this matcher and drops it, so a snapshot would do — but the file + /// watcher holds one for the life of the server, and `git add -f` rewrites + /// only the index, which is hidden. No ignore source changes, nothing + /// republishes the matcher, and a frozen cache would keep hiding a file + /// that git now tracks until the hourly reconcile. + #[test] + fn the_tracked_exemption_reloads_when_the_git_index_changes() { + let dir = ignorecase_fixture(true, &["src/main.rs", "src/Kept.TXT"]); + let root = dir.path(); + let matcher = crate::gitignore::matcher_from_ignore_paths( + root, + std::slice::from_ref(&root.join(".gitignore")), + &[], + ) + .expect("the fixture has rules"); + + // Untracked, and `*.txt` matches it once case is ignored. + assert!(matcher.is_ignored(Path::new("src/Gone.TXT"), false)); + + fake_git_repo(root, true, &["src/main.rs", "src/Kept.TXT", "src/Gone.TXT"]); + + assert!( + !matcher.is_ignored(Path::new("src/Gone.TXT"), false), + "a file git now tracks must stop being hidden without rebuilding \ + the matcher" + ); + + // And back: `git rm --cached` is the same problem in reverse, where a + // stale cache keeps indexing a file the walk has started hiding. + fake_git_repo(root, true, &["src/main.rs"]); + assert!(matcher.is_ignored(Path::new("src/Kept.TXT"), false)); + } + #[test] fn no_ignore_turns_the_whole_thing_off() { let dir = ignorecase_fixture(true, &["src/main.rs"]);