diff --git a/Cargo.lock b/Cargo.lock index 575f327..e6ef78a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1046,7 +1046,7 @@ checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" [[package]] name = "tgrep-cli" -version = "1.0.2" +version = "1.0.3" dependencies = [ "anyhow", "assert_cmd", @@ -1072,7 +1072,7 @@ dependencies = [ [[package]] name = "tgrep-core" -version = "1.0.2" +version = "1.0.3" dependencies = [ "anyhow", "criterion", diff --git a/Cargo.toml b/Cargo.toml index 8e27680..426b619 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ members = ["tgrep-core", "tgrep-cli"] resolver = "2" [workspace.package] -version = "1.0.2" +version = "1.0.3" edition = "2024" license = "MIT" repository = "https://github.com/microsoft/tgrep" diff --git a/tgrep-cli/Cargo.toml b/tgrep-cli/Cargo.toml index 3dfd3ae..eeaa75f 100644 --- a/tgrep-cli/Cargo.toml +++ b/tgrep-cli/Cargo.toml @@ -31,7 +31,7 @@ memmap2 = "0.9" memchr = "2" [target.'cfg(windows)'.dependencies] -windows-sys = { version = "0.61", features = ["Win32_System_ProcessStatus", "Win32_System_SystemInformation", "Win32_System_Threading"] } +windows-sys = { version = "0.61", features = ["Win32_Storage_FileSystem", "Win32_System_ProcessStatus", "Win32_System_SystemInformation", "Win32_System_Threading"] } [target.'cfg(unix)'.dependencies] libc = "0.2" diff --git a/tgrep-cli/src/serve.rs b/tgrep-cli/src/serve.rs index 9ab5549..a2272b0 100644 --- a/tgrep-cli/src/serve.rs +++ b/tgrep-cli/src/serve.rs @@ -10,7 +10,7 @@ use std::path::{Path, PathBuf}; use std::sync::atomic::Ordering; use std::sync::{Arc, Mutex, RwLock}; use std::thread; -use std::time::{Duration, Instant}; +use std::time::{Duration, Instant, SystemTime}; use fs2::FileExt; use lru::LruCache; @@ -290,12 +290,93 @@ struct ServerState { ignore_rules_dirty: std::sync::atomic::AtomicBool, /// Ensures a burst of ignore-file events uses at most one refresh worker. ignore_refresh_scheduled: std::sync::atomic::AtomicBool, + /// Set when events were lost, cleared by the next subscription sync, which + /// then re-issues every subscription instead of trusting its own records. + /// + /// A dropped directory-removal event leaves the registry recording a watch + /// the kernel has already released. Nothing later contradicts that record: + /// a path recreated at the same location is both wanted and believed + /// watched, so every sync skips it and it never reports again. Only + /// overflow can produce that state, so only overflow pays for the repair. + watch_resubscribe: std::sync::atomic::AtomicBool, + /// The ignore files the published matcher was built from. + /// + /// A recovery scan can spot an ignore file that *arrived* during its window + /// by its mtime, but a deleted one leaves nothing behind to notice. That is + /// the more damaging direction: the matcher keeps enforcing rules whose + /// source is gone, so an entire subtree stays unsubscribed and unindexed + /// until something else forces a rebuild. Keeping the source list lets the + /// scan test for it directly, at one stat per ignore file per scan. + ignore_sources: RwLock>, + /// What the published matcher actually read, per ignore source: a hash of + /// the bytes, keyed by the file's path relative to `root`. + /// + /// A pathname is not evidence about contents, and neither is metadata. An + /// existing `.gitignore` can be replaced after the matcher was built by one + /// restored from an archive — same length, mtime preserved by the restore, + /// so the path is still a known source and nothing about its metadata has + /// moved. Neither test in [`changed_ignore_rules_in`] would fire, and the + /// subtree would be indexed under rules that were never read. Comparing + /// against what was read closes that. + /// + /// Also holds an entry for the target of any source reached through a + /// symlink, when that target is itself under `root`. Following links is + /// what the walker does, so the matcher's contents come from the target — + /// but an edit to the target does not touch the link, and no event names a + /// path whose basename is `.gitignore`. The target's own entry is what + /// lets that event be recognised. A target outside `root` is not watched at + /// all, so for those the reconcile stays the backstop. + ignore_source_stamps: RwLock, + /// Serializes the whole check-read-commit cycle in [`reindex_file`]. + /// + /// `snapshot_gate` is held for *read* by everything that indexes a file, so + /// the watcher worker and a recovery scan can be inside `reindex_file` for + /// the same path at once. Both then see the same old stamp, both read, and + /// whichever commits last wins — which is not necessarily the one that read + /// the newer content. The losing write is already consumed, so the stale + /// version survives until the next reconcile. + /// + /// Taken per file rather than per scan, so a recovery pass and the watcher + /// interleave instead of one waiting out the other. It is never held across + /// anything but one file's read, and searches do not take it at all. + reindex_lock: Mutex<()>, + /// Paths the watcher saw while `indexing` was set, kept so they can be + /// replayed once the build publishes, each with whether its original event + /// could have introduced a directory. + /// + /// Events that arrive during a build cannot be applied — the stamps do not + /// describe the index yet, so every path would read as changed — but they + /// are the only record that those paths moved. The build's own walk misses + /// anything written to a directory it has already passed, and on a + /// whole-subtree backend there is no per-directory recovery scan to fall + /// back on, so discarding them leaves the change invisible until the hourly + /// reconcile. + /// + /// The flag has to be carried rather than reconstructed. Replaying + /// everything as a create would put every recorded path through + /// `watch_new_subtree`, and a recursive `chmod` or a checkout fires a + /// metadata-only modify per directory — so a tree that produced no new + /// directories at all would be walked and force-resubscribed once per + /// recorded path, which is quadratic over a deep checkout. + /// + /// `None` means the buffer overflowed and the paths were dropped; the + /// replay then falls back to a full stale refresh, which is slower but + /// complete. Bounded because a build can run for minutes on a large + /// repository and a checkout or a build tree churning underneath it is + /// unbounded. + deferred_events: Mutex>>, /// Progress: number of files indexed so far. index_progress: std::sync::atomic::AtomicU64, /// Total files discovered for indexing. index_total: std::sync::atomic::AtomicU64, /// True when file watching is enabled for this server. watch_enabled: bool, + /// The live watcher and the directories it is subscribed to. + /// + /// Held here rather than by `run` because the subscription set is not + /// fixed: publishing a new ignore matcher renarrows it, and a directory + /// created after startup has to be subscribed as it appears. + watch_registry: Mutex>, /// Directories to exclude from indexing. exclude_dirs: Vec, /// Disable all source-control ignore files for every server discovery path. @@ -567,9 +648,15 @@ pub fn run(root: &Path, index_path: Option<&Path>, options: ServeOptions<'_>) -> gitignore_pending: std::sync::atomic::AtomicBool::new(!no_watch && !no_ignore), ignore_rules_dirty: std::sync::atomic::AtomicBool::new(false), ignore_refresh_scheduled: std::sync::atomic::AtomicBool::new(false), + watch_resubscribe: std::sync::atomic::AtomicBool::new(false), + ignore_sources: RwLock::new(Vec::new()), + ignore_source_stamps: RwLock::new(IgnoreStamps::new()), + reindex_lock: Mutex::new(()), + deferred_events: Mutex::new(Some(std::collections::HashMap::new())), index_progress: std::sync::atomic::AtomicU64::new(0), index_total: std::sync::atomic::AtomicU64::new(0), watch_enabled: !no_watch, + watch_registry: Mutex::new(None), exclude_dirs: exclude_dirs.to_vec(), no_ignore, no_require_git, @@ -656,14 +743,13 @@ pub fn run(root: &Path, index_path: Option<&Path>, options: ServeOptions<'_>) -> } // Start file watcher (unless --no-watch) - let _watcher = if no_watch { + if no_watch { eprintln!("[trace] file watcher disabled (--no-watch)"); - None } else { let watcher_state = Arc::clone(&state); let watcher_root = root.clone(); - start_file_watcher(watcher_state, &watcher_root, watcher_queue_cap) - }; + start_file_watcher(watcher_state, &watcher_root, watcher_queue_cap); + } // Set up graceful shutdown let shutdown_index_dir = index_dir.clone(); @@ -744,14 +830,206 @@ fn build_stale_matcher( matcher } -/// Commit matcher and index semantics together while the stale refresh holds -/// `snapshot_gate`. `None` is a legitimate matcher when no rules exist. -fn commit_stale_matcher( - state: &ServerState, - matcher: Option, -) { +/// What an ignore source contained when the matcher read it: its length and a +/// hash of its bytes, keyed by its path relative to the served root. +/// +/// A digest rather than metadata, because metadata does not answer the +/// question. `rsync -a`, `tar -x` and a restore from an archive all preserve +/// mtime, and two different sets of rules can easily be the same number of +/// bytes — at which point a size-and-mtime pair is identical across a +/// replacement and the scan accepts a matcher built from rules that are gone. +/// +/// Never persisted, so the hash only has to be stable within a run. +type IgnoreStamps = std::collections::HashMap; + +/// The length and content hash of `path`, following links. +/// +/// `None` when it cannot be read, which is treated as "not what was read": +/// a source that has become unreadable has stopped contributing the rules the +/// matcher is enforcing, and that is a change. +fn ignore_digest_of(path: &Path) -> Option<(u64, u64)> { + use std::hash::{Hash, Hasher}; + + // The whole file, as the matcher builder reads it. These are rule files: + // a few hundred bytes each in practice, and already in the page cache from + // the walk that found them. + let bytes = std::fs::read(path).ok()?; + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + bytes.hash(&mut hasher); + Some((bytes.len() as u64, hasher.finish())) +} + +/// How far an mtime may lag the write it records. +/// +/// Two seconds, which covers the coarsest granularity still in use: FAT and +/// its descendants store modification times in two-second units, HFS+ and +/// ext3 in whole seconds. Used to widen comparisons against a wall-clock +/// instant, which has no such rounding, so a write cannot be dated before a +/// moment it actually followed. +const MTIME_GRANULARITY: Duration = Duration::from_secs(2); + +/// The ignore files a matcher was built from, as one list. +/// +/// Root-level `p4ignore.ini` is a separate source from the walker's point of +/// view — it is applied as its own filter rather than collected with the +/// gitignore files — but deleting it invalidates the published rules exactly +/// the same way, so it belongs in the list. +/// +/// So do the sources that sit *outside* the served tree: parent-directory +/// `.ignore` / `.gitignore` files and the repository's `info/exclude`. The +/// walk that found everything else never visits them, so without this nothing +/// would notice one being deleted — and a rule with no source left keeps being +/// enforced, holding a subtree unsubscribed and unindexed until an unrelated +/// rebuild happens along. Only files that exist are listed; a path that was +/// never there is not a source that went missing. +/// +/// The user's global ignore file is deliberately absent: the `ignore` crate +/// resolves it through git's config precedence and does not hand back the path +/// it chose, and guessing wrong would report a source as vanished on every +/// scan. +fn ignore_sources_of( + root: &Path, + gitignore_files: &[PathBuf], + ignore_files: &[PathBuf], + no_require_git: bool, +) -> Vec { + let mut sources = Vec::with_capacity(gitignore_files.len() + ignore_files.len() + 1); + sources.extend_from_slice(gitignore_files); + sources.extend_from_slice(ignore_files); + let p4 = root.join(tgrep_core::gitignore::P4IGNORE_FILENAME); + if p4.is_file() { + sources.push(p4); + } + sources.extend( + tgrep_core::gitignore::ancestor_ignore_paths(root, no_require_git) + .into_iter() + .map(|(path, _)| path), + ); + sources.extend(tgrep_core::gitignore::repo_exclude_path(root)); + sources +} + +/// Read every source so a later scan can ask whether the file it finds is the +/// one the matcher read, rather than merely whether something of that name is +/// there. +/// +/// A source reached through a symlink gets a second entry under its target's +/// own relative path, when the target is under `root`. The read follows links +/// either way, so both entries describe the contents that were read. +/// +/// Taken here rather than inside the matcher builder because the `ignore` +/// crate opens these files itself and does not hand back what it read. That +/// leaves a window between its read and this one, which is what the mtime test +/// in [`changed_ignore_rules_in`] is for. +fn ignore_stamps_of(root: &Path, sources: &[PathBuf]) -> IgnoreStamps { + let canonical_root = std::fs::canonicalize(root).ok(); + let mut stamps = IgnoreStamps::with_capacity(sources.len()); + let mut record = |path: &Path, base: &Path| { + // A source above the root — a parent `.gitignore`, the repository's + // `info/exclude` — has no path relative to it. Key it by its own full + // path rather than dropping it: these keys only have to tell one source + // from another within a single run, and `changed_ignore_rules_in` looks + // up by relative path, so an absolute key is simply never hit there. + // Dropped, the digest comparison in `publish_ignore_matcher` would be + // blind to a source it is being asked to watch. + let key = match path.strip_prefix(base) { + Ok(rel) => rel.to_string_lossy().replace('\\', "/"), + Err(_) => path.to_string_lossy().replace('\\', "/"), + }; + if let Some(digest) = ignore_digest_of(path) { + stamps.insert(key, digest); + } + }; + for source in sources { + record(source, root); + let is_link = std::fs::symlink_metadata(source).is_ok_and(|m| m.file_type().is_symlink()); + if !is_link { + continue; + } + if let Some(canonical_root) = canonical_root.as_ref() + && let Ok(target) = std::fs::canonicalize(source) + && target.starts_with(canonical_root) + { + record(&target, canonical_root); + } + } + stamps +} + +/// Publish a new ignore matcher and bring everything that depends on it up to +/// date. `None` is a legitimate matcher when no rules exist. +/// +/// Callers on the stale path hold `snapshot_gate` for write, which is what +/// makes the matcher swap and the index decisions around it atomic from the +/// watcher's point of view. +/// +/// Returns the directories that were newly subscribed to as a result. Those +/// were unwatched while the caller's walk ran, so anything written to them in +/// that window produced no event and appears in no walk result. Callers pass +/// them to [`reindex_files_in`] once `state.file_stamps` describes the index +/// they just published. +/// +/// The returned directories must be paired with a timestamp the *caller* +/// captured before the walk that produced `matcher`, and handed to +/// [`reindex_files_in`] as its `since`. This function cannot supply it: the +/// subscription walk below starts later, so an ignore file written between the +/// caller's walk and this point would predate any timestamp taken here and be +/// read as already accounted for by rules that never saw it. +/// +/// `sources` are the ignore files `matcher` was built from. They are recorded +/// so a recovery scan can notice one being deleted — which no test against +/// what is on disk can see, a deleted file leaving nothing to read — and read +/// so it can also notice one being replaced, which neither a pathname nor a +/// timestamp can show. +/// +/// The matcher is built *here*, from `build`, rather than being handed in +/// already made. The stamps have to describe the bytes the matcher actually +/// read, and the read happens inside the build — `GitignoreBuilder::add` opens +/// each source itself. Stamping afterwards alone would record whatever is on +/// disk when the build finishes, so an mtime-preserving atomic replace during +/// the build would leave the matcher enforcing the old rules while the stamps +/// swore they were current, and every later check — pathname, timestamp, +/// digest — would agree that nothing needed rereading. Taking the digests on +/// both sides of the build turns that into something visible: if a source +/// moved underneath it, the published matcher is marked stale and a refresh is +/// scheduled. It is still published, because the alternative is no matcher at +/// all, which means indexing ignored paths until the refresh lands. +#[must_use = "newly watched directories need a recovery scan or writes race the subscription"] +fn publish_ignore_matcher( + state: &Arc, + root: &Path, + sources: Vec, + build: impl FnOnce() -> Option, +) -> Vec { + let before = ignore_stamps_of(root, &sources); + let matcher = build(); + let stamps = ignore_stamps_of(root, &sources); + let raced = stamps != before; + + *state.ignore_source_stamps.write().unwrap() = stamps; + *state.ignore_sources.write().unwrap() = sources; *state.gitignore.write().unwrap() = matcher; state.gitignore_pending.store(false, Ordering::SeqCst); + // New rules mean a different set of directories worth hearing about: + // a tightened rule releases the subscriptions under it, and a relaxed + // one takes subscriptions for the tree it used to hide. + let newly_watched = sync_watch_registrations(state, root).0; + + if raced { + // After the publish, so the refresh runs against the matcher and the + // subscriptions this call just established rather than racing them. + // The refresh rewalks and republishes, and a filesystem that has + // stopped moving produces matching digests next time, so this + // converges rather than looping. + eprintln!( + "[trace] warning: an ignore rules file changed while the matcher was \ + being built; scheduling a refresh" + ); + state.ignore_rules_dirty.store(true, Ordering::SeqCst); + schedule_ignore_rules_refresh(Arc::clone(state), root.to_path_buf()); + } + + newly_watched } fn handle_connection(stream: TcpStream, state: &ServerState) -> Result<()> { @@ -1482,11 +1760,7 @@ fn handle_reload(id: Option, state: &ServerState) -> String { } } -fn start_file_watcher( - state: Arc, - root: &Path, - queue_cap: usize, -) -> Option { +fn start_file_watcher(state: Arc, root: &Path, queue_cap: usize) -> bool { use std::sync::mpsc::{RecvTimeoutError, TrySendError}; let root_path = root.to_path_buf(); @@ -1501,6 +1775,7 @@ fn start_file_watcher( let overflowed = Arc::new(std::sync::atomic::AtomicBool::new(false)); let callback_overflow = Arc::clone(&overflowed); + let callback_state = Arc::clone(&state); let mut watcher = match notify::recommended_watcher( move |result: std::result::Result| match result { Ok(event) => match tx.try_send(event) { @@ -1512,32 +1787,82 @@ fn start_file_watcher( // trying to replay an unknown number of lost events. Err(TrySendError::Full(_)) => { callback_overflow.store(true, Ordering::SeqCst); + callback_state + .watch_resubscribe + .store(true, Ordering::SeqCst); } Err(TrySendError::Disconnected(_)) => {} }, - // Surface these. A dropped ReadDirectoryChangesW buffer looks - // exactly like "the watcher stopped working" from the outside, - // and silence makes it impossible to tell apart from a bug in - // our own filtering. - Err(e) => eprintln!("[trace] warning: file watcher error: {e}"), + // A native drop is the same loss as a full channel, and the OS + // will not say what it lost — inotify's `IN_Q_OVERFLOW` and a + // dropped `ReadDirectoryChangesW` buffer both arrive here with no + // paths attached. Reconcile on them too: reporting without + // recovering left exactly one of the two overflow paths handled, + // and it was the one the kernel does not use. + // + // Surfaced as well. A dropped buffer looks exactly like "the + // watcher stopped working" from the outside, and silence makes it + // impossible to tell apart from a bug in our own filtering. + Err(e) => { + eprintln!("[trace] warning: file watcher error: {e}"); + callback_overflow.store(true, Ordering::SeqCst); + callback_state + .watch_resubscribe + .store(true, Ordering::SeqCst); + } }, ) { Ok(w) => w, Err(e) => { eprintln!("[trace] warning: failed to start file watcher: {e}"); - return None; + return false; } }; - if let Err(e) = watcher.watch(root, RecursiveMode::Recursive) { + // The root is always subscribed. On a whole-subtree backend that single + // recursive subscription is the entire watch set; on a per-directory + // backend it is the anchor, and `sync_watch_registrations` below adds the + // descendants the ignore rules allow. + let root_mode = if PER_DIRECTORY_WATCHES { + RecursiveMode::NonRecursive + } else { + RecursiveMode::Recursive + }; + if let Err(e) = watcher.watch(root, root_mode) { eprintln!("[trace] warning: failed to watch directory: {e}"); - return None; + return false; + } + + *state.watch_registry.lock().unwrap() = Some(WatchRegistry { + watcher, + root: root.to_path_buf(), + watched: std::iter::once(root.to_path_buf()).collect(), + }); + + // Subscribing to the descendants needs the ignore matcher, and on a warm + // start it is still being built on another thread. Skipping the sync here + // costs nothing: events are dropped while `gitignore_pending` is set, and + // the publish that clears it runs this same sync. Subscribing first and + // narrowing afterwards would mean briefly holding exactly the watches this + // is meant to avoid — on a repo big enough to exhaust the inotify budget, + // long enough to fail. + if state.gitignore_pending.load(Ordering::SeqCst) { + eprintln!("[trace] watcher subscriptions deferred until the ignore matcher is ready"); + } else { + // This can be the first subscription pass the repository ever gets: + // the stale check runs on a thread spawned before this function, so it + // can publish while `watch_registry` is still `None` and take no + // subscriptions at all. Its walk is then already over by the time we + // subscribe here, and nothing else revisits the tree until the hourly + // reconcile — so the recovery scan is not optional on this path. + let (newly_watched, since) = sync_watch_registrations(&state, root); + spawn_recovery_scan(&state, root, newly_watched, since); } let worker_state = Arc::clone(&state); let worker_root = root_path; let worker_index_dir = state.index_dir.clone(); - std::thread::Builder::new() + if std::thread::Builder::new() .name("tgrep-watcher".into()) .spawn(move || { loop { @@ -1579,14 +1904,19 @@ fn start_file_watcher( } } }) - .ok()?; + .is_err() + { + eprintln!("[trace] warning: failed to start the watcher worker thread"); + *state.watch_registry.lock().unwrap() = None; + return false; + } state .watcher_active .store(true, std::sync::atomic::Ordering::Relaxed); eprintln!("[trace] file watcher started"); - Some(watcher) + true } /// Decide whether the file watcher should skip a path entirely. @@ -1617,6 +1947,30 @@ fn should_skip_watcher_path( rel_path: &str, exclude_dirs: &[String], gitignore: Option<&tgrep_core::gitignore::IgnoreMatcher>, +) -> bool { + should_skip_watcher_entry(rel_path, exclude_dirs, gitignore, false) +} + +/// [`should_skip_watcher_path`] for a path known to be a directory. +/// +/// Two rules read differently for a directory. `--exclude` names apply to the +/// final segment as well, because the walker drops the whole subtree when the +/// entry it is looking at *is* the excluded directory. And the gitignore +/// matcher is told it is matching a directory, so a directory-only rule like +/// `build/` matches — as a file path, `build` does not. +fn should_skip_watcher_dir( + rel_path: &str, + exclude_dirs: &[String], + gitignore: Option<&tgrep_core::gitignore::IgnoreMatcher>, +) -> bool { + should_skip_watcher_entry(rel_path, exclude_dirs, gitignore, true) +} + +fn should_skip_watcher_entry( + rel_path: &str, + exclude_dirs: &[String], + gitignore: Option<&tgrep_core::gitignore::IgnoreMatcher>, + is_dir: bool, ) -> bool { // Single streaming pass over path components — no Vec allocation // on the hot watcher path. The hidden-component check applies to @@ -1632,23 +1986,21 @@ fn should_skip_watcher_path( if seg.starts_with('.') { return true; } - // Ancestor (i.e. not the last segment) — apply exclude_dirs. - if segments.peek().is_some() - && !exclude_dirs.is_empty() - && exclude_dirs.iter().any(|d| d == seg) - { + // An ancestor is always a directory; the final segment is one only + // when the caller says so. + let segment_is_dir = segments.peek().is_some() || is_dir; + if segment_is_dir && !exclude_dirs.is_empty() && exclude_dirs.iter().any(|d| d == seg) { return true; } } // Gitignore check (if a matcher is available). if let Some(gi) = gitignore { - // We don't know whether the path is a dir or a file here — for - // the watcher's purposes we treat all events as "file" matches. - // Notify usually fires per-file events anyway, and gitignore - // rules that target dirs would have already skipped the dir's - // contents via `matched_path_or_any_parents`. - if gi.is_ignored(Path::new(rel_path), false) { + // For a file event we don't know whether the path is a dir, so we + // treat it as a file. Notify usually fires per-file events anyway, + // and gitignore rules that target dirs would have already skipped + // the dir's contents via `matched_path_or_any_parents`. + if gi.is_ignored(Path::new(rel_path), is_dir) { return true; } } @@ -1656,1216 +2008,2212 @@ fn should_skip_watcher_path( false } +/// Whether a changed path is an ignore-rules source, i.e. one whose contents +/// feed the matcher published in `ServerState::gitignore`. +/// +/// `.gitignore` and `.ignore` are matched by file name at any depth, because +/// [`tgrep_core::gitignore::matcher_from_ignore_paths`] anchors nested files of +/// both kinds. `.ignore` must be included even though it is git-agnostic — +/// leaving it out meant a `.ignore` written while the server was live never +/// refreshed the matcher, so the watcher kept indexing files the rule excluded. +/// +/// `p4ignore.ini` stays root-scoped, mirroring the walker, which only reads the +/// root-level file. fn is_ignore_rules_file(root: &Path, path: &Path) -> bool { - path.file_name().and_then(|name| name.to_str()) == Some(".gitignore") - || path == root.join(tgrep_core::gitignore::P4IGNORE_FILENAME) + let name = path.file_name().and_then(|name| name.to_str()); + matches!( + name, + Some(tgrep_core::gitignore::GITIGNORE_FILENAME) + | Some(tgrep_core::gitignore::DOT_IGNORE_FILENAME) + ) || path == root.join(tgrep_core::gitignore::P4IGNORE_FILENAME) } -fn schedule_ignore_rules_refresh(state: Arc, root: PathBuf) { - if state - .ignore_refresh_scheduled - .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) - .is_err() - { - return; - } +/// Whether this platform's `notify` backend takes one OS subscription per +/// directory rather than a single recursive one for the whole tree. +/// +/// inotify has no recursive mode. `RecursiveMode::Recursive` makes notify walk +/// the tree itself and spend one watch descriptor per directory, so every +/// ignored directory costs a descriptor from the per-user +/// `fs.inotify.max_user_watches` budget purely to deliver events we then throw +/// away. Worse, notify's registration loop propagates the first failure, so a +/// repo whose ignored build output exhausts the budget makes `watch()` return +/// an error and the server loses its watcher entirely. +/// +/// ReadDirectoryChangesW (Windows) and FSEvents (macOS) subscribe once for the +/// whole subtree, so there is no per-directory registration to withhold. On +/// those platforms filtering on delivery is the only lever available, and +/// [`should_skip_watcher_path`] remains it. +/// +/// Deliberately limited to the backends we can exercise in CI. kqueue and +/// `PollWatcher` are per-path too, but nothing here builds or tests them. +const PER_DIRECTORY_WATCHES: bool = cfg!(any(target_os = "linux", target_os = "android")); - thread::spawn(move || { - loop { - if state.ignore_rules_dirty.swap(false, Ordering::SeqCst) { - // The stale refresh walks the tree anyway and republishes the - // matcher from that walk, so the reload costs one traversal - // rather than a rebuild plus a re-scan. - if !background_refresh_stale(&state, &root, &state.index_dir, true) { - state.ignore_rules_dirty.store(true, Ordering::SeqCst); - thread::sleep(Duration::from_secs(1)); - } - } +/// Whether `path` is a directory in its own right rather than a symlink to one. +/// +/// [`Path::is_dir`] follows links, so it answers "does this lead to a +/// directory", which is the wrong question here: the walker does not follow +/// symlinks, so a symlinked directory is not part of the indexed tree. Treating +/// one as a directory would subscribe to and index its target — possibly a tree +/// outside `root` entirely, and possibly a cycle. +fn is_real_dir(path: &Path) -> bool { + std::fs::symlink_metadata(path).is_ok_and(|meta| meta.file_type().is_dir()) +} - state - .ignore_refresh_scheduled - .store(false, Ordering::SeqCst); - if !state.ignore_rules_dirty.load(Ordering::SeqCst) - || state - .ignore_refresh_scheduled - .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) - .is_err() - { - break; - } +/// Whether `path` is a directory reached from `root` without crossing a symlink. +/// +/// [`is_real_dir`] only inspects the last component, which answers the wrong +/// question for a path assembled from an event: `root/a/b` is a perfectly real +/// directory while `a` is a symlink pointing anywhere on the machine. The +/// walker never descends through `a`, so nothing under it belongs to the served +/// tree, yet a `Create` for `root/a/b` would subscribe to it and enumerate and +/// index whatever is inside — a watch descriptor per directory of a tree that +/// is not ours, and file content filed under paths that do not lead to it. +/// +/// `root` itself is the trusted anchor and is not tested. It may legitimately +/// be reached through a link (`tgrep serve /var/tmp/...` on macOS is the common +/// case), and refusing it would leave nothing watchable at all. This is the +/// same contract [`open_within_root`] works to: containment is established +/// relative to the root that was served, not against the real filesystem. +/// +/// Component-by-component with `symlink_metadata`, so the answer is a snapshot +/// rather than a guarantee — a link swapped in afterwards is not visible here. +/// That residual window is what the post-registration re-check in +/// [`WatchRegistry::subscribe`] and `open_within_root`'s no-follow descent +/// exist to bound. +fn is_contained_dir(root: &Path, path: &Path) -> bool { + let Ok(rel) = path.strip_prefix(root) else { + return false; + }; + let mut cursor = root.to_path_buf(); + for component in rel.components() { + // `..` would climb back out of the tree and `.` cannot appear in a path + // built from an event; anything but a plain name is not a descent. + let std::path::Component::Normal(name) = component else { + return false; + }; + cursor.push(name); + if !is_real_dir(&cursor) { + return false; } - }); + } + true } -fn handle_fs_event(state: &Arc, root: &Path, event: &Event) { - use tgrep_core::meta::FileStamp; +/// The watcher plus the set of directories it is currently subscribed to. +/// +/// Only meaningful when [`PER_DIRECTORY_WATCHES`] is true; elsewhere `watched` +/// holds just the root, which is subscribed recursively. +struct WatchRegistry { + watcher: RecommendedWatcher, + /// The served root. Subscriptions are only ever taken for directories + /// reachable from it without crossing a symlink; see + /// [`WatchRegistry::contained`]. + root: PathBuf, + watched: std::collections::HashSet, +} - let dominated_kinds = matches!( - event.kind, - EventKind::Create(_) | EventKind::Modify(_) | EventKind::Remove(_) - ); - if !dominated_kinds { - return; +impl WatchRegistry { + /// Subscribe to every directory in `desired` that is not already + /// subscribed, leaving existing subscriptions alone. + /// + /// Returns the directories that were newly subscribed to. A single + /// directory that cannot be subscribed is reported and skipped rather than + /// failing the whole call: the watcher is still useful for everything + /// else, and giving up on the entire tree is exactly the failure mode this + /// registration exists to avoid. + fn add_all<'a>(&mut self, desired: impl IntoIterator) -> Vec { + self.subscribe(desired, false) } - let ignore_rules_changed = !state.no_ignore - && event - .paths - .iter() - .any(|path| is_ignore_rules_file(root, path)); - if ignore_rules_changed { - state.ignore_rules_dirty.store(true, Ordering::SeqCst); - if state.indexing.load(Ordering::SeqCst) { - return; + /// Subscribe to every directory in `dirs`, re-issuing the subscription even + /// for ones already recorded as watched. + /// + /// For directories that have just appeared, where membership in `watched` + /// proves nothing. The kernel drops an inotify watch by itself when its + /// directory is deleted or moved away, and nothing tells us the descriptor + /// is gone — so a path recreated at the same location would look + /// subscribed while receiving no events at all. [`Self::add_all`] would + /// skip it, and so would every later [`Self::sync`], since the path is in + /// `desired` *and* in `watched`: the entry stays poisoned until the server + /// restarts. Re-adding is cheap and idempotent (`inotify_add_watch` + /// returns the existing descriptor), so the doubt is worth paying for. + /// + /// Returns only the directories that were not previously recorded, so a + /// caller's notion of "newly watched" keeps its meaning. + fn resubscribe_all<'a>(&mut self, dirs: impl IntoIterator) -> Vec { + self.subscribe(dirs, true) + } + + /// Whether `dir` is still a directory the served tree actually contains. + /// + /// Every entry in `watched` was checked by this method before it went in, + /// so a directory whose parent is already watched (or is the root) inherits + /// that proof and only its own last component needs testing. That short + /// circuit is what keeps the startup sync affordable: the full walk in + /// [`is_contained_dir`] costs one `symlink_metadata` per level, and paying + /// it for each of forty thousand directories at depth ten would be four + /// hundred thousand syscalls to re-derive what the previous entry proved. + /// The sync feeds directories shallowest-first for exactly this reason, so + /// the cheap path is the one nearly every call takes; anything arriving + /// out of order still gets the full walk and the right answer. + fn contained(&self, dir: &Path) -> bool { + match dir.parent() { + Some(parent) if parent == self.root || self.watched.contains(parent) => { + is_real_dir(dir) + } + _ => is_contained_dir(&self.root, dir), } - schedule_ignore_rules_refresh(Arc::clone(state), root.to_path_buf()); - return; } - // Skip ordinary file events while the initial background index build is in - // progress. The indexer will pick up those files itself. - if state.indexing.load(Ordering::SeqCst) { - return; + fn subscribe<'a>( + &mut self, + desired: impl IntoIterator, + force: bool, + ) -> Vec { + let mut added = Vec::new(); + let mut failures = 0; + // Iterating `desired` and testing membership is deliberate: a + // `difference` would be proportional to the whole watched set, and + // this runs per newly created directory on repositories where that set + // is tens of thousands of entries. + for dir in desired { + let known = self.watched.contains(dir); + if known && !force { + continue; + } + match self.watcher.watch(dir, RecursiveMode::NonRecursive) { + Ok(()) => { + // notify's inotify backend registers without + // `IN_DONT_FOLLOW`, so the descriptor lands on whatever the + // name resolves to at that instant — and the no-follow + // check that qualified this directory happened earlier, in + // the walk. A checkout or a rename can replace it with a + // symlink in between, leaving the descriptor watching an + // inode outside the root while `watched` records an + // in-root name as covered. + // + // Re-checking after the fact catches that: if the name is + // no longer a real directory, the registration is undone + // and the entry is not recorded, so a later `sync` retries + // it rather than treating a poisoned subscription as live. + // + // This narrows the window rather than closing it — notify + // takes a path, not a handle, so a swap that is reverted + // before this check is undetectable through its API. What + // that costs is bounded: it is missed *events* on a real + // directory, which the periodic reconcile picks up, and + // never misplaced content, since `open_within_root` + // establishes containment from the handle it reads. + if !self.contained(dir) { + let _ = self.watcher.unwatch(dir); + if known { + self.watched.remove(dir); + } + continue; + } + if !known { + self.watched.insert(dir.clone()); + added.push(dir.clone()); + } + } + Err(e) => { + // A forced re-add that fails means the directory is gone + // again; drop the entry so a later attempt can retry it + // rather than trusting a descriptor that does not exist. + if known { + self.watched.remove(dir); + } + // One line per call, not per directory: exhausting the + // inotify budget fails thousands of these at once. + if failures == 0 { + eprintln!( + "[trace] warning: could not watch {}: {e} \ + (continuing with the directories that succeeded)", + dir.display() + ); + } + failures += 1; + } + } + } + if failures > 1 { + eprintln!("[trace] warning: {failures} directories could not be watched"); + } + added } - // Acquire the snapshot gate up-front for the whole event. While a - // flush/auto-save is publishing (writer holds it), no reindex - // *work* — file I/O, trigram extraction, even the [trace] line — - // should happen, both for correctness (no overlay mutation between - // snapshot and prune) and to avoid spending CPU/IO on work that - // would just block the watcher thread anyway. We hold it for read - // so multiple events can proceed concurrently outside any flush. - let _gate = state.snapshot_gate.read().unwrap(); + /// Whether `path` is already subscribed. + /// + /// For deciding whether a directory found during a recovery scan is one the + /// sync already knew about or one that appeared after it — the latter has + /// to be picked up explicitly, since a non-recursive subscription on its + /// parent says nothing about it. + fn is_watched(&self, path: &Path) -> bool { + self.watched.contains(path) + } - // Stay off the index until the initial ignore matcher exists. This check - // must happen *under* the gate: during startup the stale walk holds the - // write side, so an event waits and is applied after publication instead - // of being dropped after the walk may already have visited its path. - if state.gitignore_pending.load(Ordering::SeqCst) { - return; + /// Drop a path that no longer exists from the subscription set. + /// + /// The kernel has already released the descriptor if the directory was + /// deleted; the `unwatch` is for the moved-away case, where it is still + /// live and now pointing outside the tree. What matters either way is + /// clearing `watched`, so that if the path comes back, it is treated as + /// the new directory it is instead of an already-subscribed one. + /// + /// Cheap by design: one hash lookup per removal event, because deleting a + /// tree delivers one event per directory in it and anything proportional + /// to the whole watched set would turn that into quadratic work. + /// Descendants left behind by a move are pruned by the next + /// [`Self::sync`], which no longer finds them under the root. + fn forget(&mut self, path: &Path) { + if self.watched.remove(path) { + let _ = self.watcher.unwatch(path); + } } - for path in &event.paths { - // Skip the index directory itself - if path - .to_string_lossy() - .contains(&format!("{}.tgrep", std::path::MAIN_SEPARATOR)) - { - continue; + /// Bring the subscription set in line with `desired`, subscribing to + /// directories that are newly relevant and dropping ones that are not. + /// + /// `force` re-issues the subscription for directories already recorded as + /// watched. Only needed after events were lost: a directory removal that + /// never arrived leaves the kernel's descriptor gone and this registry's + /// entry intact, and a path recreated there is then in `desired` *and* in + /// `watched`, so an ordinary sync skips it forever. Off by default because + /// re-registering costs a syscall per directory, and a monorepo reconcile + /// would pay forty thousand of them for a doubt only overflow raises. + /// + /// Returns `(added, removed)`. Only for a set that describes the whole + /// tree — anything absent from `desired` is unsubscribed. To subscribe to + /// a subtree without disturbing the rest, use [`Self::add_all`]. + fn sync( + &mut self, + desired: &std::collections::HashSet, + force: bool, + ) -> (Vec, usize) { + let stale: Vec = self.watched.difference(desired).cloned().collect(); + let mut removed = 0; + for dir in stale { + // Best effort. inotify drops a descriptor by itself when the + // directory is deleted, so "not found" is an expected outcome + // here, not an error worth reporting. + let _ = self.watcher.unwatch(&dir); + self.watched.remove(&dir); + removed += 1; } - let rel_path = match path.strip_prefix(root) { - Ok(p) => p.to_string_lossy().replace('\\', "/"), - Err(_) => continue, + // Shallowest first, because `desired` is a `HashSet` and hands its + // contents out in whatever order hashing produced. [`Self::contained`] + // establishes containment cheaply by leaning on the parent already + // being watched; a child that arrives before its parent gets no such + // proof and walks every ancestor instead. Unordered, that is the + // common case rather than the exception, and it turns the startup + // sync from one `symlink_metadata` per directory into one per level + // per directory — on a monorepo, hundreds of thousands of syscalls in + // the path that exists to make startup cheap. Sorting by depth is + // enough: a parent is always strictly shallower than its children. + let mut ordered: Vec<&PathBuf> = desired.iter().collect(); + ordered.sort_by_key(|dir| dir.components().count()); + + let added = if force { + self.resubscribe_all(ordered) + } else { + self.add_all(ordered) }; + (added, removed) + } +} - // Mirror the walker's filtering so the watcher does not reindex - // files the initial walk would have skipped — most notably - // hidden directories like `.git/`, which fire frequent - // `index.lock`/HEAD/refs writes during normal git operations. - let should_skip = { - let gitignore = state.gitignore.read().unwrap(); - should_skip_watcher_path(&rel_path, &state.exclude_dirs, gitignore.as_ref()) - }; - if should_skip { +/// Every directory at or below `start` whose contents the watcher needs to +/// hear about. +/// +/// A per-directory subscription reports the files directly inside it, so the +/// set is "`start`, plus every descendant directory the indexer would walk +/// into". Ignored directories are pruned along with their subtrees, which is +/// the whole point: the tree under `target/` or `node_modules/` is usually +/// most of the directories in a repo. +/// +/// `root` is the repository root and is only used to build the relative paths +/// the ignore rules are written against; `start` is where the walk begins. +/// They differ when a subtree that appeared at runtime is being subscribed, +/// and conflating them would match every rule at the wrong anchor. +/// +/// `start` itself is always included — callers are responsible for not asking +/// about a directory that is ignored. +/// +/// Symlinked directories are not descended into, matching the walker (the +/// `ignore` crate does not follow links by default). That also keeps a +/// symlink cycle from turning this into an infinite walk. +fn watchable_dirs( + root: &Path, + start: &Path, + exclude_dirs: &[String], + gitignore: Option<&tgrep_core::gitignore::IgnoreMatcher>, +) -> std::collections::HashSet { + let mut found = std::collections::HashSet::new(); + found.insert(start.to_path_buf()); + + let mut stack = vec![start.to_path_buf()]; + while let Some(dir) = stack.pop() { + let Ok(entries) = std::fs::read_dir(&dir) else { + // An unreadable directory is not a reason to abandon the rest of + // the tree; the periodic reconcile is what catches what we miss. continue; + }; + for entry in entries.flatten() { + if !entry.file_type().is_ok_and(|t| t.is_dir()) { + continue; + } + let path = entry.path(); + let Ok(rel) = path.strip_prefix(root) else { + continue; + }; + let rel = rel.to_string_lossy().replace('\\', "/"); + if should_skip_watcher_dir(&rel, exclude_dirs, gitignore) { + continue; + } + stack.push(path.clone()); + found.insert(path); } + } + found +} - let is_remove = matches!(event.kind, EventKind::Remove(_)) || !path.exists(); +/// The directories that must be subscribed to for the ignore sources +/// themselves to be observable, beyond the ones the rules allow. +/// +/// A `.gitignore` symlinked to `build/shared-rules` contributes the *target's* +/// contents, and [`handle_fs_event`] already recognises an event naming that +/// target rather than a name rules usually go by. But only if one arrives: on a +/// per-directory backend nothing subscribes to `build/` when the rules hide it, +/// so the edit that changes what the matcher enforces produces no event at all, +/// and the matcher stays stale until the hourly reconcile — the one case where +/// the source of the rules is invisible to the rules' own watcher. +/// +/// One watch on the target's own directory, not its subtree: this is about +/// seeing a single file that the matcher was built from, not about indexing +/// anything under it. `should_skip_watcher_path` still discards everything else +/// delivered from there, and the target itself is matched by path against the +/// recorded stamps before any of that filtering runs. +/// +/// Targets outside `root` are deliberately not covered. Watching them would +/// mean subscribing outside the tree the server was asked to serve, and the +/// periodic reconcile remains the backstop there. +fn ignore_target_dirs(root: &Path, sources: &[PathBuf]) -> std::collections::HashSet { + let mut dirs = std::collections::HashSet::new(); + let Ok(canonical_root) = std::fs::canonicalize(root) else { + return dirs; + }; + for source in sources { + if !std::fs::symlink_metadata(source).is_ok_and(|m| m.file_type().is_symlink()) { + continue; + } + let Ok(target) = std::fs::canonicalize(source) else { + continue; + }; + let Ok(rel) = target.strip_prefix(&canonical_root) else { + continue; + }; + // Re-anchored on `root` as given rather than kept canonical: the + // registry compares paths literally, and a `\\?\` or `/private` prefix + // would register a second subscription for a directory already watched. + if let Some(parent) = root.join(rel).parent() { + dirs.insert(parent.to_path_buf()); + } + } + dirs +} - if is_remove { - // notify can deliver Remove events for transient/unknown paths - // (e.g. a build tool's temp file). Suppress the noisy log line - // for those, but still apply the delete unconditionally — if - // `file_stamps` is missing/out-of-date (e.g. first run after - // an older index), skipping the delete entirely would leave - // stale entries for files that no longer exist. - let known_path = state.file_stamps.read().unwrap().contains_key(&rel_path); - if known_path { - eprintln!("[trace] reindex: removed {rel_path}"); +/// Recompute the watcher's subscriptions against the ignore rules in force. +/// +/// Called when the watcher starts and every time the ignore matcher is +/// published, so relaxing a rule subscribes to the tree it used to hide and +/// tightening one drops it. +/// +/// Returns the directories that were newly subscribed to, and the moment the +/// walk behind that decision began. Until a directory is subscribed it cannot +/// report anything, so a file written to one of these between the walk and the +/// subscription is in neither the walk's results nor any event. The caller is +/// expected to hand both to [`reindex_files_in`] once its own bookkeeping is +/// settled; the timestamp bounds that window, which is what lets the scan tell +/// an ignore-rules file that landed inside it from the thousands that were +/// already there and are already reflected in the matcher. +fn sync_watch_registrations(state: &ServerState, root: &Path) -> (Vec, SystemTime) { + // Before the early returns as well as the walk: a caller that gets no + // directories back still gets a usable bound. + let since = SystemTime::now(); + if !PER_DIRECTORY_WATCHES { + return (Vec::new(), since); + } + let mut registry = state.watch_registry.lock().unwrap(); + let Some(registry) = registry.as_mut() else { + // The watcher has not started yet. It syncs once as it comes up, so + // there is nothing to do and nothing to remember. + return (Vec::new(), since); + }; + + let start = Instant::now(); + let mut desired = { + let gitignore = state.gitignore.read().unwrap(); + watchable_dirs(root, root, &state.exclude_dirs, gitignore.as_ref()) + }; + if !state.no_ignore { + let sources = state.ignore_sources.read().unwrap(); + desired.extend(ignore_target_dirs(root, &sources)); + } + let total = desired.len(); + // Consumed here, so one overflow buys one forced pass rather than making + // every later reconcile re-register the whole tree. + let force = state.watch_resubscribe.swap(false, Ordering::SeqCst); + let (added, removed) = registry.sync(&desired, force); + if !added.is_empty() || removed > 0 { + eprintln!( + "[trace] watcher subscriptions: {total} directories \ + (+{}, -{removed}) in {:.1}ms", + added.len(), + start.elapsed().as_secs_f64() * 1000.0 + ); + } + (added, since) +} + +/// The first ignore-rules file in `dirs` that the published matcher did not +/// read, that has been replaced since it did, or that has been edited since +/// `since`. +/// +/// Probing by name rather than inspecting listings, for three reasons. It is +/// how the walker itself discovers these files, so the two agree by +/// construction. `Path::is_file` follows symlinks, so a symlinked `.gitignore` +/// — which carries rules exactly like a real one — is seen, where +/// `DirEntry::file_type` does not resolve it and the entry gets dropped as "not +/// a regular file". And it is ordering-independent: `read_dir` offers no +/// ordering, and on macOS `.gitignore` routinely comes back *after* its +/// siblings, so a per-entry check indexes part of a directory under the stale +/// rules before it ever reaches the file that changes them. Answering for the +/// whole scan up front closes that window across directories as well. +/// +/// Three tests, because none alone is enough. +/// +/// Absence from the published sources is the exact question for an arrival — +/// this file did not feed the matcher in force — and it catches one however old +/// the file says it is, which matters because `git checkout`, `tar -x` and +/// `rsync -a` all restore mtimes from what they unpack and would sail past a +/// recency test. +/// +/// A stamp mismatch answers the same question for a file that was *already* a +/// source: a pathname proves nothing about contents, and neither does +/// metadata. `rsync -a` and `tar -x` preserve mtime, and two different sets of +/// rules are easily the same length, so the comparison is against a hash of +/// what was actually read. +/// +/// The mtime window then covers the gap the digests cannot: they are taken +/// when the matcher is published, which is after the builder read these files, +/// so a write landing between the two is recorded as if it had been read. +/// +/// That window is widened by [`MTIME_GRANULARITY`] at the near end, because +/// `since` is a wall-clock instant with nanosecond precision and an mtime is +/// not. HFS+ and ext3 store whole seconds, FAT-derived filesystems two, so a +/// write that happens after `since` can be stamped before it and read as +/// historical. Over-triggering costs one rewalk that finds nothing; the slack +/// is bounded, so a file whose mtime keeps qualifying stops doing so as later +/// scans take later timestamps. +/// +/// The far end is bounded too. On a network mount whose server clock runs +/// ahead of ours, every recently touched file carries a future mtime and would +/// pass a one-sided test — on every scan, including the one at the end of the +/// refresh this schedules, which walks the whole repository and then arms the +/// next. Treating a future mtime as skew rather than as an edit keeps that +/// loop closed. +fn changed_ignore_rules_in( + root: &Path, + dirs: &[PathBuf], + known_sources: &IgnoreStamps, + since: SystemTime, +) -> Option<(String, &'static str)> { + let since = since.checked_sub(MTIME_GRANULARITY).unwrap_or(since); + let mut probed: std::collections::HashSet = std::collections::HashSet::new(); + for dir in dirs { + let mut candidates = vec![ + dir.join(tgrep_core::gitignore::GITIGNORE_FILENAME), + dir.join(tgrep_core::gitignore::DOT_IGNORE_FILENAME), + ]; + // Root-scoped, mirroring the walker, which only reads the root file. + if dir == root { + candidates.push(root.join(tgrep_core::gitignore::P4IGNORE_FILENAME)); + } + for candidate in candidates { + if !probed.insert(candidate.clone()) || !candidate.is_file() { + continue; } - // gate acquired at the function level — the entire event - // is processed atomically with respect to flush/auto-save. - state.index.write().unwrap().live.delete_file(&rel_path); - state.file_stamps.write().unwrap().remove(&rel_path); - if let Ok(mut cache) = state.cache.write() { - cache.pop(&rel_path); + let Ok(rel) = candidate.strip_prefix(root) else { + continue; + }; + let rel = rel.to_string_lossy().replace('\\', "/"); + let Some(read_as) = known_sources.get(&rel) else { + return Some((rel, "not a known source")); + }; + if ignore_digest_of(&candidate).as_ref() != Some(read_as) { + return Some((rel, "not the file the matcher read")); + } + // Follows links, matching how the digest was taken. + let Ok(meta) = std::fs::metadata(&candidate) else { + continue; + }; + if meta + .modified() + .is_ok_and(|m| m >= since && m <= SystemTime::now()) + { + return Some((rel, "modified")); } - continue; } + } + None +} - if !path.is_file() { - continue; - } +/// Re-check the files directly inside `dirs`, indexing the ones that changed, +/// dropping the ones that are gone, and subscribing to subdirectories that +/// appeared while the subscriptions were being established. +/// Used to close the gap between a walk and the subscriptions that follow it: +/// [`reindex_file`] compares stamps first, so for a tree that did not change +/// under us this costs one `metadata` call per file and indexes nothing. +/// +/// `since` is when the walk behind `dirs` began — the start of the window this +/// is closing. It is only consulted for ignore-rules files, where "did this +/// arrive after the matcher was decided" cannot be answered from the stamps: +/// the dot-prefixed ones are hidden, so they are never indexed and never have +/// one. The opposite case — one that was *deleted* in the window — is handled +/// separately, from `state.ignore_sources`, since a deleted file leaves nothing +/// to stat. +/// +/// Callers must already hold `snapshot_gate`, and `state.file_stamps` must +/// already describe the index as published — a merge that replaces the stamps +/// afterwards would both discard what this records and make every file here +/// look changed. +fn reindex_files_in(state: &Arc, root: &Path, dirs: &[PathBuf], since: SystemTime) { + if dirs.is_empty() { + return; + } + let start = Instant::now(); - // Compute the file's current stamp and skip if it matches what we - // last indexed. notify on Windows in particular fires Modify events - // for atime/attribute updates, opens, etc. — re-indexing on those - // would re-read large files, churn the live overlay, and produce a - // misleading "modified" trace for files that didn't actually change. - let current = match std::fs::metadata(path) { - Ok(m) => FileStamp { - mtime: m - .modified() - .ok() - .and_then(|t| t.duration_since(std::time::SystemTime::UNIX_EPOCH).ok()) - .map(|d| d.as_secs()) - .unwrap_or(0), - size: m.len(), - }, - Err(_) => continue, + // An ignore file that was deleted during the window leaves nothing to stat, + // so no test over what is on disk can see it. It is also the more damaging + // direction: rules that no longer have a source keep being enforced, so the + // subtree they hide stays unsubscribed and unindexed until an unrelated + // rebuild happens along. Checking the sources the published matcher was + // built from catches it at one stat apiece, once per scan. + let known_sources: IgnoreStamps = if state.no_ignore { + IgnoreStamps::new() + } else { + let vanished = { + let sources = state.ignore_sources.read().unwrap(); + // `is_file` follows links, matching how the walker collected these + // (`ignore_files_in` qualifies candidates with `Path::is_file`) — a + // symlinked source whose target is gone has stopped contributing + // rules just as surely as a deleted one. + // + // `is_file` rather than `exists`: a source replaced by a directory, + // a FIFO or a socket still exists, but the walker would no longer + // collect it and a rebuild would no longer read it. Testing only + // for absence leaves the matcher enforcing rules from a file that + // is not a file any more, with nothing else able to notice — the + // digest check below only runs for candidates the scan walks past, + // and a rule file that has become a directory is not one of them. + sources.iter().find(|p| !p.is_file()).cloned() }; - if state.file_stamps.read().unwrap().get(&rel_path) == Some(¤t) { - continue; + if let Some(gone) = vanished { + state.ignore_rules_dirty.store(true, Ordering::SeqCst); + schedule_ignore_rules_refresh(Arc::clone(state), root.to_path_buf()); + eprintln!( + "[trace] watcher: ignore rules source {} is gone or no longer a file; \ + deferring to a refresh", + gone.display() + ); + return; } + state.ignore_source_stamps.read().unwrap().clone() + }; - // Read contents and extract trigrams OUTSIDE the index write lock - // so a concurrent search (which needs a read lock) is not blocked - // on our file I/O and trigram parsing. Windows' SRWLock is - // writer-preferring: a single waiting writer here would otherwise - // stall every subsequent search request. - let data = match std::fs::read(path) { - Ok(d) => d, - Err(_) => continue, + // Before a single file is indexed: an ignore-rules file that landed in this + // window was not seen by the walk that built the matcher in force, so every + // file in this scan would be judged by rules that do not know about it, and + // whatever was wrongly indexed would stay until something touched it again. + if !state.no_ignore + && let Some((rel, why)) = changed_ignore_rules_in(root, dirs, &known_sources, since) + { + // Abandon the scan: the refresh rewalks and republishes, which covers + // these directories properly. + state.ignore_rules_dirty.store(true, Ordering::SeqCst); + schedule_ignore_rules_refresh(Arc::clone(state), root.to_path_buf()); + eprintln!( + "[trace] watcher: ignore rules changed during recovery ({rel}, {why}); \ + deferring to a refresh" + ); + return; + } + + // Directories whose listing succeeded, and the files those listings + // contained, for the removal sweep at the end. Directory names are kept + // apart from the files: a directory that vanished with its contents leaves + // indexed paths whose own parent was never enumerated, and its absence + // from its parent's listing is the only evidence there is. + let mut swept: std::collections::HashSet = std::collections::HashSet::new(); + let mut present: std::collections::HashSet = std::collections::HashSet::new(); + let mut present_dirs: std::collections::HashSet = std::collections::HashSet::new(); + let mut unreadable_dirs: Vec = Vec::new(); + + for dir in dirs { + let Ok(rel_dir) = dir.strip_prefix(root) else { + continue; }; - let text = tgrep_core::encoding::decode_for_index(&data); - let is_binary = tgrep_core::trigram::is_binary(&text); - let per_tri = if is_binary { - None - } else { - Some(tgrep_core::live::LiveIndex::compute_trigram_masks(&text)) + let rel_dir = rel_dir.to_string_lossy().replace('\\', "/"); + let Ok(entries) = std::fs::read_dir(dir) else { + // No listing means no evidence, and the sweep below must not treat + // silence as absence. Whether the directory is gone or merely + // unreadable is decided later, from its parent's listing. + unreadable_dirs.push(rel_dir); + continue; }; + let mut subdirs: Vec = Vec::new(); + // A per-entry error is as much a gap in the evidence as a failed + // listing: the name it would have yielded is simply absent from + // `present`, and the sweep below would read that as a deletion. The + // entry is skipped either way, but the directory then does not get to + // claim it was enumerated. + let mut listing_complete = true; + for entry in entries { + let Ok(entry) = entry else { + listing_complete = false; + continue; + }; + let path = entry.path(); + let Ok(rel) = path.strip_prefix(root) else { + continue; + }; + let rel = rel.to_string_lossy().replace('\\', "/"); + let Ok(file_type) = entry.file_type() else { + // Unclassifiable, so nothing can be concluded about it — + // least of all that it is gone. + present.insert(rel); + continue; + }; + // `DirEntry::file_type` does not follow symlinks, so a symlinked + // file or directory is neither indexed nor descended into, which + // is what the walker does with `follow_links(false)`. + if file_type.is_dir() { + present_dirs.insert(rel); + subdirs.push(path); + continue; + } + if !file_type.is_file() { + continue; + } + present.insert(rel.clone()); - eprintln!("[trace] reindex: modified {rel_path}"); - // gate acquired at the function level — the commit + stamp - // update is processed atomically with respect to flush/auto-save. - { - let mut index = state.index.write().unwrap(); - match per_tri { - Some(per_tri) => index.live.commit_upsert(&rel_path, per_tri), - None => index.live.delete_file(&rel_path), + let skip = { + let gitignore = state.gitignore.read().unwrap(); + should_skip_watcher_path(&rel, &state.exclude_dirs, gitignore.as_ref()) + }; + if !skip { + reindex_file(state, &path, &rel); } } - state - .file_stamps - .write() - .unwrap() - .insert(rel_path.clone(), current); - if let Ok(mut cache) = state.cache.write() { - cache.pop(&rel_path); + if listing_complete { + swept.insert(rel_dir); + } + + // A directory created in the same window is in neither `dirs` (the + // walk did not see it) nor any event (its parent's subscription is + // non-recursive, so notify does not extend to it), and would stay + // invisible until the hourly reconcile. + // + // Only the ones not already subscribed: at startup `dirs` is every + // directory in the repository and each one is a subdirectory of + // another, so descending into all of them would re-walk the tree once + // per level. The membership test reduces that to a hash lookup apiece. + if !subdirs.is_empty() { + let unwatched: Vec = { + let mut registry = state.watch_registry.lock().unwrap(); + match registry.as_mut() { + // Filtered under the lock but subscribed outside it: + // `watch_new_subtree` takes the same lock, and it is not + // reentrant. + Some(registry) => subdirs + .into_iter() + .filter(|p| !registry.is_watched(p)) + .collect(), + None => Vec::new(), + } + }; + for subdir in &unwatched { + watch_new_subtree(state, root, subdir); + } } } + + // A directory that could not be listed is either gone or merely + // unreadable, and only its parent's listing can tell the two apart. The + // ones proven absent stand in for every file under them: those files have + // a parent nothing enumerated, so the per-directory evidence above says + // nothing about them at all, and a subtree deleted or moved away in this + // window would keep answering searches until the hourly reconcile. + let vanished_dirs: std::collections::HashSet = unreadable_dirs + .into_iter() + .filter(|rel| { + let parent = rel.rsplit_once('/').map_or("", |(dir, _)| dir); + swept.contains(parent) && !present_dirs.contains(rel) && !present.contains(rel) + }) + .collect(); + + sweep_removed_files(state, &swept, &present, &vanished_dirs); + + eprintln!( + "[trace] watcher: rechecked {} newly watched directories in {:.1}ms", + dirs.len(), + start.elapsed().as_secs_f64() * 1000.0 + ); } -/// Whether a scheduled reconcile should run now. +/// Drop index entries for files that were deleted while subscriptions were +/// being established. /// -/// Split out from the loop so the schedule can be exercised without waiting -/// hours for it. -fn reconcile_due(since_last: Duration, quiet_for: Duration, busy: bool) -> bool { - // Indexing and flushing are already rewriting the index, and a reconcile - // takes the snapshot gate for its whole walk-and-merge. Let them finish; - // the next tick is a minute away. - if busy { - return false; +/// The counterpart to the indexing pass in [`reindex_files_in`]: a file removed +/// in that window produced no event either, and unlike a modified one nothing +/// later brings it back to the watcher's attention, so it keeps answering +/// searches until the hourly reconcile. +/// +/// `swept` holds the relative directories whose listing succeeded — a failed +/// `read_dir` proves nothing and must not be read as an empty directory — and +/// `present` every file those listings contained, regardless of ignore rules or +/// eligibility. Filtering `present` would delete entries for files that are +/// still on disk and were indexed under a laxer configuration. +/// +/// `vanished_dirs` are directories that could not be listed *and* were absent +/// from a parent listing that did succeed. Their descendants cannot be judged +/// by `swept` and `present`, which only speak for a file's immediate parent: a +/// directory deleted or moved away whole leaves indexed paths whose parent was +/// never enumerated, and no event names them either — a removal delivers one +/// event for the directory, and a move away delivers nothing at all for what +/// was inside it. Anything under one of these is swept on the strength of the +/// directory's absence. +/// +/// Candidates come from everything that can answer a search, not from +/// `file_stamps` alone. A stamp is not a precondition for being searchable: +/// `filestamps.json` is optional by design — missing or unreadable leaves the +/// map empty, and a build that predates a given file's stamp leaves it partial +/// — while the reader still holds that file's content. Sweeping only what has +/// a stamp would then delete nothing at all, and the deleted files would keep +/// answering searches until the hourly reconcile. +/// +/// Reader paths already hidden by a tombstone are skipped. `delete_file` +/// tombstones unconditionally and counts a mutation for it, so re-deleting +/// them would make every scan over a directory with deletions in it look like +/// fresh churn and pull flushes forward for no reason. +/// +/// The caller must already hold `snapshot_gate`. +fn sweep_removed_files( + state: &ServerState, + swept: &std::collections::HashSet, + present: &std::collections::HashSet, + vanished_dirs: &std::collections::HashSet, +) { + if swept.is_empty() { + return; } - if since_last >= RECONCILE_DEADLINE { - return true; + // One pass per source rather than a lookup per swept directory: at startup + // both sides of this span the whole repository, and anything proportional + // to their product would not finish. + let missing = |rel: &str| { + let parent = rel.rsplit_once('/').map_or("", |(dir, _)| dir); + if swept.contains(parent) && !present.contains(rel) { + return true; + } + // Walking ancestors costs one lookup per level, so it is done only + // when something actually vanished — which is rare, while this closure + // runs once per indexed path in the repository. + if vanished_dirs.is_empty() { + return false; + } + let mut ancestor = parent; + loop { + if vanished_dirs.contains(ancestor) { + return true; + } + match ancestor.rsplit_once('/') { + Some((next, _)) => ancestor = next, + None => return false, + } + } + }; + let mut gone: std::collections::HashSet = { + let stamps = state.file_stamps.read().unwrap(); + stamps.keys().filter(|rel| missing(rel)).cloned().collect() + }; + { + let index = state.index.read().unwrap(); + gone.extend(index.reader_paths_matching(|rel| missing(rel) && !index.live.is_deleted(rel))); + gone.extend( + index + .live + .overlay_paths() + .into_iter() + .filter(|rel| missing(rel)), + ); } - since_last >= RECONCILE_INTERVAL && quiet_for >= RECONCILE_QUIET_PERIOD -} - -/// Periodically compare the whole tree against the index, so a change the -/// watcher never heard about cannot stay wrong indefinitely. -/// -/// See [`RECONCILE_INTERVAL`] for why this is needed at all. It is deliberately -/// unhurried: it defers to indexing, to flushing, and to a server that is -/// being queried, and it does nothing at all on a tree that has not drifted — -/// the walk finds no differences and returns without touching the index. -fn periodic_reconcile_loop(state: Arc, root: PathBuf, index_dir: PathBuf) { - let mut last = Instant::now(); - loop { - thread::sleep(RECONCILE_POLL); - - let busy = state.indexing.load(Ordering::SeqCst) || state.flushing.load(Ordering::SeqCst); - if !reconcile_due(last.elapsed(), state.quiet_for(), busy) { - continue; + if gone.is_empty() { + return; + } + // Per candidate, under the same lock `reindex_file` takes, and re-checked + // against the filesystem rather than against the listing that produced + // `gone`. That listing is from earlier in the scan; a file recreated since + // then has already had its create event consumed by the watcher, so + // deleting it here on the strength of a stale observation would lose it + // until the next reconcile — and there is nothing left to replay. + // + // The lock is what makes the recheck mean anything: without it the file + // could be reindexed between the check and the delete, which is the same + // bug one instruction later. + let mut dropped = 0usize; + for rel in &gone { + let _reindex = lock_reindex(state); + // Through the same containment contract `reindex_file` opens under, + // not a bare `symlink_metadata`. That call refuses to follow only the + // *final* component: a directory that vanished and came back as a link + // to somewhere else makes `root/gone-dir/a.rs` resolve to a perfectly + // ordinary file outside the tree, and reading that as "it is back" + // keeps the stale in-root entry forever — nothing under a link is + // walked or watched, so no later event corrects it. + // + // Transient failures preserve, as they do in `reindex_file`: a + // descriptor limit or a sharing violation says nothing about whether + // the path belongs in the index, and the next reconcile will ask again. + match open_within_root(&state.root, &state.root.join(rel)) { + Ok(file) => match file.metadata() { + // Back, and reachable without leaving the tree. + Ok(meta) if meta.file_type().is_file() => continue, + // There, but not something the index should hold. + Ok(_) => {} + Err(_) => continue, + }, + Err(e) if proves_ineligible(&e) => {} + Err(_) => continue, } - - // Restart the interval before the walk rather than after it. On a large - // repository the reconcile itself takes a while, and timing from its - // completion would push each one further out than the last. - last = Instant::now(); - eprintln!("[trace] periodic reconcile: looking for changes the watcher missed"); - // Same comparison the startup check makes, and for the same reason: a - // lost event is a stamp that disagrees with the filesystem, a file with - // no stamp, or a stamp with no file, and all three fall out of that. - // Comparing index *membership* as well would additionally re-add any - // file whose stamp says indexed but which the reader does not hold — - // a publication bug rather than a lost event, and one that on an hourly - // timer would rebuild the whole index every hour if it ever misfired. - if !background_refresh_stale(&state, &root, &index_dir, false) { - // It declined — an unreadable directory, or a walk that raced a - // delete. The index is untouched and correct as far as it goes, - // and the next interval tries again. - eprintln!("[trace] periodic reconcile: declined, keeping the current index"); + state.index.write().unwrap().live.delete_file(rel); + state.file_stamps.write().unwrap().remove(rel); + if let Ok(mut cache) = state.cache.write() { + cache.pop(rel); } + dropped += 1; + } + if dropped > 0 { + eprintln!( + "[trace] watcher: dropped {dropped} file(s) removed while subscriptions were \ + being established" + ); } } -fn auto_save_loop(state: Arc) { - let mut last_save = Instant::now(); - - loop { - thread::sleep(Duration::from_secs(60)); - - // Don't auto-save while background indexing or a bulk flush is - // active — those paths handle their own publication and an - // auto-save fired in parallel would just snapshot the same - // overlay redundantly. - if state.indexing.load(Ordering::SeqCst) || state.flushing.load(Ordering::SeqCst) { - continue; +/// Index a directory that has just appeared and anything already inside it, +/// subscribing to it as well on a per-directory backend. +/// +/// Two separate reasons to be here, and they apply on different platforms. +/// Non-recursive subscriptions are not extended by notify — it only auto-adds +/// watches beneath a watch that was registered as recursive — so on Linux a new +/// directory has to be subscribed here or its contents are invisible. And on +/// every backend, a directory that arrives already populated (a `mv` from +/// outside the root, a checkout, an unpacked archive) reports itself and +/// nothing else: the kernel does not enumerate what moved in. Both leave files +/// that appear in no walk and no event. +/// +/// Files that landed between the directory's creation and its subscription +/// would be missed by definition, so the same pass indexes what it finds. +/// The caller must already hold `snapshot_gate`. +/// +/// The descent subscribes to each level *before* reading it. Reading first +/// leaves a window in which a child created in between is in neither place: +/// not in what this pass enumerates, and not yet able to report itself. That +/// window is small but it is exactly the one a checkout or a build fills, and +/// anything lost in it stays invisible until the hourly reconcile. +fn watch_new_subtree(state: &Arc, root: &Path, dir: &Path) { + // `is_dir` follows symlinks; the walker does not. Refuse a symlinked + // directory here so we never subscribe to, or index, a tree the indexer + // would not have walked into — and check every level, not just the last, + // since a real directory inside a symlinked one is just as far outside the + // served tree as the link itself. + if !is_contained_dir(root, dir) { + return; + } + let Ok(rel_dir) = dir.strip_prefix(root) else { + return; + }; + let rel_dir = rel_dir.to_string_lossy().replace('\\', "/"); + // The event that brought us here was filtered with file semantics, so a + // `build/`-style rule that only ever matches directories has not been + // applied to this path yet. Re-check before subscribing to it. + if !rel_dir.is_empty() { + let gitignore = state.gitignore.read().unwrap(); + if should_skip_watcher_dir(&rel_dir, &state.exclude_dirs, gitignore.as_ref()) { + return; } + } - let dirty = { - let index = state.index.read().unwrap(); - index.live.dirty_count() - }; - - let elapsed = last_save.elapsed(); - if dirty >= state.auto_save_mutations || (dirty > 0 && elapsed >= AUTO_SAVE_INTERVAL) { - let save_start = Instant::now(); - eprintln!("[trace] auto-save: {dirty} mutations, saving..."); + let mut level = vec![dir.to_path_buf()]; + let mut seen: std::collections::HashSet = std::collections::HashSet::new(); + let mut files: Vec<(PathBuf, String)> = Vec::new(); + let mut found_ignore_rules = false; + + 'descend: while !level.is_empty() { + // Subscribing is per-directory work. On a recursive backend the root's + // one subscription already covers everything below it, and taking a + // watch per directory there would be the exhaustion this whole pass + // exists to avoid — so only the enumeration below runs on those + // platforms. + if PER_DIRECTORY_WATCHES { + let mut registry = state.watch_registry.lock().unwrap(); + let Some(registry) = registry.as_mut() else { + return; + }; + // Additive, not a sync: this covers only the new subtree, and + // `sync` would read everything outside it as stale and unsubscribe + // from the entire rest of the repository. Staying additive also + // matters at scale — a monorepo can hold tens of thousands of + // watched directories, and materialising a union for every newly + // created directory would be quadratic over a checkout. + // + // Forced, because these directories have just appeared: a path + // recreated where a watched one used to be is still recorded as + // watched, but the kernel dropped its descriptor when the original + // went away. + registry.resubscribe_all(level.iter()); + } - // Hold the gate through delta build → publish → prune so watcher - // mutations cannot race publication. Recheck after acquiring it: - // another publisher may have drained the overlay while we waited. - let _gate = state.snapshot_gate.write().unwrap(); - if !state.index.read().unwrap().live.has_pending_changes() { + // The registry lock is released before any `read_dir`, so a large + // subtree does not hold it across the I/O for a whole level. + let mut next = Vec::new(); + for subdir in level.drain(..) { + if !seen.insert(subdir.clone()) { continue; } - - let stamps = state.file_stamps.read().unwrap().clone(); - if stream_merge_stale_changes(&state, &[], &[], &[], &stamps, "auto-save", false) { - last_save = Instant::now(); - eprintln!( - "[trace] auto-save complete in {:.1}s", - save_start.elapsed().as_secs_f64() - ); + let Ok(entries) = std::fs::read_dir(&subdir) else { + continue; + }; + for entry in entries.flatten() { + let Ok(file_type) = entry.file_type() else { + continue; + }; + let path = entry.path(); + let Ok(rel) = path.strip_prefix(root) else { + continue; + }; + let rel = rel.to_string_lossy().replace('\\', "/"); + // A subtree that arrives whole — a clone, a `mv`, a branch + // switch — can carry its own ignore rules. Those files are + // dot-prefixed, so the scan below would silently drop them and + // index the rest of the subtree against rules that do not know + // about them. + // + // Ahead of the type dispatch, and following links: the walker + // collects rule files with `Path::is_file`, which resolves + // symlinks, whereas `DirEntry::file_type` does not — so a + // symlinked `.gitignore` fell between the two branches below + // and was never noticed, despite carrying rules the walker + // would read. + // + // Abandon the descent immediately rather than finishing it. + // Everything gathered from here on is discarded by the refresh + // anyway, and the rules that are about to be published are the + // ones that decide whether these directories should be watched + // at all — continuing would subscribe to every level of, say, a + // `node_modules/` that was just moved into place, which on + // Linux is a watch descriptor apiece and the exhaustion this + // pass exists to avoid. The refresh's `sync` would prune them, + // but only after they had already been taken. + if !state.no_ignore && is_ignore_rules_file(root, &path) && path.is_file() { + found_ignore_rules = true; + break 'descend; + } + // `DirEntry::file_type` does not follow symlinks, so a + // symlinked directory is neither descended into nor indexed. + if file_type.is_dir() { + let skip = { + let gitignore = state.gitignore.read().unwrap(); + should_skip_watcher_dir(&rel, &state.exclude_dirs, gitignore.as_ref()) + }; + if !skip { + next.push(path); + } + } else if file_type.is_file() { + let skip = { + let gitignore = state.gitignore.read().unwrap(); + should_skip_watcher_path(&rel, &state.exclude_dirs, gitignore.as_ref()) + }; + if !skip { + files.push((path, rel)); + } + } } } + level = next; + } + + if found_ignore_rules { + // Indexing now would apply the wrong rules to the whole subtree, and + // anything wrongly indexed would stay until something touched it + // again. The refresh rewalks and republishes, which covers these files + // correctly; it runs on its own thread and takes `snapshot_gate` + // there, so scheduling it while we hold the gate is safe. + state.ignore_rules_dirty.store(true, Ordering::SeqCst); + schedule_ignore_rules_refresh(Arc::clone(state), root.to_path_buf()); + return; + } + + for (path, rel) in &files { + reindex_file(state, path, rel); } } -/// Check whether `path` passes the glob filter list. -/// -/// Glob semantics: -/// - Patterns starting with `!` are **exclusion** patterns (path must NOT match). -/// - All other patterns are **inclusion** patterns (path must match at least one). -/// - If only exclusion patterns are present, the path passes unless it matches -/// an exclusion. -/// - If inclusion patterns are present, the path must match at least one AND -/// must not match any exclusion. -fn json_rpc_result(id: Option, result: serde_json::Value) -> String { - serde_json::json!({ - "jsonrpc": "2.0", - "result": result, - "id": id.unwrap_or(serde_json::Value::Null), - }) - .to_string() -} +fn schedule_ignore_rules_refresh(state: Arc, root: PathBuf) { + if state + .ignore_refresh_scheduled + .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) + .is_err() + { + return; + } -fn json_rpc_error(id: Option, code: i32, message: &str) -> String { - serde_json::json!({ - "jsonrpc": "2.0", - "error": { - "code": code, - "message": message, - }, - "id": id.unwrap_or(serde_json::Value::Null), - }) - .to_string() -} + thread::spawn(move || { + loop { + if state.ignore_rules_dirty.swap(false, Ordering::SeqCst) { + // Wait out a build first. `background_index_build` publishes its + // matcher — and so reaches here — while it is still only + // part-way through Phase 2, and it holds `snapshot_gate` for + // none of that. The refresh would take the gate uncontended, + // replace `file_stamps` wholesale from its own walk, and then + // have the build overwrite them again from a walk that predates + // the new rules: an index and a stamp map describing two + // different trees, with no scan left to notice. + // + // A wait rather than a lock, so it cannot deadlock against the + // build; and nothing is lost by waiting, because the build is + // still walking the tree the refresh would walk. + while state.indexing.load(Ordering::SeqCst) { + thread::sleep(Duration::from_millis(200)); + } + // The stale refresh walks the tree anyway and republishes the + // matcher from that walk, so the reload costs one traversal + // rather than a rebuild plus a re-scan. + if !background_refresh_stale(&state, &root, &state.index_dir, true) { + state.ignore_rules_dirty.store(true, Ordering::SeqCst); + thread::sleep(Duration::from_secs(1)); + } + } -/// Create a minimal empty on-disk index so HybridIndex::open() succeeds. -/// The actual data will be populated into the LiveIndex in the background. -fn create_empty_index(index_dir: &Path) -> Result<()> { - use tgrep_core::meta::IndexMeta; - // Own the directory precondition here rather than leaving it to each - // caller: the recovery path in `reset_to_empty_index` runs after a failed - // build, which is exactly when the directory is least likely to be intact. - std::fs::create_dir_all(index_dir)?; - // Empty lookup.bin, index.bin, files.bin - std::fs::write(index_dir.join("lookup.bin"), b"")?; - std::fs::write(index_dir.join("index.bin"), b"")?; - std::fs::write(index_dir.join("files.bin"), b"")?; - let mut meta = IndexMeta::new("", 0, 0); - meta.complete = false; // empty skeleton — not a complete index - meta.save(index_dir)?; - Ok(()) + state + .ignore_refresh_scheduled + .store(false, Ordering::SeqCst); + if !state.ignore_rules_dirty.load(Ordering::SeqCst) + || state + .ignore_refresh_scheduled + .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) + .is_err() + { + break; + } + } + }); } -/// Detect files that changed while the server was not running. -/// Compares stored filestamps against current filesystem metadata, then upserts -/// changed/new files and removes deleted files from the LiveIndex. -fn classify_file_changes( - current_meta: &[tgrep_core::walker::FileMeta], - old_stamps: &std::collections::HashMap, - indexed_paths: &std::collections::HashSet, - compare_index_membership: bool, -) -> (Vec, Vec, Vec) { - use tgrep_core::meta::FileStamp; - - let mut current_set = std::collections::HashSet::with_capacity(current_meta.len()); - let mut changed = Vec::new(); - let mut added = Vec::new(); - - for fm in current_meta { - current_set.insert(fm.relative_path.clone()); - let stamp = FileStamp { - mtime: fm.mtime, - size: fm.size, - }; - if compare_index_membership && !indexed_paths.contains(&fm.relative_path) { - added.push(fm.relative_path.clone()); - continue; - } - match old_stamps.get(&fm.relative_path) { - Some(old) if *old == stamp => {} - Some(_) => changed.push(fm.relative_path.clone()), - None => added.push(fm.relative_path.clone()), - } +/// Remember the paths in an event that arrived mid-build, for replay once the +/// build publishes. Returns whether it did — a `false` means the build finished +/// underneath us and the caller should handle the event normally. +/// +/// The event cannot be applied now: until `indexing` clears, `file_stamps` does +/// not describe the index, so every path would compare as changed and the +/// watcher would re-read the repository alongside the build already reading it. +/// It cannot simply be dropped either — see [`ServerState::deferred_events`]. +/// +/// `indexing` is re-read *under the buffer lock*, and that is what makes the +/// handoff safe. The caller's own check is only a hint: between it and this +/// call the build can finish and [`replay_deferred_events`] can swap the buffer +/// out, and an insert landing after that swap is in a set nothing will ever +/// look at again. Because the replay cannot swap until `indexing` is false, and +/// cannot swap without this lock, seeing `indexing` set while holding it proves +/// the swap has not happened yet. +/// +/// Capped, and the cap discards the whole set rather than truncating it: a +/// partial set is indistinguishable from a complete one at replay time, and +/// silently recovering nine tenths of a checkout is worse than knowing to fall +/// back on a full refresh. +fn defer_events_during_build(state: &ServerState, event: &Event) -> bool { + /// A checkout of the Linux kernel is ~90k files. Above this the fallback + /// refresh is cheaper than the replay would be anyway. + const MAX_DEFERRED: usize = 100_000; + + let Ok(mut deferred) = state.deferred_events.lock() else { + return false; + }; + if !state.indexing.load(Ordering::SeqCst) { + return false; } - - let mut seen = std::collections::HashSet::new(); - let deleted = old_stamps - .keys() - .chain(indexed_paths) - .filter(|path| seen.insert(path.as_str())) - .filter(|path| !current_set.contains(path.as_str())) - .cloned() - .collect(); - - (changed, added, deleted) + let Some(paths) = deferred.as_mut() else { + // Already overflowed, so the fallback refresh will cover this path too. + return true; + }; + if paths.len().saturating_add(event.paths.len()) > MAX_DEFERRED { + *deferred = None; + eprintln!( + "[trace] warning: too many file changes during the initial index build to replay \ + individually; a full reconcile will run instead" + ); + return true; + } + // Only these kinds can put a directory somewhere, and only they should + // trigger a subtree walk on replay. See `ServerState::deferred_events`. + let introduces_dir = matches!( + event.kind, + EventKind::Create(_) | EventKind::Modify(notify::event::ModifyKind::Name(_)) + ); + for path in &event.paths { + // A path seen both ways keeps the stronger claim: a directory that was + // created and then chmod'd still needs its subtree picked up. + let entry = paths.entry(path.clone()).or_insert(false); + *entry |= introduces_dir; + } + true } -/// Apply a stale diff without materializing the existing index in heap. +/// Apply the events that arrived during the build, now that it has published. /// -/// The ordinary incremental flush uses `HybridIndex::full_snapshot`, whose -/// memory is proportional to every posting already on disk. That is especially -/// harmful when a newer tgrep first opens an index built with an older file-size -/// cap: every formerly-oversized file appears as new at once. Build new and -/// replacement files into a bounded external-sort delta, then stream it together -/// with the old index while filtering replaced and deleted reader entries. +/// Replayed as synthetic events rather than handled directly so they go through +/// exactly the filtering an ordinary event gets — ignore rules, the exclude +/// list, the index directory, new-subtree subscription. The kind is +/// reconstructed from the flag recorded with each path so the directory gate +/// still holds; beyond that gate, creations and modifications take the same +/// route, and `handle_fs_event` decides removal from whether the path still +/// exists, so one kind of each class covers arrivals, edits and deletions +/// alike. /// -/// The caller holds `snapshot_gate` across the metadata walk and this merge, so -/// the walk's exact path set is newer than every live entry captured here. -fn stream_merge_stale_changes( - state: &Arc, - changed: &[String], - added: &[String], - deleted: &[String], - stamps: &std::collections::HashMap, - operation: &str, - authoritative_membership: bool, -) -> bool { - let root = &state.root; - let index_dir = &state.index_dir; - let (reader, overlay_paths, tombstone_paths) = { - let index = state.index.read().unwrap(); - ( - index.reader_arc(), - index.live.overlay_paths(), - index.live.tombstone_paths(), - ) +/// The caller must *not* hold `snapshot_gate`; `handle_fs_event` takes it. +fn replay_deferred_events(state: &Arc, root: &Path) { + let deferred = match state.deferred_events.lock() { + // Leaves an empty map behind, so anything deferred by a later build + // (a reconcile sets `indexing` again) is collected from scratch. The + // caller has already waited out `indexing`, and `defer_events_during_ + // build` re-reads it under this same lock, so nothing can be inserted + // into the old map after this point. + Ok(mut guard) => guard.replace(std::collections::HashMap::new()), + Err(_) => return, + }; + let Some(paths) = deferred else { + // Overflowed. A stale refresh rewalks the tree and diffs it against the + // index, which covers every path the replay would have, and it is what + // already runs when ignore rules change mid-build. + eprintln!("[trace] watcher: reconciling after too many changes during the initial build"); + let state = Arc::clone(state); + let root = root.to_path_buf(); + if thread::Builder::new() + .name("tgrep-deferred-reconcile".into()) + .spawn(move || { + if !background_refresh_stale(&state, &root, &state.index_dir, true) { + eprintln!( + "[trace] warning: the post-build reconcile did not complete; changes made \ + during the build wait for the next one" + ); + } + }) + .is_err() + { + eprintln!("[trace] warning: could not start the post-build reconcile"); + } + return; }; + if paths.is_empty() { + return; + } - state.flushing.store(true, Ordering::SeqCst); let start = Instant::now(); + let count = paths.len(); + for (path, introduces_dir) in paths { + let kind = if introduces_dir { + EventKind::Create(notify::event::CreateKind::Any) + } else { + EventKind::Modify(notify::event::ModifyKind::Data( + notify::event::DataChange::Any, + )) + }; + handle_fs_event( + state, + root, + &Event { + kind, + paths: vec![path], + attrs: Default::default(), + }, + ); + } eprintln!( - "[trace] {operation}: building a memory-bounded delta \ - ({} changed, {} new, {} deleted)...", - changed.len(), - added.len(), - deleted.len() + "[trace] watcher: replayed {count} change(s) deferred during the initial build in {:.1}ms", + start.elapsed().as_secs_f64() * 1000.0 ); +} - // Keep work directories inside the locked index directory. Sibling names - // collide when two independent indexes share a parent directory. - let delta_dir = index_dir.join(".stale-delta"); - let staging_dir = index_dir.join(".stale-merge"); - let _ = std::fs::remove_dir_all(&delta_dir); - let _ = std::fs::remove_dir_all(&staging_dir); - - // Fold in every live mutation. A stale walk also treats its exact path set - // as authoritative, removing reader entries missing from the walk; an - // auto-save cannot do that because its filestamps may be incomplete. - let mut seen = std::collections::HashSet::new(); - let mut candidates: Vec = changed - .iter() - .chain(added) - .chain(deleted) - .chain(overlay_paths.iter()) - .chain(tombstone_paths.iter()) - .filter(|path| seen.insert((*path).clone())) - .cloned() - .collect(); - if authoritative_membership { - candidates.extend( - reader - .all_paths() - .iter() - .filter(|path| !stamps.contains_key(path.as_str())) - .filter(|path| seen.insert((*path).clone())) - .cloned(), +/// For the publish sites that cannot scan inline. Until a directory is +/// subscribed it cannot report a write, and the walk that decided to subscribe +/// to it may have passed it before that write happened, so a file landing in +/// that window is in neither place and would wait for the hourly reconcile. +/// +/// Spawned because at startup this is every directory in the repository and +/// the callers are on paths that must not block: `start_file_watcher` has to +/// get its worker running, and an index build must not stop to stat the tree +/// it is already reading. +/// +/// Waits out `indexing` first. During a build the stamps do not describe the +/// index yet, so every file would look changed and the scan would re-read the +/// whole repository alongside the build that is already doing it. That wait is +/// also what makes this the right place to replay the events the build made the +/// watcher discard, which is why it runs even on a backend that has nothing +/// per-directory to recover. +fn spawn_recovery_scan( + state: &Arc, + root: &Path, + dirs: Vec, + since: SystemTime, +) { + let state = Arc::clone(state); + let root = root.to_path_buf(); + let spawned = thread::Builder::new() + .name("tgrep-watch-recovery".into()) + .spawn(move || { + while state.indexing.load(Ordering::SeqCst) { + thread::sleep(Duration::from_millis(200)); + } + // Before the gate, not under it: this takes `snapshot_gate` for + // read itself, and std's `RwLock` may deadlock on a recursive read + // if a writer queues up in between. + replay_deferred_events(&state, &root); + + let dirs = recovery_scan_dirs(&state, &root, dirs); + if dirs.is_empty() { + return; + } + // Read, not write: this does exactly what `handle_fs_event` does, + // and that runs under the read side. Taking it at all is what + // keeps the stamp check and the index update from interleaving + // with a flush. + let _gate = state.snapshot_gate.read().unwrap(); + reindex_files_in(&state, &root, &dirs, since); + }); + if spawned.is_err() { + eprintln!( + "[trace] warning: could not start the watcher recovery scan; \ + files written while subscriptions were being established will \ + wait for the next reconcile" ); } - let desired_paths: Vec = candidates - .iter() - .filter(|path| stamps.contains_key(path.as_str())) - .cloned() - .collect(); - let files: Vec = desired_paths.iter().map(|path| root.join(path)).collect(); - // Every candidate is either removed or replaced by the delta. Including a - // genuinely new path is harmless because it has no reader entry to filter. - let removed: std::collections::HashSet = candidates.iter().cloned().collect(); - - let mut published_stamps = stamps.clone(); +} - let result = (|| -> Result { - let build = || { - builder::build_index_for_files( - root, - &delta_dir, - &files, - builder::DEFAULT_INDEX_BUFFER_BYTES, - ) - }; - let outcome = match rayon::ThreadPoolBuilder::new() - .num_threads(state.index_threads) - .thread_name(|i| format!("tgrep-stale-index-{i}")) - .build() - { - Ok(pool) => pool.install(build)?, - Err(_) => build()?, - }; - let delta_count = outcome.indexed; +/// The directories a recovery scan should recheck, given the ones a +/// subscription sync reported as newly watched. +/// +/// The root is added because it is never in that list: it is subscribed as the +/// watcher starts, before any matcher exists, so every later sync sees it as +/// already watched. Nothing else covers it — a file written to the top level +/// while a build walk was deeper in the tree produced no event the build could +/// use and no event the watcher would keep — and it costs one directory +/// listing. +/// +/// Empty on a whole-subtree backend, where there are no per-directory +/// subscriptions to have raced and `reindex_files_in`'s pickup of unwatched +/// subdirectories would take exactly the per-directory watches that backend +/// exists to avoid. +fn recovery_scan_dirs(state: &ServerState, root: &Path, mut dirs: Vec) -> Vec { + if !PER_DIRECTORY_WATCHES || !state.watch_enabled { + return Vec::new(); + } + if !dirs.iter().any(|d| d == root) { + dirs.push(root.to_path_buf()); + } + dirs +} - // Withhold stamps for files the delta could not read. A published stamp - // means "indexed at this version", so keeping one for a skipped file - // would hide it from every later reconcile and make the miss permanent. - // Dropping the stamp leaves it looking new, so the next pass retries it. - // - // Record what the file looked like when it failed, so a permanent - // failure is retried when the file changes rather than on every pass. - // See `ServerState::unreadable`. - { - let mut memo = state.unreadable.write().unwrap(); - // Anything this delta was asked to build is settled: either it was - // read, or it is in `outcome.unreadable` and re-recorded below. - for path in changed.iter().chain(added).chain(deleted) { - memo.remove(path); - } - for path in &outcome.unreadable { - let rel = path +fn handle_fs_event(state: &Arc, root: &Path, event: &Event) { + let dominated_kinds = matches!( + event.kind, + EventKind::Create(_) | EventKind::Modify(_) | EventKind::Remove(_) + ); + if !dominated_kinds { + return; + } + + // Two ways an event can carry a rules change. The obvious one is a path the + // walker would read as a rules file, recognised by name. + // + // The other is a path the published matcher actually read through a + // symlink. `ignore_files_in` uses `Path::is_file`, which follows links, so + // a `.gitignore` symlinked to `shared-rules` contributes the *target's* + // contents — but editing the target produces an event naming `shared-rules`, + // whose basename means nothing to `is_ignore_rules_file`, and touches + // nothing whose name does. Recognising the paths that were read, and not + // just the names rules usually go by, is what closes that. + // + // Only targets inside `root` can appear here, and [`ignore_target_dirs`] is + // what makes them observable: their directory is subscribed to even when the + // rules hide it, precisely so this lookup has an event to run against. For a + // target outside the root none arrives and the periodic reconcile remains + // the backstop. + let ignore_rules_changed = !state.no_ignore && { + let stamps = state.ignore_source_stamps.read().unwrap(); + event.paths.iter().any(|path| { + is_ignore_rules_file(root, path) + || path .strip_prefix(root) - .unwrap_or(path) - .to_string_lossy() - .replace('\\', "/"); - if let Some(stamp) = published_stamps.remove(&rel) { - memo.insert(rel, stamp); - } - } - } - if !outcome.unreadable.is_empty() { - eprintln!( - "[trace] {} file(s) were unreadable during the delta build; \ - their stamps are withheld so a later reconcile retries them", - outcome.unreadable.len() - ); + .is_ok_and(|rel| stamps.contains_key(&rel.to_string_lossy().replace('\\', "/"))) + }) + }; + if ignore_rules_changed { + state.ignore_rules_dirty.store(true, Ordering::SeqCst); + if state.indexing.load(Ordering::SeqCst) { + return; } + schedule_ignore_rules_refresh(Arc::clone(state), root.to_path_buf()); + return; + } - let delta = tgrep_core::reader::IndexReader::open(&delta_dir)?; - if delta.num_files() != delta_count { - anyhow::bail!( - "delta reopened with {} files after writing {delta_count}", - delta.num_files() - ); + // Skip ordinary file events while the initial background index build is in + // progress. The indexer will pick up those files itself — but only for the + // parts of the tree it has not reached yet, so remember these and replay + // them once it publishes. + // + // The load is a fast path that keeps the mutex out of the common case; the + // decision is made under the lock, since the build can finish between the + // two and an event deferred after that is never replayed. + if state.indexing.load(Ordering::SeqCst) && defer_events_during_build(state, event) { + return; + } + + // Acquire the snapshot gate up-front for the whole event. While a + // flush/auto-save is publishing (writer holds it), no reindex + // *work* — file I/O, trigram extraction, even the [trace] line — + // should happen, both for correctness (no overlay mutation between + // snapshot and prune) and to avoid spending CPU/IO on work that + // would just block the watcher thread anyway. We hold it for read + // so multiple events can proceed concurrently outside any flush. + let _gate = state.snapshot_gate.read().unwrap(); + + // Stay off the index until the initial ignore matcher exists. This check + // must happen *under* the gate: during startup the stale walk holds the + // write side, so an event waits and is applied after publication instead + // of being dropped after the walk may already have visited its path. + if state.gitignore_pending.load(Ordering::SeqCst) { + return; + } + + for path in &event.paths { + // Skip the index directory itself + if path + .to_string_lossy() + .contains(&format!("{}.tgrep", std::path::MAIN_SEPARATOR)) + { + continue; } - builder::merge_index_with_delta(root, &staging_dir, &reader, &delta, &removed, true)?; - tgrep_core::meta::write_filestamps(&published_stamps, &staging_dir)?; - let removed_reader_files = reader - .all_paths() - .iter() - .filter(|path| removed.contains(path.as_str())) - .count(); - let expected_files = reader.num_files() - removed_reader_files + delta.num_files(); - let published = publish_staged_index(state, index_dir, &staging_dir, expected_files); - if published { - // `publish_staged_index` prunes overlay entries represented by the - // new reader. Also clear reconciled entries intentionally omitted - // (newly ignored/binary/deleted) and old tombstones for files the - // delta restored. The gate guarantees none is newer than this merge. - state - .index - .write() - .unwrap() - .live - .clear_reconciled_paths(&candidates); - state - .index_progress - .store(expected_files as u64, Ordering::Relaxed); - state - .index_total - .store(expected_files as u64, Ordering::Relaxed); + let rel_path = match path.strip_prefix(root) { + Ok(p) => p.to_string_lossy().replace('\\', "/"), + Err(_) => continue, + }; + + // Mirror the walker's filtering so the watcher does not reindex + // files the initial walk would have skipped — most notably + // hidden directories like `.git/`, which fire frequent + // `index.lock`/HEAD/refs writes during normal git operations. + let should_skip = { + let gitignore = state.gitignore.read().unwrap(); + should_skip_watcher_path(&rel_path, &state.exclude_dirs, gitignore.as_ref()) + }; + if should_skip { + continue; } - Ok(published) - })(); - let _ = std::fs::remove_dir_all(&delta_dir); - let _ = std::fs::remove_dir_all(&staging_dir); - state.flushing.store(false, Ordering::SeqCst); - if matches!(&result, Ok(true)) { - *state.file_stamps.write().unwrap() = published_stamps; - } - match result { - Ok(true) => { - eprintln!( - "[trace] {operation}: streamed {} changes into the index in {:.1}s", - candidates.len(), - start.elapsed().as_secs_f64() - ); - true + // Classified through the same contract the sweep uses, from one stat. + // `Path::exists` and `Path::is_file` map every metadata error to + // `false`, so a file held open by a build, a Windows sharing violation, + // or a momentary `EACCES` used to read as "gone" and "not a regular + // file" respectively — and both branches below then evicted content + // that was still perfectly valid. `reindex_file` deliberately preserves + // entries through exactly those failures, but it never got the chance: + // the drop happens here, before it is ever called. + let target = classify_event_target(&std::fs::metadata(path)); + if target == EventTarget::Unknown { + // Unreadable right now is not proof of anything. Leave what is + // indexed alone; the stale path keeps such files and retries them. + continue; } - Ok(false) => { - eprintln!( - "[trace] warning: {operation} delta could not be published; \ - keeping the old index" - ); - false + + let is_remove = matches!(event.kind, EventKind::Remove(_)) || target == EventTarget::Gone; + + if is_remove { + // A watched directory that disappears takes its descriptor with + // it, but not its entry in the registry. Clearing that entry is + // what lets the path be subscribed again if it comes back — and + // keeps `watched` from accumulating dead paths between syncs. + // Done for every removed path rather than only known directories, + // since by now there is nothing left to ask what it was; a path + // that was never watched is a single failed hash lookup. + if PER_DIRECTORY_WATCHES + && let Some(registry) = state.watch_registry.lock().unwrap().as_mut() + { + registry.forget(path); + } + // notify can deliver Remove events for transient/unknown paths + // (e.g. a build tool's temp file). Suppress the noisy log line + // for those, but still apply the delete unconditionally — if + // `file_stamps` is missing/out-of-date (e.g. first run after + // an older index), skipping the delete entirely would leave + // stale entries for files that no longer exist. + // + // Under `reindex_lock`, or a concurrent `reindex_file` that has + // already read the file's bytes commits them *after* this delete + // and resurrects a file that is gone — with a fresh stamp, so + // nothing afterwards disagrees and no further event is coming to + // correct it. The lock makes the two orderings the only two: the + // delete lands on content that was committed, or the reindex opens + // a path that is already gone and drops it. + let _reindex = lock_reindex(state); + let known_path = state.file_stamps.read().unwrap().contains_key(&rel_path); + if known_path { + eprintln!("[trace] reindex: removed {rel_path}"); + } + // gate acquired at the function level — the entire event + // is processed atomically with respect to flush/auto-save. + state.index.write().unwrap().live.delete_file(&rel_path); + state.file_stamps.write().unwrap().remove(&rel_path); + if let Ok(mut cache) = state.cache.write() { + cache.pop(&rel_path); + } + continue; } - Err(error) => { - eprintln!( - "[trace] warning: memory-bounded {operation} failed ({error}); \ - keeping the old index" - ); - false + + // `is_file` follows symlinks, so a link to a file lands in + // `reindex_file` below rather than here — deliberately: that is where + // it is recognised as ineligible and any content indexed under that + // path before it became a link is dropped. + if target != EventTarget::Regular { + // `is_real_dir` rather than `is_dir`: the latter follows symlinks, + // and a link to a directory is not something the walker descends + // into, so subscribing to and indexing its target would pull in a + // tree the index never contained — possibly outside `root`. + if is_real_dir(path) { + // Only for events that can actually introduce a directory. Any + // `Modify` would include `Modify(Metadata)`, which a recursive + // chmod or a checkout fires once per directory — and each one + // would re-walk and re-subscribe that directory's whole subtree + // on the single watcher worker, turning a linear operation into + // quadratic work. inotify announces a new directory as `Create` + // and one moved in as `Modify(Name)`; nothing else can. + let introduces_dir = matches!( + event.kind, + EventKind::Create(_) | EventKind::Modify(notify::event::ModifyKind::Name(_)) + ); + if introduces_dir { + // A directory that just appeared can already be full — a + // `mv` of a populated tree from outside the root, a + // checkout, an unpacked archive — and nothing reports the + // contents it arrived with. + // + // On a per-directory backend that is because notify does + // not extend a non-recursive watch set for us. On a + // recursive backend it is the kernel's own doing: both + // `ReadDirectoryChangesW` and FSEvents report a moved-in + // tree as one event for the directory and say nothing about + // what is inside it. Either way those files are in no walk + // and no event, and stay unsearchable until the hourly + // reconcile — so the enumeration runs on every platform and + // only the subscribing part stays per-directory. + watch_new_subtree(state, root, path); + } + continue; + } + // Neither a regular file nor a directory: a fifo, a socket, a + // device, or a symlink of any kind. An indexed `x.rs` atomically + // replaced by one of those is not a removal — `path.exists()` is + // still true and inotify may report only the rename destination — + // so nothing above catches it, and without this the old contents + // stay searchable indefinitely. + // + // Same lock as the removal above, for the same reason: a reindex + // already holding the old bytes must not commit them after this. + let _reindex = lock_reindex(state); + drop_indexed_file(state, &rel_path, "no longer a regular file"); + continue; } + + reindex_file(state, path, &rel_path); } } -/// Drop the candidates that failed to read last time and have not changed since. +/// Take the mutation lock, tolerating a previous holder's panic. /// -/// Returns the paths removed, which the caller must also keep out of the -/// published stamps — see [`stamps_for_indexed`]. -fn drop_memoized_failures( - memo: &std::collections::HashMap, - current_meta: &[tgrep_core::walker::FileMeta], - changed: &mut Vec, - added: &mut Vec, -) -> std::collections::HashSet { - if memo.is_empty() { - return std::collections::HashSet::new(); +/// Poisoning here means some other indexer panicked partway through an update, +/// not that the index is unusable. Refusing to serialise from then on would +/// turn one failure into the resurrection race this lock exists to prevent. +fn lock_reindex(state: &ServerState) -> std::sync::MutexGuard<'_, ()> { + match state.reindex_lock.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), } - // One pass over the walk to pick out the memoized paths, rather than a scan - // per candidate. - let still_failing: std::collections::HashSet = current_meta - .iter() - .filter(|fm| { - memo.get(&fm.relative_path) - .is_some_and(|a| a.mtime == fm.mtime && a.size == fm.size) - }) - .map(|fm| fm.relative_path.clone()) - .collect(); - changed.retain(|path| !still_failing.contains(path)); - added.retain(|path| !still_failing.contains(path)); - still_failing } -/// The stamps to publish for a walk, minus the files that were never built. +/// Drop everything the index holds for a path. /// -/// A published stamp means "indexed at this version". Stamping a file the delta -/// deliberately skipped would make every later reconcile see it as unchanged, -/// so it would never be indexed again — and because the stamp lands in -/// `filestamps.json`, not even a restart would recover it: every automatic -/// caller of the stale check passes `compare_index_membership = false`, which -/// is precisely the check that would have noticed the file is missing. Leaving -/// it unstamped keeps it looking new, which is what makes the retry-on-change -/// behaviour in [`drop_memoized_failures`] work at all. -fn stamps_for_indexed( - current_meta: &[tgrep_core::walker::FileMeta], - skipped: &std::collections::HashSet, -) -> std::collections::HashMap { - current_meta - .iter() - .filter(|fm| !skipped.contains(&fm.relative_path)) - .map(|fm| { - ( - fm.relative_path.clone(), - tgrep_core::meta::FileStamp { - mtime: fm.mtime, - size: fm.size, - }, - ) - }) - .collect() +/// The delete is not conditional on a stamp entry. `ServerState` accepts an +/// empty stamp map when `filestamps.json` is missing or unreadable, and the +/// reader can still hold the path in that state, so keying the delete on the +/// stamp alone would leave content the walk now rejects searchable. The stamp +/// removal is best-effort; the index and cache deletes always happen. +/// +/// The cost of that is a tombstone in the overlay for paths that were never +/// indexed — `live::delete_file` records one either way — and this runs for +/// every ineligible file a recovery scan walks past, which at startup is every +/// binary asset in the repository. An existing tombstone is therefore taken as +/// proof there is nothing left to do, which bounds that to one per distinct +/// path. The trace line, which is the part that would be actively misleading, +/// stays conditional on there having been something to drop. +/// +/// The caller must already hold `snapshot_gate` and `reindex_lock`. The lock is +/// the caller's rather than this function's because `reindex_file` calls in +/// while holding it, and a `Mutex` is not reentrant. +fn drop_indexed_file(state: &ServerState, rel_path: &str, reason: &str) { + let had_stamp = state + .file_stamps + .write() + .unwrap() + .remove(rel_path) + .is_some(); + { + let index = state.index.read().unwrap(); + if index.live.has_path(rel_path) || had_stamp { + eprintln!("[trace] reindex: dropped {rel_path} ({reason})"); + } else if index.live.is_deleted(rel_path) { + // Already tombstoned, so there is nothing to record and no reason + // to dirty the overlay again. This is the repeat case: a recovery + // scan or a chatty editor can bring the same rejected path back + // here any number of times. + return; + } + } + state.index.write().unwrap().live.delete_file(rel_path); + if let Ok(mut cache) = state.cache.write() { + cache.pop(rel_path); + } } -fn background_refresh_stale( - state: &Arc, - root: &Path, - index_dir: &Path, - compare_index_membership: bool, -) -> bool { - use tgrep_core::meta; - use tgrep_core::walker; +/// What an event's stat result says about the path it named. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum EventTarget { + /// A regular file: index it. + Regular, + /// Something that exists but is not a regular file — a directory, a fifo, a + /// socket, a device. Whatever was indexed under this path has to go. + NotRegular, + /// Proven not to be there: absent, unreachable except through a symlink, or + /// behind a component that is not a directory. + Gone, + /// Exists or not, this stat cannot say. Nothing may be concluded from it. + Unknown, +} - let _refresh = state.stale_refresh_lock.lock().unwrap(); - // Keep watcher/auto-save mutations out for the complete walk → matcher → - // merge cycle. Search queries do not take this gate and remain available. - let _gate = state.snapshot_gate.write().unwrap(); - let start = Instant::now(); - eprintln!("[trace] stale check: comparing index against filesystem..."); - - // Walk first. This single traversal feeds both the stale diff and the - // watcher's ignore matcher, and it must run before the early returns below - // so the matcher can be published on every path out of this function. - let walk = walker::walk_file_metadata( - root, - &walker::MetaWalkOptions { - exclude_dirs: state.exclude_dirs.clone(), - no_ignore: state.no_ignore, - no_require_git: state.no_require_git, - max_file_size: state.max_file_size, - }, - ); - let walk_ms = start.elapsed().as_millis(); - - // Publish the matcher immediately, before any early return can skip it. - // Every exit below is a decision about the *index*; none of them is a - // reason to leave the watcher gated. `gitignore_pending` is what keeps the - // watcher off the index until a matcher exists, so leaking it past a return - // disables the watcher permanently — and the overflow-repair path skips - // reconciling while that flag is set, so nothing recovers it either. A - // single unreadable directory, or one file whose `metadata()` lost a race - // with a delete, would be enough. - // - // Committing here rather than at each exit is invisible to the watcher: - // this function holds `snapshot_gate` for write across its whole body, and - // the only reader of `state.gitignore` takes the read side first, so no - // event can observe the matcher before this function returns either way. - commit_stale_matcher(state, build_stale_matcher(state, root, &walk)); - - if walk.skipped_error > 0 { - eprintln!( - "[trace] warning: stale check could not inspect {} filesystem entries \ - (walk: {walk_ms}ms); keeping the old index", - walk.skipped_error - ); - return false; +/// Classify the target of an event from a stat of its path. +/// +/// The `Unknown` case is the point of this. `Path::exists` and `Path::is_file` +/// fold every error into `false`, which turns "a build has this file open" and +/// "the directory was briefly unreadable" into "it is gone" — and the watcher +/// then evicts live content on the strength of it. [`proves_ineligible`] is the +/// contract that separates the two, and it is the same one the recovery sweep +/// and [`reindex_file`] answer to, so all three agree about what an I/O failure +/// is allowed to mean. +/// +/// The stat follows symlinks, which is deliberate: a link to a regular file is +/// classified `Regular` here and refused by `open_within_root` in +/// [`reindex_file`], which is where content indexed under a path that has since +/// become a link is dropped. +fn classify_event_target(meta: &std::io::Result) -> EventTarget { + match meta { + Ok(meta) if meta.is_file() => EventTarget::Regular, + Ok(_) => EventTarget::NotRegular, + Err(error) if proves_ineligible(error) => EventTarget::Gone, + Err(_) => EventTarget::Unknown, } - let current_meta = &walk.files; - - // Load stored per-file stamps from last index write - let mut old_stamps = match meta::read_filestamps(index_dir) { - Ok(s) => s, - Err(e) => { - eprintln!("[trace] stale check: no filestamps found ({e}), comparing against reader"); - std::collections::HashMap::new() - } - }; +} - // Fold in stamps the watcher recorded since the last flush. `filestamps.json` - // only advances when the index is written to disk, so mid-session it lags - // the live overlay. Comparing against the on-disk copy alone would re-index - // every file the watcher already handled since that write, and — worse — - // a file created and then deleted inside that window appears in neither the - // on-disk stamps nor the filesystem, so it would never be classified as - // deleted and would linger in the index. The in-memory stamps are the - // fresher record of what the index actually holds, so they win. - for (path, stamp) in state.file_stamps.read().unwrap().iter() { - old_stamps.insert(path.clone(), stamp.clone()); +/// Whether a failure to open a path establishes that it does not belong in the +/// index, as opposed to merely being unreadable right now. +/// +/// The distinction decides whether the watcher drops what it has indexed. A +/// path that is gone, or that cannot be reached without traversing a symlink, +/// is genuinely ineligible and the entry has to go. A path that is locked, +/// unreadable, or lost to a descriptor limit is none of those things — dropping +/// on that would evict live content because a build held the file open for a +/// moment, and the stale path deliberately keeps unreadable files and retries +/// them later. +fn proves_ineligible(error: &std::io::Error) -> bool { + use std::io::ErrorKind; + + // `InvalidInput` and `NotADirectory` are what `open_within_root` itself + // returns for a path that escapes the root, has a non-literal component, or + // runs through something that is not a directory. + if matches!( + error.kind(), + ErrorKind::NotFound | ErrorKind::NotADirectory | ErrorKind::InvalidInput + ) { + return true; } - let indexed_paths = { - let index = state.index.read().unwrap(); - let mut paths = index.reader_paths(); - paths.extend(index.live.overlay_paths()); - paths - }; - if old_stamps.is_empty() && indexed_paths.is_empty() && current_meta.is_empty() { - eprintln!("[trace] stale check: no indexed files or filesystem files, skipping"); + // A symlink met under `O_NOFOLLOW`, at any level. Not yet a stable + // `ErrorKind`, so it has to be read from the raw code. + #[cfg(unix)] + if error.raw_os_error() == Some(libc::ELOOP) { return true; } + false +} - let (mut changed, mut added, deleted) = classify_file_changes( - current_meta, - &old_stamps, - &indexed_paths, - compare_index_membership, - ); - - // Files that failed to read last time and have not changed since are not - // worth another attempt; without this a single permanently locked file - // makes every scheduled reconcile rebuild the index. `deleted` is exempt — - // a file that is gone needs no read to evict. - let skipped_unreadable = { - let memo = state.unreadable.read().unwrap(); - drop_memoized_failures(&memo, current_meta, &mut changed, &mut added) - }; - if !skipped_unreadable.is_empty() { - eprintln!( - "[trace] stale check: {} file(s) unchanged since they last failed to \ - read; not retrying them", - skipped_unreadable.len() - ); +/// Open a file without following a final symlink, so the handle is the path +/// itself rather than wherever it points. +/// +/// Only the last component. For a path whose ancestors are not already trusted, +/// use [`open_within_root`] — which on unix has no use for this, since `openat` +/// resolves the final component the same way as every other one. +#[cfg(not(unix))] +fn open_no_follow(path: &Path) -> std::io::Result { + #[cfg(windows)] + { + use std::os::windows::fs::OpenOptionsExt; + // Opens the reparse point rather than its target. Unlike O_NOFOLLOW + // this succeeds, so the caller's `is_file` check on the handle's + // metadata is what rejects it — a reparse point is not a regular file. + const FILE_FLAG_OPEN_REPARSE_POINT: u32 = 0x0020_0000; + std::fs::OpenOptions::new() + .read(true) + .custom_flags(FILE_FLAG_OPEN_REPARSE_POINT) + .open(path) } + #[cfg(not(any(unix, windows)))] + { + std::fs::File::open(path) + } +} - let total_changes = changed.len() + added.len() + deleted.len(); - let live_pending = state.index.read().unwrap().live.has_pending_changes(); - if total_changes == 0 && !live_pending { - eprintln!( - "[trace] stale check: index is up-to-date ({} files checked in {}ms)", - current_meta.len(), - walk_ms - ); - return true; +/// Open a file under `root` without traversing a symlink at *any* level. +/// +/// Refusing to follow the final component is not enough. A path arrives here +/// from an event or a replay as a name, and `root/a/file` reads the same +/// whether `a` is a directory or a link to one — so an intermediate link is +/// enough to hand back a file outside the served tree, which is exactly the +/// containment the walker's `follow_links(false)` promises and the index's +/// contract depends on. +/// +/// `root` itself is the trust anchor and is opened normally: it is the +/// directory the user asked us to serve, so a link there is theirs to have. +/// +/// On unix this is race-free. Each component is resolved with `openat` against +/// the handle for its parent, so the name is never re-resolved and there is no +/// window in which a directory can be swapped for a link between the check and +/// the use. +#[cfg(unix)] +fn open_within_root(root: &Path, path: &Path) -> std::io::Result { + use std::io::{Error, ErrorKind}; + use std::os::fd::{AsRawFd, FromRawFd, OwnedFd}; + use std::os::unix::ffi::OsStrExt; + + let components = relative_components(root, path)?; + let mut dir: OwnedFd = std::fs::File::open(root)?.into(); + let last = components.len() - 1; + for (i, component) in components.iter().enumerate() { + let name = std::ffi::CString::new(component.as_bytes()) + .map_err(|_| Error::new(ErrorKind::InvalidInput, "path component contains a NUL"))?; + // `O_DIRECTORY` on the intermediates so a *file* in the middle of the + // path fails here rather than at the next `openat`, and `O_NOFOLLOW` on + // every one of them, including the last. `O_NONBLOCK`: see + // `open_no_follow`. + let mut flags = libc::O_RDONLY | libc::O_NOFOLLOW | libc::O_CLOEXEC | libc::O_NONBLOCK; + if i != last { + flags |= libc::O_DIRECTORY; + } + // SAFETY: `dir` is a live directory descriptor for the parent, and + // `name` is a NUL-terminated single path component that outlives the + // call. + let fd = unsafe { libc::openat(dir.as_raw_fd(), name.as_ptr(), flags) }; + if fd < 0 { + return Err(Error::last_os_error()); + } + // SAFETY: `openat` returned a fresh owned descriptor. Assigning it + // drops the previous one, closing the parent we no longer need. + dir = unsafe { OwnedFd::from_raw_fd(fd) }; } + Ok(std::fs::File::from(dir)) +} - if total_changes == 0 { - eprintln!( - "[trace] stale check: metadata is unchanged, reconciling live mutations \ - (walk: {walk_ms}ms)" - ); - } else { - eprintln!( - "[trace] stale check: {} changed, {} new, {} deleted (walk: {}ms)", - changed.len(), - added.len(), - deleted.len(), - walk_ms - ); +/// As above. Windows has no `openat`, so containment is established after the +/// fact instead of during resolution: the file is opened without following a +/// final reparse point, and the *handle* is then asked where it actually ended +/// up. Anything that is not under the root's own resolved path is refused. +/// +/// This is race-free in the way that matters. Checking each ancestor by path +/// first would only reject a junction that happened to be there at the time of +/// the check — one substituted between the check and the open would still be +/// followed. Asking the handle removes the second lookup entirely: whatever the +/// open resolved through, the answer describes the object we are actually +/// holding. +#[cfg(windows)] +fn open_within_root(root: &Path, path: &Path) -> std::io::Result { + use std::io::{Error, ErrorKind}; + + // Rejects escapes and non-literal components before anything is opened. + relative_components(root, path)?; + let file = open_no_follow(path)?; + + // The root's own resolved path, since it may itself sit under a junction or + // a substituted drive — comparing against the path as given would then + // reject every file in the tree. `canonicalize` is the same + // `GetFinalPathNameByHandleW` query underneath, so the two agree on + // verbatim prefix and casing. + let anchor = std::fs::canonicalize(root)?; + let opened = final_path_of(&file)?; + if !opened.starts_with(&anchor) { + return Err(Error::new( + ErrorKind::InvalidInput, + "path resolves outside the served root", + )); } + Ok(file) +} - let new_stamps = stamps_for_indexed(current_meta, &skipped_unreadable); +/// Where an open handle actually is, with every reparse point on the way +/// resolved. +#[cfg(windows)] +fn final_path_of(file: &std::fs::File) -> std::io::Result { + use std::os::windows::ffi::OsStringExt; + use std::os::windows::io::AsRawHandle; + use windows_sys::Win32::Storage::FileSystem::{ + FILE_NAME_NORMALIZED, GetFinalPathNameByHandleW, VOLUME_NAME_DOS, + }; - if !stream_merge_stale_changes( - state, - &changed, - &added, - &deleted, - &new_stamps, - "stale check", - true, - ) { - return false; + let handle = file.as_raw_handle() as isize; + let mut buf = vec![0u16; 512]; + loop { + // SAFETY: `handle` is a live file handle borrowed from `file`, and the + // buffer's length is passed as its capacity in `u16`s. + let needed = unsafe { + GetFinalPathNameByHandleW( + handle as _, + buf.as_mut_ptr(), + buf.len() as u32, + FILE_NAME_NORMALIZED | VOLUME_NAME_DOS, + ) + }; + if needed == 0 { + return Err(std::io::Error::last_os_error()); + } + // The return value excludes the NUL when it fits and includes it when + // it does not, so a value at or past the capacity means "too small". + if (needed as usize) < buf.len() { + buf.truncate(needed as usize); + return Ok(PathBuf::from(std::ffi::OsString::from_wide(&buf))); + } + buf.resize(needed as usize + 1, 0); } +} - if let Ok(mut cache) = state.cache.write() { - for path in changed.iter().chain(added.iter()).chain(deleted.iter()) { - cache.pop(path); +/// A fallback for platforms that are neither unix nor Windows, where there is +/// no way to do better than refusing a link that is actually there. +#[cfg(not(any(unix, windows)))] +fn open_within_root(root: &Path, path: &Path) -> std::io::Result { + use std::io::{Error, ErrorKind}; + + let components = relative_components(root, path)?; + let mut walked = root.to_path_buf(); + for component in &components[..components.len() - 1] { + walked.push(component); + let meta = std::fs::symlink_metadata(&walked)?; + if meta.file_type().is_symlink() { + return Err(Error::new( + ErrorKind::InvalidInput, + "path traverses a symlink", + )); + } + if !meta.is_dir() { + return Err(Error::new(ErrorKind::NotADirectory, "not a directory")); } } - true + open_no_follow(path) } -/// Restore a known-empty on-disk index after a failed bootstrap. +/// `path` split into the literal components below `root`. /// -/// `build_index_with_options` writes the index files in place, so a failure -/// partway through can leave truncated files that the currently mmap'd reader -/// no longer matches. Resetting gives the fallback build a clean base. -fn reset_to_empty_index(state: &ServerState, root: &Path, index_dir: &Path) { - if let Err(e) = create_empty_index(index_dir) { - eprintln!("[trace] warning: could not reset the index directory ({e})"); - return; - } - match HybridIndex::open(index_dir, root) { - Ok(empty) => { - *state.index.write().unwrap() = empty; - state.cache.write().unwrap().clear(); +/// Anything that is not a plain name — `..`, a root, a prefix — is refused +/// rather than interpreted, since resolving those is the whole business +/// [`open_within_root`] is avoiding. +fn relative_components(root: &Path, path: &Path) -> std::io::Result> { + use std::io::{Error, ErrorKind}; + + let rel = path + .strip_prefix(root) + .map_err(|_| Error::new(ErrorKind::InvalidInput, "path is outside the served root"))?; + let mut components = Vec::new(); + for component in rel.components() { + match component { + std::path::Component::Normal(name) => components.push(name.to_os_string()), + _ => { + return Err(Error::new( + ErrorKind::InvalidInput, + "path has a non-literal component", + )); + } } - Err(e) => eprintln!("[trace] warning: could not reopen an empty index ({e})"), } + if components.is_empty() { + return Err(Error::new(ErrorKind::InvalidInput, "path is the root")); + } + Ok(components) } -/// Bootstrap an empty index with the memory-bounded external merge sort. -/// -/// The incremental path below accumulates every posting in the live overlay -/// before flushing, so a cold start on a large repository holds the whole -/// index in heap — on the Linux kernel tree that peaked at ~1.5 GiB. Handing a -/// true bootstrap to the builder with [`IndexStrategy::External`] bounds peak -/// memory to the arena budget instead, and is also faster, because it writes -/// the index once rather than growing an overlay and then flushing it. -/// -/// The trade-off is that queries see an empty index until the build finishes -/// rather than a growing partial one. That is deliberate: results from a -/// fraction of the repository are misleading, and `status` already reports -/// that indexing is in progress. +/// The outcome of reading a file whose stat'd size has already been approved. +enum CappedRead { + Data(Vec), + /// The file yielded more bytes than the cap allows, whatever its size said. + TooLarge, + Failed, +} + +/// Reads a file's contents, never pulling in more than one byte past the cap. /// -/// Only used when nothing has been indexed yet. Resuming a partial index still -/// takes the incremental path, which can skip the files already on disk. +/// The size that qualified this file was stat'd before the read, and appending +/// between the two is exactly what a log or a build artifact does. An +/// unbounded read would then hold the whole of it in memory and index it past +/// the limit the user set. One byte over is enough to prove it no longer +/// qualifies, and is all that is ever read beyond the limit. +fn read_within_limit(file: &mut std::fs::File, limit: Option, capacity: usize) -> CappedRead { + use std::io::Read; + + let mut data = Vec::with_capacity(capacity); + match limit { + Some(limit) => { + if file + .take(limit.saturating_add(1)) + .read_to_end(&mut data) + .is_err() + { + return CappedRead::Failed; + } + if data.len() as u64 > limit { + return CappedRead::TooLarge; + } + } + None => { + if file.read_to_end(&mut data).is_err() { + return CappedRead::Failed; + } + } + } + CappedRead::Data(data) +} + +/// Read a file and merge it into the live index, unless its stamp says the +/// content we already indexed is current. /// -/// Returns `false` if the index could not be built and published, leaving the -/// caller to fall back. -fn bootstrap_index_build(state: &Arc, root: &Path, index_dir: &Path) -> bool { - let start = Instant::now(); - eprintln!("[trace] bootstrapping index with the external merge sort (memory-bounded)..."); +/// The caller must hold `snapshot_gate`: the read, the commit, and the stamp +/// update have to be atomic with respect to a flush or auto-save. +fn reindex_file(state: &ServerState, path: &Path, rel_path: &str) { + use tgrep_core::meta::FileStamp; - // Dropped once the build is done so the sampled peak (on platforms without - // a kernel high-water mark) covers the whole of it. Unlike the incremental - // path below, nothing here polls memory on its own. - let sampler = crate::mem::PrivatePeakSampler::start(); - let outcome = match builder::build_index_with_options( - root, - Some(index_dir), - &builder::BuildOptions { - include_hidden: false, - no_ignore: state.no_ignore, - no_require_git: state.no_require_git, - max_file_size: state.max_file_size, - exclude_dirs: state.exclude_dirs.clone(), - // Match the walk `background_index_build` would have run, and the - // dot-prefix rule `should_skip_watcher_path` applies, so the - // watcher can maintain every file this build indexes. Also makes - // the walk hand back the .gitignore paths for the matcher below. - collect_gitignore_files: true, - strategy: builder::IndexStrategy::External, - buffer_bytes: builder::DEFAULT_INDEX_BUFFER_BYTES, - }, + // Against other indexers, not against searches. The gate above is held for + // read, so without this a recovery scan and the watcher worker can both be + // here for the same path, both read, and the one that read the *older* + // content can commit last. See `ServerState::reindex_lock`. + let _reindex = lock_reindex(state); + + // One handle for the whole decision, resolved a component at a time from + // the root so no part of the path can be a symlink, and every fact below — + // type, size, mtime, bytes — read back off it. Nothing that happens to the + // path in the meantime can then make the content we index disagree with the + // metadata we judged it by, or put it outside the tree we serve. + let file = match open_within_root(&state.root, path) { + Ok(f) => f, + Err(e) if proves_ineligible(&e) => { + // Gone, or not a regular file reachable without traversing a link. + // It may still be a path we indexed before it became one, so fall + // through to the drop rather than returning. + drop_indexed_file(state, rel_path, "no longer eligible"); + return; + } + Err(_) => { + // A permission error, a Windows sharing violation, a descriptor + // limit — none of which say anything about whether the file + // belongs in the index. Dropping on those would evict live content + // because a build held the file open for a moment. The stale path + // already treats unreadable files this way, keeping what it has and + // retrying later, and the watcher should not disagree with it. + return; + } + }; + let Ok(meta) = file.metadata() else { + return; + }; + let current = FileStamp { + mtime: meta + .modified() + .ok() + .and_then(|t| t.duration_since(std::time::SystemTime::UNIX_EPOCH).ok()) + .map(|d| d.as_secs()) + .unwrap_or(0), + size: meta.len(), + }; + + // The rules `walk_file_metadata` applies, and for the same reason: the walk + // is authoritative about what belongs in the index, so anything it rejects + // must not be added here. Without this a file that grew past the cap — or + // an ineligible extension in a directory a relaxed ignore rule just exposed + // — would be read whole and indexed, and the next reconcile would silently + // delete it again. + // + // `is_file` is the third rule, and the one with teeth: the walker runs with + // `follow_links(false)`, where a symlink is neither file nor dir and is + // skipped outright. Indexing through one would put the target's bytes under + // the link's path — and the target need not be under `root` at all, so a + // link committed to a branch (or dropped in by a build) is enough to pull + // `~/.ssh/id_rsa` into an index whose whole contract is that it covers the + // served tree. On unix the open above has already failed for a link at any + // level; this is what rejects a final one on Windows, where the reparse + // point opens fine. + let eligible = meta.is_file() + && !tgrep_core::walker::is_binary_extension(path) + && !state + .max_file_size + .is_some_and(|limit| current.size > limit); + if !eligible { + // It may have been eligible when it was last indexed — a file can grow + // past the cap, and a real file can be replaced by a link to one. Drop + // what we hold so the index matches the walk rather than keeping a + // stale copy of the smaller version until the reconcile. + drop_indexed_file(state, rel_path, "no longer eligible"); + return; + } + + if state.file_stamps.read().unwrap().get(rel_path) == Some(¤t) { + return; + } + + // Read contents and extract trigrams OUTSIDE the index write lock + // so a concurrent search (which needs a read lock) is not blocked + // on our file I/O and trigram parsing. Windows' SRWLock is + // writer-preferring: a single waiting writer here would otherwise + // stall every subsequent search request. + // + // From the handle, not the path: re-opening here is what would let a + // symlink take the place of the file we just approved. + let mut file = file; + let data = match read_within_limit( + &mut file, + state.max_file_size, + current.size.min(1 << 20) as usize, ) { - Ok(outcome) => outcome, - Err(e) => { - eprintln!( - "[trace] warning: external bootstrap build failed ({e}); \ - falling back to the in-heap build" - ); - reset_to_empty_index(state, root, index_dir); - return false; + CappedRead::Data(data) => data, + CappedRead::TooLarge => { + drop_indexed_file(state, rel_path, "grew past the size limit while being read"); + return; } + CappedRead::Failed => return, + }; + let text = tgrep_core::encoding::decode_for_index(&data); + let is_binary = tgrep_core::trigram::is_binary(&text); + let per_tri = if is_binary { + None + } else { + Some(tgrep_core::live::LiveIndex::compute_trigram_masks(&text)) }; - // Publish under the snapshot gate, and clear `indexing` before releasing - // it. `handle_fs_event` only skips while `indexing` is true, so flipping - // the flag outside the gate would let a watcher event mutate the overlay - // against the reader we are in the middle of replacing. - let gate = state.snapshot_gate.write().unwrap(); - let opened = match HybridIndex::open(index_dir, root) { - Ok(index) => index, - Err(e) => { - drop(gate); - eprintln!( - "[trace] warning: bootstrapped index failed to open ({e}); \ - falling back to the in-heap build" - ); - reset_to_empty_index(state, root, index_dir); - return false; + eprintln!("[trace] reindex: modified {rel_path}"); + // Gate held by the caller — the commit + stamp update is processed + // atomically with respect to flush/auto-save. + { + let mut index = state.index.write().unwrap(); + match per_tri { + Some(per_tri) => index.live.commit_upsert(rel_path, per_tri), + None => index.live.delete_file(rel_path), } - }; - let indexed = opened.num_files() as u64; - *state.index.write().unwrap() = opened; - state.cache.write().unwrap().clear(); - state.index_total.store(indexed, Ordering::Relaxed); - state.index_progress.store(indexed, Ordering::Relaxed); - - // Everything the watcher consults must be in place before `indexing` goes - // false, since that flag is the only thing keeping `handle_fs_event` off - // the index. Without the stamps it would reindex on spurious events; - // without the matcher it would happily index gitignored paths that the - // build just skipped. - // - // Both come out of the build itself: the builder persisted filestamps.json, - // and its walk handed back the .gitignore / .ignore paths. Building the - // matcher from those is what keeps this cheap — `gitignore::build_matcher` - // would rewalk the whole tree, which cost 49 s on a 289k-file repo. - match tgrep_core::meta::read_filestamps(index_dir) { - Ok(stamps) => *state.file_stamps.write().unwrap() = stamps, - Err(e) => eprintln!( - "[trace] warning: could not load file stamps ({e}); \ - the watcher may reindex on spurious events" - ), } - if state.watch_enabled && !state.no_ignore { - let t_gi = Instant::now(); - let matcher = tgrep_core::walker::build_gitignore_matcher_from_files( - root, - &outcome.gitignore_files, - &outcome.ignore_files, - state.no_require_git, - ); - let found = matcher.is_some(); - *state.gitignore.write().unwrap() = matcher; - state.gitignore_pending.store(false, Ordering::SeqCst); - eprintln!( - "[trace] gitignore matcher built from {} file(s) in {:.1}ms{}", - outcome.gitignore_files.len(), - t_gi.elapsed().as_secs_f64() * 1000.0, - if found { "" } else { " (no rules found)" } - ); + state + .file_stamps + .write() + .unwrap() + .insert(rel_path.to_string(), current); + if let Ok(mut cache) = state.cache.write() { + cache.pop(rel_path); } +} - state.indexing.store(false, Ordering::SeqCst); - drop(gate); - - let elapsed = start.elapsed().as_secs_f64(); - drop(sampler); - match crate::mem::format_peak_memory() { - Some(peak) => eprintln!( - "[trace] bootstrap complete: {indexed} files indexed in {elapsed:.1}s \ - (peak memory {peak})" - ), - None => eprintln!("[trace] bootstrap complete: {indexed} files indexed in {elapsed:.1}s"), +/// Whether a scheduled reconcile should run now. +/// +/// Split out from the loop so the schedule can be exercised without waiting +/// hours for it. +fn reconcile_due(since_last: Duration, quiet_for: Duration, busy: bool) -> bool { + // Indexing and flushing are already rewriting the index, and a reconcile + // takes the snapshot gate for its whole walk-and-merge. Let them finish; + // the next tick is a minute away. + if busy { + return false; } - - if state.ignore_rules_dirty.load(Ordering::SeqCst) { - schedule_ignore_rules_refresh(Arc::clone(state), root.to_path_buf()); + if since_last >= RECONCILE_DEADLINE { + return true; } - true + since_last >= RECONCILE_INTERVAL && quiet_for >= RECONCILE_QUIET_PERIOD } -/// Walk the repo and populate the LiveIndex in batches in a background thread. -/// Uses rayon for parallel trigram extraction. The bulk build is held entirely -/// in the live overlay; only one final flush to disk happens once the walk -/// completes. This avoids the super-linear cost of repeatedly snapshotting an -/// ever-growing reader+overlay during indexing, and lets us release the live -/// overlay's allocations once the data is safely on disk. +/// Periodically compare the whole tree against the index, so a change the +/// watcher never heard about cannot stay wrong indefinitely. /// -/// Trade-off: a crash during the initial build loses all in-progress work -/// (no intermediate checkpoint to fall back to). The file watcher and -/// auto-save loop continue to protect ongoing changes after the initial -/// build completes. -fn background_index_build(state: &Arc, root: &Path, index_dir: &Path) { - use rayon::prelude::*; - use tgrep_core::walker::{self, WalkOptions}; - - const BATCH_SIZE: usize = 500; - - let start = Instant::now(); - eprintln!("[trace] background indexing started..."); +/// See [`RECONCILE_INTERVAL`] for why this is needed at all. It is deliberately +/// unhurried: it defers to indexing, to flushing, and to a server that is +/// being queried, and it does nothing at all on a tree that has not drifted — +/// the walk finds no differences and returns without touching the index. +fn periodic_reconcile_loop(state: Arc, root: PathBuf, index_dir: PathBuf) { + let mut last = Instant::now(); + loop { + thread::sleep(RECONCILE_POLL); - // Build skip set from existing on-disk reader (for incremental indexing) - let skip_paths = { - let index = state.index.read().unwrap(); - let paths = index.reader_paths(); - if !paths.is_empty() { - eprintln!( - "[trace] seeding from existing index ({} files already indexed)", - paths.len() - ); + let busy = state.indexing.load(Ordering::SeqCst) || state.flushing.load(Ordering::SeqCst); + if !reconcile_due(last.elapsed(), state.quiet_for(), busy) { + continue; } - paths - }; - let seeded_count = skip_paths.len() as u64; - // Nothing indexed yet: build straight to disk with bounded memory instead - // of accumulating the whole repo in the live overlay. Resuming a partial - // index falls through, since that path can skip what is already on disk. - if skip_paths.is_empty() && bootstrap_index_build(state, root, index_dir) { - return; + // Restart the interval before the walk rather than after it. On a large + // repository the reconcile itself takes a while, and timing from its + // completion would push each one further out than the last. + last = Instant::now(); + eprintln!("[trace] periodic reconcile: looking for changes the watcher missed"); + // Same comparison the startup check makes, and for the same reason: a + // lost event is a stamp that disagrees with the filesystem, a file with + // no stamp, or a stamp with no file, and all three fall out of that. + // Comparing index *membership* as well would additionally re-add any + // file whose stamp says indexed but which the reader does not hold — + // a publication bug rather than a lost event, and one that on an hourly + // timer would rebuild the whole index every hour if it ever misfired. + if !background_refresh_stale(&state, &root, &index_dir, false) { + // It declined — an unreadable directory, or a walk that raced a + // delete. The index is untouched and correct as far as it goes, + // and the next interval tries again. + eprintln!("[trace] periodic reconcile: declined, keeping the current index"); + } } +} - // Phase 1: Walk file paths (no content reads) - let t_walk = Instant::now(); - let walk = walker::walk_dir( - root, - &WalkOptions { - include_hidden: false, - no_ignore: state.no_ignore, - no_require_git: state.no_require_git, - max_file_size: state.max_file_size, - collect_gitignore_files: state.watch_enabled && !state.no_ignore, - exclude_dirs: state.exclude_dirs.clone(), - ..Default::default() - }, - ); +fn auto_save_loop(state: Arc) { + let mut last_save = Instant::now(); - if state.watch_enabled && !state.no_ignore { - let start = Instant::now(); - let matcher = walker::build_gitignore_matcher_from_files( - root, - &walk.gitignore_files, - &walk.ignore_files, - state.no_require_git, - ); - let has_matcher = matcher.is_some(); - *state.gitignore.write().unwrap() = matcher; - state.gitignore_pending.store(false, Ordering::SeqCst); - eprintln!( - "[trace] gitignore matcher built from index walk in {:.1}ms \ - ({} .gitignore + {} .ignore files{})", - start.elapsed().as_secs_f64() * 1000.0, - walk.gitignore_files.len(), - walk.ignore_files.len(), - if has_matcher { "" } else { ", no rules found" } - ); - } + loop { + thread::sleep(Duration::from_secs(60)); - // Filter out already-indexed files - let new_files: Vec<_> = if skip_paths.is_empty() { - walk.files - } else { - walk.files - .into_iter() - .filter(|path| { - let rel = path - .strip_prefix(root) - .unwrap_or(path) - .to_string_lossy() - .replace('\\', "/"); - !skip_paths.contains(&rel) - }) - .collect() - }; + // Don't auto-save while background indexing or a bulk flush is + // active — those paths handle their own publication and an + // auto-save fired in parallel would just snapshot the same + // overlay redundantly. + if state.indexing.load(Ordering::SeqCst) || state.flushing.load(Ordering::SeqCst) { + continue; + } - let new_count = new_files.len() as u64; - let total = seeded_count + new_count; - state - .index_total - .store(total, std::sync::atomic::Ordering::Relaxed); - state - .index_progress - .store(seeded_count, std::sync::atomic::Ordering::Relaxed); - eprintln!( - "[trace] walk complete: {} new files to index ({} already indexed, {} binary skipped, {} too large, {} errors) in {:.1}ms", - new_count, - seeded_count, - walk.skipped_binary, - walk.skipped_too_large, - walk.skipped_error, - t_walk.elapsed().as_secs_f64() * 1000.0 - ); + let dirty = { + let index = state.index.read().unwrap(); + index.live.dirty_count() + }; - // Phase 2: Process new files in parallel batches. - // - // Confine the CPU-heavy file-read + trigram-extraction work to a bounded - // worker pool (sized from the `--max-cpu` budget) so the initial build - // doesn't saturate every core and starve the host. Falls back to the - // global rayon pool if a dedicated pool can't be built. - let index_pool = rayon::ThreadPoolBuilder::new() - .num_threads(state.index_threads) - .thread_name(|i| format!("tgrep-index-{i}")) - .build() - .ok(); - if index_pool.is_some() { - eprintln!( - "[trace] indexing with {} worker thread(s)", - state.index_threads - ); - } + let elapsed = last_save.elapsed(); + if dirty >= state.auto_save_mutations || (dirty > 0 && elapsed >= AUTO_SAVE_INTERVAL) { + let save_start = Instant::now(); + eprintln!("[trace] auto-save: {dirty} mutations, saving..."); - let mut incremental_flushes = 0u32; - for (batch_idx, batch) in new_files.chunks(BATCH_SIZE).enumerate() { - // Parallel: read files + extract trigrams (no locks held). Run inside - // the bounded pool when available so indexing CPU stays capped. - let extract = || { - batch - .par_iter() - .filter_map(|path| { - let data = std::fs::read(path).ok()?; - let data = tgrep_core::encoding::decode_for_index(&data); - if tgrep_core::trigram::is_binary(&data) { - return None; - } - let rel_path = path - .strip_prefix(root) - .unwrap_or(path) - .to_string_lossy() - .replace('\\', "/"); - - let mut trigrams = tgrep_core::trigram::extract(&data); - let lower = data.to_ascii_lowercase(); - if lower != *data { - trigrams.extend(tgrep_core::trigram::extract(&lower)); - } - Some((rel_path, trigrams)) - }) - .collect::)>>() - }; - let batch_results: Vec<(String, Vec)> = match &index_pool { - Some(pool) => pool.install(extract), - None => extract(), - }; - - // Sequential: insert into LiveIndex (brief write lock per batch) - { - let mut index = state.index.write().unwrap(); - for (rel_path, trigrams) in batch_results { - index.live.upsert_file_with_trigrams(&rel_path, trigrams); + // Hold the gate through delta build → publish → prune so watcher + // mutations cannot race publication. Recheck after acquiring it: + // another publisher may have drained the overlay while we waited. + let _gate = state.snapshot_gate.write().unwrap(); + if !state.index.read().unwrap().live.has_pending_changes() { + continue; } - } - - let progress = - seeded_count as usize + ((batch_idx + 1) * BATCH_SIZE).min(new_count as usize); - state - .index_progress - .store(progress as u64, std::sync::atomic::Ordering::Relaxed); - if progress % 5000 < BATCH_SIZE { - eprintln!( - "[trace] indexing progress: ~{progress}/{total} files ({:.1}s elapsed)", - start.elapsed().as_secs_f64() - ); - } - - // Memory-bounded build: if the in-heap overlay has pushed memory past - // the budget, persist what we've indexed so far to disk and reclaim the - // heap before continuing. This keeps peak memory bounded (the flush - // copies existing on-disk postings verbatim from mmap rather than into - // heap) while still converging to a *complete* index — unlike simply - // stopping, which would leave a partial index. - // - // Charged against private bytes, not the working set: the overlay is - // heap, and that is what a flush can give back. Mapped index pages sit - // in the working set too but are file-backed, so counting them would - // fire the cap on memory no flush can reclaim. - if let Some(used) = crate::mem::budgeted_memory_bytes() - && used > state.memory_cap_bytes - { - eprintln!( - "[trace] memory cap reached ({} MB in use > {} MB cap) — flushing \ - overlay to disk to reclaim memory and continuing", - used / (1024 * 1024), - state.memory_cap_bytes / (1024 * 1024), - ); - if flush_append_only_overlay(state, index_dir, false, None) { - incremental_flushes += 1; - let mut index = state.index.write().unwrap(); - index.live.shrink_to_fit(); - } else { + let stamps = state.file_stamps.read().unwrap().clone(); + if stream_merge_stale_changes(&state, &[], &[], &[], &stamps, "auto-save", false) { + last_save = Instant::now(); eprintln!( - "[trace] warning: incremental flush did not reclaim memory; \ - continuing (build may still exceed the budget)" + "[trace] auto-save complete in {:.1}s", + save_start.elapsed().as_secs_f64() ); } } } +} - eprintln!( - "[trace] background indexing complete: {} total files ({} new, {} seeded, \ - {} incremental flushes) in {:.1}s", - total, - new_count, - seeded_count, - incremental_flushes, - start.elapsed().as_secs_f64() - ); +/// Check whether `path` passes the glob filter list. +/// +/// Glob semantics: +/// - Patterns starting with `!` are **exclusion** patterns (path must NOT match). +/// - All other patterns are **inclusion** patterns (path must match at least one). +/// - If only exclusion patterns are present, the path passes unless it matches +/// an exclusion. +/// - If inclusion patterns are present, the path must match at least one AND +/// must not match any exclusion. +fn json_rpc_result(id: Option, result: serde_json::Value) -> String { + serde_json::json!({ + "jsonrpc": "2.0", + "result": result, + "id": id.unwrap_or(serde_json::Value::Null), + }) + .to_string() +} - // Walk filesystem metadata BEFORE the flush so we can publish the - // resulting per-file stamps atomically with the index files. Writing - // them after a successful flush would leave a multi-minute window where - // the index looks fully published but `filestamps.json` is missing — a - // server kill in that window disables incremental stale detection on - // the next start. - let walk_meta = tgrep_core::walker::walk_file_metadata( - root, - &tgrep_core::walker::MetaWalkOptions { - exclude_dirs: state.exclude_dirs.clone(), - no_ignore: state.no_ignore, - no_require_git: state.no_require_git, - max_file_size: state.max_file_size, +fn json_rpc_error(id: Option, code: i32, message: &str) -> String { + serde_json::json!({ + "jsonrpc": "2.0", + "error": { + "code": code, + "message": message, }, - ); - let stamps: std::collections::HashMap = walk_meta - .files + "id": id.unwrap_or(serde_json::Value::Null), + }) + .to_string() +} + +/// Create a minimal empty on-disk index so HybridIndex::open() succeeds. +/// The actual data will be populated into the LiveIndex in the background. +fn create_empty_index(index_dir: &Path) -> Result<()> { + use tgrep_core::meta::IndexMeta; + // Own the directory precondition here rather than leaving it to each + // caller: the recovery path in `reset_to_empty_index` runs after a failed + // build, which is exactly when the directory is least likely to be intact. + std::fs::create_dir_all(index_dir)?; + // Empty lookup.bin, index.bin, files.bin + std::fs::write(index_dir.join("lookup.bin"), b"")?; + std::fs::write(index_dir.join("index.bin"), b"")?; + std::fs::write(index_dir.join("files.bin"), b"")?; + let mut meta = IndexMeta::new("", 0, 0); + meta.complete = false; // empty skeleton — not a complete index + meta.save(index_dir)?; + Ok(()) +} + +/// Stamps for the walked files that are actually in the index. +/// +/// The build stamps its work from a *second* traversal, taken after the content +/// walk that fed the index, so a file created between the two appears here and +/// nowhere else. Publishing a stamp for it would be a lie the rest of the +/// server believes: `reindex_file` returns early when the stamp already matches +/// what is on disk, and the periodic reconcile runs with +/// `compare_index_membership` off, so it compares stamps alone too — the file +/// would stay unsearchable until something changed it again. Withholding the +/// stamp instead makes the very next event or scan treat it as new, which is +/// what it is. +fn stamps_for_index_members( + files: Vec, + indexed: &std::collections::HashSet, +) -> std::collections::HashMap { + files .into_iter() + .filter(|fm| indexed.contains(&fm.relative_path)) .map(|fm| { ( fm.relative_path, @@ -2875,1054 +4223,3898 @@ fn background_index_build(state: &Arc, root: &Path, index_dir: &Pat }, ) }) - .collect(); - - // The in-memory build is done — surface "complete" in status now even - // though the final disk flush below can take minutes for very large - // repos. Set `flushing` *before* clearing `indexing` so the auto-save - // loop never observes both flags as false during the handoff and - // races us into a redundant parallel snapshot of the bulk overlay. - // - // Acquire the publish gate *before* clearing `indexing`. `handle_fs_event` - // only skips while `indexing` is true; once it's false a watcher event can - // run, and if it grabbed `snapshot_gate.read()` before our flush grabbed - // the write lock it could mutate the overlay in the gap between the flag - // flip and the final snapshot — updating/deleting a path already on disk in - // the reader and violating `append_overlay_to_index`'s brand-new-paths - // precondition. Holding the gate across the flip makes any such event block - // (not skip) until the flush publishes, after which it applies safely to - // the newly published reader; no event is lost. - let gate = state.snapshot_gate.write().unwrap(); - state.flushing.store(true, Ordering::SeqCst); - state.indexing.store(false, Ordering::SeqCst); - - // Final flush to disk for the bulk build. Use the same streaming - // append-only path as incremental flushes so the final complete publish - // does not materialize the whole reader+overlay in heap and violate the - // memory cap. This always publishes with `complete = true`; any - // intermediate incremental flushes published `complete = false` so a - // mid-build kill would resume rather than be treated as finished. - eprintln!("[trace] persisting final index to disk..."); - let pruned = flush_append_only_overlay_locked(state, index_dir, true, Some(&stamps)); - drop(gate); - - state.flushing.store(false, Ordering::SeqCst); - - // Refresh the in-memory file_stamps so the file watcher can recognize - // unchanged files and skip spurious notify events (e.g. atime/attribute - // updates on Windows). Done even if the flush failed — the live overlay - // already reflects what we just indexed, and the stamps describe that. - *state.file_stamps.write().unwrap() = stamps; - - // Reclaim memory held by the indexing-time live overlay — but only when - // the flush actually completed and `prune_persisted_entries` ran. If the - // flush failed, the overlay is still the source of truth and shrinking - // the indexing-sized maps would just waste the write lock with no benefit. - if pruned { - let mut index = state.index.write().unwrap(); - index.live.shrink_to_fit(); - } - - if state.ignore_rules_dirty.load(Ordering::SeqCst) { - schedule_ignore_rules_refresh(Arc::clone(state), root.to_path_buf()); - } + .collect() } -/// Memory-bounded append-only flush used during the initial bulk build. +/// Drop the stamps the build has no right to publish, because an event for +/// those paths arrived while it ran. /// -/// Unlike building a full reader+overlay snapshot in heap (which costs -/// O(total index size) memory), this streams the live overlay onto disk via -/// [`builder::append_overlay_to_index`]: the existing postings are copied -/// verbatim from the reader's mmap and never enter the heap. Peak heap stays -/// bounded to the overlay snapshot, so repeated checkpoint flushes and the -/// final complete publish keep the whole build under the memory budget. +/// The stamp map describes what the index holds, and the build derives it from +/// a metadata walk taken *after* the content walk. For a file written between +/// the two the index holds the old bytes while the stamp describes the new +/// ones, so the stamp says "current" about content that is stale. That claim is +/// load-bearing in exactly the place that should have repaired it: +/// `reindex_file` returns early on a matching stamp, so the replay of the very +/// event that reported the write reads nothing, and the reconcile behind it +/// compares the same stamps and agrees. The old content stays searchable +/// indefinitely. /// -/// Relies on the bulk-build invariant that the overlay is **append-only** -/// (watcher + auto-save suppressed while `indexing == true`), so every overlay -/// file is new and the merge is a pure append. +/// The deferred buffer already names those paths — it is what replay is about +/// to walk — so withholding their stamps costs one map lookup each and makes +/// the replay do the read it was deferred for. /// -/// Checkpoint flushes pass `complete = false`: a kill mid-build must leave the -/// index marked partial so the next start resumes indexing the remaining files. -/// The final end-of-build flush passes `complete = true` and may pass file -/// stamps to publish alongside the index. +/// When the buffer overflowed it names nothing, and nothing distinguishes the +/// files that changed from the ones that did not, so no stamp from this build +/// can be trusted and none is published. The reconcile that overflow already +/// schedules then re-reads the tree rather than believing a walk that raced +/// 100k changes. /// -/// Returns `true` if the new reader was published and the overlay pruned. -fn flush_append_only_overlay( - state: &ServerState, - index_dir: &Path, - complete: bool, - stamps: Option<&std::collections::HashMap>, -) -> bool { - // Hold the snapshot gate for the whole snapshot → publish → prune cycle. - // During the bulk build the watcher is already suppressed, but auto-save - // coordination and future-proofing make the gate the right call. - let _gate = state.snapshot_gate.write().unwrap(); - flush_append_only_overlay_locked(state, index_dir, complete, stamps) -} - -/// Body of [`flush_append_only_overlay`] that assumes `snapshot_gate` is -/// **already held for write** by the caller. Split out so the final bulk-build -/// handoff can acquire the gate *before* clearing the `indexing` flag, closing -/// the window where a filesystem event could observe `indexing == false`, take -/// the gate first, and mutate the overlay between the flag flip and the final -/// snapshot (which would break the append-only precondition). -fn flush_append_only_overlay_locked( +/// Takes the deferred lock while `snapshot_gate` is held, which is the one +/// order in use: the watcher defers *before* it takes the gate, and replay +/// releases the buffer before handling anything. +fn withhold_stamps_for_deferred( state: &ServerState, - index_dir: &Path, - complete: bool, - stamps: Option<&std::collections::HashMap>, -) -> bool { - let flush_start = Instant::now(); - - // Snapshot the overlay (bounded heap) and the current reader (cheap Arc). - let (overlay_paths, overlay_inverted, reader) = { - let index = state.index.read().unwrap(); - let (paths, inverted) = index.live.snapshot_for_disk(); - (paths, inverted, index.reader_arc()) + root: &Path, + mut stamps: std::collections::HashMap, +) -> std::collections::HashMap { + let deferred = match state.deferred_events.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), }; - if overlay_paths.is_empty() && !complete && stamps.is_none() { - return false; + let Some(paths) = deferred.as_ref() else { + eprintln!( + "[trace] warning: too many changes during the initial build to say which files the \ + walk raced; publishing no stamps so the reconcile re-reads them" + ); + return std::collections::HashMap::new(); + }; + let mut withheld = 0usize; + for path in paths.keys() { + let Ok(rel) = path.strip_prefix(root) else { + continue; + }; + let rel = rel.to_string_lossy().replace('\\', "/"); + if stamps.remove(&rel).is_some() { + withheld += 1; + } } - let num_files = reader.num_files() + overlay_paths.len(); - - let staging_dir = index_dir.with_file_name(".tgrep_flush_staging"); - let _ = std::fs::remove_dir_all(&staging_dir); - - // Stream-merge overlay onto the existing on-disk index. Incremental - // checkpoint flushes publish `complete = false`; the final bulk-build flush - // republishes the same stream with `complete = true` and stamps. - if let Err(e) = builder::append_overlay_to_index( - &state.root, - &staging_dir, - &reader, - &overlay_paths, - &overlay_inverted, - complete, - ) { - eprintln!("[trace] warning: append-only flush write failed: {e}"); - let _ = std::fs::remove_dir_all(&staging_dir); - return false; + if withheld > 0 { + eprintln!( + "[trace] watcher: {withheld} file(s) changed during the initial build; their stamps \ + are withheld so the replay re-reads them" + ); } + stamps +} - // Stage filestamps alongside the final complete index. If this fails we - // still publish the index: losing incremental stale-check state on next - // start is preferable to dropping the completed build. - if let Some(stamps) = stamps - && let Err(e) = tgrep_core::meta::write_filestamps(stamps, &staging_dir) - { - eprintln!("[trace] warning: failed to write staging filestamps: {e}"); +/// Detect files that changed while the server was not running. +/// Compares stored filestamps against current filesystem metadata, then upserts +/// changed/new files and removes deleted files from the LiveIndex. +fn classify_file_changes( + current_meta: &[tgrep_core::walker::FileMeta], + old_stamps: &std::collections::HashMap, + indexed_paths: &std::collections::HashSet, + compare_index_membership: bool, +) -> (Vec, Vec, Vec) { + use tgrep_core::meta::FileStamp; + + let mut current_set = std::collections::HashSet::with_capacity(current_meta.len()); + let mut changed = Vec::new(); + let mut added = Vec::new(); + + for fm in current_meta { + current_set.insert(fm.relative_path.clone()); + let stamp = FileStamp { + mtime: fm.mtime, + size: fm.size, + }; + if compare_index_membership && !indexed_paths.contains(&fm.relative_path) { + added.push(fm.relative_path.clone()); + continue; + } + match old_stamps.get(&fm.relative_path) { + Some(old) if *old == stamp => {} + Some(_) => changed.push(fm.relative_path.clone()), + None => added.push(fm.relative_path.clone()), + } } - let pruned = publish_staged_index(state, index_dir, &staging_dir, num_files); - eprintln!( - "[trace] append-only flush: {num_files} files on disk (complete={complete}) in {:.1}s", - flush_start.elapsed().as_secs_f64() - ); - pruned + let mut seen = std::collections::HashSet::new(); + let deleted = old_stamps + .keys() + .chain(indexed_paths) + .filter(|path| seen.insert(path.as_str())) + .filter(|path| !current_set.contains(path.as_str())) + .cloned() + .collect(); + + (changed, added, deleted) } -/// Publish a staged index directory: move the staged files into `index_dir`, -/// reopen the on-disk reader (with Windows stale-NTFS-metadata retries), -/// validate + warm it, swap it in without blocking searches, and prune the -/// now-persisted overlay entries. +/// Apply a stale diff without materializing the existing index in heap. /// -/// Shared by stale refresh and [`flush_append_only_overlay`]. The `publish_lock` -/// is held across move + open + swap so concurrent publishers cannot interleave -/// renames or swap readers out of order. `num_files` is the expected on-disk -/// file count used to reject a partially-published reader. +/// The ordinary incremental flush uses `HybridIndex::full_snapshot`, whose +/// memory is proportional to every posting already on disk. That is especially +/// harmful when a newer tgrep first opens an index built with an older file-size +/// cap: every formerly-oversized file appears as new at once. Build new and +/// replacement files into a bounded external-sort delta, then stream it together +/// with the old index while filtering replaced and deleted reader entries. /// -/// Returns `true` when the swap + prune succeeded, `false` on any failure (the -/// previous reader and the live overlay are retained as the fallback). -fn publish_staged_index( - state: &ServerState, - index_dir: &Path, - staging_dir: &Path, - num_files: usize, +/// The caller holds `snapshot_gate` across the metadata walk and this merge, so +/// the walk's exact path set is newer than every live entry captured here. +fn stream_merge_stale_changes( + state: &Arc, + changed: &[String], + added: &[String], + deleted: &[String], + stamps: &std::collections::HashMap, + operation: &str, + authoritative_membership: bool, ) -> bool { - // Held across move + open + swap so concurrent publishers (auto-save / - // background-build / watcher reindex flush) cannot interleave renames - // or swap readers out of order. Searches do not take this lock. - let _publish = state.publish_lock.lock().unwrap(); - if let Err(e) = move_staged_files(staging_dir, index_dir) { - eprintln!("[trace] warning: flush move failed: {e}"); - let _ = std::fs::remove_dir_all(staging_dir); - return false; - } + let root = &state.root; + let index_dir = &state.index_dir; + let (reader, overlay_paths, tombstone_paths) = { + let index = state.index.read().unwrap(); + ( + index.reader_arc(), + index.live.overlay_paths(), + index.live.tombstone_paths(), + ) + }; - // Open the new reader. The publish mutex is intentionally still held - // here so that move + open + swap form an atomic publish unit (no other - // publisher can interleave a rename or swap a competing reader between - // these steps). The server-wide `state.index` RwLock is NOT taken, so - // search queries continue to be served by the previous reader (whose - // `Arc` they hold) throughout this call. - // - // On Windows, NTFS metadata for a recently-renamed file can transiently - // appear stale (zero-length), causing IndexReader::open to create a - // degenerate reader with files but no trigrams. We retry a few times - // with a short backoff to ride out the transient. - let pruned = 'open: { - const READER_OPEN_RETRIES: u32 = 5; - const READER_OPEN_BACKOFF: Duration = Duration::from_millis(200); + state.flushing.store(true, Ordering::SeqCst); + let start = Instant::now(); + eprintln!( + "[trace] {operation}: building a memory-bounded delta \ + ({} changed, {} new, {} deleted)...", + changed.len(), + added.len(), + deleted.len() + ); - for attempt in 0..READER_OPEN_RETRIES { - match tgrep_core::reader::IndexReader::open(index_dir) { - Ok(new_reader) => { - let reader_files = new_reader.num_files(); - let reader_trigrams = new_reader.num_trigrams(); + // Keep work directories inside the locked index directory. Sibling names + // collide when two independent indexes share a parent directory. + let delta_dir = index_dir.join(".stale-delta"); + let staging_dir = index_dir.join(".stale-merge"); + let _ = std::fs::remove_dir_all(&delta_dir); + let _ = std::fs::remove_dir_all(&staging_dir); - if new_reader.is_degenerate() { - eprintln!( - "[trace] warning: reader has {reader_files} files but 0 trigrams \ - (attempt {}/{READER_OPEN_RETRIES}, likely stale NTFS metadata)", - attempt + 1 - ); - if attempt + 1 < READER_OPEN_RETRIES { - thread::sleep(READER_OPEN_BACKOFF * (attempt + 1)); - continue; - } - eprintln!( - "[trace] warning: degenerate reader persists after \ - {READER_OPEN_RETRIES} attempts, keeping live overlay as fallback" - ); - break 'open false; - } + // Fold in every live mutation. A stale walk also treats its exact path set + // as authoritative, removing reader entries missing from the walk; an + // auto-save cannot do that because its filestamps may be incomplete. + let mut seen = std::collections::HashSet::new(); + let mut candidates: Vec = changed + .iter() + .chain(added) + .chain(deleted) + .chain(overlay_paths.iter()) + .chain(tombstone_paths.iter()) + .filter(|path| seen.insert((*path).clone())) + .cloned() + .collect(); + if authoritative_membership { + candidates.extend( + reader + .all_paths() + .iter() + .filter(|path| !stamps.contains_key(path.as_str())) + .filter(|path| seen.insert((*path).clone())) + .cloned(), + ); + } + let desired_paths: Vec = candidates + .iter() + .filter(|path| stamps.contains_key(path.as_str())) + .cloned() + .collect(); + let files: Vec = desired_paths.iter().map(|path| root.join(path)).collect(); + // Every candidate is either removed or replaced by the delta. Including a + // genuinely new path is harmless because it has no reader entry to filter. + let removed: std::collections::HashSet = candidates.iter().cloned().collect(); - // Validate + warm the lookup mmap before swapping the - // reader in. This catches corruption (unsorted lookup - // table, out-of-bounds posting offsets) and, as a - // side-effect, pages in every byte of lookup.bin so that - // subsequent binary searches never hit cold mmap pages - // — preventing the zero-candidate failure observed on - // Windows after flush. - if let Err(msg) = new_reader.validate_lookup() { - eprintln!( - "[trace] warning: reader validation failed \ - (attempt {}/{READER_OPEN_RETRIES}): {msg}", - attempt + 1 - ); - if attempt + 1 < READER_OPEN_RETRIES { - thread::sleep(READER_OPEN_BACKOFF * (attempt + 1)); - continue; - } - eprintln!( - "[trace] warning: reader validation failed after \ - {READER_OPEN_RETRIES} attempts, keeping live overlay" - ); - break 'open false; - } + let mut published_stamps = stamps.clone(); - if reader_files >= num_files { - // Atomic swap — no outer write lock required. - state.index.read().unwrap().swap_reader(new_reader); - // Brief write lock for in-memory overlay maintenance only. - { - let mut index = state.index.write().unwrap(); - index.prune_persisted_entries(); - index.live.reset_dirty_count(); - } - eprintln!( - "[trace] flush: reader reopened ({reader_files} files, \ - {reader_trigrams} trigrams), overlay pruned" - ); - break 'open true; - } else { - eprintln!( - "[trace] warning: reader has {reader_files} files \ - (expected {num_files}), keeping live overlay as fallback" - ); - break 'open false; - } - } - Err(e) => { - if attempt + 1 < READER_OPEN_RETRIES { - eprintln!( - "[trace] warning: reader open failed (attempt {}/{READER_OPEN_RETRIES}): {e}", - attempt + 1 - ); - thread::sleep(READER_OPEN_BACKOFF * (attempt + 1)); - continue; - } - eprintln!( - "[trace] warning: failed to reopen reader after flush: {e}, \ - live overlay retained" - ); - break 'open false; + let result = (|| -> Result { + let build = || { + builder::build_index_for_files( + root, + &delta_dir, + &files, + builder::DEFAULT_INDEX_BUFFER_BYTES, + ) + }; + let outcome = match rayon::ThreadPoolBuilder::new() + .num_threads(state.index_threads) + .thread_name(|i| format!("tgrep-stale-index-{i}")) + .build() + { + Ok(pool) => pool.install(build)?, + Err(_) => build()?, + }; + let delta_count = outcome.indexed; + + // Withhold stamps for files the delta could not read. A published stamp + // means "indexed at this version", so keeping one for a skipped file + // would hide it from every later reconcile and make the miss permanent. + // Dropping the stamp leaves it looking new, so the next pass retries it. + // + // Record what the file looked like when it failed, so a permanent + // failure is retried when the file changes rather than on every pass. + // See `ServerState::unreadable`. + { + let mut memo = state.unreadable.write().unwrap(); + // Anything this delta was asked to build is settled: either it was + // read, or it is in `outcome.unreadable` and re-recorded below. + for path in changed.iter().chain(added).chain(deleted) { + memo.remove(path); + } + for path in &outcome.unreadable { + let rel = path + .strip_prefix(root) + .unwrap_or(path) + .to_string_lossy() + .replace('\\', "/"); + if let Some(stamp) = published_stamps.remove(&rel) { + memo.insert(rel, stamp); } } } - false - }; - let _ = std::fs::remove_dir_all(staging_dir); - pruned + if !outcome.unreadable.is_empty() { + eprintln!( + "[trace] {} file(s) were unreadable during the delta build; \ + their stamps are withheld so a later reconcile retries them", + outcome.unreadable.len() + ); + } + + let delta = tgrep_core::reader::IndexReader::open(&delta_dir)?; + if delta.num_files() != delta_count { + anyhow::bail!( + "delta reopened with {} files after writing {delta_count}", + delta.num_files() + ); + } + + builder::merge_index_with_delta(root, &staging_dir, &reader, &delta, &removed, true)?; + tgrep_core::meta::write_filestamps(&published_stamps, &staging_dir)?; + let removed_reader_files = reader + .all_paths() + .iter() + .filter(|path| removed.contains(path.as_str())) + .count(); + let expected_files = reader.num_files() - removed_reader_files + delta.num_files(); + let published = publish_staged_index(state, index_dir, &staging_dir, expected_files); + if published { + // `publish_staged_index` prunes overlay entries represented by the + // new reader. Also clear reconciled entries intentionally omitted + // (newly ignored/binary/deleted) and old tombstones for files the + // delta restored. The gate guarantees none is newer than this merge. + state + .index + .write() + .unwrap() + .live + .clear_reconciled_paths(&candidates); + state + .index_progress + .store(expected_files as u64, Ordering::Relaxed); + state + .index_total + .store(expected_files as u64, Ordering::Relaxed); + } + Ok(published) + })(); + + let _ = std::fs::remove_dir_all(&delta_dir); + let _ = std::fs::remove_dir_all(&staging_dir); + state.flushing.store(false, Ordering::SeqCst); + if matches!(&result, Ok(true)) { + *state.file_stamps.write().unwrap() = published_stamps; + } + match result { + Ok(true) => { + eprintln!( + "[trace] {operation}: streamed {} changes into the index in {:.1}s", + candidates.len(), + start.elapsed().as_secs_f64() + ); + true + } + Ok(false) => { + eprintln!( + "[trace] warning: {operation} delta could not be published; \ + keeping the old index" + ); + false + } + Err(error) => { + eprintln!( + "[trace] warning: memory-bounded {operation} failed ({error}); \ + keeping the old index" + ); + false + } + } +} + +/// Drop the candidates that failed to read last time and have not changed since. +/// +/// Returns the paths removed, which the caller must also keep out of the +/// published stamps — see [`stamps_for_indexed`]. +fn drop_memoized_failures( + memo: &std::collections::HashMap, + current_meta: &[tgrep_core::walker::FileMeta], + changed: &mut Vec, + added: &mut Vec, +) -> std::collections::HashSet { + if memo.is_empty() { + return std::collections::HashSet::new(); + } + // One pass over the walk to pick out the memoized paths, rather than a scan + // per candidate. + let still_failing: std::collections::HashSet = current_meta + .iter() + .filter(|fm| { + memo.get(&fm.relative_path) + .is_some_and(|a| a.mtime == fm.mtime && a.size == fm.size) + }) + .map(|fm| fm.relative_path.clone()) + .collect(); + changed.retain(|path| !still_failing.contains(path)); + added.retain(|path| !still_failing.contains(path)); + still_failing +} + +/// The stamps to publish for a walk, minus the files that were never built. +/// +/// A published stamp means "indexed at this version". Stamping a file the delta +/// deliberately skipped would make every later reconcile see it as unchanged, +/// so it would never be indexed again — and because the stamp lands in +/// `filestamps.json`, not even a restart would recover it: every automatic +/// caller of the stale check passes `compare_index_membership = false`, which +/// is precisely the check that would have noticed the file is missing. Leaving +/// it unstamped keeps it looking new, which is what makes the retry-on-change +/// behaviour in [`drop_memoized_failures`] work at all. +fn stamps_for_indexed( + current_meta: &[tgrep_core::walker::FileMeta], + skipped: &std::collections::HashSet, +) -> std::collections::HashMap { + current_meta + .iter() + .filter(|fm| !skipped.contains(&fm.relative_path)) + .map(|fm| { + ( + fm.relative_path.clone(), + tgrep_core::meta::FileStamp { + mtime: fm.mtime, + size: fm.size, + }, + ) + }) + .collect() +} + +/// Rebuild the ignore matcher and reconcile the index against the filesystem. +/// +/// Returns whether the index can be trusted afterwards. A `false` means the +/// walk or the merge failed and the stamps are not describing the index. +fn background_refresh_stale( + state: &Arc, + root: &Path, + index_dir: &Path, + compare_index_membership: bool, +) -> bool { + let _refresh = state.stale_refresh_lock.lock().unwrap(); + // Keep watcher/auto-save mutations out for the complete walk → matcher → + // merge → recovery cycle. Search queries do not take this gate and remain + // available. Held here rather than inside so the recovery scan below is + // still covered by it. + let _gate = state.snapshot_gate.write().unwrap(); + + let mut newly_watched = Vec::new(); + // Before the walk, not after the subscriptions: this bounds the window the + // recovery scan is closing, and the window opens the moment the traversal + // that decided what to subscribe to begins. + let since = SystemTime::now(); + let ok = refresh_stale_locked( + state, + root, + index_dir, + compare_index_membership, + &mut newly_watched, + ); + + // Directories that were not subscribed while the walk ran could not report + // a write, and the walk may have passed them before it happened, so a file + // created in that window is in neither place. Recheck them now that the + // subscriptions exist. + // + // Only on success, and only here at the end: a failed walk or merge leaves + // `file_stamps` describing something other than the published index, and + // `stream_merge_stale_changes` replaces the stamps wholesale, so scanning + // any earlier would both be discarded and re-read every changed file. + if ok { + reindex_files_in(state, root, &newly_watched, since); + } + ok } -/// Move index files from staging to the target directory. -/// -/// Files are published in a fixed order, with `meta.json` last. This is only a -/// convention for publication layout; it does not provide atomic publish -/// semantics or reader-side validation by itself. -/// -/// Performance note: this function runs under the server's `publish_lock` -/// (which serializes concurrent publishers) but does NOT take the -/// `state.index` write lock, so search queries continue to be served -/// throughout. Each per-file move uses `std::fs::rename` — on the same -/// volume this is an O(microseconds) directory entry update, vs -/// `std::fs::copy` which is O(file_size) and on a large `index.bin` -/// (hundreds of MB) can take tens of seconds. Staging dirs are always -/// created next to the target (same parent) so cross-volume cases should -/// not arise; if rename truly fails, the error is surfaced rather than -/// silently falling back to a slow copy (see `publish_file`). -fn move_staged_files(staging: &Path, target: &Path) -> std::io::Result<()> { - std::fs::create_dir_all(target)?; - // Data files first, meta last. - for name in &[ - "index.bin", - "lookup.bin", - "files.bin", - "filestamps.json", - "meta.json", - ] { - let src = staging.join(name); - let dst = target.join(name); - if !src.exists() { - continue; - } - publish_file(&src, &dst)?; +fn refresh_stale_locked( + state: &Arc, + root: &Path, + index_dir: &Path, + compare_index_membership: bool, + newly_watched: &mut Vec, +) -> bool { + use tgrep_core::meta; + use tgrep_core::walker; + + let start = Instant::now(); + eprintln!("[trace] stale check: comparing index against filesystem..."); + + // Walk first. This single traversal feeds both the stale diff and the + // watcher's ignore matcher, and it must run before the early returns below + // so the matcher can be published on every path out of this function. + let walk = walker::walk_file_metadata( + root, + &walker::MetaWalkOptions { + exclude_dirs: state.exclude_dirs.clone(), + no_ignore: state.no_ignore, + no_require_git: state.no_require_git, + max_file_size: state.max_file_size, + }, + ); + let walk_ms = start.elapsed().as_millis(); + + // Publish the matcher immediately, before any early return can skip it. + // Every exit below is a decision about the *index*; none of them is a + // reason to leave the watcher gated. `gitignore_pending` is what keeps the + // watcher off the index until a matcher exists, so leaking it past a return + // disables the watcher permanently — and the overflow-repair path skips + // reconciling while that flag is set, so nothing recovers it either. A + // single unreadable directory, or one file whose `metadata()` lost a race + // with a delete, would be enough. + // + // Committing here rather than at each exit is invisible to the watcher: + // the caller holds `snapshot_gate` for write across this whole body, and + // the only reader of `state.gitignore` takes the read side first, so no + // event can observe the matcher before this function returns either way. + // The caller's walk started before this publish; its timestamp is what the + // recovery scan needs, so the one the subscription sync derives is dropped. + *newly_watched = publish_ignore_matcher( + state, + root, + ignore_sources_of( + root, + &walk.gitignore_files, + &walk.ignore_files, + state.no_require_git, + ), + || build_stale_matcher(state, root, &walk), + ); + + if walk.skipped_error > 0 { + eprintln!( + "[trace] warning: stale check could not inspect {} filesystem entries \ + (walk: {walk_ms}ms); keeping the old index", + walk.skipped_error + ); + return false; + } + let current_meta = &walk.files; + + // Load stored per-file stamps from last index write + let mut old_stamps = match meta::read_filestamps(index_dir) { + Ok(s) => s, + Err(e) => { + eprintln!("[trace] stale check: no filestamps found ({e}), comparing against reader"); + std::collections::HashMap::new() + } + }; + + // Fold in stamps the watcher recorded since the last flush. `filestamps.json` + // only advances when the index is written to disk, so mid-session it lags + // the live overlay. Comparing against the on-disk copy alone would re-index + // every file the watcher already handled since that write, and — worse — + // a file created and then deleted inside that window appears in neither the + // on-disk stamps nor the filesystem, so it would never be classified as + // deleted and would linger in the index. The in-memory stamps are the + // fresher record of what the index actually holds, so they win. + for (path, stamp) in state.file_stamps.read().unwrap().iter() { + old_stamps.insert(path.clone(), stamp.clone()); + } + let indexed_paths = { + let index = state.index.read().unwrap(); + let mut paths = index.reader_paths(); + paths.extend(index.live.overlay_paths()); + paths + }; + if old_stamps.is_empty() && indexed_paths.is_empty() && current_meta.is_empty() { + eprintln!("[trace] stale check: no indexed files or filesystem files, skipping"); + return true; + } + + let (mut changed, mut added, deleted) = classify_file_changes( + current_meta, + &old_stamps, + &indexed_paths, + compare_index_membership, + ); + + // Files that failed to read last time and have not changed since are not + // worth another attempt; without this a single permanently locked file + // makes every scheduled reconcile rebuild the index. `deleted` is exempt — + // a file that is gone needs no read to evict. + let skipped_unreadable = { + let memo = state.unreadable.read().unwrap(); + drop_memoized_failures(&memo, current_meta, &mut changed, &mut added) + }; + if !skipped_unreadable.is_empty() { + eprintln!( + "[trace] stale check: {} file(s) unchanged since they last failed to \ + read; not retrying them", + skipped_unreadable.len() + ); + } + + let total_changes = changed.len() + added.len() + deleted.len(); + let live_pending = state.index.read().unwrap().live.has_pending_changes(); + if total_changes == 0 && !live_pending { + eprintln!( + "[trace] stale check: index is up-to-date ({} files checked in {}ms)", + current_meta.len(), + walk_ms + ); + return true; + } + + if total_changes == 0 { + eprintln!( + "[trace] stale check: metadata is unchanged, reconciling live mutations \ + (walk: {walk_ms}ms)" + ); + } else { + eprintln!( + "[trace] stale check: {} changed, {} new, {} deleted (walk: {}ms)", + changed.len(), + added.len(), + deleted.len(), + walk_ms + ); + } + + let new_stamps = stamps_for_indexed(current_meta, &skipped_unreadable); + + if !stream_merge_stale_changes( + state, + &changed, + &added, + &deleted, + &new_stamps, + "stale check", + true, + ) { + return false; + } + + if let Ok(mut cache) = state.cache.write() { + for path in changed.iter().chain(added.iter()).chain(deleted.iter()) { + cache.pop(path); + } + } + true +} + +/// Restore a known-empty on-disk index after a failed bootstrap. +/// +/// `build_index_with_options` writes the index files in place, so a failure +/// partway through can leave truncated files that the currently mmap'd reader +/// no longer matches. Resetting gives the fallback build a clean base. +fn reset_to_empty_index(state: &ServerState, root: &Path, index_dir: &Path) { + if let Err(e) = create_empty_index(index_dir) { + eprintln!("[trace] warning: could not reset the index directory ({e})"); + return; + } + match HybridIndex::open(index_dir, root) { + Ok(empty) => { + *state.index.write().unwrap() = empty; + state.cache.write().unwrap().clear(); + } + Err(e) => eprintln!("[trace] warning: could not reopen an empty index ({e})"), + } +} + +/// Bootstrap an empty index with the memory-bounded external merge sort. +/// +/// The incremental path below accumulates every posting in the live overlay +/// before flushing, so a cold start on a large repository holds the whole +/// index in heap — on the Linux kernel tree that peaked at ~1.5 GiB. Handing a +/// true bootstrap to the builder with [`IndexStrategy::External`] bounds peak +/// memory to the arena budget instead, and is also faster, because it writes +/// the index once rather than growing an overlay and then flushing it. +/// +/// The trade-off is that queries see an empty index until the build finishes +/// rather than a growing partial one. That is deliberate: results from a +/// fraction of the repository are misleading, and `status` already reports +/// that indexing is in progress. +/// +/// Only used when nothing has been indexed yet. Resuming a partial index still +/// takes the incremental path, which can skip the files already on disk. +/// +/// Returns `false` if the index could not be built and published, leaving the +/// caller to fall back. +fn bootstrap_index_build(state: &Arc, root: &Path, index_dir: &Path) -> bool { + let start = Instant::now(); + // Anchors the recovery window at the start of the build's traversal, which + // is the point from which writes could be missed: nothing under `root` is + // subscribed yet, and the walk below has not reached most of it. Taking it + // after the build — or letting the later subscription sync derive its own — + // would exclude everything written while the build ran, which is precisely + // the window that needs recovering. + let since = SystemTime::now(); + eprintln!("[trace] bootstrapping index with the external merge sort (memory-bounded)..."); + + // Dropped once the build is done so the sampled peak (on platforms without + // a kernel high-water mark) covers the whole of it. Unlike the incremental + // path below, nothing here polls memory on its own. + let sampler = crate::mem::PrivatePeakSampler::start(); + let outcome = match builder::build_index_with_options( + root, + Some(index_dir), + &builder::BuildOptions { + include_hidden: false, + no_ignore: state.no_ignore, + no_require_git: state.no_require_git, + max_file_size: state.max_file_size, + exclude_dirs: state.exclude_dirs.clone(), + // Match the walk `background_index_build` would have run, and the + // dot-prefix rule `should_skip_watcher_path` applies, so the + // watcher can maintain every file this build indexes. Also makes + // the walk hand back the .gitignore paths for the matcher below. + collect_gitignore_files: true, + strategy: builder::IndexStrategy::External, + buffer_bytes: builder::DEFAULT_INDEX_BUFFER_BYTES, + }, + ) { + Ok(outcome) => outcome, + Err(e) => { + eprintln!( + "[trace] warning: external bootstrap build failed ({e}); \ + falling back to the in-heap build" + ); + reset_to_empty_index(state, root, index_dir); + return false; + } + }; + + // Publish under the snapshot gate, and clear `indexing` before releasing + // it. `handle_fs_event` only skips while `indexing` is true, so flipping + // the flag outside the gate would let a watcher event mutate the overlay + // against the reader we are in the middle of replacing. + let gate = state.snapshot_gate.write().unwrap(); + let opened = match HybridIndex::open(index_dir, root) { + Ok(index) => index, + Err(e) => { + drop(gate); + eprintln!( + "[trace] warning: bootstrapped index failed to open ({e}); \ + falling back to the in-heap build" + ); + reset_to_empty_index(state, root, index_dir); + return false; + } + }; + let indexed = opened.num_files() as u64; + *state.index.write().unwrap() = opened; + state.cache.write().unwrap().clear(); + state.index_total.store(indexed, Ordering::Relaxed); + state.index_progress.store(indexed, Ordering::Relaxed); + + // Everything the watcher consults must be in place before `indexing` goes + // false, since that flag is the only thing keeping `handle_fs_event` off + // the index. Without the stamps it would reindex on spurious events; + // without the matcher it would happily index gitignored paths that the + // build just skipped. + // + // Both come out of the build itself: the builder persisted filestamps.json, + // and its walk handed back the .gitignore / .ignore paths. Building the + // matcher from those is what keeps this cheap — `gitignore::build_matcher` + // would rewalk the whole tree, which cost 49 s on a 289k-file repo. + match tgrep_core::meta::read_filestamps(index_dir) { + // Minus the paths whose events arrived while the build ran: the builder + // read those files at some point during its walk and stamped what it + // saw, so for anything written afterwards the stamp describes bytes the + // index does not hold. See `withhold_stamps_for_deferred`. + Ok(stamps) => { + *state.file_stamps.write().unwrap() = withhold_stamps_for_deferred(state, root, stamps) + } + Err(e) => eprintln!( + "[trace] warning: could not load file stamps ({e}); \ + the watcher may reindex on spurious events" + ), + } + let mut newly_watched = Vec::new(); + if state.watch_enabled && !state.no_ignore { + let t_gi = Instant::now(); + // "Newly watched" here is every directory in the repository, and the + // build's walk ran before any of them were subscribed. Deferred rather + // than skipped: the scan waits out `indexing` and then costs one + // `metadata` call per file, since the stamps this build just wrote + // describe the index exactly. + newly_watched = publish_ignore_matcher( + state, + root, + ignore_sources_of( + root, + &outcome.gitignore_files, + &outcome.ignore_files, + state.no_require_git, + ), + || { + tgrep_core::walker::build_gitignore_matcher_from_files( + root, + &outcome.gitignore_files, + &outcome.ignore_files, + state.no_require_git, + ) + }, + ); + let found = state.gitignore.read().unwrap().is_some(); + eprintln!( + "[trace] gitignore matcher built from {} file(s) in {:.1}ms{}", + outcome.gitignore_files.len(), + t_gi.elapsed().as_secs_f64() * 1000.0, + if found { "" } else { " (no rules found)" } + ); + } + // Outside that block, because it also drains the events the watcher had to + // discard while this build ran, and those pile up whether or not there are + // ignore rules to publish. + if state.watch_enabled { + spawn_recovery_scan(state, root, newly_watched, since); + } + + state.indexing.store(false, Ordering::SeqCst); + drop(gate); + + let elapsed = start.elapsed().as_secs_f64(); + drop(sampler); + match crate::mem::format_peak_memory() { + Some(peak) => eprintln!( + "[trace] bootstrap complete: {indexed} files indexed in {elapsed:.1}s \ + (peak memory {peak})" + ), + None => eprintln!("[trace] bootstrap complete: {indexed} files indexed in {elapsed:.1}s"), + } + + if state.ignore_rules_dirty.load(Ordering::SeqCst) { + schedule_ignore_rules_refresh(Arc::clone(state), root.to_path_buf()); + } + true +} + +/// Walk the repo and populate the LiveIndex in batches in a background thread. +/// Uses rayon for parallel trigram extraction. The bulk build is held entirely +/// in the live overlay; only one final flush to disk happens once the walk +/// completes. This avoids the super-linear cost of repeatedly snapshotting an +/// ever-growing reader+overlay during indexing, and lets us release the live +/// overlay's allocations once the data is safely on disk. +/// +/// Trade-off: a crash during the initial build loses all in-progress work +/// (no intermediate checkpoint to fall back to). The file watcher and +/// auto-save loop continue to protect ongoing changes after the initial +/// build completes. +fn background_index_build(state: &Arc, root: &Path, index_dir: &Path) { + use rayon::prelude::*; + use tgrep_core::walker::{self, WalkOptions}; + + const BATCH_SIZE: usize = 500; + + let start = Instant::now(); + eprintln!("[trace] background indexing started..."); + + // Build skip set from existing on-disk reader (for incremental indexing) + let skip_paths = { + let index = state.index.read().unwrap(); + let paths = index.reader_paths(); + if !paths.is_empty() { + eprintln!( + "[trace] seeding from existing index ({} files already indexed)", + paths.len() + ); + } + paths + }; + let seeded_count = skip_paths.len() as u64; + + // Nothing indexed yet: build straight to disk with bounded memory instead + // of accumulating the whole repo in the live overlay. Resuming a partial + // index falls through, since that path can skip what is already on disk. + if skip_paths.is_empty() && bootstrap_index_build(state, root, index_dir) { + return; + } + + // Phase 1: Walk file paths (no content reads) + let t_walk = Instant::now(); + // The recovery window opens with this traversal, not with the subscriptions + // it later feeds: a nested `.ignore` written after the walk read its parent + // directory but before the matcher is published is invisible to both, and a + // timestamp taken any later would date it as already accounted for. + let since = SystemTime::now(); + let walk = walker::walk_dir( + root, + &WalkOptions { + include_hidden: false, + no_ignore: state.no_ignore, + no_require_git: state.no_require_git, + max_file_size: state.max_file_size, + collect_gitignore_files: state.watch_enabled && !state.no_ignore, + exclude_dirs: state.exclude_dirs.clone(), + ..Default::default() + }, + ); + + let mut newly_watched = Vec::new(); + if state.watch_enabled && !state.no_ignore { + let start = Instant::now(); + // Subscriptions are taken here, partway through the build, so files + // written to a directory the walk has already passed are in neither + // the build's results nor any event. The scan waits for the build to + // finish before looking, because until then the stamps describe + // nothing and every file would read as changed. + newly_watched = publish_ignore_matcher( + state, + root, + ignore_sources_of( + root, + &walk.gitignore_files, + &walk.ignore_files, + state.no_require_git, + ), + || { + walker::build_gitignore_matcher_from_files( + root, + &walk.gitignore_files, + &walk.ignore_files, + state.no_require_git, + ) + }, + ); + let has_matcher = state.gitignore.read().unwrap().is_some(); + eprintln!( + "[trace] gitignore matcher built from index walk in {:.1}ms \ + ({} .gitignore + {} .ignore files{})", + start.elapsed().as_secs_f64() * 1000.0, + walk.gitignore_files.len(), + walk.ignore_files.len(), + if has_matcher { "" } else { ", no rules found" } + ); + } + // Outside that block: the scan also drains the events discarded while this + // build ran, which accumulate with or without ignore rules. + if state.watch_enabled { + spawn_recovery_scan(state, root, newly_watched, since); + } + + // Filter out already-indexed files + let new_files: Vec<_> = if skip_paths.is_empty() { + walk.files + } else { + walk.files + .into_iter() + .filter(|path| { + let rel = path + .strip_prefix(root) + .unwrap_or(path) + .to_string_lossy() + .replace('\\', "/"); + !skip_paths.contains(&rel) + }) + .collect() + }; + + let new_count = new_files.len() as u64; + let total = seeded_count + new_count; + state + .index_total + .store(total, std::sync::atomic::Ordering::Relaxed); + state + .index_progress + .store(seeded_count, std::sync::atomic::Ordering::Relaxed); + eprintln!( + "[trace] walk complete: {} new files to index ({} already indexed, {} binary skipped, {} too large, {} errors) in {:.1}ms", + new_count, + seeded_count, + walk.skipped_binary, + walk.skipped_too_large, + walk.skipped_error, + t_walk.elapsed().as_secs_f64() * 1000.0 + ); + + // Phase 2: Process new files in parallel batches. + // + // Confine the CPU-heavy file-read + trigram-extraction work to a bounded + // worker pool (sized from the `--max-cpu` budget) so the initial build + // doesn't saturate every core and starve the host. Falls back to the + // global rayon pool if a dedicated pool can't be built. + let index_pool = rayon::ThreadPoolBuilder::new() + .num_threads(state.index_threads) + .thread_name(|i| format!("tgrep-index-{i}")) + .build() + .ok(); + if index_pool.is_some() { + eprintln!( + "[trace] indexing with {} worker thread(s)", + state.index_threads + ); + } + + let mut incremental_flushes = 0u32; + for (batch_idx, batch) in new_files.chunks(BATCH_SIZE).enumerate() { + // Parallel: read files + extract trigrams (no locks held). Run inside + // the bounded pool when available so indexing CPU stays capped. + let extract = || { + batch + .par_iter() + .filter_map(|path| { + let data = std::fs::read(path).ok()?; + let data = tgrep_core::encoding::decode_for_index(&data); + if tgrep_core::trigram::is_binary(&data) { + return None; + } + let rel_path = path + .strip_prefix(root) + .unwrap_or(path) + .to_string_lossy() + .replace('\\', "/"); + + let mut trigrams = tgrep_core::trigram::extract(&data); + let lower = data.to_ascii_lowercase(); + if lower != *data { + trigrams.extend(tgrep_core::trigram::extract(&lower)); + } + Some((rel_path, trigrams)) + }) + .collect::)>>() + }; + let batch_results: Vec<(String, Vec)> = match &index_pool { + Some(pool) => pool.install(extract), + None => extract(), + }; + + // Sequential: insert into LiveIndex (brief write lock per batch) + { + let mut index = state.index.write().unwrap(); + for (rel_path, trigrams) in batch_results { + index.live.upsert_file_with_trigrams(&rel_path, trigrams); + } + } + + let progress = + seeded_count as usize + ((batch_idx + 1) * BATCH_SIZE).min(new_count as usize); + state + .index_progress + .store(progress as u64, std::sync::atomic::Ordering::Relaxed); + + if progress % 5000 < BATCH_SIZE { + eprintln!( + "[trace] indexing progress: ~{progress}/{total} files ({:.1}s elapsed)", + start.elapsed().as_secs_f64() + ); + } + + // Memory-bounded build: if the in-heap overlay has pushed memory past + // the budget, persist what we've indexed so far to disk and reclaim the + // heap before continuing. This keeps peak memory bounded (the flush + // copies existing on-disk postings verbatim from mmap rather than into + // heap) while still converging to a *complete* index — unlike simply + // stopping, which would leave a partial index. + // + // Charged against private bytes, not the working set: the overlay is + // heap, and that is what a flush can give back. Mapped index pages sit + // in the working set too but are file-backed, so counting them would + // fire the cap on memory no flush can reclaim. + if let Some(used) = crate::mem::budgeted_memory_bytes() + && used > state.memory_cap_bytes + { + eprintln!( + "[trace] memory cap reached ({} MB in use > {} MB cap) — flushing \ + overlay to disk to reclaim memory and continuing", + used / (1024 * 1024), + state.memory_cap_bytes / (1024 * 1024), + ); + if flush_append_only_overlay(state, index_dir, false, None) { + incremental_flushes += 1; + let mut index = state.index.write().unwrap(); + index.live.shrink_to_fit(); + } else { + eprintln!( + "[trace] warning: incremental flush did not reclaim memory; \ + continuing (build may still exceed the budget)" + ); + } + } + } + + eprintln!( + "[trace] background indexing complete: {} total files ({} new, {} seeded, \ + {} incremental flushes) in {:.1}s", + total, + new_count, + seeded_count, + incremental_flushes, + start.elapsed().as_secs_f64() + ); + + // Walk filesystem metadata BEFORE the flush so we can publish the + // resulting per-file stamps atomically with the index files. Writing + // them after a successful flush would leave a multi-minute window where + // the index looks fully published but `filestamps.json` is missing — a + // server kill in that window disables incremental stale detection on + // the next start. + let walk_meta = tgrep_core::walker::walk_file_metadata( + root, + &tgrep_core::walker::MetaWalkOptions { + exclude_dirs: state.exclude_dirs.clone(), + no_ignore: state.no_ignore, + no_require_git: state.no_require_git, + max_file_size: state.max_file_size, + }, + ); + let stamps: std::collections::HashMap = { + let indexed = { + let index = state.index.read().unwrap(); + let mut paths = index.reader_paths(); + paths.extend(index.live.overlay_paths()); + paths + }; + stamps_for_index_members(walk_meta.files, &indexed) + }; + + // The in-memory build is done — surface "complete" in status now even + // though the final disk flush below can take minutes for very large + // repos. Set `flushing` *before* clearing `indexing` so the auto-save + // loop never observes both flags as false during the handoff and + // races us into a redundant parallel snapshot of the bulk overlay. + // + // Acquire the publish gate *before* clearing `indexing`. `handle_fs_event` + // only skips while `indexing` is true; once it's false a watcher event can + // run, and if it grabbed `snapshot_gate.read()` before our flush grabbed + // the write lock it could mutate the overlay in the gap between the flag + // flip and the final snapshot — updating/deleting a path already on disk in + // the reader and violating `append_overlay_to_index`'s brand-new-paths + // precondition. Holding the gate across the flip makes any such event block + // (not skip) until the flush publishes, after which it applies safely to + // the newly published reader; no event is lost. + let gate = state.snapshot_gate.write().unwrap(); + state.flushing.store(true, Ordering::SeqCst); + + // Publish the stamps *before* clearing `indexing`, not after the flush. + // The recovery scan started at publish time waits for `indexing` to clear + // and then blocks on this gate, so it runs the instant the gate drops. If + // the stamps were still unpublished at that point every file it walked + // would compare as changed and it would re-read the entire repository — + // the exact work the wait exists to avoid — and the assignment would then + // overwrite the stamps it had just recorded for anything that really did + // change during the build, losing them until the next reconcile. + // + // Done even if the flush below fails: the live overlay already reflects + // what was just indexed, and the stamps describe that. + // + // Minus whatever changed underneath the walk, which the stamps would + // otherwise describe as indexed when the index holds the older bytes. + *state.file_stamps.write().unwrap() = withhold_stamps_for_deferred(state, root, stamps); + state.indexing.store(false, Ordering::SeqCst); + + // Final flush to disk for the bulk build. Use the same streaming + // append-only path as incremental flushes so the final complete publish + // does not materialize the whole reader+overlay in heap and violate the + // memory cap. This always publishes with `complete = true`; any + // intermediate incremental flushes published `complete = false` so a + // mid-build kill would resume rather than be treated as finished. + eprintln!("[trace] persisting final index to disk..."); + let pruned = { + // A read guard rather than a clone: these maps hold an entry per file + // in the repo. Nothing reachable from the flush takes this lock, and + // every other writer is behind the publish gate we hold. + let stamps = state.file_stamps.read().unwrap(); + flush_append_only_overlay_locked(state, index_dir, true, Some(&stamps)) + }; + drop(gate); + + state.flushing.store(false, Ordering::SeqCst); + + // Reclaim memory held by the indexing-time live overlay — but only when + // the flush actually completed and `prune_persisted_entries` ran. If the + // flush failed, the overlay is still the source of truth and shrinking + // the indexing-sized maps would just waste the write lock with no benefit. + if pruned { + let mut index = state.index.write().unwrap(); + index.live.shrink_to_fit(); + } + + if state.ignore_rules_dirty.load(Ordering::SeqCst) { + schedule_ignore_rules_refresh(Arc::clone(state), root.to_path_buf()); + } +} + +/// Memory-bounded append-only flush used during the initial bulk build. +/// +/// Unlike building a full reader+overlay snapshot in heap (which costs +/// O(total index size) memory), this streams the live overlay onto disk via +/// [`builder::append_overlay_to_index`]: the existing postings are copied +/// verbatim from the reader's mmap and never enter the heap. Peak heap stays +/// bounded to the overlay snapshot, so repeated checkpoint flushes and the +/// final complete publish keep the whole build under the memory budget. +/// +/// Relies on the bulk-build invariant that the overlay is **append-only** +/// (watcher + auto-save suppressed while `indexing == true`), so every overlay +/// file is new and the merge is a pure append. +/// +/// Checkpoint flushes pass `complete = false`: a kill mid-build must leave the +/// index marked partial so the next start resumes indexing the remaining files. +/// The final end-of-build flush passes `complete = true` and may pass file +/// stamps to publish alongside the index. +/// +/// Returns `true` if the new reader was published and the overlay pruned. +fn flush_append_only_overlay( + state: &ServerState, + index_dir: &Path, + complete: bool, + stamps: Option<&std::collections::HashMap>, +) -> bool { + // Hold the snapshot gate for the whole snapshot → publish → prune cycle. + // During the bulk build the watcher is already suppressed, but auto-save + // coordination and future-proofing make the gate the right call. + let _gate = state.snapshot_gate.write().unwrap(); + flush_append_only_overlay_locked(state, index_dir, complete, stamps) +} + +/// Body of [`flush_append_only_overlay`] that assumes `snapshot_gate` is +/// **already held for write** by the caller. Split out so the final bulk-build +/// handoff can acquire the gate *before* clearing the `indexing` flag, closing +/// the window where a filesystem event could observe `indexing == false`, take +/// the gate first, and mutate the overlay between the flag flip and the final +/// snapshot (which would break the append-only precondition). +fn flush_append_only_overlay_locked( + state: &ServerState, + index_dir: &Path, + complete: bool, + stamps: Option<&std::collections::HashMap>, +) -> bool { + let flush_start = Instant::now(); + + // Snapshot the overlay (bounded heap) and the current reader (cheap Arc). + let (overlay_paths, overlay_inverted, reader) = { + let index = state.index.read().unwrap(); + let (paths, inverted) = index.live.snapshot_for_disk(); + (paths, inverted, index.reader_arc()) + }; + if overlay_paths.is_empty() && !complete && stamps.is_none() { + return false; + } + let num_files = reader.num_files() + overlay_paths.len(); + + let staging_dir = index_dir.with_file_name(".tgrep_flush_staging"); + let _ = std::fs::remove_dir_all(&staging_dir); + + // Stream-merge overlay onto the existing on-disk index. Incremental + // checkpoint flushes publish `complete = false`; the final bulk-build flush + // republishes the same stream with `complete = true` and stamps. + if let Err(e) = builder::append_overlay_to_index( + &state.root, + &staging_dir, + &reader, + &overlay_paths, + &overlay_inverted, + complete, + ) { + eprintln!("[trace] warning: append-only flush write failed: {e}"); + let _ = std::fs::remove_dir_all(&staging_dir); + return false; + } + + // Stage filestamps alongside the final complete index. If this fails we + // still publish the index: losing incremental stale-check state on next + // start is preferable to dropping the completed build. + if let Some(stamps) = stamps + && let Err(e) = tgrep_core::meta::write_filestamps(stamps, &staging_dir) + { + eprintln!("[trace] warning: failed to write staging filestamps: {e}"); + } + + let pruned = publish_staged_index(state, index_dir, &staging_dir, num_files); + eprintln!( + "[trace] append-only flush: {num_files} files on disk (complete={complete}) in {:.1}s", + flush_start.elapsed().as_secs_f64() + ); + pruned +} + +/// Publish a staged index directory: move the staged files into `index_dir`, +/// reopen the on-disk reader (with Windows stale-NTFS-metadata retries), +/// validate + warm it, swap it in without blocking searches, and prune the +/// now-persisted overlay entries. +/// +/// Shared by stale refresh and [`flush_append_only_overlay`]. The `publish_lock` +/// is held across move + open + swap so concurrent publishers cannot interleave +/// renames or swap readers out of order. `num_files` is the expected on-disk +/// file count used to reject a partially-published reader. +/// +/// Returns `true` when the swap + prune succeeded, `false` on any failure (the +/// previous reader and the live overlay are retained as the fallback). +fn publish_staged_index( + state: &ServerState, + index_dir: &Path, + staging_dir: &Path, + num_files: usize, +) -> bool { + // Held across move + open + swap so concurrent publishers (auto-save / + // background-build / watcher reindex flush) cannot interleave renames + // or swap readers out of order. Searches do not take this lock. + let _publish = state.publish_lock.lock().unwrap(); + if let Err(e) = move_staged_files(staging_dir, index_dir) { + eprintln!("[trace] warning: flush move failed: {e}"); + let _ = std::fs::remove_dir_all(staging_dir); + return false; + } + + // Open the new reader. The publish mutex is intentionally still held + // here so that move + open + swap form an atomic publish unit (no other + // publisher can interleave a rename or swap a competing reader between + // these steps). The server-wide `state.index` RwLock is NOT taken, so + // search queries continue to be served by the previous reader (whose + // `Arc` they hold) throughout this call. + // + // On Windows, NTFS metadata for a recently-renamed file can transiently + // appear stale (zero-length), causing IndexReader::open to create a + // degenerate reader with files but no trigrams. We retry a few times + // with a short backoff to ride out the transient. + let pruned = 'open: { + const READER_OPEN_RETRIES: u32 = 5; + const READER_OPEN_BACKOFF: Duration = Duration::from_millis(200); + + for attempt in 0..READER_OPEN_RETRIES { + match tgrep_core::reader::IndexReader::open(index_dir) { + Ok(new_reader) => { + let reader_files = new_reader.num_files(); + let reader_trigrams = new_reader.num_trigrams(); + + if new_reader.is_degenerate() { + eprintln!( + "[trace] warning: reader has {reader_files} files but 0 trigrams \ + (attempt {}/{READER_OPEN_RETRIES}, likely stale NTFS metadata)", + attempt + 1 + ); + if attempt + 1 < READER_OPEN_RETRIES { + thread::sleep(READER_OPEN_BACKOFF * (attempt + 1)); + continue; + } + eprintln!( + "[trace] warning: degenerate reader persists after \ + {READER_OPEN_RETRIES} attempts, keeping live overlay as fallback" + ); + break 'open false; + } + + // Validate + warm the lookup mmap before swapping the + // reader in. This catches corruption (unsorted lookup + // table, out-of-bounds posting offsets) and, as a + // side-effect, pages in every byte of lookup.bin so that + // subsequent binary searches never hit cold mmap pages + // — preventing the zero-candidate failure observed on + // Windows after flush. + if let Err(msg) = new_reader.validate_lookup() { + eprintln!( + "[trace] warning: reader validation failed \ + (attempt {}/{READER_OPEN_RETRIES}): {msg}", + attempt + 1 + ); + if attempt + 1 < READER_OPEN_RETRIES { + thread::sleep(READER_OPEN_BACKOFF * (attempt + 1)); + continue; + } + eprintln!( + "[trace] warning: reader validation failed after \ + {READER_OPEN_RETRIES} attempts, keeping live overlay" + ); + break 'open false; + } + + if reader_files >= num_files { + // Atomic swap — no outer write lock required. + state.index.read().unwrap().swap_reader(new_reader); + // Brief write lock for in-memory overlay maintenance only. + { + let mut index = state.index.write().unwrap(); + index.prune_persisted_entries(); + index.live.reset_dirty_count(); + } + eprintln!( + "[trace] flush: reader reopened ({reader_files} files, \ + {reader_trigrams} trigrams), overlay pruned" + ); + break 'open true; + } else { + eprintln!( + "[trace] warning: reader has {reader_files} files \ + (expected {num_files}), keeping live overlay as fallback" + ); + break 'open false; + } + } + Err(e) => { + if attempt + 1 < READER_OPEN_RETRIES { + eprintln!( + "[trace] warning: reader open failed (attempt {}/{READER_OPEN_RETRIES}): {e}", + attempt + 1 + ); + thread::sleep(READER_OPEN_BACKOFF * (attempt + 1)); + continue; + } + eprintln!( + "[trace] warning: failed to reopen reader after flush: {e}, \ + live overlay retained" + ); + break 'open false; + } + } + } + false + }; + let _ = std::fs::remove_dir_all(staging_dir); + pruned +} + +/// Move index files from staging to the target directory. +/// +/// Files are published in a fixed order, with `meta.json` last. This is only a +/// convention for publication layout; it does not provide atomic publish +/// semantics or reader-side validation by itself. +/// +/// Performance note: this function runs under the server's `publish_lock` +/// (which serializes concurrent publishers) but does NOT take the +/// `state.index` write lock, so search queries continue to be served +/// throughout. Each per-file move uses `std::fs::rename` — on the same +/// volume this is an O(microseconds) directory entry update, vs +/// `std::fs::copy` which is O(file_size) and on a large `index.bin` +/// (hundreds of MB) can take tens of seconds. Staging dirs are always +/// created next to the target (same parent) so cross-volume cases should +/// not arise; if rename truly fails, the error is surfaced rather than +/// silently falling back to a slow copy (see `publish_file`). +fn move_staged_files(staging: &Path, target: &Path) -> std::io::Result<()> { + std::fs::create_dir_all(target)?; + // Data files first, meta last. + for name in &[ + "index.bin", + "lookup.bin", + "files.bin", + "filestamps.json", + "meta.json", + ] { + let src = staging.join(name); + let dst = target.join(name); + if !src.exists() { + continue; + } + publish_file(&src, &dst)?; + } + Ok(()) +} + +/// Publish a single staged file at `src` to `dst`. +/// +/// Uses `std::fs::rename`, which on the same volume is an O(microseconds) +/// directory entry update — this is the property that keeps the server's +/// index write lock from being held for the duration of a multi-hundred-MB +/// file copy (which previously blocked all search queries). +/// +/// On Windows, transient sharing violations (`ERROR_SHARING_VIOLATION` = 32, +/// `ERROR_LOCK_VIOLATION` = 33) can occur after dropping an mmap (cache +/// manager / AV / indexers may briefly hold a reference), so retry only +/// those specific error codes for a short window. All other errors fail +/// fast — a broader retry surface would needlessly extend the publish +/// window for non-transient failures. +/// +/// Deliberately does NOT fall back to `std::fs::copy` on persistent failure: +/// the caller holds the index write lock and a multi-hundred-MB copy is +/// exactly the pathology we are fixing. Staging is always created next to +/// the target, so cross-volume cases should not arise; if rename truly +/// cannot succeed, surfacing the error lets the caller abort cleanly +/// rather than silently regress search latency. +/// Context wrapper that preserves the original `std::io::Error` as the +/// `source()` of the returned error so callers can downcast through the +/// chain to inspect `raw_os_error()` for diagnostics. +#[derive(Debug)] +struct PublishError { + ctx: String, + source: std::io::Error, +} + +impl std::fmt::Display for PublishError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // Include the underlying error in the formatted message for + // human-readable logging; structured access remains via `source()`. + write!(f, "{}: {}", self.ctx, self.source) + } +} + +impl std::error::Error for PublishError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(&self.source) + } +} + +fn publish_file(src: &Path, dst: &Path) -> std::io::Result<()> { + const RENAME_RETRIES: u32 = 30; + const RENAME_BACKOFF: Duration = Duration::from_millis(50); + // Windows error codes that can transiently occur when another handle + // (mmap section, AV scanner, indexer) still references the target file: + // ERROR_SHARING_VIOLATION = 32 + // ERROR_LOCK_VIOLATION = 33 + // Other errors (NotFound, permission/ACL issues, disk full, …) are + // structural and should fail fast so we don't extend the publish window. + #[cfg(windows)] + const TRANSIENT_WIN_ERRORS: &[i32] = &[32, 33]; + + let mut last_err: Option = None; + for attempt in 0..RENAME_RETRIES { + match std::fs::rename(src, dst) { + Ok(()) => return Ok(()), + Err(e) => { + #[cfg(windows)] + let transient = + matches!(e.raw_os_error(), Some(c) if TRANSIENT_WIN_ERRORS.contains(&c)); + #[cfg(not(windows))] + let transient = false; + + if !transient || attempt + 1 == RENAME_RETRIES { + // Wrap with a context error that preserves the original + // `std::io::Error` as the `source()` of the returned + // error, so callers can downcast through the chain to + // recover `raw_os_error()` for diagnostics. + let ctx = format!( + "publish_file: rename({}, {}) failed after {} attempt(s)", + src.display(), + dst.display(), + attempt + 1, + ); + let kind = e.kind(); + return Err(std::io::Error::new(kind, PublishError { ctx, source: e })); + } + last_err = Some(e); + thread::sleep(RENAME_BACKOFF); + } + } + } + // Unreachable: the loop either returns Ok, or returns Err on the last + // iteration. Defensive return preserves the last error. + Err(last_err.unwrap_or_else(|| { + std::io::Error::other("publish_file: rename retries exhausted with no error recorded") + })) +} + +fn ctrlc_handler(handler: F) { + #[cfg(windows)] + { + use std::sync::OnceLock; + static HANDLER: OnceLock> = OnceLock::new(); + HANDLER.get_or_init(|| Box::new(handler)); + + unsafe extern "system" fn console_handler(_ctrl_type: u32) -> i32 { + if let Some(h) = HANDLER.get() { + h(); + } + 1 // TRUE - we handled the event + } + + unsafe extern "system" { + fn SetConsoleCtrlHandler( + handler: unsafe extern "system" fn(u32) -> i32, + add: i32, + ) -> i32; + } + + // SAFETY: SetConsoleCtrlHandler is a stable Win32 API. The handler function + // is extern "system" with correct signature, and HANDLER is 'static. + unsafe { + SetConsoleCtrlHandler(console_handler, 1); + } + } + + #[cfg(not(windows))] + { + use std::sync::OnceLock; + static HANDLER: OnceLock> = OnceLock::new(); + HANDLER.get_or_init(|| Box::new(handler)); + + unsafe extern "C" fn signal_handler(_sig: std::ffi::c_int) { + if let Some(h) = HANDLER.get() { + h(); + } + } + + unsafe extern "C" { + fn signal(sig: std::ffi::c_int, handler: unsafe extern "C" fn(std::ffi::c_int)); + } + + // SAFETY: signal() is a POSIX API. The handler has the correct extern "C" + // signature, and HANDLER is 'static. SIGINT (2) is valid on all Unix. + // SIGINT = 2 on all Unix platforms + unsafe { + signal(2, signal_handler); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + fn cached(len: usize) -> Arc { + // `String::from_utf8` preserves the Vec's capacity, so `heap_bytes` + // is exactly `len` and the assertions below can use round numbers. + let mut bytes = Vec::with_capacity(len); + bytes.resize(len, b'a'); + Arc::new(DecodedFile { + text: String::from_utf8(bytes).unwrap(), + fixups: Default::default(), + }) + } + + #[test] + fn content_cache_evicts_to_stay_under_byte_budget() { + let mut cache = ContentCache::new(CACHE_CAPACITY, 1000, 1000); + for i in 0..10 { + cache.put(format!("f{i}"), cached(200)); + } + assert!(cache.byte_len() <= 1000, "bytes = {}", cache.byte_len()); + assert_eq!(cache.len(), 5); + // Oldest entries went first; the newest survive. + assert!(cache.peek("f0").is_none()); + assert!(cache.peek("f9").is_some()); + } + + #[test] + fn content_cache_refuses_oversized_entries() { + let mut cache = ContentCache::new(CACHE_CAPACITY, 1000, 100); + cache.put("small".into(), cached(50)); + cache.put("huge".into(), cached(500)); + assert!(cache.peek("huge").is_none(), "oversized entry was admitted"); + // Admitting it would also have evicted the useful entry. + assert!(cache.peek("small").is_some()); + assert_eq!(cache.byte_len(), 50); + } + + /// The byte total must stay exact across every path that removes an entry, + /// including the entry-count eviction that `LruCache` performs internally. + #[test] + fn content_cache_byte_accounting_stays_exact() { + let mut cache = ContentCache::new(2, u64::MAX, u64::MAX); + cache.put("a".into(), cached(100)); + cache.put("b".into(), cached(100)); + assert_eq!(cache.byte_len(), 200); + + // Capacity is 2, so this evicts "a" inside the LRU. + cache.put("c".into(), cached(100)); + assert_eq!(cache.len(), 2); + assert_eq!(cache.byte_len(), 200, "count-eviction leaked bytes"); + + // Replacing an existing key must not double-count. + cache.put("c".into(), cached(300)); + assert_eq!(cache.len(), 2); + assert_eq!(cache.byte_len(), 400); + + cache.pop("c"); + assert_eq!(cache.byte_len(), 100); + cache.clear(); + assert_eq!(cache.byte_len(), 0); + assert_eq!(cache.len(), 0); + } + + #[test] + fn content_cache_touch_promotes_recency() { + let mut cache = ContentCache::new(CACHE_CAPACITY, 300, 300); + cache.put("a".into(), cached(100)); + cache.put("b".into(), cached(100)); + cache.touch("a"); + // "b" is now least-recently-used, so it is the one that goes. + cache.put("c".into(), cached(100)); + cache.put("d".into(), cached(100)); + assert!(cache.peek("a").is_some(), "touched entry was evicted first"); + assert!(cache.peek("b").is_none()); + } + + /// A `ServerState` over an empty index, for exercising the stale path + /// directly. Mirrors the defaults `run` uses with a watcher and ignore + /// rules enabled, which is the configuration `gitignore_pending` gates. + fn test_server_state(root: &Path, index_dir: &Path) -> Arc { + create_empty_index(index_dir).expect("create empty index"); + let hybrid = HybridIndex::open(index_dir, root).expect("open empty index"); + Arc::new(ServerState { + index: RwLock::new(hybrid), + cache: RwLock::new(ContentCache::new( + CACHE_CAPACITY, + CACHE_MAX_BYTES, + CACHE_MAX_ENTRY_BYTES, + )), + root: root.to_path_buf(), + watcher_active: std::sync::atomic::AtomicBool::new(false), + indexing: std::sync::atomic::AtomicBool::new(false), + flushing: std::sync::atomic::AtomicBool::new(false), + gitignore_pending: std::sync::atomic::AtomicBool::new(true), + ignore_rules_dirty: std::sync::atomic::AtomicBool::new(false), + ignore_refresh_scheduled: std::sync::atomic::AtomicBool::new(false), + watch_resubscribe: std::sync::atomic::AtomicBool::new(false), + ignore_sources: RwLock::new(Vec::new()), + ignore_source_stamps: RwLock::new(IgnoreStamps::new()), + reindex_lock: Mutex::new(()), + deferred_events: Mutex::new(Some(std::collections::HashMap::new())), + index_progress: std::sync::atomic::AtomicU64::new(0), + index_total: std::sync::atomic::AtomicU64::new(0), + watch_enabled: true, + watch_registry: Mutex::new(None), + exclude_dirs: Vec::new(), + no_ignore: false, + no_require_git: false, + max_file_size: None, + index_dir: index_dir.to_path_buf(), + publish_lock: Mutex::new(()), + file_stamps: RwLock::new(Default::default()), + snapshot_gate: RwLock::new(()), + stale_refresh_lock: Mutex::new(()), + gitignore: RwLock::new(None), + memory_cap_bytes: 16 * 1024 * 1024 * 1024, + index_threads: 1, + auto_save_mutations: 0, + unreadable: RwLock::new(std::collections::HashMap::new()), + started: Instant::now(), + last_search_ms: std::sync::atomic::AtomicU64::new(0), + }) + } + + /// A binary marker's offset is a position in the file, not in the repaired + /// text. + /// + /// Lossy decoding widens every invalid byte to a three-byte U+FFFD, so a + /// NUL preceded by invalid UTF-8 sits further along the searched text than + /// it does on disk. `rg 15.2.0` reports the on-disk byte (verified: this + /// fixture gives `found "\0" byte around offset 9`). + /// + /// The mapping has to happen here because the client never reads a file the + /// server searched, so it has no fixups of its own to map with. + #[test] + fn binary_marker_offset_is_mapped_back_to_the_source_bytes() { + let mut bytes = vec![0xFF, 0xFF]; + bytes.extend_from_slice(b"needle\n"); + bytes.push(0); + bytes.extend_from_slice(b"tail\n"); + let nul_on_disk = bytes.iter().position(|&b| b == 0).unwrap(); + assert_eq!(nul_on_disk, 9, "fixture changed"); + + let file = DecodedFile::new(bytes, tgrep_core::encoding::EncodingMode::Auto); + assert_eq!( + file.text.as_bytes().iter().position(|&b| b == 0), + Some(13), + "the fixture must actually shift the offset, or this proves nothing" + ); + + let matcher = crate::matching::build_search_matcher( + &["needle".to_string()], + &crate::matching::MatcherConfig::default(), + ) + .unwrap(); + + let rows = search_file_matches("f.txt", &file, &matcher, &SearchOpts::default()).unwrap(); + + let marker = rows + .iter() + .find(|r| r["type"] == "binary") + .expect("a file containing a NUL is reported as binary"); + assert_eq!( + marker["offset"].as_u64(), + Some(nul_on_disk as u64), + "offset must be the byte on disk (9), not the decoded position \ + (13): {marker}" + ); + } + + /// `reset_to_empty_index` runs after a failed build, when the index + /// directory is least likely to be intact, so it must not assume the + /// directory survived. + #[test] + fn create_empty_index_makes_its_own_directory() { + let tmp = TempDir::new().unwrap(); + let index_dir = tmp.path().join("missing").join("idx"); + assert!(!index_dir.exists()); + + create_empty_index(&index_dir).expect("should create the directory it writes into"); + + assert!(index_dir.join("lookup.bin").is_file()); + assert!(index_dir.join("index.bin").is_file()); + assert!(index_dir.join("files.bin").is_file()); + // A caller that already created the directory must still succeed. + create_empty_index(&index_dir).expect("should be idempotent"); + } + + /// A truncated index left by a failed build must be replaced, not reused. + #[test] + fn create_empty_index_replaces_partially_written_files() { + let tmp = TempDir::new().unwrap(); + let index_dir = tmp.path().join("idx"); + std::fs::create_dir_all(&index_dir).unwrap(); + std::fs::write(index_dir.join("lookup.bin"), b"truncated garbage").unwrap(); + + create_empty_index(&index_dir).unwrap(); + + assert_eq!( + std::fs::read(index_dir.join("lookup.bin")).unwrap().len(), + 0 + ); + HybridIndex::open(&index_dir, tmp.path()).expect("reset index should reopen cleanly"); + } + + #[test] + fn skip_watcher_path_skips_dot_components() { + let no_exclude: Vec = Vec::new(); + // A leading dot dir is the canonical case (.git, .hg, .svn, ...). + assert!(should_skip_watcher_path( + ".git/index.lock", + &no_exclude, + None + )); + assert!(should_skip_watcher_path(".git/HEAD", &no_exclude, None)); + assert!(should_skip_watcher_path( + ".hg/store/data", + &no_exclude, + None + )); + // A dot component anywhere in the path skips, not just the leading one. + assert!(should_skip_watcher_path( + "src/.cache/build.tmp", + &no_exclude, + None + )); + assert!(should_skip_watcher_path( + "a/b/.hidden/c.txt", + &no_exclude, + None + )); + } + + #[test] + fn skip_watcher_path_keeps_non_hidden_paths() { + let no_exclude: Vec = Vec::new(); + assert!(!should_skip_watcher_path("src/main.rs", &no_exclude, None)); + assert!(!should_skip_watcher_path("README.md", &no_exclude, None)); + // A dot mid-segment (e.g. "foo.bar") is NOT a hidden component — + // only segments that *start* with `.` are hidden. + assert!(!should_skip_watcher_path("src/foo.bar", &no_exclude, None)); + assert!(!should_skip_watcher_path("a/b/c", &no_exclude, None)); + } + + #[test] + fn skip_watcher_path_honors_exclude_dirs() { + let exclude = vec!["target".to_string(), "node_modules".to_string()]; + // Excluded name as an ancestor directory => skip (matches what the + // walker would do — it skips the whole subtree). + assert!(should_skip_watcher_path("target/debug/foo", &exclude, None)); + assert!(should_skip_watcher_path( + "node_modules/react/index.js", + &exclude, + None + )); + assert!(should_skip_watcher_path("a/target/b", &exclude, None)); + // Substring match should NOT trigger — "targets" != "target". + assert!(!should_skip_watcher_path("targets/foo", &exclude, None)); + // Unrelated paths are not skipped. + assert!(!should_skip_watcher_path("src/main.rs", &exclude, None)); + } + + #[test] + fn skip_watcher_path_does_not_match_basename_against_exclude_dirs() { + // A regular file whose basename happens to equal an excluded + // directory name (e.g. a file literally called `vendor` at the + // repo root, or `src/target`) is still indexed by the walker — + // walker only treats `exclude_dirs` as directory subtree filters. + // The watcher must match that, otherwise the in-memory index and + // the on-disk index would disagree. + let exclude = vec!["target".to_string(), "vendor".to_string()]; + assert!(!should_skip_watcher_path("vendor", &exclude, None)); + assert!(!should_skip_watcher_path("src/target", &exclude, None)); + assert!(!should_skip_watcher_path("a/b/vendor", &exclude, None)); + } + + #[test] + fn skip_watcher_path_handles_dot_segments_and_empty() { + let no_exclude: Vec = Vec::new(); + // `.` and `..` are not "hidden" components — they're path-relative + // markers and should not trigger a skip on their own. + assert!(!should_skip_watcher_path("./foo.txt", &no_exclude, None)); + assert!(!should_skip_watcher_path("a/./b", &no_exclude, None)); + assert!(!should_skip_watcher_path("a/../b", &no_exclude, None)); + // An empty rel_path (root-level event) shouldn't panic or skip. + assert!(!should_skip_watcher_path("", &no_exclude, None)); + } + + #[test] + fn skip_watcher_path_honors_gitignore_matcher() { + // Build the matcher via the public tgrep-core helper so this test + // also exercises the shared loading logic. + let tmp = TempDir::new().unwrap(); + // `.gitignore` is git-gated, matching the indexing walk, so the + // matcher only picks it up inside a repo. + std::fs::create_dir(tmp.path().join(".git")).unwrap(); + let gi_path = tmp.path().join(".gitignore"); + std::fs::write(&gi_path, "*.log\ntarget/\n").unwrap(); + let gi = tgrep_core::gitignore::build_matcher(tmp.path()) + .expect("matcher should build from a non-empty .gitignore"); + + let no_exclude: Vec = Vec::new(); + // Files matched by the gitignore are skipped. + assert!(should_skip_watcher_path( + "build/output.log", + &no_exclude, + Some(&gi) + )); + assert!(should_skip_watcher_path( + "target/release/foo", + &no_exclude, + Some(&gi) + )); + // Files NOT matched by the gitignore are not skipped. + assert!(!should_skip_watcher_path( + "src/main.rs", + &no_exclude, + Some(&gi) + )); + assert!(!should_skip_watcher_path( + "README.md", + &no_exclude, + Some(&gi) + )); + } + + #[test] + fn watch_registry_add_all_is_additive_but_sync_prunes() { + // These two must not be confused. `watch_new_subtree` learns only + // about the subtree that just appeared, so if it went through `sync` + // every directory outside that subtree would look stale and the server + // would unsubscribe from the entire rest of the repository — turning a + // new folder into a silent, total loss of file watching. + let tmp = TempDir::new().unwrap(); + let a = tmp.path().join("a"); + let b = tmp.path().join("b"); + let c = tmp.path().join("c"); + for dir in [&a, &b, &c] { + std::fs::create_dir(dir).unwrap(); + } + + let watcher = notify::recommended_watcher(|_: notify::Result| {}).unwrap(); + let mut registry = WatchRegistry { + watcher, + root: tmp.path().to_path_buf(), + watched: std::collections::HashSet::new(), + }; + + let added = registry.add_all(&[a.clone(), b.clone()]); + assert_eq!(added.len(), 2); + + // Adding a subtree leaves existing subscriptions untouched, and + // re-adding one already present is a no-op rather than a duplicate. + let added = registry.add_all(&[b.clone(), c.clone()]); + assert_eq!( + added, + vec![c.clone()], + "b was already watched and must not be re-added" + ); + assert_eq!( + registry.watched, + [a.clone(), b.clone(), c.clone()].into_iter().collect(), + "add_all dropped a subscription outside the set it was given" + ); + + // `sync`, by contrast, is authoritative over the whole tree. + let (added, removed) = registry.sync(&[c.clone()].into_iter().collect(), false); + assert_eq!((added.len(), removed), (0, 2)); + assert_eq!(registry.watched, [c].into_iter().collect()); + } + + #[test] + fn a_recreated_directory_is_subscribed_again_rather_than_assumed_watched() { + // The kernel releases an inotify watch by itself when its directory is + // deleted or moved away, and says nothing about it. A path recreated at + // the same location therefore *looks* subscribed while receiving no + // events — and because it is in `desired` as well as in `watched`, no + // later `sync` can tell the difference either. The entry stays poisoned + // for the life of the process, so the directory silently stops being + // watched forever. + let tmp = TempDir::new().unwrap(); + let a = tmp.path().join("a"); + std::fs::create_dir(&a).unwrap(); + + let watcher = notify::recommended_watcher(|_: notify::Result| {}).unwrap(); + let mut registry = WatchRegistry { + watcher, + root: tmp.path().to_path_buf(), + watched: std::collections::HashSet::new(), + }; + assert_eq!(registry.add_all(std::slice::from_ref(&a)).len(), 1); + + // What the removal event does. Without it the entry below survives. + registry.forget(&a); + assert!( + !registry.watched.contains(&a), + "a removed directory must not be left recorded as watched" + ); + assert_eq!( + registry.add_all(std::slice::from_ref(&a)).len(), + 1, + "a directory recreated after removal must be subscribed again" + ); + + // And the belt-and-braces half: even with the entry still present — + // a move away delivers no event for the descendants it takes with it — + // a directory that has just appeared gets its subscription re-issued. + // Already-known paths are not reported as new, so the recovery scan + // does not treat the whole subtree as freshly watched. + assert!( + registry + .resubscribe_all(std::slice::from_ref(&a)) + .is_empty(), + "re-issuing a subscription must not report an existing path as new" + ); + assert!(registry.watched.contains(&a)); + } + + #[cfg(unix)] + #[test] + fn is_real_dir_rejects_a_symlink_to_a_directory() { + // `Path::is_dir` follows links, so it would report a symlinked + // directory as a directory and the watcher would subscribe to and + // index the link's target — a tree the walker never descends into, and + // one that can sit entirely outside the repository root. + let tmp = TempDir::new().unwrap(); + let real = tmp.path().join("real"); + std::fs::create_dir(&real).unwrap(); + let link = tmp.path().join("link"); + std::os::unix::fs::symlink(&real, &link).unwrap(); + + assert!(is_real_dir(&real)); + assert!( + link.is_dir(), + "precondition: is_dir follows the link, which is the trap" + ); + assert!(!is_real_dir(&link)); + assert!(!is_real_dir(&tmp.path().join("missing"))); + let file = tmp.path().join("file.txt"); + std::fs::write(&file, "x").unwrap(); + assert!(!is_real_dir(&file)); + } + + #[test] + fn watchable_dirs_prunes_ignored_and_hidden_subtrees() { + // The point of the subscription set: an ignored directory costs one + // inotify watch descriptor per directory inside it, so pruning has to + // happen before the subtree is walked, not after its events arrive. + let tmp = TempDir::new().unwrap(); + let root = tmp.path(); + std::fs::create_dir(root.join(".git")).unwrap(); + std::fs::write(root.join(".gitignore"), "build/\n").unwrap(); + + for dir in [ + "src", + "src/nested", + "build", + "build/a", + "build/a/deep", + ".git/objects", + "vendor", + "vendor/pkg", + ] { + std::fs::create_dir_all(root.join(dir)).unwrap(); + } + + let gi = tgrep_core::gitignore::build_matcher(root).expect("matcher should build"); + let exclude = vec!["vendor".to_string()]; + let dirs = watchable_dirs(root, root, &exclude, Some(&gi)); + + let rel: std::collections::HashSet = dirs + .iter() + .map(|p| { + p.strip_prefix(root) + .unwrap() + .to_string_lossy() + .replace('\\', "/") + }) + .collect(); + + // The root itself is always watched, plus the directories the indexer + // would descend into. + assert!(rel.contains(""), "root must always be watched: {rel:?}"); + assert!(rel.contains("src")); + assert!(rel.contains("src/nested")); + + // A gitignored directory and everything beneath it. + assert!( + !rel.contains("build"), + "gitignored dir was watched: {rel:?}" + ); + assert!(!rel.contains("build/a")); + assert!(!rel.contains("build/a/deep")); + + // Hidden directories, which the walker skips too. + assert!(!rel.contains(".git")); + assert!(!rel.contains(".git/objects")); + + // `--exclude` names prune the directory itself, not just its children. + assert!(!rel.contains("vendor"), "excluded dir was watched: {rel:?}"); + assert!(!rel.contains("vendor/pkg")); + } + + #[test] + fn watchable_dirs_without_a_matcher_keeps_everything_visible() { + // `--no-ignore` publishes no matcher. The subscription set must then + // be the whole tree minus hidden paths, matching what the walk indexes; + // silently narrowing it would drop events for files that ARE indexed. + let tmp = TempDir::new().unwrap(); + let root = tmp.path(); + std::fs::create_dir_all(root.join("build/a")).unwrap(); + std::fs::create_dir_all(root.join(".hidden")).unwrap(); + + let dirs = watchable_dirs(root, root, &[], None); + let rel: std::collections::HashSet = dirs + .iter() + .map(|p| { + p.strip_prefix(root) + .unwrap() + .to_string_lossy() + .replace('\\', "/") + }) + .collect(); + + assert!(rel.contains("build")); + assert!(rel.contains("build/a")); + assert!(!rel.contains(".hidden")); + } + + #[test] + fn watchable_dirs_anchors_rules_at_the_root_not_the_start_directory() { + // A subtree that appears at runtime is walked from itself, but the + // ignore rules are written against paths relative to the repository + // root. Walking with the subtree as the anchor would test "nested" + // against a rule meant for "src/nested" and prune the wrong things. + let tmp = TempDir::new().unwrap(); + let root = tmp.path(); + std::fs::create_dir(root.join(".git")).unwrap(); + std::fs::write(root.join(".gitignore"), "src/fresh/skipped/\n/keep/\n").unwrap(); + + for dir in ["src/fresh/skipped", "src/fresh/keep", "src/fresh/kept"] { + std::fs::create_dir_all(root.join(dir)).unwrap(); + } + + let gi = tgrep_core::gitignore::build_matcher(root).expect("matcher should build"); + let dirs = watchable_dirs(root, &root.join("src/fresh"), &[], Some(&gi)); + let rel: std::collections::HashSet = dirs + .iter() + .map(|p| { + p.strip_prefix(root) + .unwrap() + .to_string_lossy() + .replace('\\', "/") + }) + .collect(); + + assert!( + rel.contains("src/fresh"), + "start dir must be watched: {rel:?}" + ); + assert!(rel.contains("src/fresh/kept")); + assert!( + !rel.contains("src/fresh/skipped"), + "a root-anchored rule was not applied: {rel:?}" + ); + // `keep/` is anchored at the root, so it must NOT prune + // `src/fresh/keep` just because the walk started at `src/fresh`. + assert!( + rel.contains("src/fresh/keep"), + "a root-anchored rule was applied at the wrong depth: {rel:?}" + ); + } + + /// A transient stat failure is not a deletion. `Path::exists` said it was, + /// which meant one `EACCES` — or a Windows sharing violation from a build + /// holding the file open — tombstoned content that was still valid, and + /// bypassed the preservation policy `reindex_file` applies to exactly those + /// errors. + #[test] + fn an_unreadable_path_is_not_treated_as_a_deletion() { + use std::io::{Error, ErrorKind}; + + let unreadable: std::io::Result = + Err(Error::from(ErrorKind::PermissionDenied)); + assert_eq!( + classify_event_target(&unreadable), + EventTarget::Unknown, + "a locked or unreadable file must leave the index alone" + ); + + // Windows reports a file opened without FILE_SHARE_* this way, and it + // is the single most common way a stat fails on a live repository. + let sharing: std::io::Result = Err(Error::from_raw_os_error(32)); + assert_eq!(classify_event_target(&sharing), EventTarget::Unknown); + + // The other direction still has to work, or a real deletion is never + // applied. + let absent: std::io::Result = Err(Error::from(ErrorKind::NotFound)); + assert_eq!(classify_event_target(&absent), EventTarget::Gone); + + let tmp = TempDir::new().unwrap(); + let file = tmp.path().join("real.rs"); + std::fs::write(&file, "fn main() {}").unwrap(); + assert_eq!( + classify_event_target(&std::fs::metadata(&file)), + EventTarget::Regular + ); + assert_eq!( + classify_event_target(&std::fs::metadata(tmp.path())), + EventTarget::NotRegular + ); + } + + /// Create a symlink, or return `false` where the platform will not allow + /// one — an unprivileged Windows runner without Developer Mode. + fn try_symlink(target: &Path, link: &Path) -> bool { + #[cfg(unix)] + { + std::os::unix::fs::symlink(target, link).is_ok() + } + #[cfg(windows)] + { + std::os::windows::fs::symlink_file(target, link).is_ok() + } + #[cfg(not(any(unix, windows)))] + { + let _ = (target, link); + false + } + } + + /// The matcher reads a symlinked source through the link, so an edit to the + /// target changes the rules in force. `handle_fs_event` recognises an event + /// naming that target — but on a per-directory backend no event ever + /// arrived, because the directory holding it is one the rules hide and + /// nothing subscribed to it. + #[test] + fn a_rule_file_symlinked_into_a_hidden_directory_is_still_watched() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path(); + std::fs::create_dir(root.join("build")).unwrap(); + std::fs::write(root.join("build").join("shared-rules"), "target/\n").unwrap(); + if !try_symlink( + &root.join("build").join("shared-rules"), + &root.join(".gitignore"), + ) { + return; + } + + let sources = vec![root.join(".gitignore")]; + let dirs = ignore_target_dirs(root, &sources); + + assert!( + dirs.contains(&root.join("build")), + "the directory holding a rule file the matcher read must be watched \ + even when the rules hide it: {dirs:?}" + ); + } + + /// A source that is a plain file needs nothing extra, and one whose target + /// is outside the root must not pull a subscription outside the tree the + /// server was asked to serve. + #[test] + fn ordinary_and_outside_rule_files_add_no_subscriptions() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path(); + std::fs::write(root.join(".gitignore"), "target/\n").unwrap(); + assert!(ignore_target_dirs(root, &[root.join(".gitignore")]).is_empty()); + + let outside = TempDir::new().unwrap(); + std::fs::write(outside.path().join("rules"), "target/\n").unwrap(); + if !try_symlink(&outside.path().join("rules"), &root.join(".ignore")) { + return; + } + assert!( + ignore_target_dirs(root, &[root.join(".ignore")]).is_empty(), + "a target outside the root must not be subscribed to" + ); + } + + #[test] + fn skip_watcher_dir_applies_directory_semantics() { + // A directory-only gitignore rule (`build/`) does not match the path + // `build` when it is tested as a file, which is why the subscription + // set needs its own dir-aware entry point. + let tmp = TempDir::new().unwrap(); + std::fs::create_dir(tmp.path().join(".git")).unwrap(); + std::fs::write(tmp.path().join(".gitignore"), "build/\n").unwrap(); + let gi = tgrep_core::gitignore::build_matcher(tmp.path()).expect("matcher should build"); + + assert!(should_skip_watcher_dir("build", &[], Some(&gi))); + assert!(!should_skip_watcher_path("build", &[], Some(&gi))); + + // `--exclude` prunes the named directory itself... + let exclude = vec!["target".to_string()]; + assert!(should_skip_watcher_dir("target", &exclude, None)); + assert!(should_skip_watcher_dir("src/target", &exclude, None)); + // ...but a *file* of that name is still indexed, so it must not be + // skipped. This is the invariant `should_skip_watcher_path` already + // held, and sharing an implementation must not have changed it. + assert!(!should_skip_watcher_path("target", &exclude, None)); + assert!(!should_skip_watcher_path("src/target", &exclude, None)); + } + + #[test] + fn identifies_live_ignore_rule_changes() { + let root = Path::new("workspace"); + assert!(is_ignore_rules_file( + root, + Path::new("workspace/nested/.gitignore") + )); + assert!(is_ignore_rules_file( + root, + Path::new("workspace/p4ignore.ini") + )); + // `.ignore` is a first-class ignore source (and, unlike `.gitignore`, + // applies outside a git repo), so live edits to it must refresh the + // matcher too — at the root and nested. + assert!(is_ignore_rules_file(root, Path::new("workspace/.ignore"))); + assert!(is_ignore_rules_file( + root, + Path::new("workspace/nested/.ignore") + )); + assert!(!is_ignore_rules_file( + root, + Path::new("workspace/nested/p4ignore.ini") + )); + assert!(!is_ignore_rules_file( + root, + Path::new("workspace/.git/info/exclude") + )); + assert!(!is_ignore_rules_file( + root, + Path::new("workspace/src/main.rs") + )); + } + + #[test] + fn ignore_reconcile_compares_against_actual_indexed_paths() { + use std::collections::{HashMap, HashSet}; + use tgrep_core::meta::FileStamp; + use tgrep_core::walker::FileMeta; + + let current = vec![ + FileMeta { + relative_path: "kept.txt".to_string(), + mtime: 1, + size: 10, + }, + FileMeta { + relative_path: "newly-unignored.txt".to_string(), + mtime: 2, + size: 20, + }, + ]; + let stamps = HashMap::from([ + ("kept.txt".to_string(), FileStamp { mtime: 1, size: 10 }), + ( + "newly-unignored.txt".to_string(), + FileStamp { mtime: 2, size: 20 }, + ), + ]); + let indexed = HashSet::from(["kept.txt".to_string(), "newly-ignored.txt".to_string()]); + + let (changed, added, deleted) = classify_file_changes(¤t, &stamps, &indexed, true); + assert!(changed.is_empty()); + assert_eq!(added, vec!["newly-unignored.txt"]); + assert_eq!(deleted, vec!["newly-ignored.txt"]); + } + + #[test] + fn stale_classification_uses_reader_paths_for_deletions_and_case_renames() { + use std::collections::{HashMap, HashSet}; + use tgrep_core::walker::FileMeta; + + let current = vec![FileMeta { + relative_path: "case.txt".to_string(), + mtime: 1, + size: 10, + }]; + let indexed = HashSet::from(["Case.txt".to_string(), "reader-only.txt".to_string()]); + + let (changed, added, mut deleted) = + classify_file_changes(¤t, &HashMap::new(), &indexed, false); + assert!(changed.is_empty()); + assert_eq!(added, vec!["case.txt"]); + deleted.sort(); + assert_eq!(deleted, vec!["Case.txt", "reader-only.txt"]); + } + + /// The reconcile waits for the server to go quiet, but not forever. + #[test] + fn a_scheduled_reconcile_defers_to_a_busy_server_but_not_indefinitely() { + let long_quiet = RECONCILE_QUIET_PERIOD + Duration::from_secs(1); + let just_queried = Duration::from_secs(1); + + // Before the interval, nothing runs however idle the server is. + assert!(!reconcile_due( + RECONCILE_INTERVAL - Duration::from_secs(1), + long_quiet, + false + )); + // After it, an idle server reconciles. + assert!(reconcile_due(RECONCILE_INTERVAL, long_quiet, false)); + // A server mid-query waits for a gap... + assert!(!reconcile_due(RECONCILE_INTERVAL, just_queried, false)); + // ...but a server that is *always* mid-query would otherwise never + // reconcile at all, which is the failure this exists to prevent. + assert!(reconcile_due(RECONCILE_DEADLINE, just_queried, false)); + // Indexing and flushing outrank even the deadline: they are rewriting + // the index already, and the next tick is a minute away. + assert!(!reconcile_due(RECONCILE_DEADLINE, long_quiet, true)); + } + + /// A file that cannot be read must not make every reconcile rebuild. + /// + /// Its stamp is deliberately withheld so it looks new and gets retried. + /// Left at that, a file locked by another process would be "new" on every + /// pass, and a reconcile on a timer would rewrite the whole index once an + /// hour, forever, to re-attempt a read that fails the same way each time. + #[test] + fn a_file_that_stays_unreadable_is_not_retried_until_it_changes() { + use tgrep_core::meta::FileStamp; + use tgrep_core::walker::FileMeta; + + let memo = std::collections::HashMap::from([( + "locked.bin".to_string(), + FileStamp { + mtime: 100, + size: 5, + }, + )]); + + let retried = |mtime: u64, size: u64| { + let current = vec![FileMeta { + relative_path: "locked.bin".to_string(), + mtime, + size, + }]; + let (mut changed, mut added, _) = classify_file_changes( + ¤t, + &std::collections::HashMap::new(), + &std::collections::HashSet::new(), + false, + ); + let skipped = drop_memoized_failures(&memo, ¤t, &mut changed, &mut added); + (changed.len() + added.len(), skipped) + }; + + // Unchanged since the failed read: leave it alone. + assert_eq!(retried(100, 5).0, 0); + // Touched: worth another look. + assert_eq!(retried(200, 5).0, 1); + // Resized: likewise. + assert_eq!(retried(100, 6).0, 1); + } + + /// Skipping a file must not also mark it indexed. + /// + /// The two halves have to agree. `drop_memoized_failures` keeps a file out + /// of the delta, so nothing reads it and nothing writes postings for it; if + /// the stamps published alongside that delta still claimed it was indexed at + /// its current mtime and size, the next reconcile would classify it as + /// unchanged and skip it for a completely different reason — one that never + /// clears, because the memo is not involved and the stamp outlives the + /// process in `filestamps.json`. A file that failed to read once would be + /// invisible to search forever, with no error and no way back short of + /// deleting the index. + #[test] + fn a_skipped_file_is_left_unstamped_so_the_next_pass_still_sees_it() { + use tgrep_core::meta::FileStamp; + use tgrep_core::walker::FileMeta; + + let current = vec![ + FileMeta { + relative_path: "locked.bin".to_string(), + mtime: 100, + size: 5, + }, + FileMeta { + relative_path: "src/main.rs".to_string(), + mtime: 100, + size: 9, + }, + ]; + let memo = std::collections::HashMap::from([( + "locked.bin".to_string(), + FileStamp { + mtime: 100, + size: 5, + }, + )]); + + let (mut changed, mut added, _) = classify_file_changes( + ¤t, + &std::collections::HashMap::new(), + &std::collections::HashSet::new(), + false, + ); + let skipped = drop_memoized_failures(&memo, ¤t, &mut changed, &mut added); + assert!(skipped.contains("locked.bin")); + + let stamps = stamps_for_indexed(¤t, &skipped); + assert!( + !stamps.contains_key("locked.bin"), + "published a stamp for a file that was never read: {stamps:?}" + ); + assert!(stamps.contains_key("src/main.rs"), "{stamps:?}"); + + // And with that stamp withheld, a later pass over an unchanged tree + // still offers the file up rather than treating it as up-to-date. + let (changed, added, _) = + classify_file_changes(¤t, &stamps, &std::collections::HashSet::new(), false); + assert_eq!(changed.len() + added.len(), 1); + assert!(added.contains(&"locked.bin".to_string())); + } + + /// A walk error must not leave the watcher gated forever. + /// + /// `gitignore_pending` is what keeps the watcher off the index until a + /// matcher exists, and the stale check owns clearing it. It also refuses to + /// touch the index when the walk could not inspect every entry, because + /// unseen files would be misclassified as deleted. Those two are separate + /// decisions: taking the second one used to skip the first, so one + /// unreadable directory — or one file whose `metadata()` lost a race with a + /// delete — silently disabled the watcher for the life of the process, and + /// the overflow-repair path would not reconcile either, since it defers to + /// the pending matcher. + /// + /// Unix-only because it needs a directory the process genuinely cannot + /// read, which has no portable equivalent on Windows. + #[cfg(unix)] + #[test] + fn a_walk_error_still_publishes_the_watcher_matcher() { + use std::os::unix::fs::PermissionsExt; + + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + // `.gitignore` is git-gated, matching the indexing walk. + std::fs::create_dir(root.join(".git")).unwrap(); + std::fs::write(root.join(".gitignore"), "*.log\n").unwrap(); + std::fs::write(root.join("src.rs"), "fn main() {}\n").unwrap(); + + let unreadable = root.join("locked"); + std::fs::create_dir(&unreadable).unwrap(); + std::fs::write(unreadable.join("inner.rs"), "fn inner() {}\n").unwrap(); + std::fs::set_permissions(&unreadable, std::fs::Permissions::from_mode(0o000)).unwrap(); + if std::fs::read_dir(&unreadable).is_ok() { + // Running as root, so permissions prove nothing. Restore and skip. + std::fs::set_permissions(&unreadable, std::fs::Permissions::from_mode(0o755)).unwrap(); + return; + } + + let index_dir = root.join(".tgrep"); + std::fs::create_dir(&index_dir).unwrap(); + let state = test_server_state(&root, &index_dir); + state.gitignore_pending.store(true, Ordering::SeqCst); + + let ok = background_refresh_stale(&state, &root, &index_dir, false); + + // Restore before any assertion so a failure still leaves a removable dir. + std::fs::set_permissions(&unreadable, std::fs::Permissions::from_mode(0o755)).unwrap(); + + assert!( + !ok, + "a walk that could not inspect every entry must keep the old index" + ); + assert!( + !state.gitignore_pending.load(Ordering::SeqCst), + "the watcher gate must be released even when the index is left alone" + ); + assert!( + state.gitignore.read().unwrap().is_some(), + "the matcher the walk did find must still be published" + ); + } + + /// The metadata the eligibility check uses and the bytes that get indexed + /// have to describe the same object, which means one handle. + #[test] + fn open_within_root_reads_a_regular_file() { + use std::io::Read; + + let tmp = TempDir::new().unwrap(); + std::fs::create_dir(tmp.path().join("src")).unwrap(); + let path = tmp.path().join("src").join("plain.rs"); + std::fs::write(&path, "fn main() {}\n").unwrap(); + + let file = open_within_root(tmp.path(), &path).expect("a regular file opens"); + let meta = file.metadata().expect("metadata off the handle"); + assert!(meta.is_file()); + assert_eq!(meta.len(), 13); + + let mut data = String::new(); + (&file).read_to_string(&mut data).unwrap(); + assert_eq!(data, "fn main() {}\n"); + } + + /// A symlink must not open as its target, or a link committed to a branch + /// would pull a file from outside the served root into the index. + #[cfg(unix)] + #[test] + fn open_within_root_refuses_a_symlinked_file() { + let outside = TempDir::new().unwrap(); + let target = outside.path().join("secret.txt"); + std::fs::write(&target, "sensitive\n").unwrap(); + + let root = TempDir::new().unwrap(); + let link = root.path().join("link.txt"); + std::os::unix::fs::symlink(&target, &link).unwrap(); + + assert!( + open_within_root(root.path(), &link).is_err(), + "the link must not open as its target" + ); + } + + /// And neither must a symlink anywhere *above* the file: `root/a/file` + /// reads the same whether `a` is a directory or a link to one, so guarding + /// only the last component still lets a whole tree in from outside. + #[cfg(unix)] + #[test] + fn open_within_root_refuses_a_symlinked_ancestor() { + let outside = TempDir::new().unwrap(); + std::fs::write(outside.path().join("secret.txt"), "sensitive\n").unwrap(); + + let root = TempDir::new().unwrap(); + let link = root.path().join("a"); + std::os::unix::fs::symlink(outside.path(), &link).unwrap(); + let through_link = link.join("secret.txt"); + + // The file at the end of that path is a perfectly ordinary file, and + // opening it by name works — which is the point. + assert!(std::fs::File::open(&through_link).is_ok()); + assert!( + open_within_root(root.path(), &through_link).is_err(), + "an intermediate symlink must not be traversed" + ); } - Ok(()) -} -/// Publish a single staged file at `src` to `dst`. -/// -/// Uses `std::fs::rename`, which on the same volume is an O(microseconds) -/// directory entry update — this is the property that keeps the server's -/// index write lock from being held for the duration of a multi-hundred-MB -/// file copy (which previously blocked all search queries). -/// -/// On Windows, transient sharing violations (`ERROR_SHARING_VIOLATION` = 32, -/// `ERROR_LOCK_VIOLATION` = 33) can occur after dropping an mmap (cache -/// manager / AV / indexers may briefly hold a reference), so retry only -/// those specific error codes for a short window. All other errors fail -/// fast — a broader retry surface would needlessly extend the publish -/// window for non-transient failures. -/// -/// Deliberately does NOT fall back to `std::fs::copy` on persistent failure: -/// the caller holds the index write lock and a multi-hundred-MB copy is -/// exactly the pathology we are fixing. Staging is always created next to -/// the target, so cross-volume cases should not arise; if rename truly -/// cannot succeed, surfacing the error lets the caller abort cleanly -/// rather than silently regress search latency. -/// Context wrapper that preserves the original `std::io::Error` as the -/// `source()` of the returned error so callers can downcast through the -/// chain to inspect `raw_os_error()` for diagnostics. -#[derive(Debug)] -struct PublishError { - ctx: String, - source: std::io::Error, -} + /// Nothing may be resolved that could climb back out of the root. + #[test] + fn open_within_root_refuses_paths_that_escape_or_are_not_literal() { + let tmp = TempDir::new().unwrap(); + std::fs::create_dir(tmp.path().join("src")).unwrap(); + std::fs::write(tmp.path().join("src").join("a.rs"), "x\n").unwrap(); -impl std::fmt::Display for PublishError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - // Include the underlying error in the formatted message for - // human-readable logging; structured access remains via `source()`. - write!(f, "{}: {}", self.ctx, self.source) + assert!( + open_within_root(tmp.path(), tmp.path()).is_err(), + "the root" + ); + assert!( + open_within_root(tmp.path(), &tmp.path().join("..").join("a.rs")).is_err(), + "a parent component" + ); + assert!( + open_within_root(&tmp.path().join("src"), &tmp.path().join("src")).is_err(), + "outside the given root" + ); } -} -impl std::error::Error for PublishError { - fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { - Some(&self.source) + /// A burst big enough to be worth replaying individually is not worth + /// replaying individually. The buffer gives up as a whole, because a + /// truncated set looks exactly like a complete one at replay time. + #[cfg(unix)] + #[test] + fn deferring_more_changes_than_the_cap_gives_up_on_the_whole_set() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + // The function only defers while a build is running; it now reports + // back so the caller can handle an event that arrived after one ended. + state.indexing.store(true, Ordering::SeqCst); + + let small = Event { + kind: EventKind::Create(notify::event::CreateKind::Any), + paths: vec![root.join("a.rs")], + attrs: Default::default(), + }; + assert!(defer_events_during_build(&state, &small)); + assert_eq!( + state + .deferred_events + .lock() + .unwrap() + .as_ref() + .unwrap() + .len(), + 1 + ); + + let flood = Event { + kind: EventKind::Create(notify::event::CreateKind::Any), + paths: (0..100_000) + .map(|i| root.join(format!("f{i}.rs"))) + .collect(), + attrs: Default::default(), + }; + assert!(defer_events_during_build(&state, &flood)); + assert!( + state.deferred_events.lock().unwrap().is_none(), + "an overflowing burst must mark the buffer unusable, not truncate it" + ); + + // And stays given up on, rather than resuming a partial record. + assert!(defer_events_during_build(&state, &small)); + assert!(state.deferred_events.lock().unwrap().is_none()); } -} -fn publish_file(src: &Path, dst: &Path) -> std::io::Result<()> { - const RENAME_RETRIES: u32 = 30; - const RENAME_BACKOFF: Duration = Duration::from_millis(50); - // Windows error codes that can transiently occur when another handle - // (mmap section, AV scanner, indexer) still references the target file: - // ERROR_SHARING_VIOLATION = 32 - // ERROR_LOCK_VIOLATION = 33 - // Other errors (NotFound, permission/ACL issues, disk full, …) are - // structural and should fail fast so we don't extend the publish window. - #[cfg(windows)] - const TRANSIENT_WIN_ERRORS: &[i32] = &[32, 33]; + /// Events seen while a build ran are not applied then, but they are the + /// only record that those paths moved: the build's walk misses anything + /// written to a directory it already passed. + #[cfg(unix)] + #[test] + fn changes_deferred_during_a_build_are_applied_once_it_publishes() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + // As it is once a build publishes: no rules to wait for, nothing + // indexing. + state.gitignore_pending.store(false, Ordering::SeqCst); - let mut last_err: Option = None; - for attempt in 0..RENAME_RETRIES { - match std::fs::rename(src, dst) { - Ok(()) => return Ok(()), - Err(e) => { - #[cfg(windows)] - let transient = - matches!(e.raw_os_error(), Some(c) if TRANSIENT_WIN_ERRORS.contains(&c)); - #[cfg(not(windows))] - let transient = false; + let path = root.join("late.rs"); + std::fs::write(&path, "fn written_during_the_build() {}\n").unwrap(); + + state.indexing.store(true, Ordering::SeqCst); + handle_fs_event( + &state, + &root, + &Event { + kind: EventKind::Create(notify::event::CreateKind::Any), + paths: vec![path.clone()], + attrs: Default::default(), + }, + ); + assert!( + !state.index.read().unwrap().live.has_path("late.rs"), + "an event during a build must not touch the index" + ); - if !transient || attempt + 1 == RENAME_RETRIES { - // Wrap with a context error that preserves the original - // `std::io::Error` as the `source()` of the returned - // error, so callers can downcast through the chain to - // recover `raw_os_error()` for diagnostics. - let ctx = format!( - "publish_file: rename({}, {}) failed after {} attempt(s)", - src.display(), - dst.display(), - attempt + 1, - ); - let kind = e.kind(); - return Err(std::io::Error::new(kind, PublishError { ctx, source: e })); - } - last_err = Some(e); - thread::sleep(RENAME_BACKOFF); - } - } + state.indexing.store(false, Ordering::SeqCst); + replay_deferred_events(&state, &root); + + assert!( + state.index.read().unwrap().live.has_path("late.rs"), + "the deferred change must be applied once the build is done" + ); + assert!( + state + .deferred_events + .lock() + .unwrap() + .as_ref() + .is_some_and(|p| p.is_empty()), + "the buffer must be drained, and left usable for the next build" + ); } - // Unreachable: the loop either returns Ok, or returns Err on the last - // iteration. Defensive return preserves the last error. - Err(last_err.unwrap_or_else(|| { - std::io::Error::other("publish_file: rename retries exhausted with no error recorded") - })) -} -fn ctrlc_handler(handler: F) { - #[cfg(windows)] - { - use std::sync::OnceLock; - static HANDLER: OnceLock> = OnceLock::new(); - HANDLER.get_or_init(|| Box::new(handler)); + /// A metadata-only change to a directory is not a claim that anything new + /// is under it. Replaying every deferred path as a creation would turn a + /// recursive `chmod` during a build into one subtree walk per directory. + #[cfg(unix)] + #[test] + fn a_deferred_metadata_change_does_not_replay_as_a_subtree_arrival() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + state.gitignore_pending.store(false, Ordering::SeqCst); - unsafe extern "system" fn console_handler(_ctrl_type: u32) -> i32 { - if let Some(h) = HANDLER.get() { - h(); - } - 1 // TRUE - we handled the event - } + let dir = root.join("vendor"); + std::fs::create_dir(&dir).unwrap(); + std::fs::write(dir.join("deep.rs"), "fn deep() {}\n").unwrap(); + + state.indexing.store(true, Ordering::SeqCst); + handle_fs_event( + &state, + &root, + &Event { + kind: EventKind::Modify(notify::event::ModifyKind::Metadata( + notify::event::MetadataKind::Permissions, + )), + paths: vec![dir.clone()], + attrs: Default::default(), + }, + ); + assert_eq!( + state + .deferred_events + .lock() + .unwrap() + .as_ref() + .unwrap() + .get(&dir), + Some(&false), + "a metadata modify must not be recorded as introducing a directory" + ); - unsafe extern "system" { - fn SetConsoleCtrlHandler( - handler: unsafe extern "system" fn(u32) -> i32, - add: i32, - ) -> i32; - } + state.indexing.store(false, Ordering::SeqCst); + replay_deferred_events(&state, &root); - // SAFETY: SetConsoleCtrlHandler is a stable Win32 API. The handler function - // is extern "system" with correct signature, and HANDLER is 'static. - unsafe { - SetConsoleCtrlHandler(console_handler, 1); + // The replay reconstructs a modify, which stops at the directory gate. + // Had it reconstructed a create, `watch_new_subtree` would have walked + // in and indexed the file below. + assert!( + !state.index.read().unwrap().live.has_path("vendor/deep.rs"), + "a metadata modify must not trigger a subtree walk on replay" + ); + } + + /// A file that cannot be opened right now is not a file that stopped + /// belonging in the index. Evicting on a transient error would drop live + /// content because something else held the file open for a moment. + #[cfg(unix)] + #[test] + fn an_unreadable_file_keeps_its_indexed_content() { + use std::os::unix::fs::PermissionsExt; + + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + state.gitignore_pending.store(false, Ordering::SeqCst); + + let path = root.join("locked.rs"); + std::fs::write(&path, "fn readable() {}\n").unwrap(); + reindex_file(&state, &path, "locked.rs"); + assert!(state.index.read().unwrap().live.has_path("locked.rs")); + + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o000)).unwrap(); + if std::fs::File::open(&path).is_ok() { + // Running as root, where the mode is advisory. Nothing to test. + return; } + reindex_file(&state, &path, "locked.rs"); + + assert!( + !state.index.read().unwrap().live.is_deleted("locked.rs"), + "an unreadable file must keep what was already indexed for it" + ); } - #[cfg(not(windows))] - { - use std::sync::OnceLock; - static HANDLER: OnceLock> = OnceLock::new(); - HANDLER.get_or_init(|| Box::new(handler)); + /// Deleting an ignore file is invisible to a scan that looks for *arrivals* + /// by mtime, and leaves rules in force whose source is gone — so the + /// published sources are checked directly. + #[cfg(unix)] + #[test] + fn a_recovery_scan_notices_an_ignore_file_that_was_deleted() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + state.gitignore_pending.store(false, Ordering::SeqCst); + + // Published as a source, then removed behind the matcher's back. + let rules = root.join(".gitignore"); + std::fs::write(&rules, "build/\n").unwrap(); + *state.ignore_sources.write().unwrap() = vec![rules.clone()]; + std::fs::remove_file(&rules).unwrap(); + + std::fs::write(root.join("kept.rs"), "fn kept() {}\n").unwrap(); + // Pretend a refresh worker is already running, so the scan's request + // for one is coalesced into it instead of spawning a real rewalk that + // would race the assertions below. + state.ignore_refresh_scheduled.store(true, Ordering::SeqCst); + reindex_files_in( + &state, + &root, + std::slice::from_ref(&root), + SystemTime::UNIX_EPOCH, + ); + + assert!( + state.ignore_rules_dirty.load(Ordering::SeqCst), + "a vanished ignore source must schedule a refresh" + ); + assert!( + !state.index.read().unwrap().live.has_path("kept.rs"), + "the scan must abandon rather than index under rules it knows are stale" + ); + } + + /// The invariant the incomplete-listing fix relies on: a directory that is + /// not in `swept` has proved nothing about the names under it, so the + /// sweep must leave them alone. + /// + /// The wiring above it — withholding a directory whose `read_dir` iterator + /// yielded an error — has no test of its own, because a per-entry + /// `readdir` failure cannot be induced portably. This pins the half that + /// makes withholding sufficient. + #[cfg(unix)] + #[test] + fn sweep_removed_files_only_deletes_from_directories_it_enumerated() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + + let path = root.join("d").join("a.rs"); + std::fs::create_dir(root.join("d")).unwrap(); + std::fs::write(&path, "fn a() {}\n").unwrap(); + reindex_file(&state, &path, "d/a.rs"); + assert!(state.index.read().unwrap().live.has_path("d/a.rs")); + + // What `reindex_files_in` produces when an entry in `d` failed to + // enumerate: the file is missing from `present`, and `d` is therefore + // withheld from `swept`. + let swept = std::collections::HashSet::new(); + let present = std::collections::HashSet::new(); + let no_vanished = std::collections::HashSet::new(); + sweep_removed_files(&state, &swept, &present, &no_vanished); + assert!( + !state.index.read().unwrap().live.is_deleted("d/a.rs"), + "an unenumerated directory must not tombstone the files under it" + ); + + // And with the listing complete, the same absence is a deletion — once + // the file is actually gone, which the sweep now confirms itself. + std::fs::remove_file(&path).unwrap(); + let swept = std::collections::HashSet::from(["d".to_string()]); + sweep_removed_files(&state, &swept, &present, &no_vanished); + assert!( + state.index.read().unwrap().live.is_deleted("d/a.rs"), + "a fully enumerated directory must sweep what it no longer contains" + ); + } - unsafe extern "C" fn signal_handler(_sig: std::ffi::c_int) { - if let Some(h) = HANDLER.get() { - h(); - } - } + /// The listing that decides what to sweep is from earlier in the scan. A + /// file recreated since then has already had its event consumed, so + /// deleting it on that stale evidence loses it until the next reconcile. + #[cfg(unix)] + #[test] + fn the_sweep_does_not_delete_a_file_that_came_back() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); - unsafe extern "C" { - fn signal(sig: std::ffi::c_int, handler: unsafe extern "C" fn(std::ffi::c_int)); - } + let path = root.join("d").join("a.rs"); + std::fs::create_dir(root.join("d")).unwrap(); + std::fs::write(&path, "fn a() {}\n").unwrap(); + reindex_file(&state, &path, "d/a.rs"); + assert!(state.index.read().unwrap().live.has_path("d/a.rs")); - // SAFETY: signal() is a POSIX API. The handler has the correct extern "C" - // signature, and HANDLER is 'static. SIGINT (2) is valid on all Unix. - // SIGINT = 2 on all Unix platforms - unsafe { - signal(2, signal_handler); - } + // `d` enumerated cleanly and did not contain `a.rs` at the time — but + // it is back on disk by the time the sweep runs. + let swept = std::collections::HashSet::from(["d".to_string()]); + let present = std::collections::HashSet::new(); + let no_vanished = std::collections::HashSet::new(); + sweep_removed_files(&state, &swept, &present, &no_vanished); + + assert!( + !state.index.read().unwrap().live.is_deleted("d/a.rs"), + "a file that exists again must not be swept on a stale listing" + ); } -} -#[cfg(test)] -mod tests { - use super::*; - use tempfile::TempDir; + /// A vanished directory that comes back as a link to somewhere else has + /// not brought its files back: nothing under a link is walked, watched or + /// indexed. The recheck that decides "it is here again" has to answer that + /// under the same containment contract indexing does, or the stale in-root + /// entry is kept and no later event ever corrects it. + #[cfg(unix)] + #[test] + fn a_path_that_returns_through_a_symlinked_ancestor_is_still_swept() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path().join("root"); + std::fs::create_dir(&root).unwrap(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); - fn cached(len: usize) -> Arc { - // `String::from_utf8` preserves the Vec's capacity, so `heap_bytes` - // is exactly `len` and the assertions below can use round numbers. - let mut bytes = Vec::with_capacity(len); - bytes.resize(len, b'a'); - Arc::new(DecodedFile { - text: String::from_utf8(bytes).unwrap(), - fixups: Default::default(), - }) + let dir = root.join("d"); + std::fs::create_dir(&dir).unwrap(); + std::fs::write(dir.join("a.rs"), "fn ours() {}\n").unwrap(); + reindex_file(&state, &dir.join("a.rs"), "d/a.rs"); + assert!(state.index.read().unwrap().live.has_path("d/a.rs")); + + // The scan listed the root, did not find `d`, and is about to sweep + // what was under it. In between, `d` comes back — as a link to a tree + // that is not ours, holding a file by the same name. + let outside = tmp.path().join("elsewhere"); + std::fs::create_dir(&outside).unwrap(); + std::fs::write(outside.join("a.rs"), "fn theirs() {}\n").unwrap(); + std::fs::remove_dir_all(&dir).unwrap(); + std::os::unix::fs::symlink(&outside, &dir).unwrap(); + assert!( + root.join("d").join("a.rs").is_file(), + "the fixture must look like a returning file, or it proves nothing" + ); + + let swept = std::collections::HashSet::from([String::new()]); + let present = std::collections::HashSet::new(); + let vanished = std::collections::HashSet::from(["d".to_string()]); + sweep_removed_files(&state, &swept, &present, &vanished); + + assert!( + state.index.read().unwrap().live.is_deleted("d/a.rs"), + "a file reachable only through a symlink has not come back" + ); } + /// An indexed file replaced in place by something that is not a regular + /// file is not a removal — the path still exists — but its contents are no + /// longer there to be found. + #[cfg(unix)] #[test] - fn content_cache_evicts_to_stay_under_byte_budget() { - let mut cache = ContentCache::new(CACHE_CAPACITY, 1000, 1000); - for i in 0..10 { - cache.put(format!("f{i}"), cached(200)); - } - assert!(cache.byte_len() <= 1000, "bytes = {}", cache.byte_len()); - assert_eq!(cache.len(), 5); - // Oldest entries went first; the newest survive. - assert!(cache.peek("f0").is_none()); - assert!(cache.peek("f9").is_some()); + fn a_file_replaced_by_a_fifo_loses_its_indexed_content() { + use std::os::unix::ffi::OsStrExt; + + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + state.gitignore_pending.store(false, Ordering::SeqCst); + + let path = root.join("x.rs"); + std::fs::write(&path, "fn was_a_real_file() {}\n").unwrap(); + reindex_file(&state, &path, "x.rs"); + assert!(state.index.read().unwrap().live.has_path("x.rs")); + + std::fs::remove_file(&path).unwrap(); + let name = std::ffi::CString::new(path.as_os_str().as_bytes()).unwrap(); + // SAFETY: `name` is a NUL-terminated path that outlives the call. + assert_eq!(unsafe { libc::mkfifo(name.as_ptr(), 0o644) }, 0); + + handle_fs_event( + &state, + &root, + &Event { + kind: EventKind::Modify(notify::event::ModifyKind::Name( + notify::event::RenameMode::To, + )), + paths: vec![path.clone()], + attrs: Default::default(), + }, + ); + + assert!( + state.index.read().unwrap().live.is_deleted("x.rs"), + "content indexed before the path became a fifo must not stay searchable" + ); } + /// A removal must not land while another thread is midway through indexing + /// the same path, or the reindex commits bytes it read earlier and + /// resurrects a file that is gone — with a fresh stamp, so nothing + /// afterwards disagrees and no further event is coming to correct it. + #[cfg(unix)] #[test] - fn content_cache_refuses_oversized_entries() { - let mut cache = ContentCache::new(CACHE_CAPACITY, 1000, 100); - cache.put("small".into(), cached(50)); - cache.put("huge".into(), cached(500)); - assert!(cache.peek("huge").is_none(), "oversized entry was admitted"); - // Admitting it would also have evicted the useful entry. - assert!(cache.peek("small").is_some()); - assert_eq!(cache.byte_len(), 50); + fn a_removal_waits_for_an_in_flight_reindex() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + state.gitignore_pending.store(false, Ordering::SeqCst); + + let path = root.join("x.rs"); + std::fs::write(&path, "fn indexed() {}\n").unwrap(); + reindex_file(&state, &path, "x.rs"); + assert!(state.index.read().unwrap().live.has_path("x.rs")); + + std::fs::remove_file(&path).unwrap(); + + // Stands in for a `reindex_file` that has read the old bytes and not + // yet committed them: it holds exactly this lock across that window. + let held = state.reindex_lock.lock().unwrap(); + + let worker = { + let state = Arc::clone(&state); + let root = root.clone(); + let path = path.clone(); + std::thread::spawn(move || { + handle_fs_event( + &state, + &root, + &Event { + kind: EventKind::Remove(notify::event::RemoveKind::File), + paths: vec![path], + attrs: Default::default(), + }, + ); + }) + }; + + // The delete has to wait its turn. Without the lock it lands + // immediately, which is the ordering that loses the file. + std::thread::sleep(std::time::Duration::from_millis(300)); + assert!( + !state.index.read().unwrap().live.is_deleted("x.rs"), + "a removal must not mutate the index while an indexer holds the lock" + ); + + drop(held); + worker.join().unwrap(); + assert!( + state.index.read().unwrap().live.is_deleted("x.rs"), + "and it must still apply once the lock is free" + ); } - /// The byte total must stay exact across every path that removes an entry, - /// including the entry-count eviction that `LruCache` performs internally. + /// `git checkout`, `tar -x` and `rsync -a` all restore mtimes from what + /// they unpack, so a nested ignore file can arrive carrying a timestamp + /// from months ago. A recency test cannot see that; absence from the + /// published sources can. + #[cfg(unix)] #[test] - fn content_cache_byte_accounting_stays_exact() { - let mut cache = ContentCache::new(2, u64::MAX, u64::MAX); - cache.put("a".into(), cached(100)); - cache.put("b".into(), cached(100)); - assert_eq!(cache.byte_len(), 200); + fn an_arriving_ignore_file_with_a_preserved_mtime_still_refreshes_the_matcher() { + use std::os::unix::ffi::OsStrExt; - // Capacity is 2, so this evicts "a" inside the LRU. - cache.put("c".into(), cached(100)); - assert_eq!(cache.len(), 2); - assert_eq!(cache.byte_len(), 200, "count-eviction leaked bytes"); + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + state.gitignore_pending.store(false, Ordering::SeqCst); - // Replacing an existing key must not double-count. - cache.put("c".into(), cached(300)); - assert_eq!(cache.len(), 2); - assert_eq!(cache.byte_len(), 400); + // The matcher in force was built without it. + *state.ignore_sources.write().unwrap() = Vec::new(); - cache.pop("c"); - assert_eq!(cache.byte_len(), 100); - cache.clear(); - assert_eq!(cache.byte_len(), 0); - assert_eq!(cache.len(), 0); + let sub = root.join("sub"); + std::fs::create_dir(&sub).unwrap(); + let rules = sub.join(".gitignore"); + std::fs::write(&rules, "kept.rs\n").unwrap(); + std::fs::write(sub.join("kept.rs"), "fn kept() {}\n").unwrap(); + + // Backdated well outside any plausible scan window. + let name = std::ffi::CString::new(rules.as_os_str().as_bytes()).unwrap(); + let stamp = libc::timeval { + tv_sec: 1_000_000, + tv_usec: 0, + }; + let times = [stamp, stamp]; + // SAFETY: `name` is NUL-terminated and `times` is a two-element array, + // both outliving the call. + assert_eq!(unsafe { libc::utimes(name.as_ptr(), times.as_ptr()) }, 0); + + state.ignore_refresh_scheduled.store(true, Ordering::SeqCst); + let since = SystemTime::now() - std::time::Duration::from_secs(60); + reindex_files_in(&state, &root, std::slice::from_ref(&sub), since); + + assert!( + state.ignore_rules_dirty.load(Ordering::SeqCst), + "an ignore file the matcher never read must schedule a refresh, \ + however old its mtime is" + ); + assert!( + !state.index.read().unwrap().live.has_path("sub/kept.rs"), + "the scan must abandon rather than index under rules it knows are stale" + ); } + /// The walker collects rule files with `Path::is_file`, which follows + /// links, so a symlinked `.gitignore` contributes rules like any other — + /// but `DirEntry::file_type` does not follow links, and the scan used to + /// drop those entries before ever asking what they were. + #[cfg(unix)] #[test] - fn content_cache_touch_promotes_recency() { - let mut cache = ContentCache::new(CACHE_CAPACITY, 300, 300); - cache.put("a".into(), cached(100)); - cache.put("b".into(), cached(100)); - cache.touch("a"); - // "b" is now least-recently-used, so it is the one that goes. - cache.put("c".into(), cached(100)); - cache.put("d".into(), cached(100)); - assert!(cache.peek("a").is_some(), "touched entry was evicted first"); - assert!(cache.peek("b").is_none()); + fn a_symlinked_ignore_file_is_still_seen_by_a_recovery_scan() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + state.gitignore_pending.store(false, Ordering::SeqCst); + + let sub = root.join("sub"); + std::fs::create_dir(&sub).unwrap(); + let target = root.join("shared-rules"); + std::fs::write(&target, "kept.rs\n").unwrap(); + std::os::unix::fs::symlink(&target, sub.join(".gitignore")).unwrap(); + std::fs::write(sub.join("kept.rs"), "fn kept() {}\n").unwrap(); + + // Recent enough that the mtime window alone would catch a *regular* + // file here: what this pins is that a symlink gets that far at all. + state.ignore_refresh_scheduled.store(true, Ordering::SeqCst); + let since = SystemTime::now() - std::time::Duration::from_secs(60); + reindex_files_in(&state, &root, std::slice::from_ref(&sub), since); + + assert!( + state.ignore_rules_dirty.load(Ordering::SeqCst), + "a symlinked ignore file carries rules and must schedule a refresh" + ); + assert!( + !state.index.read().unwrap().live.has_path("sub/kept.rs"), + "the scan must abandon rather than index under rules it knows are stale" + ); } - /// A `ServerState` over an empty index, for exercising the stale path - /// directly. Mirrors the defaults `run` uses with a watcher and ignore - /// rules enabled, which is the configuration `gitignore_pending` gates. + /// `read_dir` promises no ordering, and the rules a scan must respect can + /// live in a directory it has not reached yet — so every directory in the + /// scan is asked before any file in it is indexed. #[cfg(unix)] - fn test_server_state(root: &Path, index_dir: &Path) -> Arc { - create_empty_index(index_dir).expect("create empty index"); - let hybrid = HybridIndex::open(index_dir, root).expect("open empty index"); - Arc::new(ServerState { - index: RwLock::new(hybrid), - cache: RwLock::new(ContentCache::new( - CACHE_CAPACITY, - CACHE_MAX_BYTES, - CACHE_MAX_ENTRY_BYTES, - )), - root: root.to_path_buf(), - watcher_active: std::sync::atomic::AtomicBool::new(false), - indexing: std::sync::atomic::AtomicBool::new(false), - flushing: std::sync::atomic::AtomicBool::new(false), - gitignore_pending: std::sync::atomic::AtomicBool::new(true), - ignore_rules_dirty: std::sync::atomic::AtomicBool::new(false), - ignore_refresh_scheduled: std::sync::atomic::AtomicBool::new(false), - index_progress: std::sync::atomic::AtomicU64::new(0), - index_total: std::sync::atomic::AtomicU64::new(0), - watch_enabled: true, - exclude_dirs: Vec::new(), - no_ignore: false, - no_require_git: false, - max_file_size: None, - index_dir: index_dir.to_path_buf(), - publish_lock: Mutex::new(()), - file_stamps: RwLock::new(Default::default()), - snapshot_gate: RwLock::new(()), - stale_refresh_lock: Mutex::new(()), - gitignore: RwLock::new(None), - memory_cap_bytes: 16 * 1024 * 1024 * 1024, - index_threads: 1, - auto_save_mutations: 0, - unreadable: RwLock::new(std::collections::HashMap::new()), - started: Instant::now(), - last_search_ms: std::sync::atomic::AtomicU64::new(0), - }) + #[test] + fn a_scan_checks_every_directory_for_rules_before_indexing_any_file() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + state.gitignore_pending.store(false, Ordering::SeqCst); + *state.ignore_sources.write().unwrap() = Vec::new(); + + // The rules are in `b`, the file is in `a`, and `a` is scanned first. + let a = root.join("a"); + let b = root.join("b"); + std::fs::create_dir(&a).unwrap(); + std::fs::create_dir(&b).unwrap(); + std::fs::write(a.join("keep.rs"), "fn keep() {}\n").unwrap(); + std::fs::write(b.join(".gitignore"), "keep.rs\n").unwrap(); + + state.ignore_refresh_scheduled.store(true, Ordering::SeqCst); + let since = SystemTime::now() - std::time::Duration::from_secs(60); + reindex_files_in(&state, &root, &[a, b], since); + + assert!( + state.ignore_rules_dirty.load(Ordering::SeqCst), + "the scan must notice rules that live later in its own list" + ); + assert!( + !state.index.read().unwrap().live.has_path("a/keep.rs"), + "nothing may be indexed before every directory has been asked for rules" + ); } - /// A binary marker's offset is a position in the file, not in the repaired - /// text. - /// - /// Lossy decoding widens every invalid byte to a three-byte U+FFFD, so a - /// NUL preceded by invalid UTF-8 sits further along the searched text than - /// it does on disk. `rg 15.2.0` reports the on-disk byte (verified: this - /// fixture gives `found "\0" byte around offset 9`). - /// - /// The mapping has to happen here because the client never reads a file the - /// server searched, so it has no fixups of its own to map with. + /// A path is not evidence about contents. An archive restore can put a + /// different `.gitignore` at a path the matcher already read, carrying an + /// mtime older than the scan window — known name, untouched by the clock, + /// and yet not the file whose rules are being enforced. #[test] - fn binary_marker_offset_is_mapped_back_to_the_source_bytes() { - let mut bytes = vec![0xFF, 0xFF]; - bytes.extend_from_slice(b"needle\n"); - bytes.push(0); - bytes.extend_from_slice(b"tail\n"); - let nul_on_disk = bytes.iter().position(|&b| b == 0).unwrap(); - assert_eq!(nul_on_disk, 9, "fixture changed"); + fn a_rule_file_swapped_for_an_older_one_is_not_taken_on_faith() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + state.gitignore_pending.store(false, Ordering::SeqCst); - let file = DecodedFile::new(bytes, tgrep_core::encoding::EncodingMode::Auto); - assert_eq!( - file.text.as_bytes().iter().position(|&b| b == 0), - Some(13), - "the fixture must actually shift the offset, or this proves nothing" + let rules = root.join(".gitignore"); + std::fs::write(&rules, "nothing-at-all.rs\n").unwrap(); + std::fs::write(root.join("keep.rs"), "fn keep() {}\n").unwrap(); + *state.ignore_source_stamps.write().unwrap() = + ignore_stamps_of(&root, std::slice::from_ref(&rules)); + *state.ignore_sources.write().unwrap() = vec![rules.clone()]; + + // Far enough ahead that the mtime window cannot fire for anything on + // disk: what is left is the question of whether the file is the one + // that was read. + let since = SystemTime::now() + std::time::Duration::from_secs(3600); + state.ignore_refresh_scheduled.store(true, Ordering::SeqCst); + reindex_files_in(&state, &root, std::slice::from_ref(&root), since); + assert!( + !state.ignore_rules_dirty.load(Ordering::SeqCst), + "the file the matcher read must not be reported as changed" + ); + assert!( + state.index.read().unwrap().live.has_path("keep.rs"), + "and the scan must get on with its work" ); - let matcher = crate::matching::build_search_matcher( - &["needle".to_string()], - &crate::matching::MatcherConfig::default(), - ) - .unwrap(); + // Same path, same age as far as the window is concerned, different + // rules. + std::fs::write(&rules, "keep.rs\n").unwrap(); + reindex_files_in(&state, &root, std::slice::from_ref(&root), since); + assert!( + state.ignore_rules_dirty.load(Ordering::SeqCst), + "a source that is no longer the file that was read must schedule a refresh" + ); + } - let rows = search_file_matches("f.txt", &file, &matcher, &SearchOpts::default()).unwrap(); + /// A `.gitignore` symlinked to `shared-rules` contributes the target's + /// contents, because the walker follows links. Editing the target is + /// therefore a rules change — but it touches nothing named like one. + #[cfg(unix)] + #[test] + fn an_edit_to_a_symlinked_rule_files_target_schedules_a_refresh() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + state.gitignore_pending.store(false, Ordering::SeqCst); + + let target = root.join("shared-rules"); + std::fs::write(&target, "keep.rs\n").unwrap(); + let link = root.join(".gitignore"); + std::os::unix::fs::symlink(&target, &link).unwrap(); + + let stamps = ignore_stamps_of(&root, std::slice::from_ref(&link)); + assert!( + stamps.contains_key("shared-rules"), + "the file the rules were actually read from has to be recorded too" + ); + *state.ignore_source_stamps.write().unwrap() = stamps; + + // Stop at the flag: what is being pinned is that the event is + // recognised, not what the refresh then does. + state.indexing.store(true, Ordering::SeqCst); + handle_fs_event( + &state, + &root, + &Event { + kind: EventKind::Modify(notify::event::ModifyKind::Data( + notify::event::DataChange::Any, + )), + paths: vec![target], + attrs: Default::default(), + }, + ); - let marker = rows - .iter() - .find(|r| r["type"] == "binary") - .expect("a file containing a NUL is reported as binary"); - assert_eq!( - marker["offset"].as_u64(), - Some(nul_on_disk as u64), - "offset must be the byte on disk (9), not the decoded position \ - (13): {marker}" + assert!( + state.ignore_rules_dirty.load(Ordering::SeqCst), + "an edit to the file a rules symlink resolves to is a rules change, \ + whatever the path is called" ); } - /// `reset_to_empty_index` runs after a failed build, when the index - /// directory is least likely to be intact, so it must not assume the - /// directory survived. + /// An mtime is not a wall-clock instant. Whole-second (HFS+, ext3) or + /// two-second (FAT) granularity can date a write before the moment it + /// actually followed, and a source edited between the walk and the + /// publication carries a stamp that matches it — so the window is the only + /// test left, and it has to allow for the rounding. #[test] - fn create_empty_index_makes_its_own_directory() { + fn a_rule_file_stamped_a_second_early_is_still_inside_the_window() { let tmp = TempDir::new().unwrap(); - let index_dir = tmp.path().join("missing").join("idx"); - assert!(!index_dir.exists()); + let root = tmp.path().to_path_buf(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + state.gitignore_pending.store(false, Ordering::SeqCst); - create_empty_index(&index_dir).expect("should create the directory it writes into"); + let rules = root.join(".gitignore"); + std::fs::write(&rules, "nothing-at-all.rs\n").unwrap(); + *state.ignore_source_stamps.write().unwrap() = + ignore_stamps_of(&root, std::slice::from_ref(&rules)); + *state.ignore_sources.write().unwrap() = vec![rules.clone()]; + let stamped = std::fs::metadata(&rules).unwrap().modified().unwrap(); + state.ignore_refresh_scheduled.store(true, Ordering::SeqCst); + + // Far outside any rounding: this one really is history. + let old = stamped + std::time::Duration::from_secs(3600); + reindex_files_in(&state, &root, std::slice::from_ref(&root), old); + assert!( + !state.ignore_rules_dirty.load(Ordering::SeqCst), + "a source last written an hour before the walk is not a change" + ); - assert!(index_dir.join("lookup.bin").is_file()); - assert!(index_dir.join("index.bin").is_file()); - assert!(index_dir.join("files.bin").is_file()); - // A caller that already created the directory must still succeed. - create_empty_index(&index_dir).expect("should be idempotent"); + // Within the granularity of a coarse filesystem's clock: the file may + // well have been written after the walk began and been rounded down. + let rounded = stamped + std::time::Duration::from_secs(1); + reindex_files_in(&state, &root, std::slice::from_ref(&root), rounded); + assert!( + state.ignore_rules_dirty.load(Ordering::SeqCst), + "an mtime that sits just under the window has to be treated as recent" + ); } - /// A truncated index left by a failed build must be replaced, not reused. + /// A stamp is not what makes a file searchable — the index is. + /// `filestamps.json` is optional, and a partial or absent map used to mean + /// the sweep had no candidates and deleted files kept answering searches. #[test] - fn create_empty_index_replaces_partially_written_files() { + fn the_sweep_drops_a_deleted_file_that_never_had_a_stamp() { let tmp = TempDir::new().unwrap(); - let index_dir = tmp.path().join("idx"); - std::fs::create_dir_all(&index_dir).unwrap(); - std::fs::write(index_dir.join("lookup.bin"), b"truncated garbage").unwrap(); + let root = tmp.path().to_path_buf(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + state.gitignore_pending.store(false, Ordering::SeqCst); - create_empty_index(&index_dir).unwrap(); + let path = root.join("seeded.rs"); + std::fs::write(&path, "fn seeded() {}\n").unwrap(); + let gate = state.snapshot_gate.read().unwrap(); + reindex_file(&state, &path, "seeded.rs"); + assert!(state.index.read().unwrap().live.has_path("seeded.rs")); + + // Indexed, and searchable, but with nothing in the stamp map to say so + // — as after a seed whose stamps could not be read. + state.file_stamps.write().unwrap().remove("seeded.rs"); + std::fs::remove_file(&path).unwrap(); + + let swept: std::collections::HashSet = [String::new()].into_iter().collect(); + let no_vanished = std::collections::HashSet::new(); + sweep_removed_files( + &state, + &swept, + &std::collections::HashSet::new(), + &no_vanished, + ); + drop(gate); - assert_eq!( - std::fs::read(index_dir.join("lookup.bin")).unwrap().len(), - 0 + assert!( + state.index.read().unwrap().live.is_deleted("seeded.rs"), + "a file that is gone from disk must stop answering searches whether or \ + not it had a stamp" ); - HybridIndex::open(&index_dir, tmp.path()).expect("reset index should reopen cleanly"); } + /// Metadata is not content either. `rsync -a` and `tar -x` preserve mtime, + /// and two different sets of rules are easily the same length — so a + /// size-and-mtime pair is identical across the replacement and only the + /// bytes tell them apart. + #[cfg(unix)] #[test] - fn skip_watcher_path_skips_dot_components() { - let no_exclude: Vec = Vec::new(); - // A leading dot dir is the canonical case (.git, .hg, .svn, ...). - assert!(should_skip_watcher_path( - ".git/index.lock", - &no_exclude, - None - )); - assert!(should_skip_watcher_path(".git/HEAD", &no_exclude, None)); - assert!(should_skip_watcher_path( - ".hg/store/data", - &no_exclude, - None - )); - // A dot component anywhere in the path skips, not just the leading one. - assert!(should_skip_watcher_path( - "src/.cache/build.tmp", - &no_exclude, - None - )); - assert!(should_skip_watcher_path( - "a/b/.hidden/c.txt", - &no_exclude, - None - )); - } + fn a_rule_file_swapped_for_one_of_the_same_size_and_age_is_still_caught() { + use std::os::unix::ffi::OsStrExt; - #[test] - fn skip_watcher_path_keeps_non_hidden_paths() { - let no_exclude: Vec = Vec::new(); - assert!(!should_skip_watcher_path("src/main.rs", &no_exclude, None)); - assert!(!should_skip_watcher_path("README.md", &no_exclude, None)); - // A dot mid-segment (e.g. "foo.bar") is NOT a hidden component — - // only segments that *start* with `.` are hidden. - assert!(!should_skip_watcher_path("src/foo.bar", &no_exclude, None)); - assert!(!should_skip_watcher_path("a/b/c", &no_exclude, None)); - } + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + state.gitignore_pending.store(false, Ordering::SeqCst); - #[test] - fn skip_watcher_path_honors_exclude_dirs() { - let exclude = vec!["target".to_string(), "node_modules".to_string()]; - // Excluded name as an ancestor directory => skip (matches what the - // walker would do — it skips the whole subtree). - assert!(should_skip_watcher_path("target/debug/foo", &exclude, None)); - assert!(should_skip_watcher_path( - "node_modules/react/index.js", - &exclude, - None - )); - assert!(should_skip_watcher_path("a/target/b", &exclude, None)); - // Substring match should NOT trigger — "targets" != "target". - assert!(!should_skip_watcher_path("targets/foo", &exclude, None)); - // Unrelated paths are not skipped. - assert!(!should_skip_watcher_path("src/main.rs", &exclude, None)); + let rules = root.join(".gitignore"); + let name = std::ffi::CString::new(rules.as_os_str().as_bytes()).unwrap(); + // Whole seconds, so it survives a round trip through `utimes`, which + // takes microseconds while ext4 stores nanoseconds. + let stamp = libc::timeval { + tv_sec: 1_000_000, + tv_usec: 0, + }; + let times = [stamp, stamp]; + // SAFETY: `name` is NUL-terminated and `times` is a two-element array, + // both outliving each call. + let backdate = || assert_eq!(unsafe { libc::utimes(name.as_ptr(), times.as_ptr()) }, 0); + + std::fs::write(&rules, "aaa.rs\n").unwrap(); + backdate(); + let before = std::fs::metadata(&rules).unwrap(); + *state.ignore_source_stamps.write().unwrap() = + ignore_stamps_of(&root, std::slice::from_ref(&rules)); + *state.ignore_sources.write().unwrap() = vec![rules.clone()]; + + // Different rules, same seven bytes, and the restore puts the old + // timestamp back. + std::fs::write(&rules, "bbb.rs\n").unwrap(); + backdate(); + let after = std::fs::metadata(&rules).unwrap(); + assert_eq!(before.len(), after.len()); + assert_eq!(before.modified().unwrap(), after.modified().unwrap()); + + state.ignore_refresh_scheduled.store(true, Ordering::SeqCst); + let since = SystemTime::now() + std::time::Duration::from_secs(3600); + reindex_files_in(&state, &root, std::slice::from_ref(&root), since); + assert!( + state.ignore_rules_dirty.load(Ordering::SeqCst), + "rules that were swapped for different ones of the same size and age \ + must still be noticed" + ); } + /// A rule file that has become a directory, a FIFO or a socket is as gone + /// as a deleted one: the walker would no longer collect it and a rebuild + /// would no longer read it, so the rules it contributed are being enforced + /// by nothing. `exists` says otherwise, and nothing downstream corrects it + /// — the digest check skips candidates that are not files. #[test] - fn skip_watcher_path_does_not_match_basename_against_exclude_dirs() { - // A regular file whose basename happens to equal an excluded - // directory name (e.g. a file literally called `vendor` at the - // repo root, or `src/target`) is still indexed by the walker — - // walker only treats `exclude_dirs` as directory subtree filters. - // The watcher must match that, otherwise the in-memory index and - // the on-disk index would disagree. - let exclude = vec!["target".to_string(), "vendor".to_string()]; - assert!(!should_skip_watcher_path("vendor", &exclude, None)); - assert!(!should_skip_watcher_path("src/target", &exclude, None)); - assert!(!should_skip_watcher_path("a/b/vendor", &exclude, None)); - } + fn a_rule_file_replaced_by_a_directory_counts_as_gone() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + state.gitignore_pending.store(false, Ordering::SeqCst); - #[test] - fn skip_watcher_path_handles_dot_segments_and_empty() { - let no_exclude: Vec = Vec::new(); - // `.` and `..` are not "hidden" components — they're path-relative - // markers and should not trigger a skip on their own. - assert!(!should_skip_watcher_path("./foo.txt", &no_exclude, None)); - assert!(!should_skip_watcher_path("a/./b", &no_exclude, None)); - assert!(!should_skip_watcher_path("a/../b", &no_exclude, None)); - // An empty rel_path (root-level event) shouldn't panic or skip. - assert!(!should_skip_watcher_path("", &no_exclude, None)); + let rules = root.join(".gitignore"); + std::fs::write(&rules, "aaa.rs\n").unwrap(); + *state.ignore_source_stamps.write().unwrap() = + ignore_stamps_of(&root, std::slice::from_ref(&rules)); + *state.ignore_sources.write().unwrap() = vec![rules.clone()]; + + std::fs::remove_file(&rules).unwrap(); + std::fs::create_dir(&rules).unwrap(); + + // Keeps the refresh this schedules from clearing the flag underneath + // the assertion. + state.ignore_refresh_scheduled.store(true, Ordering::SeqCst); + let since = SystemTime::now() + Duration::from_secs(3600); + reindex_files_in(&state, &root, std::slice::from_ref(&root), since); + assert!( + state.ignore_rules_dirty.load(Ordering::SeqCst), + "a source that is no longer a file must be treated as gone" + ); } + /// Containment is a property of the whole path, not of its last component. + /// + /// `root/link/inner` is a perfectly real directory while `link` is a + /// symlink to somewhere else entirely. The walker never descends through + /// the link, so nothing under it is part of the served tree — subscribing + /// to it spends a watch descriptor per directory of a tree that is not + /// ours, which on a large linked-in tree is the inotify exhaustion this + /// registration exists to avoid. + #[cfg(unix)] #[test] - fn skip_watcher_path_honors_gitignore_matcher() { - // Build the matcher via the public tgrep-core helper so this test - // also exercises the shared loading logic. + fn a_directory_below_a_symlinked_one_is_not_subscribed_to() { let tmp = TempDir::new().unwrap(); - // `.gitignore` is git-gated, matching the indexing walk, so the - // matcher only picks it up inside a repo. - std::fs::create_dir(tmp.path().join(".git")).unwrap(); - let gi_path = tmp.path().join(".gitignore"); - std::fs::write(&gi_path, "*.log\ntarget/\n").unwrap(); - let gi = tgrep_core::gitignore::build_matcher(tmp.path()) - .expect("matcher should build from a non-empty .gitignore"); + let root = tmp.path().join("root"); + let outside = tmp.path().join("outside"); + std::fs::create_dir(&root).unwrap(); + std::fs::create_dir_all(outside.join("inner")).unwrap(); + std::os::unix::fs::symlink(&outside, root.join("link")).unwrap(); - let no_exclude: Vec = Vec::new(); - // Files matched by the gitignore are skipped. - assert!(should_skip_watcher_path( - "build/output.log", - &no_exclude, - Some(&gi) - )); - assert!(should_skip_watcher_path( - "target/release/foo", - &no_exclude, - Some(&gi) - )); - // Files NOT matched by the gitignore are not skipped. - assert!(!should_skip_watcher_path( - "src/main.rs", - &no_exclude, - Some(&gi) - )); - assert!(!should_skip_watcher_path( - "README.md", - &no_exclude, - Some(&gi) - )); - } + let escaped = root.join("link").join("inner"); + assert!( + is_real_dir(&escaped), + "the fixture must be a real directory, or it proves nothing" + ); + assert!(!is_contained_dir(&root, &escaped)); - #[test] - fn identifies_live_ignore_rule_changes() { - let root = Path::new("workspace"); - assert!(is_ignore_rules_file( - root, - Path::new("workspace/nested/.gitignore") - )); - assert!(is_ignore_rules_file( - root, - Path::new("workspace/p4ignore.ini") - )); - assert!(!is_ignore_rules_file( - root, - Path::new("workspace/nested/p4ignore.ini") - )); - assert!(!is_ignore_rules_file( - root, - Path::new("workspace/.git/info/exclude") - )); - assert!(!is_ignore_rules_file( - root, - Path::new("workspace/src/main.rs") - )); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + state.gitignore_pending.store(false, Ordering::SeqCst); + let watcher = notify::recommended_watcher(|_: notify::Result| {}).unwrap(); + *state.watch_registry.lock().unwrap() = Some(WatchRegistry { + watcher, + root: root.clone(), + watched: std::iter::once(root.clone()).collect(), + }); + + let _gate = state.snapshot_gate.read().unwrap(); + watch_new_subtree(&state, &root, &escaped); + + let registry = state.watch_registry.lock().unwrap(); + assert!( + !registry.as_ref().unwrap().watched.contains(&escaped), + "a directory reached through a symlink must not be subscribed to" + ); } + /// A stamp is a claim about the index, not about the filesystem. + /// + /// The build stamps from a second traversal that runs after the one that + /// fed the index, so a file created between them is on disk and in no + /// index. Stamping it makes every later check agree it is up to date: + /// `reindex_file` returns early on a matching stamp, and the periodic + /// reconcile compares stamps alone. The file would never be searchable. #[test] - fn ignore_reconcile_compares_against_actual_indexed_paths() { - use std::collections::{HashMap, HashSet}; - use tgrep_core::meta::FileStamp; + fn stamps_are_published_only_for_what_the_build_indexed() { use tgrep_core::walker::FileMeta; - let current = vec![ + let files = vec![ FileMeta { - relative_path: "kept.txt".to_string(), + relative_path: "indexed.rs".to_string(), mtime: 1, size: 10, }, FileMeta { - relative_path: "newly-unignored.txt".to_string(), + relative_path: "arrived_between_the_walks.rs".to_string(), mtime: 2, size: 20, }, ]; - let stamps = HashMap::from([ - ("kept.txt".to_string(), FileStamp { mtime: 1, size: 10 }), - ( - "newly-unignored.txt".to_string(), - FileStamp { mtime: 2, size: 20 }, - ), - ]); - let indexed = HashSet::from(["kept.txt".to_string(), "newly-ignored.txt".to_string()]); + let indexed = std::iter::once("indexed.rs".to_string()).collect(); - let (changed, added, deleted) = classify_file_changes(¤t, &stamps, &indexed, true); - assert!(changed.is_empty()); - assert_eq!(added, vec!["newly-unignored.txt"]); - assert_eq!(deleted, vec!["newly-ignored.txt"]); + let stamps = stamps_for_index_members(files, &indexed); + assert!(stamps.contains_key("indexed.rs")); + assert!( + !stamps.contains_key("arrived_between_the_walks.rs"), + "a file the build never indexed must not be stamped as if it had been" + ); + } + + /// The matcher reads its sources itself, inside the build. A replace that + /// lands while it is reading leaves it enforcing the old rules, and stamps + /// taken afterwards describe the new file — so pathname, timestamp and + /// digest all agree there is nothing to reread, and the stale rules stay in + /// force until something unrelated rebuilds them. + #[test] + fn a_rule_file_rewritten_during_the_build_marks_the_matcher_stale() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + let rules = root.join(".gitignore"); + std::fs::write(&rules, "aaa.rs\n").unwrap(); + state.ignore_refresh_scheduled.store(true, Ordering::SeqCst); + + // A build nothing raced publishes without complaint. Asserted first, or + // the test below passes for a matcher that is always stale. + let quiet = publish_ignore_matcher(&state, &root, vec![rules.clone()], || None); + assert!(quiet.is_empty()); + assert!(!state.ignore_rules_dirty.load(Ordering::SeqCst)); + + let newly = publish_ignore_matcher(&state, &root, vec![rules.clone()], || { + // The checkout that lands while the builder is reading. + std::fs::write(&rules, "bbb.rs\n").unwrap(); + None + }); + assert!(newly.is_empty()); + assert!( + state.ignore_rules_dirty.load(Ordering::SeqCst), + "a matcher built over a moving source must be marked stale" + ); + } + + /// A directory that vanished whole leaves indexed files whose own parent + /// was never enumerated. `swept`/`present` only speak for a file's + /// immediate parent, so without the vanished-directory evidence every one + /// of those files stays searchable — and no event names them either: a + /// move away delivers nothing at all for what was inside. + #[test] + fn a_directory_that_went_away_whole_takes_its_files_with_it() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + + let dir = root.join("d"); + let nested = dir.join("deep"); + std::fs::create_dir_all(&nested).unwrap(); + std::fs::write(dir.join("a.rs"), "fn a() {}\n").unwrap(); + std::fs::write(nested.join("b.rs"), "fn b() {}\n").unwrap(); + reindex_file(&state, &dir.join("a.rs"), "d/a.rs"); + reindex_file(&state, &nested.join("b.rs"), "d/deep/b.rs"); + assert!(state.index.read().unwrap().live.has_path("d/a.rs")); + + // Still there, merely unlistable from the scan's point of view: the + // directory it was named in enumerated it, so silence proves nothing. + { + let _gate = state.snapshot_gate.read().unwrap(); + let since = SystemTime::now() + Duration::from_secs(3600); + reindex_files_in(&state, &root, std::slice::from_ref(&root), since); + } + assert!( + !state.index.read().unwrap().live.is_deleted("d/a.rs"), + "a directory that is still on disk must not sweep its files" + ); + + // Gone, and the scan is told to recheck it — which is what a recovery + // scan over a newly watched directory does. Its parent lists cleanly + // and does not contain it. + std::fs::remove_dir_all(&dir).unwrap(); + { + let _gate = state.snapshot_gate.read().unwrap(); + let since = SystemTime::now() + Duration::from_secs(3600); + reindex_files_in(&state, &root, &[root.clone(), dir.clone()], since); + } + let index = state.index.read().unwrap(); + assert!( + index.live.is_deleted("d/a.rs"), + "a file directly inside a vanished directory must be swept" + ); + assert!( + index.live.is_deleted("d/deep/b.rs"), + "and so must one further down, whose parent vanished with it" + ); } + /// `desired` is a set, so the order it iterates in is not the order the + /// tree is in. Subscribing a child before its parent makes the containment + /// check walk every level from the root for each one, which is the cost + /// the parent-first fast path exists to avoid on a tree with 40k + /// directories in it. #[test] - fn stale_classification_uses_reader_paths_for_deletions_and_case_renames() { - use std::collections::{HashMap, HashSet}; - use tgrep_core::walker::FileMeta; + fn subscriptions_are_established_from_the_root_down() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let deep = root.join("a").join("b").join("c"); + std::fs::create_dir_all(&deep).unwrap(); + + let watcher = notify::recommended_watcher(|_: notify::Result| {}).unwrap(); + let mut registry = WatchRegistry { + watcher, + root: root.clone(), + watched: std::collections::HashSet::new(), + }; - let current = vec![FileMeta { - relative_path: "case.txt".to_string(), - mtime: 1, - size: 10, - }]; - let indexed = HashSet::from(["Case.txt".to_string(), "reader-only.txt".to_string()]); + let desired: std::collections::HashSet = [ + deep.clone(), + root.join("a").join("b"), + root.join("a"), + root.clone(), + ] + .into_iter() + .collect(); + let (added, _removed) = registry.sync(&desired, false); - let (changed, added, mut deleted) = - classify_file_changes(¤t, &HashMap::new(), &indexed, false); - assert!(changed.is_empty()); - assert_eq!(added, vec!["case.txt"]); - deleted.sort(); - assert_eq!(deleted, vec!["Case.txt", "reader-only.txt"]); + assert_eq!(added.len(), 4); + let depths: Vec = added.iter().map(|d| d.components().count()).collect(); + assert!( + depths.windows(2).all(|w| w[0] <= w[1]), + "a parent must be subscribed before anything under it, got {added:?}" + ); + assert!(registry.watched.contains(&deep)); } - /// The reconcile waits for the server to go quiet, but not forever. + /// After an overflow the registry's records are not evidence. A directory + /// removal that was dropped leaves the kernel's descriptor released and the + /// entry here intact, and an ordinary sync skips anything it already + /// believes it watches — so the entry never gets corrected and a path + /// recreated there reports nothing for the life of the server. #[test] - fn a_scheduled_reconcile_defers_to_a_busy_server_but_not_indefinitely() { - let long_quiet = RECONCILE_QUIET_PERIOD + Duration::from_secs(1); - let just_queried = Duration::from_secs(1); + fn a_forced_sync_retires_a_subscription_the_kernel_already_dropped() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let gone = root.join("gone"); + std::fs::create_dir(&gone).unwrap(); + + let watcher = notify::recommended_watcher(|_: notify::Result| {}).unwrap(); + let mut registry = WatchRegistry { + watcher, + root: root.clone(), + watched: std::collections::HashSet::new(), + }; + assert_eq!(registry.add_all(std::slice::from_ref(&gone)).len(), 1); - // Before the interval, nothing runs however idle the server is. - assert!(!reconcile_due( - RECONCILE_INTERVAL - Duration::from_secs(1), - long_quiet, - false - )); - // After it, an idle server reconciles. - assert!(reconcile_due(RECONCILE_INTERVAL, long_quiet, false)); - // A server mid-query waits for a gap... - assert!(!reconcile_due(RECONCILE_INTERVAL, just_queried, false)); - // ...but a server that is *always* mid-query would otherwise never - // reconcile at all, which is the failure this exists to prevent. - assert!(reconcile_due(RECONCILE_DEADLINE, just_queried, false)); - // Indexing and flushing outrank even the deadline: they are rewriting - // the index already, and the next tick is a minute away. - assert!(!reconcile_due(RECONCILE_DEADLINE, long_quiet, true)); + // The removal event that would have called `forget` was one of the + // ones the overflow ate. + std::fs::remove_dir(&gone).unwrap(); + let desired: std::collections::HashSet = std::iter::once(gone.clone()).collect(); + + registry.sync(&desired, false); + assert!( + registry.is_watched(&gone), + "the fixture must reproduce the poisoned entry, or it proves nothing" + ); + + registry.sync(&desired, true); + assert!( + !registry.is_watched(&gone), + "a forced sync must re-issue the subscription and drop the entry \ + when it fails, rather than trusting a descriptor that is gone" + ); } - /// A file that cannot be read must not make every reconcile rebuild. - /// - /// Its stamp is deliberately withheld so it looks new and gets retried. - /// Left at that, a file locked by another process would be "new" on every - /// pass, and a reconcile on a timer would rewrite the whole index once an - /// hour, forever, to re-attempt a read that fails the same way each time. + /// A directory can arrive already full — a `mv` from outside the root, a + /// checkout, an unpacked archive — and nothing reports what came with it. + /// Linux needs the descent to subscribe; Windows and macOS get the whole + /// move as a single event for the directory and no per-file events at all. + /// The indexing half therefore has to run everywhere, not only where the + /// subscribing half does. #[test] - fn a_file_that_stays_unreadable_is_not_retried_until_it_changes() { - use tgrep_core::meta::FileStamp; - use tgrep_core::walker::FileMeta; + fn a_populated_directory_that_arrives_whole_is_indexed_on_every_platform() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + state.gitignore_pending.store(false, Ordering::SeqCst); - let memo = std::collections::HashMap::from([( - "locked.bin".to_string(), - FileStamp { - mtime: 100, - size: 5, - }, - )]); + let watcher = notify::recommended_watcher(|_: notify::Result| {}).unwrap(); + *state.watch_registry.lock().unwrap() = Some(WatchRegistry { + watcher, + root: root.clone(), + watched: std::iter::once(root.clone()).collect(), + }); - let retried = |mtime: u64, size: u64| { - let current = vec![FileMeta { - relative_path: "locked.bin".to_string(), - mtime, - size, - }]; - let (mut changed, mut added, _) = classify_file_changes( - ¤t, - &std::collections::HashMap::new(), - &std::collections::HashSet::new(), - false, - ); - let skipped = drop_memoized_failures(&memo, ¤t, &mut changed, &mut added); - (changed.len() + added.len(), skipped) - }; + // Built somewhere else and moved in, so no event ever described its + // contents. + let staging = tmp.path().join("staging"); + std::fs::create_dir_all(staging.join("deep")).unwrap(); + std::fs::write(staging.join("top.rs"), "fn top() {}\n").unwrap(); + std::fs::write(staging.join("deep").join("low.rs"), "fn low() {}\n").unwrap(); + let moved = root.join("moved"); + std::fs::rename(&staging, &moved).unwrap(); + + handle_fs_event( + &state, + &root, + &Event { + kind: EventKind::Create(notify::event::CreateKind::Any), + paths: vec![moved.clone()], + attrs: Default::default(), + }, + ); - // Unchanged since the failed read: leave it alone. - assert_eq!(retried(100, 5).0, 0); - // Touched: worth another look. - assert_eq!(retried(200, 5).0, 1); - // Resized: likewise. - assert_eq!(retried(100, 6).0, 1); + let index = state.index.read().unwrap(); + assert!( + index.live.has_path("moved/top.rs"), + "a file that moved in with its directory must be indexed" + ); + assert!( + index.live.has_path("moved/deep/low.rs"), + "and so must one further down" + ); } - /// Skipping a file must not also mark it indexed. - /// - /// The two halves have to agree. `drop_memoized_failures` keeps a file out - /// of the delta, so nothing reads it and nothing writes postings for it; if - /// the stamps published alongside that delta still claimed it was indexed at - /// its current mtime and size, the next reconcile would classify it as - /// unchanged and skip it for a completely different reason — one that never - /// clears, because the memo is not involved and the stamp outlives the - /// process in `filestamps.json`. A file that failed to read once would be - /// invisible to search forever, with no error and no way back short of - /// deleting the index. + /// Parent-directory rule files and the repository's `info/exclude` are + /// enforced by the published matcher but sit outside the walk that finds + /// everything else, so nothing would notice one being deleted — and rules + /// with no source left keep hiding a subtree from the index. #[test] - fn a_skipped_file_is_left_unstamped_so_the_next_pass_still_sees_it() { - use tgrep_core::meta::FileStamp; - use tgrep_core::walker::FileMeta; + fn ignore_sources_include_the_rules_that_live_outside_the_tree() { + let tmp = TempDir::new().unwrap(); + let repo = tmp.path().join("repo"); + let root = repo.join("sub"); + std::fs::create_dir_all(root.join("nested")).unwrap(); + std::fs::create_dir_all(repo.join(".git").join("info")).unwrap(); - let current = vec![ - FileMeta { - relative_path: "locked.bin".to_string(), - mtime: 100, - size: 5, - }, - FileMeta { - relative_path: "src/main.rs".to_string(), - mtime: 100, - size: 9, - }, - ]; - let memo = std::collections::HashMap::from([( - "locked.bin".to_string(), - FileStamp { - mtime: 100, - size: 5, - }, - )]); + std::fs::write(repo.join(".git").join("info").join("exclude"), "*.tmp\n").unwrap(); + std::fs::write(repo.join(".gitignore"), "build/\n").unwrap(); + std::fs::write(root.join(".gitignore"), "target/\n").unwrap(); - let (mut changed, mut added, _) = classify_file_changes( - ¤t, - &std::collections::HashMap::new(), - &std::collections::HashSet::new(), - false, + let sources = ignore_sources_of(&root, &[root.join(".gitignore")], &[], false); + assert!( + sources.contains(&repo.join(".gitignore")), + "a parent .gitignore the matcher enforces must be tracked: {sources:?}" ); - let skipped = drop_memoized_failures(&memo, ¤t, &mut changed, &mut added); - assert!(skipped.contains("locked.bin")); - - let stamps = stamps_for_indexed(¤t, &skipped); assert!( - !stamps.contains_key("locked.bin"), - "published a stamp for a file that was never read: {stamps:?}" + sources.contains(&repo.join(".git").join("info").join("exclude")), + "so must the repository's own exclude file: {sources:?}" ); - assert!(stamps.contains_key("src/main.rs"), "{stamps:?}"); + // Every listed source must exist, or the vanished-source check treats a + // path that was never there as one that just disappeared and schedules + // a refresh on every single scan. + for source in &sources { + assert!( + source.is_file(), + "listed a source that is not there: {source:?}" + ); + } - // And with that stamp withheld, a later pass over an unchanged tree - // still offers the file up rather than treating it as up-to-date. - let (changed, added, _) = - classify_file_changes(¤t, &stamps, &std::collections::HashSet::new(), false); - assert_eq!(changed.len() + added.len(), 1); - assert!(added.contains(&"locked.bin".to_string())); + // And they are digested, rather than silently dropped for having no + // path relative to the served root. + let stamps = ignore_stamps_of(&root, &sources); + assert_eq!( + stamps.len(), + sources.len(), + "every source must be stamped: {sources:?} -> {stamps:?}" + ); } - /// A walk error must not leave the watcher gated forever. - /// - /// `gitignore_pending` is what keeps the watcher off the index until a - /// matcher exists, and the stale check owns clearing it. It also refuses to - /// touch the index when the walk could not inspect every entry, because - /// unseen files would be misclassified as deleted. Those two are separate - /// decisions: taking the second one used to skip the first, so one - /// unreadable directory — or one file whose `metadata()` lost a race with a - /// delete — silently disabled the watcher for the life of the process, and - /// the overflow-repair path would not reconcile either, since it defers to - /// the pending matcher. - /// - /// Unix-only because it needs a directory the process genuinely cannot - /// read, which has no portable equivalent on Windows. - #[cfg(unix)] + /// `background_index_build` publishes its matcher while it is still + /// part-way through Phase 2 and holds `snapshot_gate` for none of that. A + /// refresh scheduled from that publish would take the gate uncontended and + /// replace `file_stamps` from its own walk, only for the build to overwrite + /// them from a walk that predates the new rules — an index and a stamp map + /// describing two different trees, with no scan left to notice. #[test] - fn a_walk_error_still_publishes_the_watcher_matcher() { - use std::os::unix::fs::PermissionsExt; - + fn an_ignore_refresh_waits_for_a_running_build() { let tmp = TempDir::new().unwrap(); let root = tmp.path().to_path_buf(); - // `.gitignore` is git-gated, matching the indexing walk. - std::fs::create_dir(root.join(".git")).unwrap(); - std::fs::write(root.join(".gitignore"), "*.log\n").unwrap(); - std::fs::write(root.join("src.rs"), "fn main() {}\n").unwrap(); + let index_dir = root.join(".tgrep"); + std::fs::write(root.join("a.rs"), "fn a() {}\n").unwrap(); + let state = test_server_state(&root, &index_dir); - let unreadable = root.join("locked"); - std::fs::create_dir(&unreadable).unwrap(); - std::fs::write(unreadable.join("inner.rs"), "fn inner() {}\n").unwrap(); - std::fs::set_permissions(&unreadable, std::fs::Permissions::from_mode(0o000)).unwrap(); - if std::fs::read_dir(&unreadable).is_ok() { - // Running as root, so permissions prove nothing. Restore and skip. - std::fs::set_permissions(&unreadable, std::fs::Permissions::from_mode(0o755)).unwrap(); - return; + state.indexing.store(true, Ordering::SeqCst); + state.gitignore_pending.store(true, Ordering::SeqCst); + state.ignore_rules_dirty.store(true, Ordering::SeqCst); + schedule_ignore_rules_refresh(Arc::clone(&state), root.clone()); + + // Publishing a matcher is the first thing the refresh does that anyone + // outside it can see, so `gitignore_pending` still being set is proof + // that it has not started. + thread::sleep(Duration::from_millis(400)); + assert!( + state.gitignore_pending.load(Ordering::SeqCst), + "a refresh must not run against an index that is still being built" + ); + + state.indexing.store(false, Ordering::SeqCst); + let deadline = Instant::now() + Duration::from_secs(30); + while state.gitignore_pending.load(Ordering::SeqCst) && Instant::now() < deadline { + thread::sleep(Duration::from_millis(20)); } + assert!( + !state.gitignore_pending.load(Ordering::SeqCst), + "and must run once the build is done, rather than waiting forever" + ); + } + + /// Stamps describe what the index holds. A file written while the build + /// ran is read by the metadata walk that produces them but not by the + /// content walk that fed the index, so its stamp says "current" about + /// bytes that are already stale — and `reindex_file` returns early on a + /// matching stamp, so the replay of the very event that reported the write + /// reads nothing. + #[test] + fn a_file_written_during_the_build_is_not_stamped_by_it() { + use tgrep_core::meta::FileStamp; + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); let index_dir = root.join(".tgrep"); - std::fs::create_dir(&index_dir).unwrap(); let state = test_server_state(&root, &index_dir); - state.gitignore_pending.store(true, Ordering::SeqCst); - let ok = background_refresh_stale(&state, &root, &index_dir, false); + let stamps: std::collections::HashMap = [ + ("quiet.rs".to_string(), FileStamp { mtime: 1, size: 10 }), + ("racy.rs".to_string(), FileStamp { mtime: 2, size: 20 }), + ] + .into_iter() + .collect(); - // Restore before any assertion so a failure still leaves a removable dir. - std::fs::set_permissions(&unreadable, std::fs::Permissions::from_mode(0o755)).unwrap(); + state + .deferred_events + .lock() + .unwrap() + .as_mut() + .unwrap() + .insert(root.join("racy.rs"), false); + let published = withhold_stamps_for_deferred(&state, &root, stamps.clone()); assert!( - !ok, - "a walk that could not inspect every entry must keep the old index" + published.contains_key("quiet.rs"), + "a file nothing touched keeps its stamp, or the build re-reads the repository" ); assert!( - !state.gitignore_pending.load(Ordering::SeqCst), - "the watcher gate must be released even when the index is left alone" + !published.contains_key("racy.rs"), + "a file whose event is waiting to be replayed must not be stamped as indexed" ); + + // Overflowed: the buffer names nothing, so nothing in the map can be + // told apart from what changed underneath it. + *state.deferred_events.lock().unwrap() = None; assert!( - state.gitignore.read().unwrap().is_some(), - "the matcher the walk did find must still be published" + withhold_stamps_for_deferred(&state, &root, stamps).is_empty(), + "with the buffer overflowed no stamp from this build can be trusted" + ); + } + + /// A file that grows in between must not be read into memory without + /// bound, nor indexed past the cap. + #[test] + fn a_read_stops_one_byte_past_the_cap() { + let tmp = TempDir::new().unwrap(); + let path = tmp.path().join("grew.txt"); + std::fs::write(&path, "x".repeat(4096)).unwrap(); + + // Stat said 16 bytes; the file is 4096 by the time it is read. + let mut file = std::fs::File::open(&path).unwrap(); + assert!(matches!( + read_within_limit(&mut file, Some(64), 16), + CappedRead::TooLarge + )); + + let mut file = std::fs::File::open(&path).unwrap(); + assert!( + matches!(read_within_limit(&mut file, None, 16), CappedRead::Data(d) if d.len() == 4096), + "no cap means no bound" + ); + + std::fs::write(&path, "small").unwrap(); + let mut file = std::fs::File::open(&path).unwrap(); + assert!( + matches!(read_within_limit(&mut file, Some(64), 5), CappedRead::Data(d) if d == b"small"), + "a file within the cap reads whole" + ); + + // Exactly at the cap is still within it. + std::fs::write(&path, "x".repeat(64)).unwrap(); + let mut file = std::fs::File::open(&path).unwrap(); + assert!( + matches!(read_within_limit(&mut file, Some(64), 64), CappedRead::Data(d) if d.len() == 64) + ); + } + + /// The size gate is checked against a fresh stat on every visit, so a file + /// that outgrew the cap since it was indexed loses what the index holds + /// rather than keeping the smaller version until the reconcile. (The + /// growth this pins is between visits — growth *during* a read is what + /// `read_within_limit` above covers.) + #[cfg(unix)] + #[test] + fn a_file_that_outgrew_the_cap_between_visits_loses_its_indexed_content() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let index_dir = root.join(".tgrep"); + let mut state = test_server_state(&root, &index_dir); + Arc::get_mut(&mut state).unwrap().max_file_size = Some(64); + + let path = root.join("grows.rs"); + std::fs::write(&path, "fn small_enough() {}\n").unwrap(); + reindex_file(&state, &path, "grows.rs"); + assert!( + state.index.read().unwrap().live.has_path("grows.rs"), + "a file under the cap should index normally" + ); + + std::fs::write(&path, "x".repeat(4096)).unwrap(); + reindex_file(&state, &path, "grows.rs"); + assert!( + state.index.read().unwrap().live.is_deleted("grows.rs"), + "content past the cap must not stay searchable" ); } diff --git a/tgrep-cli/tests/watcher_dot_ignore.rs b/tgrep-cli/tests/watcher_dot_ignore.rs index 652141c..914dc5d 100644 --- a/tgrep-cli/tests/watcher_dot_ignore.rs +++ b/tgrep-cli/tests/watcher_dot_ignore.rs @@ -101,22 +101,21 @@ fn wait_for_match(port: u16, pattern: &str, timeout: Duration) -> bool { } } -#[test] -fn watcher_honors_dot_ignore_and_still_indexes_new_files() { - let dir = TempDir::new().unwrap(); - let root = dir.path(); - let index_dir = root.join(".tgrep_test_index"); - - // No `.git` here: `.ignore` must apply on its own. - fs::write(root.join(".ignore"), "secret/\n").unwrap(); - fs::create_dir_all(root.join("secret")).unwrap(); - fs::create_dir_all(root.join("src")).unwrap(); - fs::write( - root.join("src").join("lib.rs"), - "fn seeded() { let normal_source_marker = 1; }\n", - ) - .unwrap(); +/// Poll until `pattern` stops being searchable, returning whether it went away. +fn wait_for_no_match(port: u16, pattern: &str, timeout: Duration) -> bool { + let start = Instant::now(); + loop { + if search_matches(port, pattern) == 0 { + return true; + } + if start.elapsed() > timeout { + return false; + } + thread::sleep(Duration::from_millis(100)); + } +} +fn build_index(root: &Path, index_dir: &Path) { let status = Command::new(tgrep_bin()) .args([ "index", @@ -127,7 +126,9 @@ fn watcher_honors_dot_ignore_and_still_indexes_new_files() { .status() .expect("failed to run tgrep index"); assert!(status.success(), "initial index build failed"); +} +fn spawn_server(root: &Path, index_dir: &Path) -> ServerGuard { let child = Command::new(tgrep_bin()) .args([ "serve", @@ -139,7 +140,27 @@ fn watcher_honors_dot_ignore_and_still_indexes_new_files() { .stdout(std::process::Stdio::null()) .spawn() .expect("failed to start tgrep serve"); - let _server = ServerGuard { child }; + ServerGuard { child } +} + +#[test] +fn watcher_honors_dot_ignore_and_still_indexes_new_files() { + let dir = TempDir::new().unwrap(); + let root = dir.path(); + let index_dir = root.join(".tgrep_test_index"); + + // No `.git` here: `.ignore` must apply on its own. + fs::write(root.join(".ignore"), "secret/\n").unwrap(); + fs::create_dir_all(root.join("secret")).unwrap(); + fs::create_dir_all(root.join("src")).unwrap(); + fs::write( + root.join("src").join("lib.rs"), + "fn seeded() { let normal_source_marker = 1; }\n", + ) + .unwrap(); + + build_index(root, &index_dir); + let _server = spawn_server(root, &index_dir); let port = wait_for_port(&index_dir); @@ -176,3 +197,71 @@ fn watcher_honors_dot_ignore_and_still_indexes_new_files() { "watcher indexed a file under a directory excluded by .ignore" ); } + +/// A `.ignore` written *after* the server is live must refresh the ignore +/// rules, exactly as a `.gitignore` write does. +/// +/// The startup matcher is built from the walk that the initial index used, so +/// a `.ignore` that already exists is honored for free — which is what the +/// test above covers. Rules that appear later only take effect if the watcher +/// recognizes the `.ignore` write as an ignore-rules change and schedules the +/// refresh; when it does not, the stale matcher stays published and the +/// already-indexed content under the newly excluded directory stays +/// searchable indefinitely (the periodic reconcile is on an hourly timer). +/// +/// The fixture seeds the ignored file *before* indexing, so the assertion is a +/// transition — searchable, then not — rather than a fixed sleep racing the +/// refresh. +#[test] +fn late_dot_ignore_refreshes_the_watchers_ignore_rules() { + let dir = TempDir::new().unwrap(); + let root = dir.path(); + let index_dir = root.join(".tgrep_test_index"); + + // No `.git` here either: `.ignore` is not git-gated, so this pins the + // refresh path for the one ignore source that works outside a repo. + fs::create_dir_all(root.join("secret")).unwrap(); + fs::create_dir_all(root.join("src")).unwrap(); + fs::write( + root.join("secret").join("creds.txt"), + "late_ignored_leak_marker\n", + ) + .unwrap(); + fs::write( + root.join("src").join("lib.rs"), + "fn seeded() { let normal_source_marker = 1; }\n", + ) + .unwrap(); + + build_index(root, &index_dir); + let _server = spawn_server(root, &index_dir); + + let port = wait_for_port(&index_dir); + + assert!( + wait_for_match(port, "normal_source_marker", Duration::from_secs(30)), + "expected the seeded source file to be searchable" + ); + // Positive control: with no `.ignore` yet, the seeded file under `secret/` + // is legitimately indexed. Without this the assertion below could pass + // simply because the file was never indexed in the first place. + assert!( + wait_for_match(port, "late_ignored_leak_marker", Duration::from_secs(30)), + "expected the file under secret/ to be indexed before any .ignore exists" + ); + thread::sleep(Duration::from_secs(2)); + + fs::write(root.join(".ignore"), "secret/\n").unwrap(); + + assert!( + wait_for_no_match(port, "late_ignored_leak_marker", Duration::from_secs(60)), + "a .ignore written while the server was live never refreshed the ignore \ + rules; content under the newly excluded directory is still searchable" + ); + + // The refresh must not take the rest of the index with it. + assert!( + search_matches(port, "normal_source_marker") > 0, + "the ignore-rules refresh dropped a file that is not ignored" + ); +} diff --git a/tgrep-cli/tests/watcher_watch_registration.rs b/tgrep-cli/tests/watcher_watch_registration.rs new file mode 100644 index 0000000..e8e765e --- /dev/null +++ b/tgrep-cli/tests/watcher_watch_registration.rs @@ -0,0 +1,691 @@ +//! The watcher must not take OS subscriptions for trees it is going to ignore. +//! +//! On Linux `notify` has no recursive inotify mode: `RecursiveMode::Recursive` +//! walks the tree and spends one watch descriptor per directory. Ignored build +//! output is usually most of the directories in a repository, so subscribing to +//! it burns the per-user `fs.inotify.max_user_watches` budget on events that +//! are then discarded — and because notify propagates the first registration +//! failure, a repo large enough to exhaust that budget loses its watcher +//! entirely. +//! +//! This is observable: the kernel reports a process's inotify watches in +//! `/proc//fdinfo/`, one `inotify wd:` line per watch. The test below +//! counts them for a live server and pins that the ignored subtree is absent, +//! while asserting the watcher still works — a watcher that registered nothing +//! at all would trivially satisfy the count. +//! +//! Windows (ReadDirectoryChangesW) and macOS (FSEvents) subscribe once for the +//! whole subtree, so there is no per-directory registration to withhold and +//! nothing here applies; the count assertion is Linux-only for that reason. + +use std::fs; +use std::io::{BufRead, BufReader, Write}; +use std::net::TcpStream; +use std::path::Path; +use std::process::{Child, Command}; +use std::thread; +use std::time::{Duration, Instant}; + +use tempfile::TempDir; + +/// Directories inside the ignored tree. Large enough that a recursive +/// subscription is unmistakable in the watch count, small enough to create +/// quickly. +#[cfg(target_os = "linux")] +const IGNORED_DIRS: usize = 60; +/// Directories inside the indexed tree. +#[cfg(target_os = "linux")] +const SOURCE_DIRS: usize = 4; + +fn tgrep_bin() -> std::path::PathBuf { + assert_cmd::cargo::cargo_bin("tgrep") +} + +struct ServerGuard { + child: Child, +} + +impl Drop for ServerGuard { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} + +fn send_request(port: u16, request: &str) -> std::io::Result { + let mut stream = TcpStream::connect(format!("127.0.0.1:{port}"))?; + stream.set_read_timeout(Some(Duration::from_secs(30)))?; + writeln!(stream, "{request}")?; + stream.flush()?; + let mut reader = BufReader::new(stream); + let mut response = String::new(); + reader.read_line(&mut response)?; + Ok(response) +} + +fn search_matches(port: u16, pattern: &str) -> u64 { + let request = serde_json::json!({ + "jsonrpc": "2.0", + "method": "search", + "id": 1, + "params": { "pattern": pattern } + }) + .to_string(); + let response = send_request(port, &request).expect("search request failed"); + let value: serde_json::Value = serde_json::from_str(&response).expect("invalid JSON response"); + value + .pointer("/result/num_matches") + .and_then(|v| v.as_u64()) + .unwrap_or_else(|| panic!("missing num_matches in response: {response}")) +} + +fn wait_for_match(port: u16, pattern: &str, timeout: Duration) -> bool { + let start = Instant::now(); + loop { + if search_matches(port, pattern) > 0 { + return true; + } + if start.elapsed() > timeout { + return false; + } + thread::sleep(Duration::from_millis(100)); + } +} + +/// Wait until `pattern` stops matching. +/// +/// For content that has to be *dropped* from the index. The barrier the other +/// assertions use — a second file appearing — only proves that some later event +/// was processed, which on a backend that coalesces or reorders events (macOS) +/// says nothing about the drop. +fn wait_for_no_match(port: u16, pattern: &str, timeout: Duration) -> bool { + let start = Instant::now(); + loop { + if search_matches(port, pattern) == 0 { + return true; + } + if start.elapsed() > timeout { + return false; + } + thread::sleep(Duration::from_millis(100)); + } +} + +/// Read `(pid, port)` once the server is accepting connections. +fn wait_for_server(index_dir: &Path) -> (u32, u16) { + let serve_json = index_dir.join("serve.json"); + let start = Instant::now(); + loop { + assert!( + start.elapsed() <= Duration::from_secs(60), + "tgrep serve did not start within 60 seconds" + ); + if let Ok(data) = fs::read_to_string(&serve_json) + && let Ok(info) = serde_json::from_str::(&data) + && let Some(p) = info.get("port").and_then(|v| v.as_u64()) + && let Some(pid) = info.get("pid").and_then(|v| v.as_u64()) + && TcpStream::connect(format!("127.0.0.1:{p}")).is_ok() + { + return (pid as u32, p as u16); + } + thread::sleep(Duration::from_millis(20)); + } +} + +/// Total inotify watch descriptors held by `pid`. +/// +/// Each inotify file descriptor's `fdinfo` lists one `inotify wd:` line per +/// watch, so summing them across the process's descriptors gives the number of +/// directories it is subscribed to. +#[cfg(target_os = "linux")] +fn inotify_watch_count(pid: u32) -> usize { + let dir = format!("/proc/{pid}/fdinfo"); + let Ok(entries) = fs::read_dir(&dir) else { + panic!("could not read {dir}; /proc must be mounted for this test"); + }; + let mut total = 0; + for entry in entries.flatten() { + // Descriptors come and go while we read; a vanished one is not a + // failure, it simply holds no watches we can count. + if let Ok(contents) = fs::read_to_string(entry.path()) { + total += contents + .lines() + .filter(|line| line.starts_with("inotify wd:")) + .count(); + } + } + total +} + +#[cfg(target_os = "linux")] +#[test] +fn watcher_does_not_subscribe_to_gitignored_directories() { + let dir = TempDir::new().unwrap(); + let root = dir.path(); + let index_dir = root.join(".tgrep_test_index"); + + // A real (if empty) `.git` entry: `.gitignore` rules only apply inside a + // repository, so without it the fixture would measure the wrong thing. + fs::create_dir_all(root.join(".git")).unwrap(); + fs::write(root.join(".gitignore"), "build/\n").unwrap(); + + for i in 0..IGNORED_DIRS { + let sub = root.join("build").join(format!("out{i}")); + fs::create_dir_all(&sub).unwrap(); + fs::write(sub.join("artifact.txt"), "ignored_build_output\n").unwrap(); + } + for i in 0..SOURCE_DIRS { + let sub = root.join("src").join(format!("pkg{i}")); + fs::create_dir_all(&sub).unwrap(); + fs::write( + sub.join("lib.rs"), + format!("fn seeded{i}() {{ let normal_source_marker = {i}; }}\n"), + ) + .unwrap(); + } + + let status = Command::new(tgrep_bin()) + .args([ + "index", + root.to_str().unwrap(), + "--index-path", + index_dir.to_str().unwrap(), + ]) + .status() + .expect("failed to run tgrep index"); + assert!(status.success(), "initial index build failed"); + + let child = Command::new(tgrep_bin()) + .args([ + "serve", + "--index-path", + index_dir.to_str().unwrap(), + root.to_str().unwrap(), + ]) + .stderr(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .spawn() + .expect("failed to start tgrep serve"); + let _server = ServerGuard { child }; + + let (pid, port) = wait_for_server(&index_dir); + + // Positive control, and a wait for the watcher to be live: subscriptions + // are deferred until the ignore matcher is published, so counting before + // that would pass for the wrong reason. + assert!( + wait_for_match(port, "normal_source_marker", Duration::from_secs(30)), + "expected the seeded source files to be searchable" + ); + fs::write( + root.join("src").join("added.rs"), + "fn added() { let watcher_added_marker = 1; }\n", + ) + .unwrap(); + assert!( + wait_for_match(port, "watcher_added_marker", Duration::from_secs(30)), + "watcher never indexed a newly created ordinary file, so the watch \ + count below would be meaningless" + ); + + let watches = inotify_watch_count(pid); + + // The tree the watcher legitimately needs is the root, `src`, `src/pkg*`, + // and `.git` is hidden so it is not watched either. Allow generous slack + // for anything else in the process holding an inotify fd, but stay far + // below the ~66 a recursive subscription over this fixture would take. + let allowed = SOURCE_DIRS + 8; + assert!( + watches <= allowed, + "watcher holds {watches} inotify watches for a tree whose only \ + non-ignored directories are the root plus {SOURCE_DIRS} under src/; \ + it is subscribing to the {IGNORED_DIRS} gitignored directories under \ + build/ (expected at most {allowed})" + ); +} + +/// New directories still have to be picked up. +/// +/// Non-recursive subscriptions are not extended by notify, so a directory +/// created after startup is invisible unless the watcher subscribes to it as +/// it appears. This runs everywhere: on a whole-subtree backend it simply +/// re-confirms the existing behaviour, which is the point — the two paths must +/// agree. +#[test] +fn watcher_indexes_files_in_directories_created_after_startup() { + let dir = TempDir::new().unwrap(); + let root = dir.path(); + let index_dir = root.join(".tgrep_test_index"); + + fs::create_dir_all(root.join(".git")).unwrap(); + fs::write(root.join(".gitignore"), "build/\n").unwrap(); + fs::create_dir_all(root.join("src")).unwrap(); + fs::write( + root.join("src").join("lib.rs"), + "fn seeded() { let normal_source_marker = 1; }\n", + ) + .unwrap(); + + let status = Command::new(tgrep_bin()) + .args([ + "index", + root.to_str().unwrap(), + "--index-path", + index_dir.to_str().unwrap(), + ]) + .status() + .expect("failed to run tgrep index"); + assert!(status.success(), "initial index build failed"); + + let child = Command::new(tgrep_bin()) + .args([ + "serve", + "--index-path", + index_dir.to_str().unwrap(), + root.to_str().unwrap(), + ]) + .stderr(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .spawn() + .expect("failed to start tgrep serve"); + let _server = ServerGuard { child }; + + let (_pid, port) = wait_for_server(&index_dir); + assert!( + wait_for_match(port, "normal_source_marker", Duration::from_secs(30)), + "expected the seeded source file to be searchable" + ); + thread::sleep(Duration::from_secs(2)); + + // A whole new subtree, several levels deep, written in one go. The files + // land immediately after their directories, which is exactly the race the + // subscription pass has to close. + let nested = root.join("src").join("fresh").join("deeper"); + fs::create_dir_all(&nested).unwrap(); + fs::write( + nested.join("new.rs"), + "fn fresh() { let nested_new_dir_marker = 2; }\n", + ) + .unwrap(); + + assert!( + wait_for_match(port, "nested_new_dir_marker", Duration::from_secs(30)), + "watcher never indexed a file created in a directory that did not \ + exist when the server started" + ); + + // ...and a directory created inside the ignored tree stays ignored. + let ignored = root.join("build").join("fresh"); + fs::create_dir_all(&ignored).unwrap(); + fs::write(ignored.join("out.txt"), "new_ignored_dir_marker\n").unwrap(); + thread::sleep(Duration::from_secs(3)); + assert_eq!( + search_matches(port, "new_ignored_dir_marker"), + 0, + "watcher indexed a file in a directory created under a gitignored path" + ); +} + +/// A subtree that arrives already populated can carry its own ignore rules. +/// +/// A clone, a `git mv`, a branch switch or an unpacked archive all land this +/// way: the directory and everything under it appear in one step, so there is +/// no moment at which the `.gitignore` inside it is seen on its own. The +/// recovery pass skips dot-prefixed files, so without explicitly looking for +/// ignore rules it would index the subtree against rules that never mentioned +/// it — and the wrongly indexed files would stay until something touched them +/// again or the hourly reconcile came round. +#[test] +fn watcher_honors_ignore_rules_inside_a_subtree_that_arrives_whole() { + let dir = TempDir::new().unwrap(); + let root = dir.path(); + let index_dir = root.join(".tgrep_test_index"); + + fs::create_dir_all(root.join(".git")).unwrap(); + fs::write(root.join(".gitignore"), "build/\n").unwrap(); + fs::create_dir_all(root.join("src")).unwrap(); + fs::write( + root.join("src").join("lib.rs"), + "fn seeded() { let normal_source_marker = 1; }\n", + ) + .unwrap(); + + let status = Command::new(tgrep_bin()) + .args([ + "index", + root.to_str().unwrap(), + "--index-path", + index_dir.to_str().unwrap(), + ]) + .status() + .expect("failed to run tgrep index"); + assert!(status.success(), "initial index build failed"); + + let child = Command::new(tgrep_bin()) + .args([ + "serve", + "--index-path", + index_dir.to_str().unwrap(), + root.to_str().unwrap(), + ]) + .stderr(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .spawn() + .expect("failed to start tgrep serve"); + let _server = ServerGuard { child }; + + let (_pid, port) = wait_for_server(&index_dir); + assert!( + wait_for_match(port, "normal_source_marker", Duration::from_secs(30)), + "expected the seeded source file to be searchable" + ); + thread::sleep(Duration::from_secs(2)); + + // Staged under a dot-prefixed name so the watcher ignores it while it is + // being built, and on the same filesystem so the move below is atomic. + let staging = root.join(".staging"); + fs::create_dir_all(&staging).unwrap(); + fs::write(staging.join(".gitignore"), "secret.txt\n").unwrap(); + fs::write(staging.join("secret.txt"), "moved_subtree_secret_marker\n").unwrap(); + fs::write( + staging.join("keep.rs"), + "fn keep() { let moved_subtree_keep_marker = 3; }\n", + ) + .unwrap(); + + fs::rename(&staging, root.join("vendor")).unwrap(); + + assert!( + wait_for_match(port, "moved_subtree_keep_marker", Duration::from_secs(60)), + "watcher never indexed the non-ignored file in a subtree that arrived whole" + ); + assert_eq!( + search_matches(port, "moved_subtree_secret_marker"), + 0, + "watcher indexed a file excluded by a .gitignore that arrived inside \ + the same subtree, so the subtree was indexed against stale rules" + ); +} + +/// A directory removed and recreated at the same path must still be watched. +/// +/// The kernel drops an inotify watch when its directory goes away and does not +/// say so, leaving the path recorded as watched with no descriptor behind it. +/// Nothing later can tell that entry from a live one — it is in the desired set +/// *and* in the watched set — so without explicitly clearing it on removal the +/// directory stops being watched for the life of the process. +/// +/// `rm -rf build && mkdir build`, a branch switch, and a `git clean` all do +/// exactly this. +#[test] +fn watcher_rewatches_a_directory_that_is_removed_and_recreated() { + let dir = TempDir::new().unwrap(); + let root = dir.path(); + let index_dir = root.join(".tgrep_test_index"); + + fs::create_dir_all(root.join(".git")).unwrap(); + fs::write(root.join(".gitignore"), "build/\n").unwrap(); + fs::create_dir_all(root.join("src").join("gen")).unwrap(); + fs::write( + root.join("src").join("lib.rs"), + "fn seeded() { let normal_source_marker = 1; }\n", + ) + .unwrap(); + fs::write( + root.join("src").join("gen").join("old.rs"), + "fn old() { let pre_delete_marker = 2; }\n", + ) + .unwrap(); + + let status = Command::new(tgrep_bin()) + .args([ + "index", + root.to_str().unwrap(), + "--index-path", + index_dir.to_str().unwrap(), + ]) + .status() + .expect("failed to run tgrep index"); + assert!(status.success(), "initial index build failed"); + + let child = Command::new(tgrep_bin()) + .args([ + "serve", + "--index-path", + index_dir.to_str().unwrap(), + root.to_str().unwrap(), + ]) + .stderr(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .spawn() + .expect("failed to start tgrep serve"); + let _server = ServerGuard { child }; + + let (_pid, port) = wait_for_server(&index_dir); + assert!( + wait_for_match(port, "pre_delete_marker", Duration::from_secs(30)), + "expected the seeded file in src/gen to be searchable" + ); + thread::sleep(Duration::from_secs(2)); + + let gen_dir = root.join("src").join("gen"); + fs::remove_dir_all(&gen_dir).unwrap(); + thread::sleep(Duration::from_secs(2)); + fs::create_dir_all(&gen_dir).unwrap(); + + // Deliberately after the recreation has been processed. A file written in + // the same breath would be picked up by the subscription pass's own scan + // and would say nothing about whether the watch itself was re-established. + thread::sleep(Duration::from_secs(3)); + fs::write( + gen_dir.join("new.rs"), + "fn regenerated() { let post_recreate_marker = 3; }\n", + ) + .unwrap(); + + assert!( + wait_for_match(port, "post_recreate_marker", Duration::from_secs(30)), + "watcher never saw a file written to a directory that was removed and \ + recreated, so its subscription was not re-established" + ); +} + +/// The walk decides what belongs in the index; the watcher must agree with it. +/// +/// `should_skip_watcher_path` only filters by location — excludes, ignore +/// rules, hidden paths. It says nothing about the two rules the walker applies +/// per file: binary extensions are skipped, and so is anything over +/// `--max-filesize`. A file arriving through the watcher therefore used to be +/// indexed even when a walk of the very same tree would have rejected it, so +/// the index disagreed with itself depending on whether a file was present at +/// startup or written afterwards — and the next reconcile silently deleted it. +/// +/// Both rejections are asserted alongside an eligible file written at the same +/// moment. Without that control the test would pass just as happily against a +/// watcher that had stopped indexing anything at all. +#[test] +fn watcher_applies_the_same_file_eligibility_rules_as_the_walker() { + let dir = TempDir::new().unwrap(); + let root = dir.path(); + let index_dir = root.join(".tgrep_test_index"); + + fs::create_dir_all(root.join(".git")).unwrap(); + fs::create_dir_all(root.join("src")).unwrap(); + fs::write( + root.join("src").join("lib.rs"), + "fn seeded() { let seeded_source_marker = 1; }\n", + ) + .unwrap(); + // Small enough to be indexed now, and grown past the cap below. + fs::write( + root.join("src").join("grows.rs"), + "fn grows() { let outgrew_the_cap_marker = 5; }\n", + ) + .unwrap(); + + let status = Command::new(tgrep_bin()) + .args([ + "index", + root.to_str().unwrap(), + "--index-path", + index_dir.to_str().unwrap(), + "--max-filesize", + "2K", + ]) + .status() + .expect("failed to run tgrep index"); + assert!(status.success(), "initial index build failed"); + + let child = Command::new(tgrep_bin()) + .args([ + "serve", + "--index-path", + index_dir.to_str().unwrap(), + "--max-filesize", + "2K", + root.to_str().unwrap(), + ]) + .stderr(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .spawn() + .expect("failed to start tgrep serve"); + let _server = ServerGuard { child }; + + let (_pid, port) = wait_for_server(&index_dir); + assert!( + wait_for_match(port, "seeded_source_marker", Duration::from_secs(30)), + "expected the seeded file to be searchable" + ); + + let src = root.join("src"); + // Text content, so nothing but the extension can keep it out. + fs::write( + src.join("asset.png"), + "fn decoy() { let binary_extension_marker = 2; }\n", + ) + .unwrap(); + // Same marker, pushed past the 2K cap by padding. + let mut oversized = String::from("fn big() { let oversized_file_marker = 3; }\n"); + oversized.push_str(&"// padding\n".repeat(400)); + fs::write(src.join("huge.rs"), oversized).unwrap(); + // Already in the index, and now over the cap: what was indexed has to go, + // not just stop being updated. + let mut grown = String::from("fn grows() { let outgrew_the_cap_marker = 5; }\n"); + grown.push_str(&"// padding\n".repeat(400)); + fs::write(src.join("grows.rs"), grown).unwrap(); + fs::write( + src.join("extra.rs"), + "fn extra() { let eligible_file_marker = 4; }\n", + ) + .unwrap(); + + assert!( + wait_for_match(port, "eligible_file_marker", Duration::from_secs(30)), + "the eligible file written next to the rejected ones was never indexed, \ + so this test cannot say anything about the rejections" + ); + + assert_eq!( + search_matches(port, "binary_extension_marker"), + 0, + "the watcher indexed a file whose extension the walker rejects" + ); + assert_eq!( + search_matches(port, "oversized_file_marker"), + 0, + "the watcher indexed a file larger than --max-filesize" + ); + assert!( + wait_for_no_match(port, "outgrew_the_cap_marker", Duration::from_secs(30)), + "a file that grew past --max-filesize kept its indexed content" + ); +} + +#[cfg(unix)] +#[test] +fn watcher_does_not_index_through_symlinks() { + let dir = TempDir::new().unwrap(); + let root = dir.path(); + let index_dir = root.join(".tgrep_test_index"); + + // Deliberately outside the served root: this stands in for anything a + // symlink committed to a branch could point at. + let outside = TempDir::new().unwrap(); + let secret = outside.path().join("secret.txt"); + fs::write(&secret, "fn leak() { let outside_root_marker = 1; }\n").unwrap(); + + fs::create_dir_all(root.join(".git")).unwrap(); + fs::create_dir_all(root.join("src")).unwrap(); + fs::write( + root.join("src").join("lib.rs"), + "fn seeded() { let seeded_source_marker = 1; }\n", + ) + .unwrap(); + // Indexed as a real file first, then replaced by a link below. + fs::write( + root.join("src").join("swapped.rs"), + "fn swapped() { let replaced_by_symlink_marker = 2; }\n", + ) + .unwrap(); + + let status = Command::new(tgrep_bin()) + .args([ + "index", + root.to_str().unwrap(), + "--index-path", + index_dir.to_str().unwrap(), + ]) + .status() + .expect("failed to run tgrep index"); + assert!(status.success(), "initial index build failed"); + + let child = Command::new(tgrep_bin()) + .args([ + "serve", + "--index-path", + index_dir.to_str().unwrap(), + root.to_str().unwrap(), + ]) + .stderr(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .spawn() + .expect("failed to start tgrep serve"); + let _server = ServerGuard { child }; + + let (_pid, port) = wait_for_server(&index_dir); + assert!( + wait_for_match(port, "seeded_source_marker", Duration::from_secs(30)), + "expected the seeded file to be searchable" + ); + assert_eq!( + search_matches(port, "replaced_by_symlink_marker"), + 1, + "expected the file that is about to be replaced to start out indexed" + ); + + let src = root.join("src"); + std::os::unix::fs::symlink(&secret, src.join("link.rs")).unwrap(); + fs::remove_file(src.join("swapped.rs")).unwrap(); + std::os::unix::fs::symlink(&secret, src.join("swapped.rs")).unwrap(); + fs::write( + src.join("extra.rs"), + "fn extra() { let eligible_file_marker = 3; }\n", + ) + .unwrap(); + + assert!( + wait_for_match(port, "eligible_file_marker", Duration::from_secs(30)), + "the eligible file written alongside the symlinks was never indexed, \ + so this test cannot say anything about the symlinks" + ); + + assert!( + wait_for_no_match(port, "replaced_by_symlink_marker", Duration::from_secs(30)), + "a real file replaced by a symlink kept its old content in the index" + ); + assert_eq!( + search_matches(port, "outside_root_marker"), + 0, + "the watcher followed a symlink and indexed content from outside the served root" + ); +} diff --git a/tgrep-core/src/git_index.rs b/tgrep-core/src/git_index.rs index f8fbdac..2b01e6e 100644 --- a/tgrep-core/src/git_index.rs +++ b/tgrep-core/src/git_index.rs @@ -84,7 +84,7 @@ fn normalise(path: &str) -> String { /// /// `.git` is usually a directory, but is a file holding `gitdir: ` in a /// linked worktree or a submodule. -fn git_dir(repo_root: &Path) -> Option { +pub(crate) fn git_dir(repo_root: &Path) -> Option { let dot_git = repo_root.join(".git"); let meta = std::fs::metadata(&dot_git).ok()?; if meta.is_dir() { @@ -144,9 +144,13 @@ pub fn ignores_case(repo_root: &Path) -> bool { /// /// Returns `None` when there is no readable index, which the caller must treat /// as "exempt nothing" only where that is the safe direction. +/// The index file a repository's tracked-file set is read from. +pub(crate) fn index_path(repo_root: &Path) -> Option { + Some(git_dir(repo_root)?.join("index")) +} + pub fn load_tracked(repo_root: &Path) -> Option { - let git_dir = git_dir(repo_root)?; - let bytes = std::fs::read(git_dir.join("index")).ok()?; + let bytes = std::fs::read(index_path(repo_root)?).ok()?; parse_index(&bytes) } diff --git a/tgrep-core/src/gitignore.rs b/tgrep-core/src/gitignore.rs index e32bb34..173dea4 100644 --- a/tgrep-core/src/gitignore.rs +++ b/tgrep-core/src/gitignore.rs @@ -10,7 +10,7 @@ use crate::walker::walker_thread_count; use ignore::WalkBuilder; -use std::path::Path; +use std::path::{Path, PathBuf}; pub const P4IGNORE_FILENAME: &str = "p4ignore.ini"; @@ -94,6 +94,19 @@ pub struct IgnoreMatcher { /// Repository-local `.git/info/exclude`, below global rules in precedence. repo_exclude: Option<(String, Gitignore)>, global: Gitignore, + /// The directory the relative paths handed to [`Self::is_ignored`] are + /// relative to, needed to rebuild the absolute path + /// [`CaseInsensitiveIgnore`] matches against. + root: std::path::PathBuf, + /// Git's `core.ignorecase` narrowing, when the repository asks for it. + /// + /// The indexing walk applies this as a `filter_entry` alongside the + /// case-sensitive rules, so a matcher without it answers a different + /// question than the walk did. That gap is not academic: on a Windows + /// enlistment it left the watcher subscribing to, and indexing, a 13.4 GiB + /// build artifact the walk had already excluded — and every event under it + /// re-added a file the next stale check then evicted. + ignorecase: Option, } impl IgnoreMatcher { @@ -117,9 +130,10 @@ impl IgnoreMatcher { nested: Vec<(String, IgnoreKind, Gitignore)>, global: Gitignore, ) -> Option { - Self::with_all_sources(local, false, nested, Vec::new(), None, global) + Self::with_all_sources(local, false, nested, Vec::new(), None, global, None) } + #[allow(clippy::too_many_arguments)] fn with_all_sources( local: Gitignore, local_is_filter: bool, @@ -127,6 +141,7 @@ impl IgnoreMatcher { ancestors: Vec<(String, IgnoreKind, Gitignore)>, repo_exclude: Option<(String, Gitignore)>, global: Gitignore, + ignorecase: Option, ) -> Option { let mut nested: Vec = nested .into_iter() @@ -164,14 +179,20 @@ impl IgnoreMatcher { || !nested.is_empty() || !ancestors.is_empty() || repo_exclude.is_some() - || !global.is_empty()) + || !global.is_empty() + || ignorecase.is_some()) .then_some(Self { + // `GitignoreBuilder::new(root)` records `root`, and every caller + // builds `local` from the served root, so this is that root + // without threading it through three more signatures. + root: local.path().to_path_buf(), local, local_is_filter, nested, ancestors, repo_exclude, global, + ignorecase, }) } @@ -271,11 +292,21 @@ impl IgnoreMatcher { if standard_decision == Some(true) { return true; } - self.local_is_filter + if self.local_is_filter && self .local .matched_path_or_any_parents(path, is_dir) .is_ignore() + { + return true; + } + // Applied last, and to whitelisted paths too, because that is where the + // indexing walk applies it: a `filter_entry` rejection is not undone by + // a whitelist rule. Both passes must agree on what the tree contains, + // or the watcher indexes a file the stale check immediately evicts. + self.ignorecase + .as_ref() + .is_some_and(|ignorecase| ignorecase.excludes(&self.root.join(rel), is_dir)) } } @@ -416,7 +447,36 @@ pub struct CaseInsensitiveIgnore { /// Loaded on the first path this matcher actually claims, which for most /// repositories is never. Reading it costs 163 ms and ~30 MB on a /// 299k-file repository, and nothing at all if no rule ever matches. - tracked: std::sync::OnceLock>, + /// + /// Reloaded when the index it was read from changes. A walk builds this + /// matcher, uses it and drops it, so a snapshot would do; the file watcher + /// holds one for the life of the server, and there a `git add -f` or a + /// `git rm --cached` rewrites only `.git/index` — which is hidden, so no + /// ignore source changes and nothing republishes the matcher. Frozen, the + /// exemption would keep answering from the tracked set as it stood at + /// startup, and the watcher would disagree with a fresh walk about which + /// files exist until the hourly reconcile. + tracked: std::sync::RwLock, +} + +/// The tracked-file set, together with the identity of the index it came from. +#[derive(Default)] +struct TrackedCache { + /// `None` until something is loaded; `Some` even when the load failed, so + /// an unreadable index is not retried on every path. + loaded_from: Option>, + tracked: Option, +} + +/// Modification time and length of the index, which is what identifies it. +/// +/// git replaces the index by renaming `index.lock` over it, so any rewrite +/// lands as a new mtime — the same pair git's own racy-index handling relies +/// on. Two rewrites inside one filesystem mtime tick that leave the length +/// unchanged are the gap, and the periodic reconcile is what closes it. +fn index_identity(repo_root: &Path) -> Option<(std::time::SystemTime, u64)> { + let meta = std::fs::metadata(crate::git_index::index_path(repo_root)?).ok()?; + Some((meta.modified().ok()?, meta.len())) } impl CaseInsensitiveIgnore { @@ -460,8 +520,8 @@ impl CaseInsensitiveIgnore { if use_gitignore { let _ = builder.add(repo_root.join(GITIGNORE_FILENAME)); } - if use_exclude { - let _ = builder.add(repo_root.join(".git").join("info").join("exclude")); + if use_exclude && let Some(exclude) = repo_exclude_path(&repo_root) { + let _ = builder.add(exclude); } let matcher = builder.build().ok()?; if matcher.is_empty() { @@ -470,7 +530,7 @@ impl CaseInsensitiveIgnore { Some(Self { matcher, repo_root, - tracked: std::sync::OnceLock::new(), + tracked: std::sync::RwLock::new(TrackedCache::default()), }) } @@ -487,21 +547,45 @@ impl CaseInsensitiveIgnore { return false; } // Only now is the index worth reading. - let Some(tracked) = self - .tracked - .get_or_init(|| crate::git_index::load_tracked(&self.repo_root)) - else { - // No readable index means no way to tell tracked from untracked. - // Excluding could hide real source, so decline instead. + let relative = relative.to_string_lossy(); + self.tracked_hides(&relative, is_dir) + } + + /// Whether the tracked-file set leaves `relative` hidden. + /// + /// `false` when the index cannot be read: with no way to tell tracked from + /// untracked, excluding could hide real source, so it declines instead. + fn tracked_hides(&self, relative: &str, is_dir: bool) -> bool { + let identity = index_identity(&self.repo_root); + { + let cache = self.tracked.read().unwrap(); + if cache.loaded_from.as_ref() == Some(&identity) { + return Self::hides(cache.tracked.as_ref(), relative, is_dir); + } + } + let mut cache = self.tracked.write().unwrap(); + // Another thread may have reloaded it while this one waited. + if cache.loaded_from.as_ref() != Some(&identity) { + cache.tracked = crate::git_index::load_tracked(&self.repo_root); + cache.loaded_from = Some(identity); + } + Self::hides(cache.tracked.as_ref(), relative, is_dir) + } + + fn hides( + tracked: Option<&crate::git_index::TrackedFiles>, + relative: &str, + is_dir: bool, + ) -> bool { + let Some(tracked) = tracked else { return false; }; - let relative = relative.to_string_lossy(); if is_dir { // A rule matching a directory does not hide tracked files inside // it, so the walk still has to descend. - !tracked.contains_any_under(&relative) + !tracked.contains_any_under(relative) } else { - !tracked.contains(&relative) + !tracked.contains(relative) } } } @@ -531,6 +615,62 @@ fn git_repo_root(root: &Path) -> Option<&Path> { root.ancestors().find(|dir| dir.join(".git").exists()) } +/// The `info/exclude` file whose rules apply to `root`, if there is one. +/// +/// Not `.git/info/exclude`. In a linked worktree or a submodule `.git` is a +/// file holding a `gitdir:` pointer, and that directory in turn holds a +/// `commondir` naming the repository every worktree shares — which is where the +/// one `info/exclude` lives. `WalkBuilder` resolves that chain, so a matcher +/// that stopped at the literal path enforced different rules than the walk in +/// exactly the layouts where the two differ, and the watcher would index a file +/// the next stale check evicts. +pub fn repo_exclude_path(root: &Path) -> Option { + let git_dir = crate::git_index::git_dir(git_repo_root(root)?)?; + let common = match std::fs::read_to_string(git_dir.join("commondir")) { + Ok(target) => { + let target = target.trim(); + let path = Path::new(target); + if target.is_empty() { + git_dir + } else if path.is_absolute() { + path.to_path_buf() + } else { + git_dir.join(path) + } + } + Err(_) => git_dir, + }; + let path = common.join("info").join("exclude"); + path.is_file().then_some(path) +} + +/// The parent-directory `.ignore` / `.gitignore` files that apply to `root`, +/// closest directory first, paired with the directory that anchors them. +/// +/// `WalkBuilder` applies `.ignore` files from every ancestor. Its git boundary +/// is independent: with the default `require_git`, ancestor `.gitignore` files +/// stop after the nearest repository root; with `--no-require-git` they +/// continue to the filesystem root. +pub fn ancestor_ignore_paths(root: &Path, no_require_git: bool) -> Vec<(PathBuf, IgnoreKind)> { + let repo_root = git_repo_root(root); + let mut found = Vec::new(); + for dir in root.ancestors().skip(1) { + for (kind, path, enabled) in [ + (IgnoreKind::DotIgnore, dir.join(DOT_IGNORE_FILENAME), true), + ( + IgnoreKind::GitIgnore, + dir.join(GITIGNORE_FILENAME), + no_require_git || repo_root.is_some_and(|repo| dir.starts_with(repo)), + ), + ] { + if enabled && path.is_file() { + found.push((path, kind)); + } + } + } + found +} + /// `.gitignore` and `.git/info/exclude` are **git-gated** to match the indexing /// walk (`WalkBuilder`'s `require_git` default): they apply only when `root` is /// inside a git repository, detected by scanning `root` and its ancestors for a @@ -602,41 +742,27 @@ pub fn matcher_from_ignore_paths_with_options( } } - // WalkBuilder applies `.ignore` files from every ancestor. Its git boundary - // is independent: with the default `require_git`, ancestor `.gitignore` - // files stop after the nearest repository root; with `--no-require-git` - // they continue to the filesystem root. + // WalkBuilder applies `.ignore` files from every ancestor, with its own + // git boundary — see [`ancestor_ignore_paths`]. let mut ancestors = Vec::new(); - for dir in root.ancestors().skip(1) { + for (path, kind) in ancestor_ignore_paths(root, no_require_git) { + let Some(dir) = path.parent() else { + continue; + }; let prefix = root .strip_prefix(dir) .unwrap_or(root) .to_string_lossy() .replace('\\', "/"); - for (kind, path, enabled) in [ - (IgnoreKind::DotIgnore, dir.join(DOT_IGNORE_FILENAME), true), - ( - IgnoreKind::GitIgnore, - dir.join(GITIGNORE_FILENAME), - no_require_git || repo_root.is_some_and(|repo| dir.starts_with(repo)), - ), - ] { - if !enabled || !path.is_file() { - continue; - } - let mut builder = GitignoreBuilder::new(dir); - let _ = builder.add(&path); - if let Ok(matcher) = builder.build() { - ancestors.push((prefix.clone(), kind, matcher)); - } + let mut builder = GitignoreBuilder::new(dir); + let _ = builder.add(&path); + if let Ok(matcher) = builder.build() { + ancestors.push((prefix, kind, matcher)); } } let repo_exclude = repo_root.and_then(|repo| { - let path = repo.join(".git").join("info").join("exclude"); - if !path.is_file() { - return None; - } + let path = repo_exclude_path(root)?; let mut builder = GitignoreBuilder::new(repo); let _ = builder.add(&path); let matcher = builder.build().ok()?; @@ -654,7 +780,21 @@ pub fn matcher_from_ignore_paths_with_options( } else { GitignoreBuilder::new(root).build().ok()? }; - IgnoreMatcher::with_all_sources(local, true, nested, ancestors, repo_exclude, global) + // The same narrowing `walker::walk_dir` and `walk_file_metadata` apply as a + // `filter_entry`. Serving takes no `--no-ignore-vcs` / `--no-ignore-exclude` + // / `--no-ignore-parent`, so the flags the walk was built with are the + // defaults; `no_ignore` is handled by the caller, which does not build a + // matcher at all in that case. + let ignorecase = CaseInsensitiveIgnore::new(root, true, true, true); + IgnoreMatcher::with_all_sources( + local, + true, + nested, + ancestors, + repo_exclude, + global, + ignorecase, + ) } /// Convenience wrapper for callers that only have `.gitignore` paths. @@ -886,6 +1026,75 @@ mod tests { assert!(build_matcher(tmp.path()).is_none()); } + #[test] + fn the_repository_exclude_is_found_through_a_worktree_pointer() { + // In a linked worktree `.git` is a file naming the worktree's own git + // directory, which in turn names the repository every worktree shares + // — and that is where the one `info/exclude` lives. `WalkBuilder` + // resolves the whole chain, so a matcher that stopped at the literal + // `.git/info/exclude` enforced different rules than the walk did, and + // the watcher indexed files the next stale check evicted. + let tmp = tempfile::tempdir().unwrap(); + let common = tmp.path().join("main").join(".git"); + std::fs::create_dir_all(common.join("info")).unwrap(); + std::fs::write(common.join("info").join("exclude"), "*.secret\n").unwrap(); + + let worktree_git = common.join("worktrees").join("wt"); + std::fs::create_dir_all(&worktree_git).unwrap(); + std::fs::write( + worktree_git.join("commondir"), + format!("{}\n", common.display()), + ) + .unwrap(); + + let worktree = tmp.path().join("wt"); + std::fs::create_dir_all(&worktree).unwrap(); + std::fs::write( + worktree.join(".git"), + format!("gitdir: {}\n", worktree_git.display()), + ) + .unwrap(); + + assert_eq!( + repo_exclude_path(&worktree).as_deref(), + Some(common.join("info").join("exclude").as_path()), + "the exclude file has to be reached through the pointer chain" + ); + + let matcher = matcher_from_ignore_paths(&worktree, &[], &[]) + .expect("the exclude file supplies rules"); + assert!(matcher.is_ignored(Path::new("keys.secret"), false)); + assert!(!matcher.is_ignored(Path::new("main.rs"), false)); + } + + #[test] + fn a_relative_commondir_resolves_against_the_worktree_git_dir() { + let tmp = tempfile::tempdir().unwrap(); + let common = tmp.path().join(".git"); + std::fs::create_dir_all(common.join("info")).unwrap(); + std::fs::write(common.join("info").join("exclude"), "*.bin\n").unwrap(); + + // What git actually writes: a path relative to the worktree's git dir. + let worktree_git = common.join("worktrees").join("wt"); + std::fs::create_dir_all(&worktree_git).unwrap(); + std::fs::write(worktree_git.join("commondir"), "../..\n").unwrap(); + + let worktree = tmp.path().join("wt"); + std::fs::create_dir_all(&worktree).unwrap(); + std::fs::write( + worktree.join(".git"), + format!("gitdir: {}\n", worktree_git.display()), + ) + .unwrap(); + + let found = repo_exclude_path(&worktree).expect("resolved through commondir"); + assert!( + std::fs::canonicalize(&found).unwrap() + == std::fs::canonicalize(common.join("info").join("exclude")).unwrap(), + "got {found:?}" + ); + } + #[test] fn local_whitelist_overrides_global_ignore() { use ignore::gitignore::GitignoreBuilder; diff --git a/tgrep-core/src/hybrid.rs b/tgrep-core/src/hybrid.rs index 24197ff..bf96237 100644 --- a/tgrep-core/src/hybrid.rs +++ b/tgrep-core/src/hybrid.rs @@ -293,6 +293,21 @@ impl HybridIndex { self.reader().all_paths().iter().cloned().collect() } + /// The reader paths `keep` accepts. + /// + /// For callers that want a few of them — everything under one directory, + /// say. [`Self::reader_paths`] allocates a copy of every path in the index + /// to answer that, which at repository scale is the bulk of the cost and + /// all of it wasted. + pub fn reader_paths_matching(&self, mut keep: impl FnMut(&str) -> bool) -> Vec { + self.reader() + .all_paths() + .iter() + .filter(|path| keep(path)) + .cloned() + .collect() + } + /// Number of files in the on-disk reader. pub fn reader_file_count(&self) -> usize { self.reader().num_files() diff --git a/tgrep-core/src/walker.rs b/tgrep-core/src/walker.rs index 2af0a04..cc5be0b 100644 --- a/tgrep-core/src/walker.rs +++ b/tgrep-core/src/walker.rs @@ -158,7 +158,11 @@ impl Default for WalkOptions { } /// Check if a file extension indicates a binary format. -fn is_binary_extension(path: &Path) -> bool { +/// +/// Public so the watcher can apply the same rule the walk does. A file the +/// walk rejected here must not be inserted into the index by an incremental +/// update, or the two disagree about what the index contains. +pub fn is_binary_extension(path: &Path) -> bool { path.extension() .and_then(|e| e.to_str()) .is_some_and(|ext| { @@ -1413,6 +1417,95 @@ mod tests { ); } + /// The watcher cannot walk per event, so it asks the same question as a + /// point query. If the two disagree the watcher subscribes to and indexes a + /// tree the walk excluded, and the next stale check evicts every file it + /// added — on the enlistment this came from, a 13.4 GiB build artifact + /// making up 71% of the corpus, re-read and re-evicted on every pass. + #[test] + fn the_point_query_matcher_hides_exactly_what_the_walk_hides() { + let dir = ignorecase_fixture(true, &["src/main.rs", "src/Kept.TXT"]); + let root = dir.path(); + let matcher = crate::gitignore::matcher_from_ignore_paths( + root, + std::slice::from_ref(&root.join(".gitignore")), + &[], + ) + .expect("the fixture has rules"); + + assert!( + matcher.is_ignored(Path::new("qlogs"), true), + "a directory the walk prunes must not be subscribed to" + ); + assert!( + matcher.is_ignored(Path::new("qlogs/artifact.rs"), false), + "nor may a file inside it be indexed" + ); + assert!( + matcher.is_ignored(Path::new("src/Gone.TXT"), false), + "an untracked file the rule matches once case is ignored" + ); + // The tracked-file exemption comes with it, or the watcher would drop + // events for files git never hides. + assert!( + !matcher.is_ignored(Path::new("src/Kept.TXT"), false), + "a tracked file must stay visible" + ); + assert!(!matcher.is_ignored(Path::new("src/main.rs"), false)); + } + + #[test] + fn the_point_query_matcher_follows_the_case_sensitivity_gate() { + // The other direction, which is the one that loses files: a repository + // that distinguishes case must not have anything hidden from it. + let dir = ignorecase_fixture(false, &["src/main.rs", "src/Kept.TXT"]); + let root = dir.path(); + let matcher = crate::gitignore::matcher_from_ignore_paths( + root, + std::slice::from_ref(&root.join(".gitignore")), + &[], + ) + .expect("the fixture has rules"); + + assert!(!matcher.is_ignored(Path::new("qlogs"), true)); + assert!(!matcher.is_ignored(Path::new("qlogs/artifact.rs"), false)); + assert!(!matcher.is_ignored(Path::new("src/Gone.TXT"), false)); + } + + /// The exemption is answered from a cached read of `.git/index`. A walk + /// builds this matcher and drops it, so a snapshot would do — but the file + /// watcher holds one for the life of the server, and `git add -f` rewrites + /// only the index, which is hidden. No ignore source changes, nothing + /// republishes the matcher, and a frozen cache would keep hiding a file + /// that git now tracks until the hourly reconcile. + #[test] + fn the_tracked_exemption_reloads_when_the_git_index_changes() { + let dir = ignorecase_fixture(true, &["src/main.rs", "src/Kept.TXT"]); + let root = dir.path(); + let matcher = crate::gitignore::matcher_from_ignore_paths( + root, + std::slice::from_ref(&root.join(".gitignore")), + &[], + ) + .expect("the fixture has rules"); + + // Untracked, and `*.txt` matches it once case is ignored. + assert!(matcher.is_ignored(Path::new("src/Gone.TXT"), false)); + + fake_git_repo(root, true, &["src/main.rs", "src/Kept.TXT", "src/Gone.TXT"]); + + assert!( + !matcher.is_ignored(Path::new("src/Gone.TXT"), false), + "a file git now tracks must stop being hidden without rebuilding \ + the matcher" + ); + + // And back: `git rm --cached` is the same problem in reverse, where a + // stale cache keeps indexing a file the walk has started hiding. + fake_git_repo(root, true, &["src/main.rs"]); + assert!(matcher.is_ignored(Path::new("src/Kept.TXT"), false)); + } + #[test] fn no_ignore_turns_the_whole_thing_off() { let dir = ignorecase_fixture(true, &["src/main.rs"]);