diff --git a/README.md b/README.md index bfcd77a..4ea329f 100644 --- a/README.md +++ b/README.md @@ -306,6 +306,13 @@ gives up waiting after four hours so a continuously busy server still reconciles. On an unchanged tree it finds nothing and leaves the index alone. `--no-watch` turns it off along with the watcher. +On Linux and Android, tgrep registers only the non-ignored directories with +inotify, avoiding watch-descriptor growth beneath ignored trees. This guarantee +is backend-specific: the implementation intentionally keeps one recursive +`ReadDirectoryChangesW` root subscription on Windows and one root FSEvents +stream on macOS, where ignored events are filtered after delivery and ignored +descendants remain watched. kqueue and `PollWatcher` are not covered. + ### Search ```bash diff --git a/tgrep-cli/src/serve.rs b/tgrep-cli/src/serve.rs index a2272b0..e96aeb0 100644 --- a/tgrep-cli/src/serve.rs +++ b/tgrep-cli/src/serve.rs @@ -52,6 +52,10 @@ const WATCHER_QUEUE_CAP: usize = 16_384; /// reconciling stale check runs. const WATCHER_IDLE_POLL: Duration = Duration::from_secs(1); +/// Git's index is hidden from the repository watcher. Poll its metadata only +/// when the case-insensitive tracked-file exemption is active. +const TRACKED_INDEX_POLL: Duration = Duration::from_secs(2); + /// How long a file the watcher never heard about can stay wrong in the index. /// /// Every mutation the index takes after the initial build arrives as an OS @@ -266,6 +270,7 @@ impl ContentCache { struct ServerState { index: RwLock, cache: RwLock, + cache_generation: std::sync::atomic::AtomicU64, root: PathBuf, watcher_active: std::sync::atomic::AtomicBool, /// True while the initial index build is in progress. @@ -290,6 +295,9 @@ struct ServerState { ignore_rules_dirty: std::sync::atomic::AtomicBool, /// Ensures a burst of ignore-file events uses at most one refresh worker. ignore_refresh_scheduled: std::sync::atomic::AtomicBool, + /// Last observed tracked-path membership. `None` means the published + /// matcher has no case-insensitive tracked-file exemption. + tracked_membership: Mutex>, /// Set when events were lost, cleared by the next subscription sync, which /// then re-issues every subscription instead of trusting its own records. /// @@ -471,8 +479,23 @@ struct ServerState { /// Milliseconds since `started` at the last search request, used by the /// periodic reconcile to stay out of the way of a server in active use. last_search_ms: std::sync::atomic::AtomicU64, + #[cfg(test)] + stale_refresh_hook: Mutex>, +} + +#[cfg(test)] +#[derive(Clone, Copy)] +enum StaleRefreshPhase { + BeforeWalk, + AfterBuildBeforeStampPublish, + AfterConcreteRead, + BeforeConcreteCommit, + AfterMatcherPublish, } +#[cfg(test)] +type StaleRefreshHook = Arc; + impl ServerState { fn note_search(&self) { self.last_search_ms @@ -639,6 +662,7 @@ pub fn run(root: &Path, index_path: Option<&Path>, options: ServeOptions<'_>) -> CACHE_MAX_BYTES, CACHE_MAX_ENTRY_BYTES, )), + cache_generation: std::sync::atomic::AtomicU64::new(0), root: root.clone(), watcher_active: std::sync::atomic::AtomicBool::new(false), indexing: std::sync::atomic::AtomicBool::new(needs_build), @@ -648,6 +672,7 @@ pub fn run(root: &Path, index_path: Option<&Path>, options: ServeOptions<'_>) -> gitignore_pending: std::sync::atomic::AtomicBool::new(!no_watch && !no_ignore), ignore_rules_dirty: std::sync::atomic::AtomicBool::new(false), ignore_refresh_scheduled: std::sync::atomic::AtomicBool::new(false), + tracked_membership: Mutex::new(None), watch_resubscribe: std::sync::atomic::AtomicBool::new(false), ignore_sources: RwLock::new(Vec::new()), ignore_source_stamps: RwLock::new(IgnoreStamps::new()), @@ -673,6 +698,8 @@ pub fn run(root: &Path, index_path: Option<&Path>, options: ServeOptions<'_>) -> unreadable: RwLock::new(std::collections::HashMap::new()), started: serve_start, last_search_ms: std::sync::atomic::AtomicU64::new(0), + #[cfg(test)] + stale_refresh_hook: Mutex::new(None), }); // Bind TCP listener on a random port @@ -806,17 +833,19 @@ fn build_stale_matcher( state: &ServerState, root: &Path, walk: &tgrep_core::walker::MetaWalkResult, + ignorecase: Option>, ) -> Option { if state.no_ignore { return None; } let start = Instant::now(); - let matcher = tgrep_core::walker::build_gitignore_matcher_from_files( + let matcher = tgrep_core::walker::build_gitignore_matcher_from_files_with_ignorecase( root, &walk.gitignore_files, &walk.ignore_files, state.no_require_git, + ignorecase, ); let has_matcher = matcher.is_some(); eprintln!( @@ -1008,7 +1037,17 @@ fn publish_ignore_matcher( *state.ignore_source_stamps.write().unwrap() = stamps; *state.ignore_sources.write().unwrap() = sources; - *state.gitignore.write().unwrap() = matcher; + // Keep the matcher and semantic baseline in one critical section. The + // matcher's tracked exemption is immutable, so this baseline describes the + // exact decisions the stale walk and watcher publication made. + let mut published = state.gitignore.write().unwrap(); + let mut membership = state.tracked_membership.lock().unwrap(); + *membership = matcher + .as_ref() + .and_then(tgrep_core::gitignore::IgnoreMatcher::tracked_membership_fingerprint); + *published = matcher; + drop(membership); + drop(published); state.gitignore_pending.store(false, Ordering::SeqCst); // New rules mean a different set of directories worth hearing about: // a tightened rule releases the subscriptions under it, and a relaxed @@ -1032,7 +1071,7 @@ fn publish_ignore_matcher( newly_watched } -fn handle_connection(stream: TcpStream, state: &ServerState) -> Result<()> { +fn handle_connection(stream: TcpStream, state: &Arc) -> Result<()> { let mut reader = BufReader::new(stream.try_clone()?); let mut writer = stream; @@ -1047,7 +1086,7 @@ fn handle_connection(stream: TcpStream, state: &ServerState) -> Result<()> { Ok(()) } -fn process_request(request: &str, state: &ServerState) -> String { +fn process_request(request: &str, state: &Arc) -> String { let req: serde_json::Value = match serde_json::from_str(request) { Ok(v) => v, Err(e) => { @@ -1447,6 +1486,7 @@ fn handle_search( }) .collect() } else { + let cache_generation = state.cache_generation.load(Ordering::SeqCst); // Phase 1: read-lock to find cache hits (peek avoids write-lock need) let mut hit_keys: Vec = Vec::new(); let mut hits: Vec<(String, Arc)> = Vec::with_capacity(candidate_info.len()); @@ -1478,17 +1518,7 @@ fn handle_search( // Phase 3: single write-lock to promote hits and insert misses if !hit_keys.is_empty() || !disk_results.is_empty() { - let mut cache = state.cache.write().unwrap(); - // Promote hit entries so LRU recency stays accurate - for key in &hit_keys { - cache.touch(key); - } - // Insert disk results, re-checking for races with other threads - for (rel_path, content) in &disk_results { - if cache.peek(rel_path).is_none() { - cache.put(rel_path.clone(), Arc::clone(content)); - } - } + update_content_cache(state, cache_generation, &hit_keys, &disk_results); } // Combine hits and disk results, preserving candidate order @@ -1554,6 +1584,43 @@ fn handle_search( json_rpc_result(id, result) } +fn update_content_cache( + state: &ServerState, + expected_generation: u64, + hit_keys: &[String], + disk_results: &[(String, Arc)], +) { + let mut cache = state.cache.write().unwrap(); + // Index mutations invalidate cached paths and advance this generation under + // the same lock. Earlier disk reads may serve their in-flight query, but + // must not repopulate stale bytes for later searches. + if state.cache_generation.load(Ordering::SeqCst) != expected_generation { + return; + } + for key in hit_keys { + cache.touch(key); + } + for (rel_path, content) in disk_results { + if cache.peek(rel_path).is_none() { + cache.put(rel_path.clone(), Arc::clone(content)); + } + } +} + +fn invalidate_cached_paths<'a>(state: &ServerState, paths: impl IntoIterator) { + let Ok(mut cache) = state.cache.write() else { + return; + }; + let mut invalidated = false; + for path in paths { + cache.pop(path); + invalidated = true; + } + if invalidated { + state.cache_generation.fetch_add(1, Ordering::SeqCst); + } +} + fn search_file_matches( rel_path: &str, file: &DecodedFile, @@ -1727,37 +1794,156 @@ fn handle_status(id: Option, state: &ServerState) -> String { json_rpc_result(id, result) } -fn handle_reload(id: Option, state: &ServerState) -> String { +fn handle_reload(id: Option, state: &Arc) -> String { let index_dir = state.index_dir.clone(); + while state.indexing.load(Ordering::SeqCst) || state.flushing.load(Ordering::SeqCst) { + thread::sleep(Duration::from_millis(50)); + } + let refresh = state.stale_refresh_lock.lock().unwrap(); + let gate = state.snapshot_gate.write().unwrap(); + { + let _deferred = match state.deferred_events.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + }; + state.indexing.store(true, Ordering::SeqCst); + } + let ignorecase = frozen_tracked_membership(state, &state.root); + #[cfg(test)] + run_stale_refresh_hook(state, StaleRefreshPhase::BeforeWalk); + let since = SystemTime::now(); - // Rebuild from disk. Uses the options form rather than `build_index` so the - // rebuild keeps the ignore semantics the server started with — a reload that - // silently changed them would rewrite the index against different rules. - if let Err(e) = builder::build_index_with_options( + // Rebuild the index and watcher matcher from the same immutable tracked + // membership. The snapshot gate keeps watcher/auto-save mutations out until + // both are published. + let staging_dir = index_dir.join(".reload-build"); + let _ = std::fs::remove_dir_all(&staging_dir); + let outcome = match builder::build_index_with_options_and_ignorecase( &state.root, - Some(&index_dir), + Some(&staging_dir), &builder::BuildOptions { no_ignore: state.no_ignore, no_require_git: state.no_require_git, max_file_size: state.max_file_size, exclude_dirs: state.exclude_dirs.clone(), + collect_gitignore_files: !state.no_ignore, ..Default::default() }, + ignorecase.clone(), ) { - return json_rpc_error(id, -32000, &format!("rebuild failed: {e}")); + Ok(outcome) => outcome, + Err(e) => { + let _ = std::fs::remove_dir_all(&staging_dir); + state.indexing.store(false, Ordering::SeqCst); + drop(gate); + drop(refresh); + replay_deferred_events(state, &state.root); + schedule_pending_ignore_refresh(state); + return json_rpc_error(id, -32000, &format!("rebuild failed: {e}")); + } + }; + #[cfg(test)] + run_stale_refresh_hook(state, StaleRefreshPhase::AfterBuildBeforeStampPublish); + + // Freeze the deferred buffer through publication. Events already captured + // have their stamps withheld; callbacks arriving now wait on this lock, + // then observe `indexing == false` and process normally after the gate drops. + let deferred = match state.deferred_events.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + }; + let stamps = match tgrep_core::meta::read_filestamps(&staging_dir) { + Ok(stamps) if state.watch_enabled => { + withhold_stamps_for_deferred_snapshot(&state.root, stamps, deferred.as_ref()) + } + // Without a watcher there is no event to identify a file that changed + // between extraction and the builder's later stamp collection. Publish + // no claims from that racy window; the serialized catch-up below then + // treats every path as changed and rebuilds it once. + Ok(_) => std::collections::HashMap::new(), + Err(e) => { + eprintln!("[trace] warning: reload could not read file stamps ({e})"); + std::collections::HashMap::new() + } + }; + if let Err(e) = tgrep_core::meta::write_filestamps(&stamps, &staging_dir) { + let _ = std::fs::remove_dir_all(&staging_dir); + state.indexing.store(false, Ordering::SeqCst); + drop(deferred); + drop(gate); + drop(refresh); + replay_deferred_events(state, &state.root); + schedule_pending_ignore_refresh(state); + return json_rpc_error(id, -32000, &format!("could not stage rebuild stamps: {e}")); + } + if !publish_reloaded_index(state, &index_dir, &staging_dir, outcome.num_files) { + state.indexing.store(false, Ordering::SeqCst); + drop(deferred); + drop(gate); + drop(refresh); + replay_deferred_events(state, &state.root); + schedule_pending_ignore_refresh(state); + return json_rpc_error(id, -32000, "rebuild publication failed"); } + let indexed = outcome.num_files as u64; + state.index_total.store(indexed, Ordering::Relaxed); + state.index_progress.store(indexed, Ordering::Relaxed); + *state.file_stamps.write().unwrap() = stamps; - // Reopen index - match HybridIndex::open(&index_dir, &state.root) { - Ok(new_index) => { - let mut index = state.index.write().unwrap(); - *index = new_index; - let mut cache = state.cache.write().unwrap(); - cache.clear(); - json_rpc_result(id, serde_json::json!({"status": "reloaded"})) + let newly_watched = if state.no_ignore { + Vec::new() + } else { + publish_ignore_matcher( + state, + &state.root, + ignore_sources_of( + &state.root, + &outcome.gitignore_files, + &outcome.ignore_files, + state.no_require_git, + ), + || { + tgrep_core::walker::build_gitignore_matcher_from_files_with_ignorecase( + &state.root, + &outcome.gitignore_files, + &outcome.ignore_files, + state.no_require_git, + ignorecase, + ) + }, + ) + }; + #[cfg(test)] + run_stale_refresh_hook(state, StaleRefreshPhase::AfterMatcherPublish); + let mut membership_changed = tracked_membership_changed(state); + state.indexing.store(false, Ordering::SeqCst); + drop(deferred); + drop(gate); + drop(refresh); + replay_deferred_events(state, &state.root); + schedule_pending_ignore_refresh(state); + + if !state.watch_enabled { + #[cfg(test)] + if membership_changed { + run_stale_refresh_hook(state, StaleRefreshPhase::BeforeWalk); } - Err(e) => json_rpc_error(id, -32000, &format!("reopen failed: {e}")), + let caught_up = catch_up_unwatched_build(state, &state.root, &state.index_dir); + if !caught_up { + schedule_tracked_membership_correction(state, &state.root, membership_changed); + return json_rpc_error( + id, + -32000, + "rebuild catch-up did not complete; reload was not authoritative", + ); + } + membership_changed = tracked_membership_changed(state); + } + if state.watch_enabled { + spawn_recovery_scan(state, &state.root, newly_watched, since); } + schedule_tracked_membership_correction(state, &state.root, membership_changed); + json_rpc_result(id, serde_json::json!({"status": "reloaded"})) } fn start_file_watcher(state: Arc, root: &Path, queue_cap: usize) -> bool { @@ -1865,7 +2051,24 @@ fn start_file_watcher(state: Arc, root: &Path, queue_cap: usize) -> if std::thread::Builder::new() .name("tgrep-watcher".into()) .spawn(move || { + let mut last_tracked_index_poll = Instant::now(); loop { + if last_tracked_index_poll.elapsed() >= TRACKED_INDEX_POLL { + last_tracked_index_poll = Instant::now(); + let changed = poll_tracked_membership_changed(&worker_state); + if changed { + eprintln!( + "[trace] Git tracked paths changed; reconciling tracked-file exemptions" + ); + worker_state + .ignore_rules_dirty + .store(true, Ordering::SeqCst); + schedule_ignore_rules_refresh( + Arc::clone(&worker_state), + worker_root.clone(), + ); + } + } match rx.recv_timeout(WATCHER_IDLE_POLL) { Ok(event) => handle_fs_event(&worker_state, &worker_root, &event), // A quiet interval means the burst has drained, so this is @@ -1919,6 +2122,64 @@ fn start_file_watcher(state: Arc, root: &Path, queue_cap: usize) -> true } +/// Detect a tracked-file exemption change without subscribing to `.git`. +/// +/// The metadata probe is enabled only while the published matcher actually +/// uses the exemption. Index metadata only decides when to reparse its paths; +/// an unchanged membership fingerprint does not schedule a repository walk. +/// Updating the observed membership before scheduling makes +/// a burst one dirty signal; the existing refresh scheduler coalesces it with +/// any ignore-source changes and serializes the full stale reconciliation. +fn tracked_membership_changed(state: &ServerState) -> bool { + let matcher = state.gitignore.read().unwrap(); + let current = matcher + .as_ref() + .and_then(tgrep_core::gitignore::IgnoreMatcher::current_tracked_membership_fingerprint); + let mut observed = state.tracked_membership.lock().unwrap(); + let changed = matches!( + (observed.as_ref(), current.as_ref()), + (Some(previous), Some(current)) if previous != current + ); + *observed = current; + changed +} + +fn poll_tracked_membership_changed(state: &ServerState) -> bool { + // A reconcile holds the write side across snapshot, walk and publication. + // Waiting here prevents a poll from committing transient A→B→A membership. + let _gate = state.snapshot_gate.read().unwrap(); + tracked_membership_changed(state) +} + +fn frozen_tracked_membership( + state: &ServerState, + root: &Path, +) -> Option> { + (!state.no_ignore) + .then(|| { + tgrep_core::gitignore::CaseInsensitiveIgnore::frozen_snapshot(root, true, true, true) + }) + .flatten() + .map(Arc::new) +} + +fn schedule_tracked_membership_correction(state: &Arc, root: &Path, changed: bool) { + if !changed { + return; + } + eprintln!("[trace] Git tracked paths changed during reconciliation; scheduling a retry"); + state.ignore_rules_dirty.store(true, Ordering::SeqCst); + schedule_ignore_rules_refresh(Arc::clone(state), root.to_path_buf()); +} + +#[cfg(test)] +fn run_stale_refresh_hook(state: &ServerState, phase: StaleRefreshPhase) { + let hook = state.stale_refresh_hook.lock().unwrap().clone(); + if let Some(hook) = hook { + hook(phase); + } +} + /// Decide whether the file watcher should skip a path entirely. /// /// Mirrors the file walker's hidden-path, `--exclude` directory filtering, @@ -2039,10 +2300,10 @@ fn is_ignore_rules_file(root: &Path, path: &Path) -> bool { /// repo whose ignored build output exhausts the budget makes `watch()` return /// an error and the server loses its watcher entirely. /// -/// ReadDirectoryChangesW (Windows) and FSEvents (macOS) subscribe once for the -/// whole subtree, so there is no per-directory registration to withhold. On -/// those platforms filtering on delivery is the only lever available, and -/// [`should_skip_watcher_path`] remains it. +/// This implementation deliberately keeps one recursive root subscription on +/// Windows (`ReadDirectoryChangesW`) and one root stream on macOS (FSEvents). +/// Ignored events are filtered after delivery there, so ignored descendants are +/// not unwatched even though the design avoids per-directory descriptor growth. /// /// Deliberately limited to the backends we can exercise in CI. kqueue and /// `PollWatcher` are per-path too, but nothing here builds or tests them. @@ -2112,6 +2373,34 @@ struct WatchRegistry { watched: std::collections::HashSet, } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum TraversalCompleteness { + Complete, + Incomplete, +} + +/// Consume one overflow repair request only when this traversal can complete it. +/// +/// Clearing before the sync means an overflow that arrives during a complete +/// pass sets the flag for another pass. Re-arming an incomplete pass preserves +/// the original request without overwriting a concurrent `true`. +fn take_force_resubscribe( + pending: &std::sync::atomic::AtomicBool, + completeness: TraversalCompleteness, +) -> bool { + let force = pending.swap(false, Ordering::SeqCst); + if force && completeness == TraversalCompleteness::Incomplete { + pending.store(true, Ordering::SeqCst); + return false; + } + force +} + +struct WatchableDirs { + dirs: std::collections::HashSet, + completeness: TraversalCompleteness, +} + impl WatchRegistry { /// Subscribe to every directory in `desired` that is not already /// subscribed, leaving existing subscriptions alone. @@ -2283,22 +2572,27 @@ impl WatchRegistry { /// would pay forty thousand of them for a doubt only overflow raises. /// /// Returns `(added, removed)`. Only for a set that describes the whole - /// tree — anything absent from `desired` is unsubscribed. To subscribe to - /// a subtree without disturbing the rest, use [`Self::add_all`]. + /// tree. `completeness` makes its authority explicit: a complete set prunes + /// anything absent from `desired`, while an incomplete traversal only adds + /// directories it proved desirable. To subscribe to a subtree without + /// disturbing the rest, use [`Self::add_all`]. fn sync( &mut self, desired: &std::collections::HashSet, + completeness: TraversalCompleteness, force: bool, ) -> (Vec, usize) { - let stale: Vec = self.watched.difference(desired).cloned().collect(); let mut removed = 0; - for dir in stale { - // Best effort. inotify drops a descriptor by itself when the - // directory is deleted, so "not found" is an expected outcome - // here, not an error worth reporting. - let _ = self.watcher.unwatch(&dir); - self.watched.remove(&dir); - removed += 1; + if completeness == TraversalCompleteness::Complete { + let stale: Vec = self.watched.difference(desired).cloned().collect(); + for dir in stale { + // Best effort. inotify drops a descriptor by itself when the + // directory is deleted, so "not found" is an expected outcome + // here, not an error worth reporting. + let _ = self.watcher.unwatch(&dir); + self.watched.remove(&dir); + removed += 1; + } } // Shallowest first, because `desired` is a `HashSet` and hands its @@ -2343,28 +2637,43 @@ impl WatchRegistry { /// Symlinked directories are not descended into, matching the walker (the /// `ignore` crate does not follow links by default). That also keeps a /// symlink cycle from turning this into an infinite walk. +/// +/// Any failed listing, entry read, or type query marks the result incomplete. +/// Its proven directories remain useful for adding subscriptions, but its +/// omissions must not be used to remove existing ones. fn watchable_dirs( root: &Path, start: &Path, exclude_dirs: &[String], gitignore: Option<&tgrep_core::gitignore::IgnoreMatcher>, -) -> std::collections::HashSet { +) -> WatchableDirs { let mut found = std::collections::HashSet::new(); + let mut completeness = TraversalCompleteness::Complete; found.insert(start.to_path_buf()); let mut stack = vec![start.to_path_buf()]; while let Some(dir) = stack.pop() { let Ok(entries) = std::fs::read_dir(&dir) else { // An unreadable directory is not a reason to abandon the rest of - // the tree; the periodic reconcile is what catches what we miss. + // the tree, but it makes the result unsafe for pruning. + completeness = TraversalCompleteness::Incomplete; continue; }; - for entry in entries.flatten() { - if !entry.file_type().is_ok_and(|t| t.is_dir()) { + for entry in entries { + let Ok(entry) = entry else { + completeness = TraversalCompleteness::Incomplete; + continue; + }; + let Ok(file_type) = entry.file_type() else { + completeness = TraversalCompleteness::Incomplete; + continue; + }; + if !file_type.is_dir() { continue; } let path = entry.path(); let Ok(rel) = path.strip_prefix(root) else { + completeness = TraversalCompleteness::Incomplete; continue; }; let rel = rel.to_string_lossy().replace('\\', "/"); @@ -2375,7 +2684,10 @@ fn watchable_dirs( found.insert(path); } } - found + WatchableDirs { + dirs: found, + completeness, + } } /// The directories that must be subscribed to for the ignore sources @@ -2458,13 +2770,13 @@ fn sync_watch_registrations(state: &ServerState, root: &Path) -> (Vec, }; if !state.no_ignore { let sources = state.ignore_sources.read().unwrap(); - desired.extend(ignore_target_dirs(root, &sources)); + desired.dirs.extend(ignore_target_dirs(root, &sources)); } - let total = desired.len(); - // Consumed here, so one overflow buys one forced pass rather than making - // every later reconcile re-register the whole tree. - let force = state.watch_resubscribe.swap(false, Ordering::SeqCst); - let (added, removed) = registry.sync(&desired, force); + // An incomplete traversal cannot prove that omitted recorded watches are + // live, so it does not get to consume the overflow repair request. + let force = take_force_resubscribe(&state.watch_resubscribe, desired.completeness); + let (added, removed) = registry.sync(&desired.dirs, desired.completeness, force); + let total = registry.watched.len(); if !added.is_empty() || removed > 0 { eprintln!( "[trace] watcher subscriptions: {total} directories \ @@ -2713,7 +3025,7 @@ fn reindex_files_in(state: &Arc, root: &Path, dirs: &[PathBuf], sin should_skip_watcher_path(&rel, &state.exclude_dirs, gitignore.as_ref()) }; if !skip { - reindex_file(state, &path, &rel); + reindex_file(state, &path, &rel, false); } } if listing_complete { @@ -2898,9 +3210,7 @@ fn sweep_removed_files( } state.index.write().unwrap().live.delete_file(rel); state.file_stamps.write().unwrap().remove(rel); - if let Ok(mut cache) = state.cache.write() { - cache.pop(rel); - } + invalidate_cached_paths(state, std::iter::once(rel.as_str())); dropped += 1; } if dropped > 0 { @@ -3066,7 +3376,7 @@ fn watch_new_subtree(state: &Arc, root: &Path, dir: &Path) { } for (path, rel) in &files { - reindex_file(state, path, rel); + reindex_file(state, path, rel, true); } } @@ -3121,6 +3431,12 @@ fn schedule_ignore_rules_refresh(state: Arc, root: PathBuf) { }); } +fn schedule_pending_ignore_refresh(state: &Arc) { + if state.ignore_rules_dirty.load(Ordering::SeqCst) { + schedule_ignore_rules_refresh(Arc::clone(state), state.root.clone()); + } +} + /// Remember the paths in an event that arrived mid-build, for replay once the /// build publishes. Returns whether it did — a `false` means the build finished /// underneath us and the caller should handle the event normally. @@ -3231,7 +3547,22 @@ fn replay_deferred_events(state: &Arc, root: &Path) { let start = Instant::now(); let count = paths.len(); + let mut missing_subtree = false; for (path, introduces_dir) in paths { + if !is_real_dir(&path) + && path.strip_prefix(root).is_ok_and(|rel| { + let rel = rel.to_string_lossy().replace('\\', "/"); + let index = state.index.read().unwrap(); + index.reader_has_descendant_path(&rel) || index.live.has_descendant_path(&rel) + }) + { + // A single directory removal/rename event names no descendants. + // Replaying it as a file event would drop only the directory path, + // leaving every indexed child searchable. One coalesced full pass + // supplies the missing subtree membership evidence. + missing_subtree = true; + continue; + } let kind = if introduces_dir { EventKind::Create(notify::event::CreateKind::Any) } else { @@ -3249,6 +3580,10 @@ fn replay_deferred_events(state: &Arc, root: &Path) { }, ); } + if missing_subtree { + state.ignore_rules_dirty.store(true, Ordering::SeqCst); + schedule_ignore_rules_refresh(Arc::clone(state), root.to_path_buf()); + } eprintln!( "[trace] watcher: replayed {count} change(s) deferred during the initial build in {:.1}ms", start.elapsed().as_secs_f64() * 1000.0 @@ -3485,9 +3820,7 @@ fn handle_fs_event(state: &Arc, root: &Path, event: &Event) { // is processed atomically with respect to flush/auto-save. state.index.write().unwrap().live.delete_file(&rel_path); state.file_stamps.write().unwrap().remove(&rel_path); - if let Ok(mut cache) = state.cache.write() { - cache.pop(&rel_path); - } + invalidate_cached_paths(state, std::iter::once(rel_path.as_str())); continue; } @@ -3545,7 +3878,7 @@ fn handle_fs_event(state: &Arc, root: &Path, event: &Event) { continue; } - reindex_file(state, path, &rel_path); + reindex_file(state, path, &rel_path, true); } } @@ -3563,19 +3896,10 @@ fn lock_reindex(state: &ServerState) -> std::sync::MutexGuard<'_, ()> { /// Drop everything the index holds for a path. /// -/// The delete is not conditional on a stamp entry. `ServerState` accepts an -/// empty stamp map when `filestamps.json` is missing or unreadable, and the -/// reader can still hold the path in that state, so keying the delete on the -/// stamp alone would leave content the walk now rejects searchable. The stamp -/// removal is best-effort; the index and cache deletes always happen. -/// -/// The cost of that is a tombstone in the overlay for paths that were never -/// indexed — `live::delete_file` records one either way — and this runs for -/// every ineligible file a recovery scan walks past, which at startup is every -/// binary asset in the repository. An existing tombstone is therefore taken as -/// proof there is nothing left to do, which bounds that to one per distinct -/// path. The trace line, which is the part that would be actively misleading, -/// stays conditional on there having been something to drop. +/// A stamp is evidence, but not a precondition: `filestamps.json` may be missing +/// while the active reader still holds the path. Conversely, a rejected file +/// absent from the reader, live overlay, and stamps was never indexed, so +/// recording a tombstone for it would dirty the overlay for no state change. /// /// The caller must already hold `snapshot_gate` and `reindex_lock`. The lock is /// the caller's rather than this function's because `reindex_file` calls in @@ -3589,20 +3913,16 @@ fn drop_indexed_file(state: &ServerState, rel_path: &str, reason: &str) { .is_some(); { let index = state.index.read().unwrap(); - if index.live.has_path(rel_path) || had_stamp { - eprintln!("[trace] reindex: dropped {rel_path} ({reason})"); - } else if index.live.is_deleted(rel_path) { - // Already tombstoned, so there is nothing to record and no reason - // to dirty the overlay again. This is the repeat case: a recovery - // scan or a chatty editor can bring the same rejected path back - // here any number of times. + if index.live.is_deleted(rel_path) { + return; + } + if !had_stamp && !index.live.has_path(rel_path) && !index.reader_has_path(rel_path) { return; } } + eprintln!("[trace] reindex: dropped {rel_path} ({reason})"); state.index.write().unwrap().live.delete_file(rel_path); - if let Ok(mut cache) = state.cache.write() { - cache.pop(rel_path); - } + invalidate_cached_paths(state, std::iter::once(rel_path)); } /// What an event's stat result says about the path it named. @@ -3882,6 +4202,44 @@ enum CappedRead { Failed, } +fn file_still_has_bytes( + root: &Path, + path: &Path, + expected_version: &tgrep_core::builder::FileVersion, + expected: &[u8], +) -> std::io::Result { + use std::io::Read; + + fn matches( + mut file: std::fs::File, + expected_version: &tgrep_core::builder::FileVersion, + expected: &[u8], + ) -> std::io::Result { + if tgrep_core::builder::file_version(&file.metadata()?) != *expected_version { + return Ok(false); + } + let mut offset = 0; + let mut buffer = [0u8; 64 * 1024]; + loop { + let read = file.read(&mut buffer)?; + if read == 0 { + break; + } + if expected.get(offset..offset + read) != Some(&buffer[..read]) { + return Ok(false); + } + offset += read; + } + Ok(offset == expected.len() + && tgrep_core::builder::file_version(&file.metadata()?) == *expected_version) + } + + if !matches(open_within_root(root, path)?, expected_version, expected)? { + return Ok(false); + } + matches(open_within_root(root, path)?, expected_version, expected) +} + /// Reads a file's contents, never pulling in more than one byte past the cap. /// /// The size that qualified this file was stat'd before the read, and appending @@ -3920,9 +4278,7 @@ fn read_within_limit(file: &mut std::fs::File, limit: Option, capacity: usi /// /// The caller must hold `snapshot_gate`: the read, the commit, and the stamp /// update have to be atomic with respect to a flush or auto-save. -fn reindex_file(state: &ServerState, path: &Path, rel_path: &str) { - use tgrep_core::meta::FileStamp; - +fn reindex_file(state: &Arc, path: &Path, rel_path: &str, force: bool) { // Against other indexers, not against searches. The gate above is held for // read, so without this a recovery scan and the watcher worker can both be // here for the same path, both read, and the one that read the *older* @@ -3950,21 +4306,20 @@ fn reindex_file(state: &ServerState, path: &Path, rel_path: &str) { // because a build held the file open for a moment. The stale path // already treats unreadable files this way, keeping what it has and // retrying later, and the watcher should not disagree with it. + if force { + retry_failed_forced_reindex(state, rel_path, "the file could not be opened"); + } return; } }; let Ok(meta) = file.metadata() else { + if force { + retry_failed_forced_reindex(state, rel_path, "the opened file could not be inspected"); + } return; }; - let current = FileStamp { - mtime: meta - .modified() - .ok() - .and_then(|t| t.duration_since(std::time::SystemTime::UNIX_EPOCH).ok()) - .map(|d| d.as_secs()) - .unwrap_or(0), - size: meta.len(), - }; + let mut version = tgrep_core::builder::file_version(&meta); + let mut current = version.stamp().clone(); // The rules `walk_file_metadata` applies, and for the same reason: the walk // is authoritative about what belongs in the index, so anything it rejects @@ -3996,7 +4351,7 @@ fn reindex_file(state: &ServerState, path: &Path, rel_path: &str) { return; } - if state.file_stamps.read().unwrap().get(rel_path) == Some(¤t) { + if !force && state.file_stamps.read().unwrap().get(rel_path) == Some(¤t) { return; } @@ -4009,7 +4364,7 @@ fn reindex_file(state: &ServerState, path: &Path, rel_path: &str) { // From the handle, not the path: re-opening here is what would let a // symlink take the place of the file we just approved. let mut file = file; - let data = match read_within_limit( + let mut data = match read_within_limit( &mut file, state.max_file_size, current.size.min(1 << 20) as usize, @@ -4019,8 +4374,94 @@ fn reindex_file(state: &ServerState, path: &Path, rel_path: &str) { drop_indexed_file(state, rel_path, "grew past the size limit while being read"); return; } - CappedRead::Failed => return, + CappedRead::Failed => { + if force { + retry_failed_forced_reindex(state, rel_path, "the file could not be read"); + } + return; + } }; + if force { + #[cfg(test)] + run_stale_refresh_hook(state, StaleRefreshPhase::AfterConcreteRead); + // A concrete event is stronger evidence than the persisted stamp. Read + // containment-safe snapshots until the bytes and full-resolution file + // version agree, then bind the final decision to a fresh read of those + // exact bytes immediately before commit. + let mut stable = false; + for _ in 0..2 { + let mut verify = match open_within_root(&state.root, path) { + Ok(file) => file, + Err(e) if proves_ineligible(&e) => { + drop_indexed_file(state, rel_path, "no longer eligible"); + return; + } + Err(_) => { + retry_failed_forced_reindex( + state, + rel_path, + "the verification handle could not be opened", + ); + return; + } + }; + let Ok(meta) = verify.metadata() else { + retry_failed_forced_reindex( + state, + rel_path, + "the verification handle could not be inspected", + ); + return; + }; + let verify_version = tgrep_core::builder::file_version(&meta); + let verify_current = verify_version.stamp().clone(); + if !meta.is_file() + || tgrep_core::walker::is_binary_extension(path) + || state + .max_file_size + .is_some_and(|limit| verify_current.size > limit) + { + drop_indexed_file(state, rel_path, "no longer eligible"); + return; + } + let verified = match read_within_limit( + &mut verify, + state.max_file_size, + verify_current.size.min(1 << 20) as usize, + ) { + CappedRead::Data(data) => data, + CappedRead::TooLarge => { + drop_indexed_file(state, rel_path, "grew past the size limit while being read"); + return; + } + CappedRead::Failed => { + retry_failed_forced_reindex(state, rel_path, "the verification read failed"); + return; + } + }; + let handle_stable = verify.metadata().is_ok_and(|metadata| { + tgrep_core::builder::file_version(&metadata) == verify_version + }); + if data == verified + && handle_stable + && file_still_has_bytes(&state.root, path, &verify_version, &verified) + .unwrap_or(false) + { + data = verified; + current = verify_current; + version = verify_version; + stable = true; + break; + } + data = verified; + current = verify_current; + version = verify_version; + } + if !stable { + retry_failed_forced_reindex(state, rel_path, "the file kept changing while read"); + return; + } + } let text = tgrep_core::encoding::decode_for_index(&data); let is_binary = tgrep_core::trigram::is_binary(&text); let per_tri = if is_binary { @@ -4028,6 +4469,12 @@ fn reindex_file(state: &ServerState, path: &Path, rel_path: &str) { } else { Some(tgrep_core::live::LiveIndex::compute_trigram_masks(&text)) }; + #[cfg(test)] + run_stale_refresh_hook(state, StaleRefreshPhase::BeforeConcreteCommit); + if force && !file_still_has_bytes(&state.root, path, &version, &data).unwrap_or(false) { + retry_failed_forced_reindex(state, rel_path, "the file changed before commit"); + return; + } eprintln!("[trace] reindex: modified {rel_path}"); // Gate held by the caller — the commit + stamp update is processed @@ -4044,9 +4491,23 @@ fn reindex_file(state: &ServerState, path: &Path, rel_path: &str) { .write() .unwrap() .insert(rel_path.to_string(), current); - if let Ok(mut cache) = state.cache.write() { - cache.pop(rel_path); - } + invalidate_cached_paths(state, std::iter::once(rel_path)); +} + +fn retry_failed_forced_reindex(state: &Arc, rel_path: &str, reason: &str) { + // In-memory stamps override the persisted map during stale comparison. + // A sentinel therefore records "this path must be read" without rewriting + // filestamps.json for a transient event failure. + state.file_stamps.write().unwrap().insert( + rel_path.to_string(), + tgrep_core::meta::FileStamp { + mtime: u64::MAX, + size: u64::MAX, + }, + ); + eprintln!("[trace] warning: {rel_path} was not reindexed because {reason}; scheduling a retry"); + state.ignore_rules_dirty.store(true, Ordering::SeqCst); + schedule_ignore_rules_refresh(Arc::clone(state), state.root.clone()); } /// Whether a scheduled reconcile should run now. @@ -4255,29 +4716,54 @@ fn stamps_for_index_members( fn withhold_stamps_for_deferred( state: &ServerState, root: &Path, - mut stamps: std::collections::HashMap, + stamps: std::collections::HashMap, ) -> std::collections::HashMap { let deferred = match state.deferred_events.lock() { Ok(guard) => guard, Err(poisoned) => poisoned.into_inner(), }; - let Some(paths) = deferred.as_ref() else { + withhold_stamps_for_deferred_snapshot(root, stamps, deferred.as_ref()) +} + +fn withhold_stamps_for_deferred_snapshot( + root: &Path, + mut stamps: std::collections::HashMap, + paths: Option<&std::collections::HashMap>, +) -> std::collections::HashMap { + let Some(paths) = paths else { eprintln!( "[trace] warning: too many changes during the initial build to say which files the \ walk raced; publishing no stamps so the reconcile re-reads them" ); return std::collections::HashMap::new(); }; - let mut withheld = 0usize; - for path in paths.keys() { + let mut exact = std::collections::HashSet::new(); + let mut directories = std::collections::HashSet::new(); + for (path, introduces_dir) in paths { let Ok(rel) = path.strip_prefix(root) else { continue; }; let rel = rel.to_string_lossy().replace('\\', "/"); - if stamps.remove(&rel).is_some() { - withheld += 1; + exact.insert(rel.clone()); + if *introduces_dir { + directories.insert(rel); } } + let before = stamps.len(); + stamps.retain(|rel, _| { + if exact.contains(rel) { + return false; + } + let mut ancestor = rel.as_str(); + while let Some((parent, _)) = ancestor.rsplit_once('/') { + if directories.contains(parent) { + return false; + } + ancestor = parent; + } + true + }); + let withheld = before - stamps.len(); if withheld > 0 { eprintln!( "[trace] watcher: {withheld} file(s) changed during the initial build; their stamps \ @@ -4414,7 +4900,7 @@ fn stream_merge_stale_changes( let mut published_stamps = stamps.clone(); - let result = (|| -> Result { + let result = (|| -> Result { let build = || { builder::build_index_for_files( root, @@ -4484,7 +4970,7 @@ fn stream_merge_stale_changes( .count(); let expected_files = reader.num_files() - removed_reader_files + delta.num_files(); let published = publish_staged_index(state, index_dir, &staging_dir, expected_files); - if published { + if published.is_published() { // `publish_staged_index` prunes overlay entries represented by the // new reader. Also clear reconciled entries intentionally omitted // (newly ignored/binary/deleted) and old tombstones for files the @@ -4506,13 +4992,17 @@ fn stream_merge_stale_changes( })(); let _ = std::fs::remove_dir_all(&delta_dir); - let _ = std::fs::remove_dir_all(&staging_dir); + if !matches!(&result, Ok(PublishStatus::RollbackFailed)) { + let _ = std::fs::remove_dir_all(&staging_dir); + } else { + eprintln!("[trace] warning: preserving {staging_dir:?} after rollback failure"); + } state.flushing.store(false, Ordering::SeqCst); - if matches!(&result, Ok(true)) { + if matches!(&result, Ok(PublishStatus::Published)) { *state.file_stamps.write().unwrap() = published_stamps; } match result { - Ok(true) => { + Ok(PublishStatus::Published) => { eprintln!( "[trace] {operation}: streamed {} changes into the index in {:.1}s", candidates.len(), @@ -4520,7 +5010,7 @@ fn stream_merge_stale_changes( ); true } - Ok(false) => { + Ok(PublishStatus::Failed) | Ok(PublishStatus::RollbackFailed) => { eprintln!( "[trace] warning: {operation} delta could not be published; \ keeping the old index" @@ -4604,12 +5094,18 @@ fn background_refresh_stale( index_dir: &Path, compare_index_membership: bool, ) -> bool { - let _refresh = state.stale_refresh_lock.lock().unwrap(); + let refresh = state.stale_refresh_lock.lock().unwrap(); // Keep watcher/auto-save mutations out for the complete walk → matcher → // merge → recovery cycle. Search queries do not take this gate and remain // available. Held here rather than inside so the recovery scan below is // still covered by it. - let _gate = state.snapshot_gate.write().unwrap(); + let gate = state.snapshot_gate.write().unwrap(); + // One immutable tracked-file exemption is shared by the walk and the + // matcher it publishes. Git-index rewrites cannot change ignore decisions + // halfway through this pass. + let ignorecase = frozen_tracked_membership(state, root); + #[cfg(test)] + run_stale_refresh_hook(state, StaleRefreshPhase::BeforeWalk); let mut newly_watched = Vec::new(); // Before the walk, not after the subscriptions: this bounds the window the @@ -4622,6 +5118,8 @@ fn background_refresh_stale( index_dir, compare_index_membership, &mut newly_watched, + ignorecase, + true, ); // Directories that were not subscribed while the walk ran could not report @@ -4636,15 +5134,45 @@ fn background_refresh_stale( if ok { reindex_files_in(state, root, &newly_watched, since); } + // Compare semantics, not index metadata. A→B→A needs no correction because + // this pass used A throughout, while A→B schedules exactly one coalesced + // refresh. Content-only staging cannot create an immediate refresh loop. + let membership_changed = tracked_membership_changed(state); + drop(gate); + drop(refresh); + schedule_tracked_membership_correction(state, root, membership_changed); ok } +fn catch_up_unwatched_build(state: &Arc, root: &Path, index_dir: &Path) -> bool { + let refresh = state.stale_refresh_lock.lock().unwrap(); + let gate = state.snapshot_gate.write().unwrap(); + let ignorecase = frozen_tracked_membership(state, root); + let mut ignored_watches = Vec::new(); + let caught_up = refresh_stale_locked( + state, + root, + index_dir, + true, + &mut ignored_watches, + ignorecase, + false, + ); + let membership_changed = tracked_membership_changed(state); + drop(gate); + drop(refresh); + schedule_tracked_membership_correction(state, root, membership_changed); + caught_up +} + fn refresh_stale_locked( state: &Arc, root: &Path, index_dir: &Path, compare_index_membership: bool, newly_watched: &mut Vec, + ignorecase: Option>, + run_test_hooks: bool, ) -> bool { use tgrep_core::meta; use tgrep_core::walker; @@ -4655,7 +5183,7 @@ fn refresh_stale_locked( // Walk first. This single traversal feeds both the stale diff and the // watcher's ignore matcher, and it must run before the early returns below // so the matcher can be published on every path out of this function. - let walk = walker::walk_file_metadata( + let walk = walker::walk_file_metadata_with_ignorecase( root, &walker::MetaWalkOptions { exclude_dirs: state.exclude_dirs.clone(), @@ -4663,6 +5191,7 @@ fn refresh_stale_locked( no_require_git: state.no_require_git, max_file_size: state.max_file_size, }, + ignorecase.clone(), ); let walk_ms = start.elapsed().as_millis(); @@ -4690,8 +5219,14 @@ fn refresh_stale_locked( &walk.ignore_files, state.no_require_git, ), - || build_stale_matcher(state, root, &walk), + || build_stale_matcher(state, root, &walk, ignorecase), ); + #[cfg(test)] + if run_test_hooks { + run_stale_refresh_hook(state, StaleRefreshPhase::AfterMatcherPublish); + } + #[cfg(not(test))] + let _ = run_test_hooks; if walk.skipped_error > 0 { eprintln!( @@ -4797,11 +5332,14 @@ fn refresh_stale_locked( return false; } - if let Ok(mut cache) = state.cache.write() { - for path in changed.iter().chain(added.iter()).chain(deleted.iter()) { - cache.pop(path); - } - } + invalidate_cached_paths( + state, + changed + .iter() + .chain(added.iter()) + .chain(deleted.iter()) + .map(String::as_str), + ); true } @@ -4817,8 +5355,11 @@ fn reset_to_empty_index(state: &ServerState, root: &Path, index_dir: &Path) { } match HybridIndex::open(index_dir, root) { Ok(empty) => { - *state.index.write().unwrap() = empty; - state.cache.write().unwrap().clear(); + let mut index = state.index.write().unwrap(); + let mut cache = state.cache.write().unwrap(); + *index = empty; + cache.clear(); + state.cache_generation.fetch_add(1, Ordering::SeqCst); } Err(e) => eprintln!("[trace] warning: could not reopen an empty index ({e})"), } @@ -4845,6 +5386,7 @@ fn reset_to_empty_index(state: &ServerState, root: &Path, index_dir: &Path) { /// caller to fall back. fn bootstrap_index_build(state: &Arc, root: &Path, index_dir: &Path) -> bool { let start = Instant::now(); + let ignorecase = frozen_tracked_membership(state, root); // Anchors the recovery window at the start of the build's traversal, which // is the point from which writes could be missed: nothing under `root` is // subscribed yet, and the walk below has not reached most of it. Taking it @@ -4858,7 +5400,7 @@ fn bootstrap_index_build(state: &Arc, root: &Path, index_dir: &Path // a kernel high-water mark) covers the whole of it. Unlike the incremental // path below, nothing here polls memory on its own. let sampler = crate::mem::PrivatePeakSampler::start(); - let outcome = match builder::build_index_with_options( + let outcome = match builder::build_index_with_options_and_ignorecase( root, Some(index_dir), &builder::BuildOptions { @@ -4875,6 +5417,7 @@ fn bootstrap_index_build(state: &Arc, root: &Path, index_dir: &Path strategy: builder::IndexStrategy::External, buffer_bytes: builder::DEFAULT_INDEX_BUFFER_BYTES, }, + ignorecase.clone(), ) { Ok(outcome) => outcome, Err(e) => { @@ -4886,6 +5429,8 @@ fn bootstrap_index_build(state: &Arc, root: &Path, index_dir: &Path return false; } }; + #[cfg(test)] + run_stale_refresh_hook(state, StaleRefreshPhase::AfterBuildBeforeStampPublish); // Publish under the snapshot gate, and clear `indexing` before releasing // it. `handle_fs_event` only skips while `indexing` is true, so flipping @@ -4905,8 +5450,13 @@ fn bootstrap_index_build(state: &Arc, root: &Path, index_dir: &Path } }; let indexed = opened.num_files() as u64; - *state.index.write().unwrap() = opened; - state.cache.write().unwrap().clear(); + { + let mut index = state.index.write().unwrap(); + let mut cache = state.cache.write().unwrap(); + *index = opened; + cache.clear(); + state.cache_generation.fetch_add(1, Ordering::SeqCst); + } state.index_total.store(indexed, Ordering::Relaxed); state.index_progress.store(indexed, Ordering::Relaxed); @@ -4920,21 +5470,35 @@ fn bootstrap_index_build(state: &Arc, root: &Path, index_dir: &Path // and its walk handed back the .gitignore / .ignore paths. Building the // matcher from those is what keeps this cheap — `gitignore::build_matcher` // would rewalk the whole tree, which cost 49 s on a 289k-file repo. - match tgrep_core::meta::read_filestamps(index_dir) { + let stamps = match tgrep_core::meta::read_filestamps(index_dir) { // Minus the paths whose events arrived while the build ran: the builder // read those files at some point during its walk and stamped what it // saw, so for anything written afterwards the stamp describes bytes the // index does not hold. See `withhold_stamps_for_deferred`. - Ok(stamps) => { - *state.file_stamps.write().unwrap() = withhold_stamps_for_deferred(state, root, stamps) + Ok(stamps) if state.watch_enabled => withhold_stamps_for_deferred(state, root, stamps), + Ok(_) => std::collections::HashMap::new(), + Err(e) => { + eprintln!( + "[trace] warning: could not load file stamps ({e}); \ + the watcher may reindex on spurious events" + ); + std::collections::HashMap::new() } - Err(e) => eprintln!( - "[trace] warning: could not load file stamps ({e}); \ - the watcher may reindex on spurious events" - ), + }; + if !state.watch_enabled + && let Err(e) = tgrep_core::meta::write_filestamps(&stamps, index_dir) + { + drop(gate); + eprintln!( + "[trace] warning: could not prepare unwatched bootstrap catch-up ({e}); \ + falling back to the in-heap build" + ); + reset_to_empty_index(state, root, index_dir); + return false; } + *state.file_stamps.write().unwrap() = stamps; let mut newly_watched = Vec::new(); - if state.watch_enabled && !state.no_ignore { + if !state.no_ignore { let t_gi = Instant::now(); // "Newly watched" here is every directory in the repository, and the // build's walk ran before any of them were subscribed. Deferred rather @@ -4951,11 +5515,12 @@ fn bootstrap_index_build(state: &Arc, root: &Path, index_dir: &Path state.no_require_git, ), || { - tgrep_core::walker::build_gitignore_matcher_from_files( + tgrep_core::walker::build_gitignore_matcher_from_files_with_ignorecase( root, &outcome.gitignore_files, &outcome.ignore_files, state.no_require_git, + ignorecase, ) }, ); @@ -4974,13 +5539,28 @@ fn bootstrap_index_build(state: &Arc, root: &Path, index_dir: &Path spawn_recovery_scan(state, root, newly_watched, since); } - state.indexing.store(false, Ordering::SeqCst); + if state.watch_enabled { + state.indexing.store(false, Ordering::SeqCst); + } drop(gate); - - let elapsed = start.elapsed().as_secs_f64(); - drop(sampler); - match crate::mem::format_peak_memory() { - Some(peak) => eprintln!( + if !state.watch_enabled { + if !catch_up_unwatched_build(state, root, index_dir) { + eprintln!( + "[trace] warning: unwatched bootstrap catch-up was incomplete; \ + falling back to the in-heap build" + ); + reset_to_empty_index(state, root, index_dir); + return false; + } + state.indexing.store(false, Ordering::SeqCst); + } + let membership_changed = tracked_membership_changed(state); + schedule_tracked_membership_correction(state, root, membership_changed); + + let elapsed = start.elapsed().as_secs_f64(); + drop(sampler); + match crate::mem::format_peak_memory() { + Some(peak) => eprintln!( "[trace] bootstrap complete: {indexed} files indexed in {elapsed:.1}s \ (peak memory {peak})" ), @@ -5036,26 +5616,28 @@ fn background_index_build(state: &Arc, root: &Path, index_dir: &Pat // Phase 1: Walk file paths (no content reads) let t_walk = Instant::now(); + let ignorecase = frozen_tracked_membership(state, root); // The recovery window opens with this traversal, not with the subscriptions // it later feeds: a nested `.ignore` written after the walk read its parent // directory but before the matcher is published is invisible to both, and a // timestamp taken any later would date it as already accounted for. let since = SystemTime::now(); - let walk = walker::walk_dir( + let walk = walker::walk_dir_with_ignorecase( root, &WalkOptions { include_hidden: false, no_ignore: state.no_ignore, no_require_git: state.no_require_git, max_file_size: state.max_file_size, - collect_gitignore_files: state.watch_enabled && !state.no_ignore, + collect_gitignore_files: !state.no_ignore, exclude_dirs: state.exclude_dirs.clone(), ..Default::default() }, + ignorecase.clone(), ); let mut newly_watched = Vec::new(); - if state.watch_enabled && !state.no_ignore { + if !state.no_ignore { let start = Instant::now(); // Subscriptions are taken here, partway through the build, so files // written to a directory the walk has already passed are in neither @@ -5072,11 +5654,12 @@ fn background_index_build(state: &Arc, root: &Path, index_dir: &Pat state.no_require_git, ), || { - walker::build_gitignore_matcher_from_files( + walker::build_gitignore_matcher_from_files_with_ignorecase( root, &walk.gitignore_files, &walk.ignore_files, state.no_require_git, + ignorecase, ) }, ); @@ -5245,6 +5828,8 @@ fn background_index_build(state: &Arc, root: &Path, index_dir: &Pat incremental_flushes, start.elapsed().as_secs_f64() ); + #[cfg(test)] + run_stale_refresh_hook(state, StaleRefreshPhase::AfterBuildBeforeStampPublish); // Walk filesystem metadata BEFORE the flush so we can publish the // resulting per-file stamps atomically with the index files. Writing @@ -5303,8 +5888,14 @@ fn background_index_build(state: &Arc, root: &Path, index_dir: &Pat // // Minus whatever changed underneath the walk, which the stamps would // otherwise describe as indexed when the index holds the older bytes. - *state.file_stamps.write().unwrap() = withhold_stamps_for_deferred(state, root, stamps); - state.indexing.store(false, Ordering::SeqCst); + *state.file_stamps.write().unwrap() = if state.watch_enabled { + withhold_stamps_for_deferred(state, root, stamps) + } else { + std::collections::HashMap::new() + }; + if state.watch_enabled { + state.indexing.store(false, Ordering::SeqCst); + } // Final flush to disk for the bulk build. Use the same streaming // append-only path as incremental flushes so the final complete publish @@ -5323,6 +5914,16 @@ fn background_index_build(state: &Arc, root: &Path, index_dir: &Pat drop(gate); state.flushing.store(false, Ordering::SeqCst); + if !state.watch_enabled { + if !catch_up_unwatched_build(state, root, index_dir) { + state.ignore_rules_dirty.store(true, Ordering::SeqCst); + eprintln!( + "[trace] warning: unwatched background-build catch-up was incomplete; \ + leaving stamps invalid for the scheduled stale check" + ); + } + state.indexing.store(false, Ordering::SeqCst); + } // Reclaim memory held by the indexing-time live overlay — but only when // the flush actually completed and `prune_persisted_entries` ran. If the @@ -5333,6 +5934,8 @@ fn background_index_build(state: &Arc, root: &Path, index_dir: &Pat index.live.shrink_to_fit(); } + let membership_changed = tracked_membership_changed(state); + schedule_tracked_membership_correction(state, root, membership_changed); if state.ignore_rules_dirty.load(Ordering::SeqCst) { schedule_ignore_rules_refresh(Arc::clone(state), root.to_path_buf()); } @@ -5423,7 +6026,7 @@ fn flush_append_only_overlay_locked( eprintln!("[trace] warning: failed to write staging filestamps: {e}"); } - let pruned = publish_staged_index(state, index_dir, &staging_dir, num_files); + let pruned = publish_staged_index(state, index_dir, &staging_dir, num_files).is_published(); eprintln!( "[trace] append-only flush: {num_files} files on disk (complete={complete}) in {:.1}s", flush_start.elapsed().as_secs_f64() @@ -5441,136 +6044,304 @@ fn flush_append_only_overlay_locked( /// renames or swap readers out of order. `num_files` is the expected on-disk /// file count used to reject a partially-published reader. /// -/// Returns `true` when the swap + prune succeeded, `false` on any failure (the -/// previous reader and the live overlay are retained as the fallback). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum PublishStatus { + Published, + Failed, + RollbackFailed, +} + +impl PublishStatus { + fn is_published(self) -> bool { + matches!(self, Self::Published) + } +} + +/// Returns the publication outcome. A rollback failure preserves the staging +/// directory so its backups remain available for recovery. fn publish_staged_index( state: &ServerState, index_dir: &Path, staging_dir: &Path, num_files: usize, -) -> bool { +) -> PublishStatus { // Held across move + open + swap so concurrent publishers (auto-save / // background-build / watcher reindex flush) cannot interleave renames // or swap readers out of order. Searches do not take this lock. let _publish = state.publish_lock.lock().unwrap(); - if let Err(e) = move_staged_files(staging_dir, index_dir) { - eprintln!("[trace] warning: flush move failed: {e}"); - let _ = std::fs::remove_dir_all(staging_dir); + let mut moved = match move_staged_files(staging_dir, index_dir) { + Ok(moved) => moved, + Err(e) => { + eprintln!("[trace] warning: flush move failed: {e}"); + return if e.rollback_failed() { + PublishStatus::RollbackFailed + } else { + PublishStatus::Failed + }; + } + }; + let Some((new_reader, reader_files, reader_trigrams)) = + open_published_reader(index_dir, num_files) + else { + return match moved.rollback() { + Ok(()) => { + let _ = std::fs::remove_dir_all(staging_dir); + PublishStatus::Failed + } + Err(e) => { + moved.preserve(); + eprintln!("[trace] warning: failed to roll back index publication: {e}"); + PublishStatus::RollbackFailed + } + }; + }; + + // Atomic swap — no outer write lock required. + state.index.read().unwrap().swap_reader(new_reader); + // Brief write lock for in-memory overlay maintenance only. + { + let mut index = state.index.write().unwrap(); + index.prune_persisted_entries(); + index.live.reset_dirty_count(); + } + moved.commit(); + eprintln!( + "[trace] flush: reader reopened ({reader_files} files, \ + {reader_trigrams} trigrams), overlay pruned" + ); + let _ = std::fs::remove_dir_all(staging_dir); + PublishStatus::Published +} + +fn publish_reloaded_index( + state: &ServerState, + index_dir: &Path, + staging_dir: &Path, + num_files: usize, +) -> bool { + let _publish = state.publish_lock.lock().unwrap(); + let mut moved = match move_staged_files(staging_dir, index_dir) { + Ok(moved) => moved, + Err(e) => { + eprintln!("[trace] warning: reload move failed: {e}"); + return false; + } + }; + let Some((new_reader, reader_files, reader_trigrams)) = + open_published_reader(index_dir, num_files) + else { + match moved.rollback() { + Ok(()) => { + let _ = std::fs::remove_dir_all(staging_dir); + } + Err(e) => { + moved.preserve(); + eprintln!("[trace] warning: failed to roll back reload publication: {e}"); + } + } return false; + }; + + // Searches take the outer index lock before consulting the cache. Holding + // both in that order makes the complete reload visible as one generation. + { + let mut index = state.index.write().unwrap(); + let mut cache = state.cache.write().unwrap(); + index.swap_reader(new_reader); + let mut reconciled = index.live.overlay_paths(); + reconciled.extend(index.live.tombstone_paths()); + index.live.clear_reconciled_paths(&reconciled); + index.live.reset_dirty_count(); + cache.clear(); + state.cache_generation.fetch_add(1, Ordering::SeqCst); } + moved.commit(); + eprintln!( + "[trace] reload: reader reopened ({reader_files} files, \ + {reader_trigrams} trigrams), overlay and cache cleared" + ); + let _ = std::fs::remove_dir_all(staging_dir); + true +} - // Open the new reader. The publish mutex is intentionally still held - // here so that move + open + swap form an atomic publish unit (no other - // publisher can interleave a rename or swap a competing reader between - // these steps). The server-wide `state.index` RwLock is NOT taken, so - // search queries continue to be served by the previous reader (whose - // `Arc` they hold) throughout this call. - // - // On Windows, NTFS metadata for a recently-renamed file can transiently - // appear stale (zero-length), causing IndexReader::open to create a - // degenerate reader with files but no trigrams. We retry a few times - // with a short backoff to ride out the transient. - let pruned = 'open: { - const READER_OPEN_RETRIES: u32 = 5; - const READER_OPEN_BACKOFF: Duration = Duration::from_millis(200); - - for attempt in 0..READER_OPEN_RETRIES { - match tgrep_core::reader::IndexReader::open(index_dir) { - Ok(new_reader) => { - let reader_files = new_reader.num_files(); - let reader_trigrams = new_reader.num_trigrams(); - - if new_reader.is_degenerate() { - eprintln!( - "[trace] warning: reader has {reader_files} files but 0 trigrams \ - (attempt {}/{READER_OPEN_RETRIES}, likely stale NTFS metadata)", - attempt + 1 - ); - if attempt + 1 < READER_OPEN_RETRIES { - thread::sleep(READER_OPEN_BACKOFF * (attempt + 1)); - continue; - } - eprintln!( - "[trace] warning: degenerate reader persists after \ - {READER_OPEN_RETRIES} attempts, keeping live overlay as fallback" - ); - break 'open false; - } +fn open_published_reader( + index_dir: &Path, + num_files: usize, +) -> Option<(tgrep_core::reader::IndexReader, usize, usize)> { + const READER_OPEN_RETRIES: u32 = 5; + const READER_OPEN_BACKOFF: Duration = Duration::from_millis(200); + + for attempt in 0..READER_OPEN_RETRIES { + match tgrep_core::reader::IndexReader::open(index_dir) { + Ok(new_reader) => { + let reader_files = new_reader.num_files(); + let reader_trigrams = new_reader.num_trigrams(); + if new_reader.is_degenerate() { + eprintln!( + "[trace] warning: reader has {reader_files} files but 0 trigrams \ + (attempt {}/{READER_OPEN_RETRIES}, likely stale NTFS metadata)", + attempt + 1 + ); + } else if let Err(msg) = new_reader.validate_lookup() { + eprintln!( + "[trace] warning: reader validation failed \ + (attempt {}/{READER_OPEN_RETRIES}): {msg}", + attempt + 1 + ); + } else if reader_files >= num_files { + return Some((new_reader, reader_files, reader_trigrams)); + } else { + eprintln!( + "[trace] warning: reader has {reader_files} files \ + (expected {num_files}), keeping live overlay as fallback" + ); + return None; + } + } + Err(e) => { + eprintln!( + "[trace] warning: reader open failed (attempt {}/{READER_OPEN_RETRIES}): {e}", + attempt + 1 + ); + } + } + if attempt + 1 < READER_OPEN_RETRIES { + thread::sleep(READER_OPEN_BACKOFF * (attempt + 1)); + } + } + eprintln!( + "[trace] warning: failed to validate reader after \ + {READER_OPEN_RETRIES} attempts, keeping the previous reader" + ); + None +} - // Validate + warm the lookup mmap before swapping the - // reader in. This catches corruption (unsorted lookup - // table, out-of-bounds posting offsets) and, as a - // side-effect, pages in every byte of lookup.bin so that - // subsequent binary searches never hit cold mmap pages - // — preventing the zero-candidate failure observed on - // Windows after flush. - if let Err(msg) = new_reader.validate_lookup() { - eprintln!( - "[trace] warning: reader validation failed \ - (attempt {}/{READER_OPEN_RETRIES}): {msg}", - attempt + 1 - ); - if attempt + 1 < READER_OPEN_RETRIES { - thread::sleep(READER_OPEN_BACKOFF * (attempt + 1)); - continue; - } - eprintln!( - "[trace] warning: reader validation failed after \ - {READER_OPEN_RETRIES} attempts, keeping live overlay" - ); - break 'open false; - } +const INDEX_FILE_NAMES: &[&str] = &[ + "index.bin", + "lookup.bin", + "files.bin", + "filestamps.json", + "meta.json", +]; + +struct StagedFileMove { + staging: PathBuf, + target: PathBuf, + backed_up: Vec<&'static str>, + published: Vec<&'static str>, + finished: bool, +} - if reader_files >= num_files { - // Atomic swap — no outer write lock required. - state.index.read().unwrap().swap_reader(new_reader); - // Brief write lock for in-memory overlay maintenance only. - { - let mut index = state.index.write().unwrap(); - index.prune_persisted_entries(); - index.live.reset_dirty_count(); - } - eprintln!( - "[trace] flush: reader reopened ({reader_files} files, \ - {reader_trigrams} trigrams), overlay pruned" - ); - break 'open true; - } else { - eprintln!( - "[trace] warning: reader has {reader_files} files \ - (expected {num_files}), keeping live overlay as fallback" - ); - break 'open false; - } +impl StagedFileMove { + fn backup_path(&self, name: &str) -> PathBuf { + self.staging.join(format!(".previous-{name}")) + } + + fn rollback(&mut self) -> std::io::Result<()> { + if self.finished { + return Ok(()); + } + let mut first_error = None; + let mut pending_published = Vec::new(); + for name in std::mem::take(&mut self.published).into_iter().rev() { + let published = self.target.join(name); + if let Err(e) = std::fs::remove_file(&published) + && e.kind() != std::io::ErrorKind::NotFound + { + if first_error.is_none() { + first_error = Some(e); + } + pending_published.push(name); + } + } + pending_published.reverse(); + self.published = pending_published; + + let mut pending_backups = Vec::new(); + for name in std::mem::take(&mut self.backed_up).into_iter().rev() { + match publish_file(&self.backup_path(name), &self.target.join(name)) { + Ok(()) => self.published.retain(|published| *published != name), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + self.published.retain(|published| *published != name); } Err(e) => { - if attempt + 1 < READER_OPEN_RETRIES { - eprintln!( - "[trace] warning: reader open failed (attempt {}/{READER_OPEN_RETRIES}): {e}", - attempt + 1 - ); - thread::sleep(READER_OPEN_BACKOFF * (attempt + 1)); - continue; + if first_error.is_none() { + first_error = Some(e); } - eprintln!( - "[trace] warning: failed to reopen reader after flush: {e}, \ - live overlay retained" - ); - break 'open false; + pending_backups.push(name); } } } - false - }; - let _ = std::fs::remove_dir_all(staging_dir); - pruned + pending_backups.reverse(); + self.backed_up = pending_backups; + if let Some(e) = first_error { + return Err(e); + } + self.finished = true; + Ok(()) + } + + fn commit(&mut self) { + self.finished = true; + } + + fn preserve(&mut self) { + // Leave every remaining backup and published file exactly where the + // failed rollback left it so an operator or later recovery can use it. + self.finished = true; + } + + fn fail(mut self, publish: std::io::Error) -> MoveStagedFilesError { + let rollback = self.rollback().err(); + if rollback.is_some() { + self.preserve(); + } + MoveStagedFilesError { publish, rollback } + } +} + +impl Drop for StagedFileMove { + fn drop(&mut self) { + if let Err(e) = self.rollback() { + eprintln!("[trace] warning: failed to roll back staged index files: {e}"); + } + } +} + +#[derive(Debug)] +struct MoveStagedFilesError { + publish: std::io::Error, + rollback: Option, +} + +impl MoveStagedFilesError { + fn rollback_failed(&self) -> bool { + self.rollback.is_some() + } +} + +impl std::fmt::Display for MoveStagedFilesError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.publish)?; + if let Some(rollback) = &self.rollback { + write!(f, "; rollback also failed: {rollback}")?; + } + Ok(()) + } +} + +impl std::error::Error for MoveStagedFilesError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(&self.publish) + } } -/// Move index files from staging to the target directory. +/// Move index files from staging to the target directory, retaining backups +/// until the caller validates and commits the new reader. /// -/// Files are published in a fixed order, with `meta.json` last. This is only a -/// convention for publication layout; it does not provide atomic publish -/// semantics or reader-side validation by itself. +/// Files are published in a fixed order, with `meta.json` last. Existing files +/// are retained in staging until reader validation succeeds, so dropping the +/// returned transaction rolls back a partial or rejected publication. /// /// Performance note: this function runs under the server's `publish_lock` /// (which serializes concurrent publishers) but does NOT take the @@ -5582,24 +6353,39 @@ fn publish_staged_index( /// created next to the target (same parent) so cross-volume cases should /// not arise; if rename truly fails, the error is surfaced rather than /// silently falling back to a slow copy (see `publish_file`). -fn move_staged_files(staging: &Path, target: &Path) -> std::io::Result<()> { - std::fs::create_dir_all(target)?; - // Data files first, meta last. - for name in &[ - "index.bin", - "lookup.bin", - "files.bin", - "filestamps.json", - "meta.json", - ] { +fn move_staged_files( + staging: &Path, + target: &Path, +) -> Result { + std::fs::create_dir_all(target).map_err(|publish| MoveStagedFilesError { + publish, + rollback: None, + })?; + let mut moved = StagedFileMove { + staging: staging.to_path_buf(), + target: target.to_path_buf(), + backed_up: Vec::new(), + published: Vec::new(), + finished: false, + }; + for &name in INDEX_FILE_NAMES { let src = staging.join(name); let dst = target.join(name); if !src.exists() { continue; } - publish_file(&src, &dst)?; + if dst.exists() { + if let Err(error) = publish_file(&dst, &moved.backup_path(name)) { + return Err(moved.fail(error)); + } + moved.backed_up.push(name); + } + if let Err(error) = publish_file(&src, &dst) { + return Err(moved.fail(error)); + } + moved.published.push(name); } - Ok(()) + Ok(moved) } /// Publish a single staged file at `src` to `dst`. @@ -5839,6 +6625,7 @@ mod tests { CACHE_MAX_BYTES, CACHE_MAX_ENTRY_BYTES, )), + cache_generation: std::sync::atomic::AtomicU64::new(0), root: root.to_path_buf(), watcher_active: std::sync::atomic::AtomicBool::new(false), indexing: std::sync::atomic::AtomicBool::new(false), @@ -5846,6 +6633,7 @@ mod tests { gitignore_pending: std::sync::atomic::AtomicBool::new(true), ignore_rules_dirty: std::sync::atomic::AtomicBool::new(false), ignore_refresh_scheduled: std::sync::atomic::AtomicBool::new(false), + tracked_membership: Mutex::new(None), watch_resubscribe: std::sync::atomic::AtomicBool::new(false), ignore_sources: RwLock::new(Vec::new()), ignore_source_stamps: RwLock::new(IgnoreStamps::new()), @@ -5871,9 +6659,19 @@ mod tests { unreadable: RwLock::new(std::collections::HashMap::new()), started: Instant::now(), last_search_ms: std::sync::atomic::AtomicU64::new(0), + stale_refresh_hook: Mutex::new(None), }) } + fn test_git(root: &Path, args: &[&str]) { + let status = std::process::Command::new("git") + .current_dir(root) + .args(args) + .status() + .expect("run git"); + assert!(status.success(), "git {args:?} failed"); + } + /// A binary marker's offset is a position in the file, not in the repaired /// text. /// @@ -6077,7 +6875,7 @@ mod tests { } #[test] - fn watch_registry_add_all_is_additive_but_sync_prunes() { + fn incomplete_watch_sync_preserves_existing_subscriptions_until_a_complete_pass() { // These two must not be confused. `watch_new_subtree` learns only // about the subtree that just appeared, so if it went through `sync` // every directory outside that subtree would look stale and the server @@ -6115,8 +6913,38 @@ mod tests { "add_all dropped a subscription outside the set it was given" ); - // `sync`, by contrast, is authoritative over the whole tree. - let (added, removed) = registry.sync(&[c.clone()].into_iter().collect(), false); + // A traversal that missed entries is additive only: absence from its + // partial result is not evidence that an existing watch became stale. + let partial: std::collections::HashSet = std::iter::once(c.clone()).collect(); + let pending = std::sync::atomic::AtomicBool::new(true); + let force = take_force_resubscribe(&pending, TraversalCompleteness::Incomplete); + assert!( + !force, + "the incomplete pass must not re-register known entries" + ); + assert!( + pending.load(Ordering::SeqCst), + "an incomplete pass must leave forced resubscription pending" + ); + let (added, removed) = registry.sync(&partial, TraversalCompleteness::Incomplete, force); + assert_eq!((added.len(), removed), (0, 0)); + assert_eq!( + registry.watched, + [a.clone(), b.clone(), c.clone()].into_iter().collect(), + "an incomplete desired set retired valid existing subscriptions" + ); + + // A later complete traversal is authoritative and may prune them. + let force = take_force_resubscribe(&pending, TraversalCompleteness::Complete); + assert!( + force, + "the next complete pass must inherit the force request" + ); + assert!( + !pending.load(Ordering::SeqCst), + "a complete forced pass consumes the request" + ); + let (added, removed) = registry.sync(&partial, TraversalCompleteness::Complete, force); assert_eq!((added.len(), removed), (0, 2)); assert_eq!(registry.watched, [c].into_iter().collect()); } @@ -6221,6 +7049,7 @@ mod tests { let dirs = watchable_dirs(root, root, &exclude, Some(&gi)); let rel: std::collections::HashSet = dirs + .dirs .iter() .map(|p| { p.strip_prefix(root) @@ -6248,79 +7077,1148 @@ mod tests { assert!(!rel.contains(".git")); assert!(!rel.contains(".git/objects")); - // `--exclude` names prune the directory itself, not just its children. - assert!(!rel.contains("vendor"), "excluded dir was watched: {rel:?}"); - assert!(!rel.contains("vendor/pkg")); + // `--exclude` names prune the directory itself, not just its children. + assert!(!rel.contains("vendor"), "excluded dir was watched: {rel:?}"); + assert!(!rel.contains("vendor/pkg")); + } + + #[test] + fn watchable_dirs_without_a_matcher_keeps_everything_visible() { + // `--no-ignore` publishes no matcher. The subscription set must then + // be the whole tree minus hidden paths, matching what the walk indexes; + // silently narrowing it would drop events for files that ARE indexed. + let tmp = TempDir::new().unwrap(); + let root = tmp.path(); + std::fs::create_dir_all(root.join("build/a")).unwrap(); + std::fs::create_dir_all(root.join(".hidden")).unwrap(); + + let dirs = watchable_dirs(root, root, &[], None); + let rel: std::collections::HashSet = dirs + .dirs + .iter() + .map(|p| { + p.strip_prefix(root) + .unwrap() + .to_string_lossy() + .replace('\\', "/") + }) + .collect(); + + assert!(rel.contains("build")); + assert!(rel.contains("build/a")); + assert!(!rel.contains(".hidden")); + } + + #[test] + fn watchable_dirs_anchors_rules_at_the_root_not_the_start_directory() { + // A subtree that appears at runtime is walked from itself, but the + // ignore rules are written against paths relative to the repository + // root. Walking with the subtree as the anchor would test "nested" + // against a rule meant for "src/nested" and prune the wrong things. + let tmp = TempDir::new().unwrap(); + let root = tmp.path(); + std::fs::create_dir(root.join(".git")).unwrap(); + std::fs::write(root.join(".gitignore"), "src/fresh/skipped/\n/keep/\n").unwrap(); + + for dir in ["src/fresh/skipped", "src/fresh/keep", "src/fresh/kept"] { + std::fs::create_dir_all(root.join(dir)).unwrap(); + } + + let gi = tgrep_core::gitignore::build_matcher(root).expect("matcher should build"); + let dirs = watchable_dirs(root, &root.join("src/fresh"), &[], Some(&gi)); + let rel: std::collections::HashSet = dirs + .dirs + .iter() + .map(|p| { + p.strip_prefix(root) + .unwrap() + .to_string_lossy() + .replace('\\', "/") + }) + .collect(); + + assert!( + rel.contains("src/fresh"), + "start dir must be watched: {rel:?}" + ); + assert!(rel.contains("src/fresh/kept")); + assert!( + !rel.contains("src/fresh/skipped"), + "a root-anchored rule was not applied: {rel:?}" + ); + // `keep/` is anchored at the root, so it must NOT prune + // `src/fresh/keep` just because the walk started at `src/fresh`. + assert!( + rel.contains("src/fresh/keep"), + "a root-anchored rule was applied at the wrong depth: {rel:?}" + ); + } + + #[test] + fn rejected_never_indexed_file_does_not_dirty_the_overlay() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + let rejected = root.join("asset.png"); + std::fs::write(&rejected, "not indexed despite textual contents\n").unwrap(); + + let _gate = state.snapshot_gate.read().unwrap(); + reindex_file(&state, &rejected, "asset.png", false); + + let index = state.index.read().unwrap(); + assert_eq!( + index.live.dirty_count(), + 0, + "a path absent from reader, overlay, and stamps must not create a tombstone" + ); + assert!(!index.live.is_deleted("asset.png")); + } + + #[test] + fn reader_path_without_a_stamp_is_still_tombstoned() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + let indexed = root.join("legacy.rs"); + std::fs::write(&indexed, "fn legacy_reader_entry() {}\n").unwrap(); + builder::build_index_for_files(&root, &index_dir, std::slice::from_ref(&indexed), 1024) + .unwrap(); + *state.index.write().unwrap() = HybridIndex::open(&index_dir, &root).unwrap(); + assert!( + state.file_stamps.read().unwrap().is_empty(), + "fixture requires a reader entry with no stamp" + ); + assert!( + state.index.read().unwrap().reader_has_path("legacy.rs"), + "fixture did not put the path in the active reader" + ); + + let _gate = state.snapshot_gate.read().unwrap(); + let _reindex = lock_reindex(&state); + drop_indexed_file(&state, "legacy.rs", "test rejection"); + + let index = state.index.read().unwrap(); + assert!( + index.live.is_deleted("legacy.rs"), + "reader membership must remain sufficient evidence to tombstone" + ); + assert_eq!(index.live.dirty_count(), 1); + } + + #[test] + fn content_only_git_index_rewrite_is_ignored_but_membership_changes_reconcile() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + + test_git(&root, &["init", "--quiet"]); + test_git(&root, &["config", "core.ignorecase", "true"]); + std::fs::write(root.join(".gitignore"), "IGNORED/\n").unwrap(); + std::fs::create_dir_all(root.join("src")).unwrap(); + std::fs::write(root.join("src/lib.rs"), "fn original() {}\n").unwrap(); + std::fs::create_dir_all(root.join("ignored")).unwrap(); + std::fs::write(root.join("ignored/forced.rs"), "fn forced() {}\n").unwrap(); + test_git(&root, &["add", "--", ".gitignore", "src/lib.rs"]); + + let state = test_server_state(&root, &root.join(".tgrep")); + let matcher = + tgrep_core::gitignore::build_matcher(&root).expect("case-insensitive matcher"); + let baseline = matcher + .tracked_membership_fingerprint() + .expect("tracked exemption active"); + *state.gitignore.write().unwrap() = Some(matcher); + *state.tracked_membership.lock().unwrap() = Some(baseline); + + std::fs::write(root.join("src/lib.rs"), "fn content_changed() {}\n").unwrap(); + test_git(&root, &["add", "--", "src/lib.rs"]); + assert!( + !tracked_membership_changed(&state), + "rewriting index metadata with the same tracked paths must not request a full scan" + ); + + std::fs::write(root.join("src/new.rs"), "fn newly_tracked() {}\n").unwrap(); + test_git(&root, &["add", "--", "src/new.rs"]); + assert!( + tracked_membership_changed(&state), + "conservative polling must notice every tracked-membership change" + ); + assert!( + !tracked_membership_changed(&state), + "an observed membership change must be coalesced" + ); + + test_git(&root, &["add", "-f", "--", "ignored/forced.rs"]); + assert!( + tracked_membership_changed(&state), + "adding a tracked-path exemption must request reconciliation" + ); + assert!( + !tracked_membership_changed(&state), + "an observed membership change must be coalesced" + ); + + test_git( + &root, + &[ + "rm", + "--cached", + "--quiet", + "--force", + "--", + "ignored/forced.rs", + ], + ); + assert!( + tracked_membership_changed(&state), + "removing a tracked-path exemption must request reconciliation" + ); + } + + #[test] + fn first_matcher_publication_uses_one_snapshot_across_git_index_aba() { + use std::sync::Barrier; + use std::sync::atomic::AtomicUsize; + + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + test_git(&root, &["init", "--quiet"]); + test_git(&root, &["config", "core.ignorecase", "true"]); + std::fs::write(root.join(".gitignore"), "IGNORED/\n").unwrap(); + std::fs::create_dir_all(root.join("src")).unwrap(); + std::fs::write(root.join("src/lib.rs"), "fn ordinary() {}\n").unwrap(); + let ignored = root.join("ignored"); + std::fs::create_dir_all(&ignored).unwrap(); + std::fs::write(ignored.join("forced.rs"), "fn aba_marker() {}\n").unwrap(); + test_git(&root, &["add", "--", ".gitignore", "src/lib.rs"]); + + let mut state = test_server_state(&root, &root.join(".tgrep")); + Arc::get_mut(&mut state).unwrap().watch_enabled = false; + state.gitignore_pending.store(false, Ordering::SeqCst); + assert!( + state.gitignore.read().unwrap().is_none(), + "the race must cover first matcher publication" + ); + + if PER_DIRECTORY_WATCHES { + let watcher = notify::recommended_watcher(|_: notify::Result| {}).unwrap(); + *state.watch_registry.lock().unwrap() = Some(WatchRegistry { + watcher, + root: root.clone(), + watched: std::iter::once(root.clone()).collect(), + }); + } + + let before_walk_entered = Arc::new(Barrier::new(2)); + let before_walk_release = Arc::new(Barrier::new(2)); + let after_publish_entered = Arc::new(Barrier::new(2)); + let after_publish_release = Arc::new(Barrier::new(2)); + let passes = Arc::new(AtomicUsize::new(0)); + let hook: StaleRefreshHook = { + let before_walk_entered = Arc::clone(&before_walk_entered); + let before_walk_release = Arc::clone(&before_walk_release); + let after_publish_entered = Arc::clone(&after_publish_entered); + let after_publish_release = Arc::clone(&after_publish_release); + let passes = Arc::clone(&passes); + Arc::new(move |phase| match phase { + StaleRefreshPhase::BeforeWalk => { + if passes.fetch_add(1, Ordering::SeqCst) == 0 { + before_walk_entered.wait(); + before_walk_release.wait(); + } + } + StaleRefreshPhase::AfterMatcherPublish => { + if passes.load(Ordering::SeqCst) == 1 { + after_publish_entered.wait(); + after_publish_release.wait(); + } + } + StaleRefreshPhase::AfterBuildBeforeStampPublish => {} + StaleRefreshPhase::AfterConcreteRead => {} + StaleRefreshPhase::BeforeConcreteCommit => {} + }) + }; + *state.stale_refresh_hook.lock().unwrap() = Some(hook); + + let refresh_state = Arc::clone(&state); + let refresh_root = root.clone(); + let index_dir = state.index_dir.clone(); + let refresh = thread::spawn(move || { + background_refresh_stale(&refresh_state, &refresh_root, &index_dir, true) + }); + + // A: untracked and hidden when the pass captures its immutable set. + before_walk_entered.wait(); + // B: the index changes, but this pass must continue using A throughout. + test_git(&root, &["add", "-f", "--", "ignored/forced.rs"]); + before_walk_release.wait(); + after_publish_entered.wait(); + assert!( + state + .gitignore + .read() + .unwrap() + .as_ref() + .unwrap() + .is_ignored(Path::new("ignored"), true), + "the published matcher must use the same A snapshot as the walk" + ); + if PER_DIRECTORY_WATCHES { + assert!( + !state + .watch_registry + .lock() + .unwrap() + .as_ref() + .unwrap() + .watched + .contains(&ignored), + "the B transition must not change subscriptions mid-pass" + ); + } + // A again: the pass already represents the final membership, so no + // corrective pass is necessary. + test_git( + &root, + &[ + "rm", + "--cached", + "--quiet", + "--force", + "--", + "ignored/forced.rs", + ], + ); + after_publish_release.wait(); + assert!( + refresh.join().unwrap(), + "the raced pass itself should finish" + ); + + thread::sleep(Duration::from_millis(100)); + assert!( + passes.load(Ordering::SeqCst) == 1, + "A→B→A should not need a retry when the first pass used A throughout" + ); + assert!( + !state.ignore_refresh_scheduled.load(Ordering::SeqCst) + && !state.ignore_rules_dirty.load(Ordering::SeqCst), + "the semantic baseline returned to A" + ); + + let matcher = state.gitignore.read().unwrap(); + assert!( + matcher + .as_ref() + .unwrap() + .is_ignored(Path::new("ignored"), true), + "the final matcher must restore the final untracked state" + ); + drop(matcher); + let index = state.index.read().unwrap(); + assert!( + !index.live.has_path("ignored/forced.rs") + && (!index.reader_has_path("ignored/forced.rs") + || index.live.is_deleted("ignored/forced.rs")), + "the immutable A walk must not index content from intermediate B" + ); + drop(index); + if PER_DIRECTORY_WATCHES { + assert!( + !state + .watch_registry + .lock() + .unwrap() + .as_ref() + .unwrap() + .watched + .contains(&ignored), + "the immutable A matcher must leave the ignored tree unsubscribed" + ); + } + } + + #[test] + fn content_only_index_churn_during_refresh_does_not_chain_reconciles() { + use std::sync::atomic::AtomicUsize; + + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + test_git(&root, &["init", "--quiet"]); + test_git(&root, &["config", "core.ignorecase", "true"]); + std::fs::write(root.join(".gitignore"), "IGNORED/\n").unwrap(); + std::fs::create_dir_all(root.join("src")).unwrap(); + std::fs::write(root.join("src/lib.rs"), "fn original() {}\n").unwrap(); + test_git(&root, &["add", "--", ".gitignore", "src/lib.rs"]); + + let state = test_server_state(&root, &root.join(".tgrep")); + state.gitignore_pending.store(false, Ordering::SeqCst); + let passes = Arc::new(AtomicUsize::new(0)); + let hook: StaleRefreshHook = { + let root = root.clone(); + let passes = Arc::clone(&passes); + Arc::new(move |phase| match phase { + StaleRefreshPhase::BeforeWalk => { + passes.fetch_add(1, Ordering::SeqCst); + } + StaleRefreshPhase::AfterMatcherPublish => { + let pass = passes.load(Ordering::SeqCst); + if pass <= 4 { + std::fs::write( + root.join("src/lib.rs"), + format!("fn content_only_{pass}() {{}}\n"), + ) + .unwrap(); + test_git(&root, &["add", "--", "src/lib.rs"]); + } + } + StaleRefreshPhase::AfterBuildBeforeStampPublish => {} + StaleRefreshPhase::AfterConcreteRead => {} + StaleRefreshPhase::BeforeConcreteCommit => {} + }) + }; + *state.stale_refresh_hook.lock().unwrap() = Some(hook); + + assert!(background_refresh_stale( + &state, + &root, + &state.index_dir, + true + )); + thread::sleep(Duration::from_millis(250)); + + assert_eq!( + passes.load(Ordering::SeqCst), + 1, + "metadata churn with unchanged relevant membership must not chain full scans" + ); + assert!( + !state.ignore_refresh_scheduled.load(Ordering::SeqCst) + && !state.ignore_rules_dirty.load(Ordering::SeqCst) + ); + } + + #[test] + fn membership_change_during_refresh_schedules_one_corrective_pass() { + use std::sync::Barrier; + use std::sync::atomic::AtomicUsize; + + let tmp = TempDir::new().unwrap(); + let root = tmp.path().to_path_buf(); + test_git(&root, &["init", "--quiet"]); + test_git(&root, &["config", "core.ignorecase", "true"]); + std::fs::write(root.join(".gitignore"), "IGNORED/\n").unwrap(); + let ignored = root.join("ignored"); + std::fs::create_dir_all(&ignored).unwrap(); + std::fs::write(ignored.join("forced.rs"), "fn newly_exempt() {}\n").unwrap(); + test_git(&root, &["add", "--", ".gitignore"]); + + let state = test_server_state(&root, &root.join(".tgrep")); + state.gitignore_pending.store(false, Ordering::SeqCst); + if PER_DIRECTORY_WATCHES { + let watcher = notify::recommended_watcher(|_: notify::Result| {}).unwrap(); + *state.watch_registry.lock().unwrap() = Some(WatchRegistry { + watcher, + root: root.clone(), + watched: std::iter::once(root.clone()).collect(), + }); + } + + let before_walk_entered = Arc::new(Barrier::new(2)); + let before_walk_release = Arc::new(Barrier::new(2)); + let passes = Arc::new(AtomicUsize::new(0)); + let hook: StaleRefreshHook = { + let before_walk_entered = Arc::clone(&before_walk_entered); + let before_walk_release = Arc::clone(&before_walk_release); + let passes = Arc::clone(&passes); + Arc::new(move |phase| { + if matches!(phase, StaleRefreshPhase::BeforeWalk) + && passes.fetch_add(1, Ordering::SeqCst) == 0 + { + before_walk_entered.wait(); + before_walk_release.wait(); + } + }) + }; + *state.stale_refresh_hook.lock().unwrap() = Some(hook); + + let refresh_state = Arc::clone(&state); + let refresh_root = root.clone(); + let index_dir = state.index_dir.clone(); + let refresh = thread::spawn(move || { + background_refresh_stale(&refresh_state, &refresh_root, &index_dir, true) + }); + + before_walk_entered.wait(); + test_git(&root, &["add", "-f", "--", "ignored/forced.rs"]); + before_walk_release.wait(); + assert!(refresh.join().unwrap()); + + let deadline = Instant::now() + Duration::from_secs(10); + while (passes.load(Ordering::SeqCst) < 2 + || state.ignore_refresh_scheduled.load(Ordering::SeqCst) + || state.ignore_rules_dirty.load(Ordering::SeqCst)) + && Instant::now() < deadline + { + thread::sleep(Duration::from_millis(10)); + } + + assert_eq!( + passes.load(Ordering::SeqCst), + 2, + "a semantic A→B change must schedule exactly one corrective pass" + ); + assert!( + !state + .gitignore + .read() + .unwrap() + .as_ref() + .unwrap() + .is_ignored(Path::new("ignored"), true) + ); + let index = state.index.read().unwrap(); + assert!( + index.reader_has_path("ignored/forced.rs") || index.live.has_path("ignored/forced.rs"), + "the corrective pass must index the newly exempt file" + ); + drop(index); + if PER_DIRECTORY_WATCHES { + assert!( + state + .watch_registry + .lock() + .unwrap() + .as_ref() + .unwrap() + .watched + .contains(&ignored), + "the corrective pass must subscribe the newly exempt directory" + ); + } + } + + #[test] + fn rpc_reload_uses_one_snapshot_across_git_index_aba() { + use std::sync::Barrier; + use std::sync::atomic::AtomicUsize; + + let tmp = TempDir::new().unwrap(); + let root = std::fs::canonicalize(tmp.path()).unwrap(); + test_git(&root, &["init", "--quiet"]); + test_git(&root, &["config", "core.ignorecase", "true"]); + std::fs::write(root.join(".gitignore"), "IGNORED/\n").unwrap(); + let ignored = root.join("ignored"); + std::fs::create_dir_all(&ignored).unwrap(); + std::fs::write(ignored.join("forced.rs"), "fn reload_aba() {}\n").unwrap(); + test_git(&root, &["add", "--", ".gitignore"]); + + let mut state = test_server_state(&root, &root.join(".tgrep")); + Arc::get_mut(&mut state).unwrap().watch_enabled = false; + state.gitignore_pending.store(false, Ordering::SeqCst); + + let before_walk_entered = Arc::new(Barrier::new(2)); + let before_walk_release = Arc::new(Barrier::new(2)); + let after_publish_entered = Arc::new(Barrier::new(2)); + let after_publish_release = Arc::new(Barrier::new(2)); + let passes = Arc::new(AtomicUsize::new(0)); + let hook: StaleRefreshHook = { + let before_walk_entered = Arc::clone(&before_walk_entered); + let before_walk_release = Arc::clone(&before_walk_release); + let after_publish_entered = Arc::clone(&after_publish_entered); + let after_publish_release = Arc::clone(&after_publish_release); + let passes = Arc::clone(&passes); + Arc::new(move |phase| match phase { + StaleRefreshPhase::BeforeWalk => { + if passes.fetch_add(1, Ordering::SeqCst) == 0 { + before_walk_entered.wait(); + before_walk_release.wait(); + } + } + StaleRefreshPhase::AfterMatcherPublish => { + if passes.load(Ordering::SeqCst) == 1 { + after_publish_entered.wait(); + after_publish_release.wait(); + } + } + StaleRefreshPhase::AfterBuildBeforeStampPublish => {} + StaleRefreshPhase::AfterConcreteRead => {} + StaleRefreshPhase::BeforeConcreteCommit => {} + }) + }; + *state.stale_refresh_hook.lock().unwrap() = Some(hook); + + let reload_state = Arc::clone(&state); + let reload = thread::spawn(move || handle_reload(None, &reload_state)); + before_walk_entered.wait(); + test_git(&root, &["add", "-f", "--", "ignored/forced.rs"]); + before_walk_release.wait(); + after_publish_entered.wait(); + test_git( + &root, + &[ + "rm", + "--cached", + "--quiet", + "--force", + "--", + "ignored/forced.rs", + ], + ); + after_publish_release.wait(); + + let response = reload.join().unwrap(); + assert!(response.contains("\"status\":\"reloaded\""), "{response}"); + thread::sleep(Duration::from_millis(100)); + assert_eq!( + passes.load(Ordering::SeqCst), + 1, + "A→B→A must not need a correction when reload used A throughout" + ); + assert!( + state + .gitignore + .read() + .unwrap() + .as_ref() + .unwrap() + .is_ignored(Path::new("ignored"), true) + ); + let index = state.index.read().unwrap(); + assert!( + !index.reader_has_path("ignored/forced.rs") + && !index.live.has_path("ignored/forced.rs"), + "reload index and matcher must both represent final A" + ); + assert!( + !state.ignore_refresh_scheduled.load(Ordering::SeqCst) + && !state.ignore_rules_dirty.load(Ordering::SeqCst) + ); + } + + #[test] + fn rpc_reload_membership_change_schedules_one_corrective_pass() { + use std::sync::Barrier; + use std::sync::atomic::AtomicUsize; + + let tmp = TempDir::new().unwrap(); + let root = std::fs::canonicalize(tmp.path()).unwrap(); + test_git(&root, &["init", "--quiet"]); + test_git(&root, &["config", "core.ignorecase", "true"]); + std::fs::write(root.join(".gitignore"), "IGNORED/\n").unwrap(); + std::fs::create_dir_all(root.join("ignored")).unwrap(); + std::fs::write( + root.join("ignored/forced.rs"), + "fn reload_membership_change() {}\n", + ) + .unwrap(); + test_git(&root, &["add", "--", ".gitignore"]); + + let mut state = test_server_state(&root, &root.join(".tgrep")); + Arc::get_mut(&mut state).unwrap().watch_enabled = false; + state.gitignore_pending.store(false, Ordering::SeqCst); + let before_walk_entered = Arc::new(Barrier::new(2)); + let before_walk_release = Arc::new(Barrier::new(2)); + let passes = Arc::new(AtomicUsize::new(0)); + let hook: StaleRefreshHook = { + let before_walk_entered = Arc::clone(&before_walk_entered); + let before_walk_release = Arc::clone(&before_walk_release); + let passes = Arc::clone(&passes); + Arc::new(move |phase| { + if matches!(phase, StaleRefreshPhase::BeforeWalk) + && passes.fetch_add(1, Ordering::SeqCst) == 0 + { + before_walk_entered.wait(); + before_walk_release.wait(); + } + }) + }; + *state.stale_refresh_hook.lock().unwrap() = Some(hook); + + let reload_state = Arc::clone(&state); + let reload = thread::spawn(move || handle_reload(None, &reload_state)); + before_walk_entered.wait(); + test_git(&root, &["add", "-f", "--", "ignored/forced.rs"]); + before_walk_release.wait(); + let response = reload.join().unwrap(); + assert!(response.contains("\"status\":\"reloaded\""), "{response}"); + + let deadline = Instant::now() + Duration::from_secs(10); + while (passes.load(Ordering::SeqCst) < 2 + || state.ignore_refresh_scheduled.load(Ordering::SeqCst) + || state.ignore_rules_dirty.load(Ordering::SeqCst)) + && Instant::now() < deadline + { + thread::sleep(Duration::from_millis(10)); + } + + assert_eq!( + passes.load(Ordering::SeqCst), + 2, + "A→B during reload must schedule exactly one corrective pass" + ); + assert!( + !state + .gitignore + .read() + .unwrap() + .as_ref() + .unwrap() + .is_ignored(Path::new("ignored"), true) + ); + let index = state.index.read().unwrap(); + assert!( + index.reader_has_path("ignored/forced.rs") || index.live.has_path("ignored/forced.rs"), + "the correction must make reload's index agree with final B" + ); + } + + #[test] + fn rpc_reload_without_watcher_repairs_change_after_extraction() { + let tmp = TempDir::new().unwrap(); + let root = std::fs::canonicalize(tmp.path()).unwrap(); + test_git(&root, &["init", "--quiet"]); + let path = root.join("raced.rs"); + std::fs::write(&path, "fn old_reload_marker() {}\n").unwrap(); + + let mut state = test_server_state(&root, &root.join(".tgrep")); + Arc::get_mut(&mut state).unwrap().watch_enabled = false; + state.gitignore_pending.store(false, Ordering::SeqCst); + let hook_path = path.clone(); + *state.stale_refresh_hook.lock().unwrap() = Some(Arc::new(move |phase| { + if matches!(phase, StaleRefreshPhase::AfterBuildBeforeStampPublish) { + std::fs::write(&hook_path, "fn new_reload_marker() {}\n").unwrap(); + } + })); + + let response = handle_reload(None, &state); + assert!(response.contains("\"status\":\"reloaded\""), "{response}"); + let result = handle_search( + None, + &serde_json::json!({"pattern": "new_reload_marker"}), + &state, + ); + assert!( + result.contains("new_reload_marker"), + "the no-watch catch-up must index the bytes written after extraction: {result}" + ); + } + + #[test] + fn unwatched_external_bootstrap_repairs_change_after_extraction() { + let tmp = TempDir::new().unwrap(); + let root = std::fs::canonicalize(tmp.path()).unwrap(); + test_git(&root, &["init", "--quiet"]); + let path = root.join("raced.rs"); + std::fs::write(&path, "fn old_bootstrap_marker() {}\n").unwrap(); + + let mut state = test_server_state(&root, &root.join(".tgrep")); + Arc::get_mut(&mut state).unwrap().watch_enabled = false; + state.indexing.store(true, Ordering::SeqCst); + state.gitignore_pending.store(false, Ordering::SeqCst); + let hook_path = path.clone(); + *state.stale_refresh_hook.lock().unwrap() = Some(Arc::new(move |phase| { + if matches!(phase, StaleRefreshPhase::AfterBuildBeforeStampPublish) { + std::fs::write(&hook_path, "fn new_bootstrap_marker() {}\n").unwrap(); + } + })); + + assert!(bootstrap_index_build(&state, &root, &state.index_dir)); + let result = handle_search( + None, + &serde_json::json!({"pattern": "new_bootstrap_marker"}), + &state, + ); + assert!( + result.contains("new_bootstrap_marker"), + "the no-watch bootstrap catch-up must index the final bytes: {result}" + ); + } + + #[test] + fn unwatched_resumed_build_repairs_change_after_extraction() { + let tmp = TempDir::new().unwrap(); + let root = std::fs::canonicalize(tmp.path()).unwrap(); + test_git(&root, &["init", "--quiet"]); + std::fs::write(root.join("seeded.rs"), "fn seeded() {}\n").unwrap(); + let mut state = test_server_state(&root, &root.join(".tgrep")); + Arc::get_mut(&mut state).unwrap().watch_enabled = false; + builder::build_index_with_options( + &root, + Some(&state.index_dir), + &builder::BuildOptions { + no_require_git: false, + ..Default::default() + }, + ) + .unwrap(); + *state.index.write().unwrap() = HybridIndex::open(&state.index_dir, &root).unwrap(); + + let path = root.join("raced.rs"); + std::fs::write(&path, "fn old_resumed_marker() {}\n").unwrap(); + state.indexing.store(true, Ordering::SeqCst); + state.gitignore_pending.store(false, Ordering::SeqCst); + let hook_path = path.clone(); + *state.stale_refresh_hook.lock().unwrap() = Some(Arc::new(move |phase| { + if matches!(phase, StaleRefreshPhase::AfterBuildBeforeStampPublish) { + std::fs::write(&hook_path, "fn new_resumed_marker() {}\n").unwrap(); + } + })); + + background_index_build(&state, &root, &state.index_dir); + let result = handle_search( + None, + &serde_json::json!({"pattern": "new_resumed_marker"}), + &state, + ); + assert!( + result.contains("new_resumed_marker"), + "the no-watch resumed-build catch-up must index the final bytes: {result}" + ); + } + + #[test] + fn rpc_reload_replays_event_received_after_extraction() { + let tmp = TempDir::new().unwrap(); + let root = std::fs::canonicalize(tmp.path()).unwrap(); + test_git(&root, &["init", "--quiet"]); + let path = root.join("raced.rs"); + std::fs::write(&path, "fn old_watched_marker() {}\n").unwrap(); + let state = test_server_state(&root, &root.join(".tgrep")); + state.gitignore_pending.store(false, Ordering::SeqCst); + let prior = root.join("prior.rs"); + state + .deferred_events + .lock() + .unwrap() + .as_mut() + .unwrap() + .insert(prior.clone(), false); + + let hook_path = path.clone(); + let hook_state = Arc::clone(&state); + *state.stale_refresh_hook.lock().unwrap() = Some(Arc::new(move |phase| { + if matches!(phase, StaleRefreshPhase::AfterBuildBeforeStampPublish) { + assert!( + hook_state + .deferred_events + .lock() + .unwrap() + .as_ref() + .unwrap() + .contains_key(&prior), + "reload must preserve events awaiting an earlier build replay" + ); + std::fs::write(&hook_path, "fn new_watched_marker() {}\n").unwrap(); + assert!(defer_events_during_build( + &hook_state, + &Event { + kind: EventKind::Modify(notify::event::ModifyKind::Data( + notify::event::DataChange::Any, + )), + paths: vec![hook_path.clone()], + attrs: Default::default(), + } + )); + } + })); + + let response = handle_reload(None, &state); + assert!(response.contains("\"status\":\"reloaded\""), "{response}"); + let result = handle_search( + None, + &serde_json::json!({"pattern": "new_watched_marker"}), + &state, + ); + assert!( + result.contains("new_watched_marker"), + "the replay must force the concrete event despite the later matching stamp: {result}" + ); + } + + #[test] + fn rpc_reload_schedules_ignore_rule_change_received_during_build() { + let tmp = TempDir::new().unwrap(); + let root = std::fs::canonicalize(tmp.path()).unwrap(); + test_git(&root, &["init", "--quiet"]); + let rules = root.join(".gitignore"); + std::fs::write(&rules, "").unwrap(); + std::fs::create_dir(root.join("ignored")).unwrap(); + std::fs::write(root.join("ignored/file.rs"), "fn should_disappear() {}\n").unwrap(); + let state = test_server_state(&root, &root.join(".tgrep")); + state.gitignore_pending.store(false, Ordering::SeqCst); + + let hook_rules = rules.clone(); + let hook_root = root.clone(); + let hook_state = Arc::clone(&state); + *state.stale_refresh_hook.lock().unwrap() = Some(Arc::new(move |phase| { + if matches!(phase, StaleRefreshPhase::AfterBuildBeforeStampPublish) { + std::fs::write(&hook_rules, "ignored/\n").unwrap(); + handle_fs_event( + &hook_state, + &hook_root, + &Event { + kind: EventKind::Modify(notify::event::ModifyKind::Data( + notify::event::DataChange::Any, + )), + paths: vec![hook_rules.clone()], + attrs: Default::default(), + }, + ); + } + })); + + let response = handle_reload(None, &state); + assert!(response.contains("\"status\":\"reloaded\""), "{response}"); + let deadline = Instant::now() + Duration::from_secs(10); + while (state.ignore_refresh_scheduled.load(Ordering::SeqCst) + || state.ignore_rules_dirty.load(Ordering::SeqCst)) + && Instant::now() < deadline + { + thread::sleep(Duration::from_millis(10)); + } + + assert!( + !state.ignore_refresh_scheduled.load(Ordering::SeqCst) + && !state.ignore_rules_dirty.load(Ordering::SeqCst), + "the rule change observed during reload must run a serialized refresh" + ); + let index = state.index.read().unwrap(); + assert!( + !index.reader_has_path("ignored/file.rs") || index.live.is_deleted("ignored/file.rs"), + "the refresh must remove content hidden by the new rule" + ); + } + + #[test] + fn concrete_event_reindexes_equal_length_rewrite_with_matching_stamp() { + let tmp = TempDir::new().unwrap(); + let root = std::fs::canonicalize(tmp.path()).unwrap(); + let path = root.join("same.rs"); + std::fs::write(&path, "fn old_marker() {}\n").unwrap(); + let state = test_server_state(&root, &root.join(".tgrep")); + state.gitignore_pending.store(false, Ordering::SeqCst); + + { + let _gate = state.snapshot_gate.read().unwrap(); + reindex_file(&state, &path, "same.rs", false); + } + std::fs::write(&path, "fn new_marker() {}\n").unwrap(); + let current = tgrep_core::meta::collect_filestamps(&root, &["same.rs".to_string()]) + .remove("same.rs") + .unwrap(); + state + .file_stamps + .write() + .unwrap() + .insert("same.rs".to_string(), current); + + // Mutation control: the speculative path still trusts the persisted + // stamp and therefore leaves the old posting set in place. + { + let _gate = state.snapshot_gate.read().unwrap(); + reindex_file(&state, &path, "same.rs", false); + } + let stale = handle_search(None, &serde_json::json!({"pattern": "new_marker"}), &state); + assert!( + !stale.contains("\"content\":\"fn new_marker"), + "the control must demonstrate that stamp-only filtering misses the rewrite" + ); + + handle_fs_event( + &state, + &root, + &Event { + kind: EventKind::Modify(notify::event::ModifyKind::Data( + notify::event::DataChange::Any, + )), + paths: vec![path], + attrs: Default::default(), + }, + ); + let repaired = handle_search(None, &serde_json::json!({"pattern": "new_marker"}), &state); + assert!( + repaired.contains("\"content\":\"fn new_marker"), + "a concrete event must bypass the coarse matching stamp: {repaired}" + ); + } + + #[test] + fn concrete_event_rejects_change_after_verification_before_commit() { + let tmp = TempDir::new().unwrap(); + let root = std::fs::canonicalize(tmp.path()).unwrap(); + let path = root.join("raced.rs"); + std::fs::write(&path, "fn old_marker() {}\n").unwrap(); + let state = test_server_state(&root, &root.join(".tgrep")); + state.gitignore_pending.store(false, Ordering::SeqCst); + { + let _gate = state.snapshot_gate.read().unwrap(); + reindex_file(&state, &path, "raced.rs", false); + } + let dirty_before = state.index.read().unwrap().live.dirty_count(); + std::fs::write(&path, "fn first_new_() {}\n").unwrap(); + + state.indexing.store(true, Ordering::SeqCst); + let changed = path.clone(); + *state.stale_refresh_hook.lock().unwrap() = Some(Arc::new(move |phase| { + if matches!(phase, StaleRefreshPhase::BeforeConcreteCommit) { + std::fs::write(&changed, "fn final_new_() {}\n").unwrap(); + } + })); + { + let _gate = state.snapshot_gate.read().unwrap(); + reindex_file(&state, &path, "raced.rs", true); + } + + assert_eq!( + state.index.read().unwrap().live.dirty_count(), + dirty_before, + "bytes invalidated after verification must not be committed" + ); + assert_eq!( + state.file_stamps.read().unwrap().get("raced.rs"), + Some(&tgrep_core::meta::FileStamp { + mtime: u64::MAX, + size: u64::MAX, + }), + "a rejected final version must remain scheduled for correction" + ); + + *state.stale_refresh_hook.lock().unwrap() = None; + state.indexing.store(false, Ordering::SeqCst); + let deadline = Instant::now() + Duration::from_secs(10); + while state.ignore_refresh_scheduled.load(Ordering::SeqCst) && Instant::now() < deadline { + thread::sleep(Duration::from_millis(10)); + } + } + + #[test] + fn rpc_reload_build_failure_preserves_active_index() { + let tmp = TempDir::new().unwrap(); + let root = std::fs::canonicalize(tmp.path()).unwrap(); + let index_dir = root.join(".tgrep"); + let state = test_server_state(&root, &index_dir); + state + .index + .write() + .unwrap() + .live + .upsert_file("active.rs", b"fn active_before_reload() {}\n"); + + std::fs::write(index_dir.join(".reload-build"), b"blocks staging directory").unwrap(); + let response = handle_reload(None, &state); + + assert!(response.contains("rebuild failed"), "{response}"); + assert!( + state.index.read().unwrap().live.has_path("active.rs"), + "a failed staged build must not disturb the active index" + ); + } + + #[test] + fn rpc_reload_waits_for_initial_build() { + use std::sync::mpsc::{RecvTimeoutError, channel}; + + let tmp = TempDir::new().unwrap(); + let root = std::fs::canonicalize(tmp.path()).unwrap(); + let state = test_server_state(&root, &root.join(".tgrep")); + state.indexing.store(true, Ordering::SeqCst); + let (entered_tx, entered_rx) = channel(); + *state.stale_refresh_hook.lock().unwrap() = Some(Arc::new(move |phase| { + if matches!(phase, StaleRefreshPhase::BeforeWalk) { + entered_tx.send(()).unwrap(); + } + })); + + let reload_state = Arc::clone(&state); + let reload = thread::spawn(move || handle_reload(None, &reload_state)); + assert!(matches!( + entered_rx.recv_timeout(Duration::from_millis(150)), + Err(RecvTimeoutError::Timeout) + )); + + state.indexing.store(false, Ordering::SeqCst); + entered_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + let response = reload.join().unwrap(); + assert!(response.contains("\"status\":\"reloaded\""), "{response}"); } #[test] - fn watchable_dirs_without_a_matcher_keeps_everything_visible() { - // `--no-ignore` publishes no matcher. The subscription set must then - // be the whole tree minus hidden paths, matching what the walk indexes; - // silently narrowing it would drop events for files that ARE indexed. + fn pre_reload_disk_read_cannot_repopulate_content_cache() { let tmp = TempDir::new().unwrap(); - let root = tmp.path(); - std::fs::create_dir_all(root.join("build/a")).unwrap(); - std::fs::create_dir_all(root.join(".hidden")).unwrap(); + let root = std::fs::canonicalize(tmp.path()).unwrap(); + let state = test_server_state(&root, &root.join(".tgrep")); + let before_reload = state.cache_generation.load(Ordering::SeqCst); + let stale = vec![( + "stale.rs".to_string(), + Arc::new(DecodedFile::new( + b"old content".to_vec(), + tgrep_core::encoding::EncodingMode::Auto, + )), + )]; - let dirs = watchable_dirs(root, root, &[], None); - let rel: std::collections::HashSet = dirs - .iter() - .map(|p| { - p.strip_prefix(root) - .unwrap() - .to_string_lossy() - .replace('\\', "/") - }) - .collect(); + invalidate_cached_paths(&state, std::iter::once("stale.rs")); + update_content_cache(&state, before_reload, &[], &stale); + assert!( + state.cache.read().unwrap().peek("stale.rs").is_none(), + "a disk read from the previous index generation must not refill the cache" + ); - assert!(rel.contains("build")); - assert!(rel.contains("build/a")); - assert!(!rel.contains(".hidden")); + let current = state.cache_generation.load(Ordering::SeqCst); + update_content_cache(&state, current, &[], &stale); + assert!(state.cache.read().unwrap().peek("stale.rs").is_some()); } #[test] - fn watchable_dirs_anchors_rules_at_the_root_not_the_start_directory() { - // A subtree that appears at runtime is walked from itself, but the - // ignore rules are written against paths relative to the repository - // root. Walking with the subtree as the anchor would test "nested" - // against a rule meant for "src/nested" and prune the wrong things. - let tmp = TempDir::new().unwrap(); - let root = tmp.path(); - std::fs::create_dir(root.join(".git")).unwrap(); - std::fs::write(root.join(".gitignore"), "src/fresh/skipped/\n/keep/\n").unwrap(); + fn tracked_membership_poll_waits_for_reconcile_publication() { + use std::sync::mpsc::{RecvTimeoutError, channel}; - for dir in ["src/fresh/skipped", "src/fresh/keep", "src/fresh/kept"] { - std::fs::create_dir_all(root.join(dir)).unwrap(); - } + let tmp = TempDir::new().unwrap(); + let root = std::fs::canonicalize(tmp.path()).unwrap(); + test_git(&root, &["init", "--quiet"]); + test_git(&root, &["config", "core.ignorecase", "true"]); + std::fs::write(root.join(".gitignore"), "IGNORED/\n").unwrap(); + std::fs::create_dir_all(root.join("ignored")).unwrap(); + std::fs::write(root.join("ignored/forced.rs"), "fn transient() {}\n").unwrap(); + test_git(&root, &["add", "--", ".gitignore"]); + + let state = test_server_state(&root, &root.join(".tgrep")); + state.gitignore_pending.store(false, Ordering::SeqCst); + assert!(background_refresh_stale( + &state, + &root, + &state.index_dir, + true + )); - let gi = tgrep_core::gitignore::build_matcher(root).expect("matcher should build"); - let dirs = watchable_dirs(root, &root.join("src/fresh"), &[], Some(&gi)); - let rel: std::collections::HashSet = dirs - .iter() - .map(|p| { - p.strip_prefix(root) - .unwrap() - .to_string_lossy() - .replace('\\', "/") - }) - .collect(); + let gate = state.snapshot_gate.write().unwrap(); + test_git(&root, &["add", "-f", "--", "ignored/forced.rs"]); + let (result_tx, result_rx) = channel(); + let poll_state = Arc::clone(&state); + let poll = thread::spawn(move || { + result_tx + .send(poll_tracked_membership_changed(&poll_state)) + .unwrap(); + }); + assert!(matches!( + result_rx.recv_timeout(Duration::from_millis(150)), + Err(RecvTimeoutError::Timeout) + )); - assert!( - rel.contains("src/fresh"), - "start dir must be watched: {rel:?}" - ); - assert!(rel.contains("src/fresh/kept")); - assert!( - !rel.contains("src/fresh/skipped"), - "a root-anchored rule was not applied: {rel:?}" - ); - // `keep/` is anchored at the root, so it must NOT prune - // `src/fresh/keep` just because the walk started at `src/fresh`. - assert!( - rel.contains("src/fresh/keep"), - "a root-anchored rule was applied at the wrong depth: {rel:?}" + test_git( + &root, + &[ + "rm", + "--cached", + "--quiet", + "--force", + "--", + "ignored/forced.rs", + ], ); + drop(gate); + assert!(!result_rx.recv_timeout(Duration::from_secs(5)).unwrap()); + poll.join().unwrap(); } /// A transient stat failure is not a deletion. `Path::exists` said it was, @@ -6790,6 +8688,148 @@ mod tests { ); } + #[cfg(unix)] + #[test] + fn concrete_event_rejects_file_swapped_to_outside_symlink_between_reads() { + let outside = TempDir::new().unwrap(); + let target = outside.path().join("outside.rs"); + std::fs::write(&target, "fn outside_file_marker() {}\n").unwrap(); + let root_dir = TempDir::new().unwrap(); + let root = std::fs::canonicalize(root_dir.path()).unwrap(); + let path = root.join("raced.rs"); + std::fs::write(&path, "fn inside_file_marker() {}\n").unwrap(); + let state = test_server_state(&root, &root.join(".tgrep")); + state.gitignore_pending.store(false, Ordering::SeqCst); + { + let _gate = state.snapshot_gate.read().unwrap(); + reindex_file(&state, &path, "raced.rs", false); + } + + let swap_path = path.clone(); + let swap_target = target.clone(); + *state.stale_refresh_hook.lock().unwrap() = Some(Arc::new(move |phase| { + if matches!(phase, StaleRefreshPhase::AfterConcreteRead) { + std::fs::remove_file(&swap_path).unwrap(); + std::os::unix::fs::symlink(&swap_target, &swap_path).unwrap(); + } + })); + handle_fs_event( + &state, + &root, + &Event { + kind: EventKind::Modify(notify::event::ModifyKind::Data( + notify::event::DataChange::Any, + )), + paths: vec![path], + attrs: Default::default(), + }, + ); + + let result = handle_search( + None, + &serde_json::json!({"pattern": "outside_file_marker"}), + &state, + ); + assert!( + !result.contains("outside_file_marker"), + "verification must not follow a replacement symlink outside the root" + ); + assert!(state.index.read().unwrap().live.is_deleted("raced.rs")); + } + + #[cfg(unix)] + #[test] + fn concrete_event_rejects_ancestor_swapped_to_outside_symlink_between_reads() { + let outside = TempDir::new().unwrap(); + std::fs::write( + outside.path().join("raced.rs"), + "fn outside_ancestor_marker() {}\n", + ) + .unwrap(); + let root_dir = TempDir::new().unwrap(); + let root = std::fs::canonicalize(root_dir.path()).unwrap(); + let dir = root.join("dir"); + std::fs::create_dir(&dir).unwrap(); + let path = dir.join("raced.rs"); + std::fs::write(&path, "fn inside_ancestor_marker() {}\n").unwrap(); + let state = test_server_state(&root, &root.join(".tgrep")); + state.gitignore_pending.store(false, Ordering::SeqCst); + { + let _gate = state.snapshot_gate.read().unwrap(); + reindex_file(&state, &path, "dir/raced.rs", false); + } + + let swap_dir = dir.clone(); + let moved_dir = root.join("original-dir"); + let outside_dir = outside.path().to_path_buf(); + *state.stale_refresh_hook.lock().unwrap() = Some(Arc::new(move |phase| { + if matches!(phase, StaleRefreshPhase::AfterConcreteRead) { + std::fs::rename(&swap_dir, &moved_dir).unwrap(); + std::os::unix::fs::symlink(&outside_dir, &swap_dir).unwrap(); + } + })); + handle_fs_event( + &state, + &root, + &Event { + kind: EventKind::Modify(notify::event::ModifyKind::Data( + notify::event::DataChange::Any, + )), + paths: vec![path], + attrs: Default::default(), + }, + ); + + let result = handle_search( + None, + &serde_json::json!({"pattern": "outside_ancestor_marker"}), + &state, + ); + assert!( + !result.contains("outside_ancestor_marker"), + "verification must not traverse a replacement ancestor symlink" + ); + assert!(state.index.read().unwrap().live.is_deleted("dir/raced.rs")); + } + + #[test] + fn concrete_event_caps_growth_between_verification_reads() { + let tmp = TempDir::new().unwrap(); + let root = std::fs::canonicalize(tmp.path()).unwrap(); + let path = root.join("grows.rs"); + std::fs::write(&path, "fn small() {}\n").unwrap(); + let mut state = test_server_state(&root, &root.join(".tgrep")); + Arc::get_mut(&mut state).unwrap().max_file_size = Some(64); + state.gitignore_pending.store(false, Ordering::SeqCst); + { + let _gate = state.snapshot_gate.read().unwrap(); + reindex_file(&state, &path, "grows.rs", false); + } + + let grow_path = path.clone(); + *state.stale_refresh_hook.lock().unwrap() = Some(Arc::new(move |phase| { + if matches!(phase, StaleRefreshPhase::AfterConcreteRead) { + std::fs::write(&grow_path, vec![b'x'; 4096]).unwrap(); + } + })); + handle_fs_event( + &state, + &root, + &Event { + kind: EventKind::Modify(notify::event::ModifyKind::Data( + notify::event::DataChange::Any, + )), + paths: vec![path], + attrs: Default::default(), + }, + ); + + assert!( + state.index.read().unwrap().live.is_deleted("grows.rs"), + "growth past the configured limit must be rejected before a second unbounded read" + ); + } + /// Nothing may be resolved that could climb back out of the root. #[test] fn open_within_root_refuses_paths_that_escape_or_are_not_literal() { @@ -6962,6 +9002,51 @@ mod tests { ); } + #[test] + fn a_deferred_directory_rename_reconciles_indexed_descendants() { + let tmp = TempDir::new().unwrap(); + let root = std::fs::canonicalize(tmp.path()).unwrap(); + let dir = root.join("removed"); + let path = dir.join("deep.rs"); + std::fs::create_dir(&dir).unwrap(); + std::fs::write(&path, "fn removed_descendant() {}\n").unwrap(); + let state = test_server_state(&root, &root.join(".tgrep")); + state.gitignore_pending.store(false, Ordering::SeqCst); + { + let _gate = state.snapshot_gate.read().unwrap(); + reindex_file(&state, &path, "removed/deep.rs", false); + } + + state.indexing.store(true, Ordering::SeqCst); + std::fs::rename(&dir, root.join("moved")).unwrap(); + handle_fs_event( + &state, + &root, + &Event { + kind: EventKind::Modify(notify::event::ModifyKind::Name( + notify::event::RenameMode::From, + )), + paths: vec![dir], + attrs: Default::default(), + }, + ); + state.indexing.store(false, Ordering::SeqCst); + replay_deferred_events(&state, &root); + + let deadline = Instant::now() + Duration::from_secs(10); + while (state.ignore_refresh_scheduled.load(Ordering::SeqCst) + || state.ignore_rules_dirty.load(Ordering::SeqCst)) + && Instant::now() < deadline + { + thread::sleep(Duration::from_millis(10)); + } + let index = state.index.read().unwrap(); + assert!( + !index.reader_has_path("removed/deep.rs") && !index.live.has_path("removed/deep.rs"), + "the coalesced reconcile must retire descendants named by no removal event" + ); + } + /// A file that cannot be opened right now is not a file that stopped /// belonging in the index. Evicting on a transient error would drop live /// content because something else held the file open for a moment. @@ -6978,7 +9063,7 @@ mod tests { let path = root.join("locked.rs"); std::fs::write(&path, "fn readable() {}\n").unwrap(); - reindex_file(&state, &path, "locked.rs"); + reindex_file(&state, &path, "locked.rs", false); assert!(state.index.read().unwrap().live.has_path("locked.rs")); std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o000)).unwrap(); @@ -6986,7 +9071,7 @@ mod tests { // Running as root, where the mode is advisory. Nothing to test. return; } - reindex_file(&state, &path, "locked.rs"); + reindex_file(&state, &path, "locked.rs", false); assert!( !state.index.read().unwrap().live.is_deleted("locked.rs"), @@ -7053,7 +9138,7 @@ mod tests { let path = root.join("d").join("a.rs"); std::fs::create_dir(root.join("d")).unwrap(); std::fs::write(&path, "fn a() {}\n").unwrap(); - reindex_file(&state, &path, "d/a.rs"); + reindex_file(&state, &path, "d/a.rs", false); assert!(state.index.read().unwrap().live.has_path("d/a.rs")); // What `reindex_files_in` produces when an entry in `d` failed to @@ -7093,7 +9178,7 @@ mod tests { let path = root.join("d").join("a.rs"); std::fs::create_dir(root.join("d")).unwrap(); std::fs::write(&path, "fn a() {}\n").unwrap(); - reindex_file(&state, &path, "d/a.rs"); + reindex_file(&state, &path, "d/a.rs", false); assert!(state.index.read().unwrap().live.has_path("d/a.rs")); // `d` enumerated cleanly and did not contain `a.rs` at the time — but @@ -7126,7 +9211,7 @@ mod tests { let dir = root.join("d"); std::fs::create_dir(&dir).unwrap(); std::fs::write(dir.join("a.rs"), "fn ours() {}\n").unwrap(); - reindex_file(&state, &dir.join("a.rs"), "d/a.rs"); + reindex_file(&state, &dir.join("a.rs"), "d/a.rs", false); assert!(state.index.read().unwrap().live.has_path("d/a.rs")); // The scan listed the root, did not find `d`, and is about to sweep @@ -7169,7 +9254,7 @@ mod tests { let path = root.join("x.rs"); std::fs::write(&path, "fn was_a_real_file() {}\n").unwrap(); - reindex_file(&state, &path, "x.rs"); + reindex_file(&state, &path, "x.rs", false); assert!(state.index.read().unwrap().live.has_path("x.rs")); std::fs::remove_file(&path).unwrap(); @@ -7210,7 +9295,7 @@ mod tests { let path = root.join("x.rs"); std::fs::write(&path, "fn indexed() {}\n").unwrap(); - reindex_file(&state, &path, "x.rs"); + reindex_file(&state, &path, "x.rs", false); assert!(state.index.read().unwrap().live.has_path("x.rs")); std::fs::remove_file(&path).unwrap(); @@ -7516,7 +9601,7 @@ mod tests { let path = root.join("seeded.rs"); std::fs::write(&path, "fn seeded() {}\n").unwrap(); let gate = state.snapshot_gate.read().unwrap(); - reindex_file(&state, &path, "seeded.rs"); + reindex_file(&state, &path, "seeded.rs", false); assert!(state.index.read().unwrap().live.has_path("seeded.rs")); // Indexed, and searchable, but with nothing in the stamp map to say so @@ -7755,8 +9840,8 @@ mod tests { std::fs::create_dir_all(&nested).unwrap(); std::fs::write(dir.join("a.rs"), "fn a() {}\n").unwrap(); std::fs::write(nested.join("b.rs"), "fn b() {}\n").unwrap(); - reindex_file(&state, &dir.join("a.rs"), "d/a.rs"); - reindex_file(&state, &nested.join("b.rs"), "d/deep/b.rs"); + reindex_file(&state, &dir.join("a.rs"), "d/a.rs", false); + reindex_file(&state, &nested.join("b.rs"), "d/deep/b.rs", false); assert!(state.index.read().unwrap().live.has_path("d/a.rs")); // Still there, merely unlistable from the scan's point of view: the @@ -7818,7 +9903,7 @@ mod tests { ] .into_iter() .collect(); - let (added, _removed) = registry.sync(&desired, false); + let (added, _removed) = registry.sync(&desired, TraversalCompleteness::Complete, false); assert_eq!(added.len(), 4); let depths: Vec = added.iter().map(|d| d.components().count()).collect(); @@ -7854,13 +9939,13 @@ mod tests { std::fs::remove_dir(&gone).unwrap(); let desired: std::collections::HashSet = std::iter::once(gone.clone()).collect(); - registry.sync(&desired, false); + registry.sync(&desired, TraversalCompleteness::Complete, false); assert!( registry.is_watched(&gone), "the fixture must reproduce the poisoned entry, or it proves nothing" ); - registry.sync(&desired, true); + registry.sync(&desired, TraversalCompleteness::Complete, true); assert!( !registry.is_watched(&gone), "a forced sync must re-issue the subscription and drop the entry \ @@ -8021,6 +10106,14 @@ mod tests { let stamps: std::collections::HashMap = [ ("quiet.rs".to_string(), FileStamp { mtime: 1, size: 10 }), ("racy.rs".to_string(), FileStamp { mtime: 2, size: 20 }), + ( + "moved/deep.rs".to_string(), + FileStamp { mtime: 3, size: 30 }, + ), + ( + "moved-aside.rs".to_string(), + FileStamp { mtime: 4, size: 40 }, + ), ] .into_iter() .collect(); @@ -8032,6 +10125,13 @@ mod tests { .as_mut() .unwrap() .insert(root.join("racy.rs"), false); + state + .deferred_events + .lock() + .unwrap() + .as_mut() + .unwrap() + .insert(root.join("moved"), true); let published = withhold_stamps_for_deferred(&state, &root, stamps.clone()); assert!( @@ -8042,6 +10142,14 @@ mod tests { !published.contains_key("racy.rs"), "a file whose event is waiting to be replayed must not be stamped as indexed" ); + assert!( + !published.contains_key("moved/deep.rs"), + "a directory event must withhold stamps for every descendant the subtree replay covers" + ); + assert!( + published.contains_key("moved-aside.rs"), + "directory-prefix matching must not withhold similarly named siblings" + ); // Overflowed: the buffer names nothing, so nothing in the map can be // told apart from what changed underneath it. @@ -8104,14 +10212,14 @@ mod tests { let path = root.join("grows.rs"); std::fs::write(&path, "fn small_enough() {}\n").unwrap(); - reindex_file(&state, &path, "grows.rs"); + reindex_file(&state, &path, "grows.rs", false); assert!( state.index.read().unwrap().live.has_path("grows.rs"), "a file under the cap should index normally" ); std::fs::write(&path, "x".repeat(4096)).unwrap(); - reindex_file(&state, &path, "grows.rs"); + reindex_file(&state, &path, "grows.rs", false); assert!( state.index.read().unwrap().live.is_deleted("grows.rs"), "content past the cap must not stay searchable" @@ -8273,7 +10381,8 @@ mod tests { write_file(&staging.join(name), name.as_bytes()); } write_file(&staging.join("ignored.txt"), b"nope"); - move_staged_files(&staging, &target).unwrap(); + let mut moved = move_staged_files(&staging, &target).unwrap(); + moved.commit(); for name in ["index.bin", "lookup.bin", "files.bin", "meta.json"] { assert_eq!(std::fs::read(target.join(name)).unwrap(), name.as_bytes()); assert!( @@ -8286,4 +10395,123 @@ mod tests { "unknown files should be left alone" ); } + + #[test] + fn staged_file_move_rolls_back_replaced_files() { + let tmp = TempDir::new().unwrap(); + let staging = tmp.path().join("staging"); + let target = tmp.path().join("target"); + std::fs::create_dir_all(&staging).unwrap(); + std::fs::create_dir_all(&target).unwrap(); + write_file(&staging.join("index.bin"), b"new"); + write_file(&target.join("index.bin"), b"old"); + + let mut moved = move_staged_files(&staging, &target).unwrap(); + assert_eq!(std::fs::read(target.join("index.bin")).unwrap(), b"new"); + moved.rollback().unwrap(); + + assert_eq!(std::fs::read(target.join("index.bin")).unwrap(), b"old"); + } + + #[test] + fn rollback_restores_backup_when_published_target_is_already_missing() { + let tmp = TempDir::new().unwrap(); + let staging = tmp.path().join("staging"); + let target = tmp.path().join("target"); + std::fs::create_dir_all(&staging).unwrap(); + std::fs::create_dir_all(&target).unwrap(); + write_file(&staging.join("index.bin"), b"new"); + write_file(&target.join("index.bin"), b"old"); + + let mut moved = move_staged_files(&staging, &target).unwrap(); + std::fs::remove_file(target.join("index.bin")).unwrap(); + moved.rollback().unwrap(); + + assert_eq!(std::fs::read(target.join("index.bin")).unwrap(), b"old"); + } + + #[test] + fn rollback_restores_backup_when_replacement_was_never_published() { + let tmp = TempDir::new().unwrap(); + let staging = tmp.path().join("staging"); + let target = tmp.path().join("target"); + std::fs::create_dir_all(&staging).unwrap(); + std::fs::create_dir_all(&target).unwrap(); + write_file(&staging.join(".previous-index.bin"), b"old"); + + let mut moved = StagedFileMove { + staging, + target: target.clone(), + backed_up: vec!["index.bin"], + published: Vec::new(), + finished: false, + }; + moved.rollback().unwrap(); + + assert_eq!(std::fs::read(target.join("index.bin")).unwrap(), b"old"); + } + + #[test] + fn move_failure_reports_rollback_failure_and_preserves_backups() { + let tmp = TempDir::new().unwrap(); + let staging = tmp.path().join("staging"); + let target = tmp.path().join("target"); + std::fs::create_dir_all(&staging).unwrap(); + std::fs::create_dir_all(&target).unwrap(); + write_file(&staging.join(".previous-index.bin"), b"old-index"); + std::fs::create_dir(target.join("index.bin")).unwrap(); + + let moved = StagedFileMove { + staging: staging.clone(), + target, + backed_up: vec!["index.bin"], + published: vec!["index.bin"], + finished: false, + }; + let error = moved.fail(std::io::Error::other("publish failed")); + + assert!(error.rollback_failed()); + assert_eq!( + std::fs::read(staging.join(".previous-index.bin")).unwrap(), + b"old-index" + ); + } + + #[test] + fn rollback_retry_does_not_delete_an_already_restored_backup() { + let tmp = TempDir::new().unwrap(); + let staging = tmp.path().join("staging"); + let target = tmp.path().join("target"); + std::fs::create_dir_all(&staging).unwrap(); + std::fs::create_dir_all(&target).unwrap(); + write_file(&staging.join(".previous-index.bin"), b"old-index"); + write_file(&staging.join(".previous-lookup.bin"), b"old-lookup"); + write_file(&target.join("index.bin"), b"new-index"); + std::fs::create_dir(target.join("lookup.bin")).unwrap(); + + let mut moved = StagedFileMove { + staging, + target: target.clone(), + backed_up: vec!["index.bin", "lookup.bin"], + published: vec!["index.bin", "lookup.bin"], + finished: false, + }; + assert!(moved.rollback().is_err()); + assert_eq!( + std::fs::read(target.join("index.bin")).unwrap(), + b"old-index" + ); + + std::fs::remove_dir(target.join("lookup.bin")).unwrap(); + moved.rollback().unwrap(); + + assert_eq!( + std::fs::read(target.join("index.bin")).unwrap(), + b"old-index" + ); + assert_eq!( + std::fs::read(target.join("lookup.bin")).unwrap(), + b"old-lookup" + ); + } } diff --git a/tgrep-cli/tests/watcher_watch_registration.rs b/tgrep-cli/tests/watcher_watch_registration.rs index e8e765e..be2f5c2 100644 --- a/tgrep-cli/tests/watcher_watch_registration.rs +++ b/tgrep-cli/tests/watcher_watch_registration.rs @@ -1,4 +1,5 @@ -//! The watcher must not take OS subscriptions for trees it is going to ignore. +//! On Linux and Android, the watcher must not take OS subscriptions for trees +//! it is going to ignore. //! //! On Linux `notify` has no recursive inotify mode: `RecursiveMode::Recursive` //! walks the tree and spends one watch descriptor per directory. Ignored build @@ -14,9 +15,13 @@ //! while asserting the watcher still works — a watcher that registered nothing //! at all would trivially satisfy the count. //! -//! Windows (ReadDirectoryChangesW) and macOS (FSEvents) subscribe once for the -//! whole subtree, so there is no per-directory registration to withhold and -//! nothing here applies; the count assertion is Linux-only for that reason. +//! This implementation intentionally uses one recursive +//! `ReadDirectoryChangesW` root subscription on Windows and one root FSEvents +//! stream on macOS. Ignored events are filtered after delivery there, so the +//! implementation avoids per-directory descriptor growth but does not leave +//! ignored descendants unwatched. kqueue and `PollWatcher` are not covered. +//! The descriptor-count assertion is Linux-only; Android has the same selective +//! registration design but is not exercised by this test target. use std::fs; use std::io::{BufRead, BufReader, Write}; @@ -111,6 +116,15 @@ fn wait_for_no_match(port: u16, pattern: &str, timeout: Duration) -> bool { } } +fn git(root: &Path, args: &[&str]) { + let status = Command::new("git") + .current_dir(root) + .args(args) + .status() + .expect("failed to run git"); + assert!(status.success(), "git {args:?} failed"); +} + /// Read `(pid, port)` once the server is accepting connections. fn wait_for_server(index_dir: &Path) -> (u32, u16) { let serve_json = index_dir.join("serve.json"); @@ -174,6 +188,7 @@ fn watcher_does_not_subscribe_to_gitignored_directories() { fs::create_dir_all(&sub).unwrap(); fs::write(sub.join("artifact.txt"), "ignored_build_output\n").unwrap(); } + for i in 0..SOURCE_DIRS { let sub = root.join("src").join(format!("pkg{i}")); fs::create_dir_all(&sub).unwrap(); @@ -244,6 +259,93 @@ fn watcher_does_not_subscribe_to_gitignored_directories() { ); } +/// The Git index is hidden and deliberately not watched, but it changes which +/// case-insensitively ignored paths Git considers tracked. The long-lived +/// watcher must reconcile both directions from the index identity alone. +#[test] +fn watcher_reconciles_forced_add_and_rm_cached_inside_an_ignored_tree() { + let dir = TempDir::new().unwrap(); + let root = dir.path(); + let index_dir = root.join(".tgrep_test_index"); + + git(root, &["init", "--quiet"]); + git(root, &["config", "core.ignorecase", "true"]); + fs::write(root.join(".gitignore"), "IGNORED/\n").unwrap(); + fs::create_dir_all(root.join("src")).unwrap(); + fs::write( + root.join("src").join("lib.rs"), + "fn seeded() { let normal_source_marker = 1; }\n", + ) + .unwrap(); + fs::create_dir_all(root.join("ignored")).unwrap(); + fs::write( + root.join("ignored").join("tracked.rs"), + "fn forced() { let forced_tracked_marker = 2; }\n", + ) + .unwrap(); + git(root, &["add", "--", ".gitignore", "src/lib.rs"]); + + let status = Command::new(tgrep_bin()) + .args([ + "index", + root.to_str().unwrap(), + "--index-path", + index_dir.to_str().unwrap(), + ]) + .status() + .expect("failed to run tgrep index"); + assert!(status.success(), "initial index build failed"); + + let child = Command::new(tgrep_bin()) + .args([ + "serve", + "--index-path", + index_dir.to_str().unwrap(), + root.to_str().unwrap(), + ]) + .stderr(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .spawn() + .expect("failed to start tgrep serve"); + let _server = ServerGuard { child }; + + let (_pid, port) = wait_for_server(&index_dir); + fs::write( + root.join("src").join("watcher-ready.rs"), + "fn ready() { let watcher_ready_marker = 3; }\n", + ) + .unwrap(); + assert!( + wait_for_match(port, "watcher_ready_marker", Duration::from_secs(30)), + "watcher did not become ready" + ); + assert_eq!(search_matches(port, "forced_tracked_marker"), 0); + + // Only .git/index changes: the file already exists under an unsubscribed + // ignored directory, so no ordinary watcher event can rescue this. + git(root, &["add", "-f", "--", "ignored/tracked.rs"]); + assert!( + wait_for_match(port, "forced_tracked_marker", Duration::from_secs(60)), + "git add -f did not restore the tracked file through reconciliation" + ); + + git( + root, + &[ + "rm", + "--cached", + "--force", + "--quiet", + "--", + "ignored/tracked.rs", + ], + ); + assert!( + wait_for_no_match(port, "forced_tracked_marker", Duration::from_secs(60)), + "git rm --cached did not prune stale indexed content" + ); +} + /// New directories still have to be picked up. /// /// Non-recursive subscriptions are not extended by notify, so a directory diff --git a/tgrep-core/src/builder.rs b/tgrep-core/src/builder.rs index 0b636e8..9759418 100644 --- a/tgrep-core/src/builder.rs +++ b/tgrep-core/src/builder.rs @@ -38,6 +38,7 @@ const LOOKUP_WRITE_CHUNK_ENTRIES: usize = 4096; /// charging mapped bytes honestly costs nothing on trees like the kernel, where /// only 110 of 94,747 files are mapped at all. const INDEX_BUILD_BATCH_BYTES: u64 = 64 * 1024 * 1024; +const MAX_OWNED_FILE_BYTES: u64 = INDEX_BUILD_BATCH_BYTES; /// Default arena budget for [`IndexStrategy::External`] before spilling. pub use crate::external::DEFAULT_BUFFER_BYTES as DEFAULT_INDEX_BUFFER_BYTES; @@ -316,6 +317,19 @@ fn batch_sizes_and_charges(files: &[std::path::PathBuf]) -> (Vec, Vec, +) -> Vec { + files + .iter() + .filter(|path| !excluded.contains(*path)) + .filter_map(|path| path.strip_prefix(root).ok()) + .map(|path| path.to_string_lossy().replace('\\', "/")) + .collect() +} + /// Smallest file worth memory-mapping instead of reading. /// /// The same tradeoff the search path makes: mapping costs a syscall pair and a @@ -337,18 +351,108 @@ const MMAP_MIN_BYTES: u64 = 1024 * 1024; /// The bytes of a file to index, either read onto the heap or borrowed from a /// memory map. enum FileBytes { - Read(Vec), + Read { + bytes: Vec, + _permit: OwnedReadPermit, + }, Mapped(memmap2::Mmap), } +struct OwnedReadBudget { + capacity: u64, + available: std::sync::Mutex, + ready: std::sync::Condvar, +} + +impl OwnedReadBudget { + fn new(bytes: u64) -> std::sync::Arc { + std::sync::Arc::new(Self { + capacity: bytes, + available: std::sync::Mutex::new(bytes), + ready: std::sync::Condvar::new(), + }) + } + + fn acquire(self: &std::sync::Arc, bytes: u64) -> OwnedReadPermit { + // One oversized fallback may exceed the ordinary budget, but it claims + // the whole budget so no other owned read can overlap it. + let charged = bytes.min(self.capacity); + let mut available = self.available.lock().unwrap(); + while *available < charged { + available = self.ready.wait(available).unwrap(); + } + *available -= charged; + OwnedReadPermit { + budget: std::sync::Arc::clone(self), + bytes: charged, + } + } +} + +struct OwnedReadPermit { + budget: std::sync::Arc, + bytes: u64, +} + +impl Drop for OwnedReadPermit { + fn drop(&mut self) { + let mut available = self.budget.available.lock().unwrap(); + *available += self.bytes; + self.budget.ready.notify_all(); + } +} + type ExtractedFile = (String, trigram::TrigramMaskMap); +/// Full-resolution identity used to validate that bytes came from one file +/// version without changing the persisted [`meta::FileStamp`] format. +#[doc(hidden)] +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FileVersion { + stamp: meta::FileStamp, + modified: Option, + created: Option, + #[cfg(unix)] + device: u64, + #[cfg(unix)] + inode: u64, + #[cfg(unix)] + change_seconds: i64, + #[cfg(unix)] + change_nanos: i64, +} + +impl FileVersion { + pub fn stamp(&self) -> &meta::FileStamp { + &self.stamp + } +} + +#[doc(hidden)] +pub fn file_version(metadata: &std::fs::Metadata) -> FileVersion { + #[cfg(unix)] + use std::os::unix::fs::MetadataExt; + FileVersion { + stamp: meta::file_stamp(metadata), + modified: metadata.modified().ok(), + created: metadata.created().ok(), + #[cfg(unix)] + device: metadata.dev(), + #[cfg(unix)] + inode: metadata.ino(), + #[cfg(unix)] + change_seconds: metadata.ctime(), + #[cfg(unix)] + change_nanos: metadata.ctime_nsec(), + } +} + impl std::ops::Deref for FileBytes { type Target = [u8]; fn deref(&self) -> &[u8] { match self { - FileBytes::Read(bytes) => bytes, + FileBytes::Read { bytes, .. } => bytes, FileBytes::Mapped(map) => map, } } @@ -357,9 +461,22 @@ impl std::ops::Deref for FileBytes { /// Get a file's bytes for indexing, mapping it when it is large enough to be /// worth avoiding the copy. /// -/// Falls back to reading whenever mapping is unavailable, so a filesystem that -/// cannot map still indexes correctly. -fn read_for_index(path: &Path, size: u64) -> std::io::Result { +/// Falls back to an owned read whenever mapping is unavailable. Ordinary reads +/// share the batch budget; an unlimited oversized fallback claims it +/// exclusively so at most one such allocation is in flight. +fn read_for_index( + path: &Path, + size: u64, + owned_budget: &std::sync::Arc, + configured_limit: Option, +) -> std::io::Result { + let owned_limit = configured_limit.unwrap_or(u64::MAX); + if size > owned_limit { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("file exceeds the configured {} byte limit", owned_limit), + )); + } if size >= MMAP_MIN_BYTES { // SAFETY: the map is read-only and owned by the returned value, which // is dropped once the file's trigrams have been extracted. A concurrent @@ -368,11 +485,92 @@ fn read_for_index(path: &Path, size: u64) -> std::io::Result { // accepts. let mapped = std::fs::File::open(path).and_then(|file| unsafe { memmap2::Mmap::map(&file) }); - if let Ok(map) = mapped { + if let Ok(map) = mapped + && map.len() as u64 <= owned_limit + { return Ok(FileBytes::Mapped(map)); } + // The file grew between the walk's stat and the map. Do not let a + // successful mmap bypass a caller-supplied max-file-size limit. } - std::fs::read(path).map(FileBytes::Read) + let permit = owned_budget.acquire(size); + match read_owned_for_index(path, size, owned_limit) { + Ok(bytes) => Ok(FileBytes::Read { + bytes, + _permit: permit, + }), + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + // The file grew after it was stat'd. Release the smaller claim + // before waiting for the retry allowance so concurrent growers + // cannot deadlock while each holds part of the batch budget. + drop(permit); + match configured_limit { + Some(limit) => { + let permit = owned_budget.acquire(limit); + read_owned_for_index(path, limit, limit) + .map(|bytes| FileBytes::Read { + bytes, + _permit: permit, + }) + .map_err(|retry| { + if retry.kind() == std::io::ErrorKind::WouldBlock { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("file exceeds the configured {} byte limit", limit), + ) + } else { + retry + } + }) + } + None => { + // No caller limit means no implicit fallback limit either. + // Claiming more than the budget takes it exclusively, so + // this unbounded read cannot overlap another owned buffer. + let permit = owned_budget.acquire(u64::MAX); + read_owned_unbounded(path, size).map(|bytes| FileBytes::Read { + bytes, + _permit: permit, + }) + } + } + } + Err(error) => Err(error), + } +} + +fn read_owned_for_index(path: &Path, size: u64, owned_limit: u64) -> std::io::Result> { + use std::io::Read; + + if size > owned_limit { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("file exceeds the configured {} byte limit", owned_limit), + )); + } + let mut file = std::fs::File::open(path)?; + let mut bytes = Vec::with_capacity(usize::try_from(size).unwrap_or(0)); + std::io::Read::by_ref(&mut file) + .take(size) + .read_to_end(&mut bytes)?; + let mut extra = [0u8; 1]; + if file.read(&mut extra)? != 0 { + return Err(std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "file grew beyond its owned-buffer reservation", + )); + } + Ok(bytes) +} + +fn read_owned_unbounded(path: &Path, expected_size: u64) -> std::io::Result> { + use std::io::Read; + + let mut file = std::fs::File::open(path)?; + let initial_capacity = expected_size.min(MAX_OWNED_FILE_BYTES); + let mut bytes = Vec::with_capacity(usize::try_from(initial_capacity).unwrap_or(0)); + file.read_to_end(&mut bytes)?; + Ok(bytes) } /// Build a trigram index, choosing how postings are accumulated. @@ -380,6 +578,22 @@ pub fn build_index_with_options( root: &Path, index_dir: Option<&Path>, opts: &BuildOptions, +) -> Result { + let root = std::fs::canonicalize(root)?; + let ignorecase = (!opts.no_ignore) + .then(|| crate::gitignore::CaseInsensitiveIgnore::new(&root, true, true, true)) + .flatten() + .map(std::sync::Arc::new); + build_index_with_options_and_ignorecase(&root, index_dir, opts, ignorecase) +} + +/// Build an index using an immutable tracked-file exemption that the caller +/// can also use for watcher matcher publication. +pub fn build_index_with_options_and_ignorecase( + root: &Path, + index_dir: Option<&Path>, + opts: &BuildOptions, + ignorecase: Option>, ) -> Result { let include_hidden = opts.include_hidden; let no_ignore = opts.no_ignore; @@ -395,7 +609,7 @@ pub fn build_index_with_options( if let Some(hint) = gitignore_gate_hint(&root, opts) { eprintln!("{hint}"); } - let walk = walker::walk_dir( + let walk = walker::walk_dir_with_ignorecase( &root, &walker::WalkOptions { include_hidden, @@ -406,6 +620,7 @@ pub fn build_index_with_options( max_file_size: opts.max_file_size, ..Default::default() }, + ignorecase, ); eprintln!( "Found {} text files ({} binary skipped, {} too large, {} errors)", @@ -439,15 +654,25 @@ pub fn build_index_with_options( // The walk already stats every entry but discards the size, so recover it // here rather than widening WalkResult into the search and serve paths. let (sizes, charges) = batch_sizes_and_charges(&walk.files); + let raced_too_large = std::sync::Mutex::new(std::collections::HashSet::new()); for range in batch_ranges(&charges, INDEX_BUILD_BATCH_BYTES) { let batch = &walk.files[range.clone()]; let batch_sizes = &sizes[range]; + let owned_budget = OwnedReadBudget::new(INDEX_BUILD_BATCH_BYTES); let batch_data: Vec = batch .par_iter() .zip(batch_sizes.par_iter()) .filter_map(|(path, &size)| { - let data = read_for_index(path, size).ok()?; + let data = match read_for_index(path, size, &owned_budget, opts.max_file_size) { + Ok(data) => data, + Err(error) => { + if error.kind() == std::io::ErrorKind::InvalidData { + raced_too_large.lock().unwrap().insert(path.clone()); + } + return None; + } + }; let text = crate::encoding::decode_for_index(&data); if trigram::is_binary(&text) { binary_skipped.fetch_add(1, std::sync::atomic::Ordering::Relaxed); @@ -501,15 +726,17 @@ pub fn build_index_with_options( } } - // Write per-file stamps for ALL walked files (including those later - // rejected as binary-by-content) so the stale check on next startup - // won't re-process unchanged files that aren't in the index. - let all_walked: Vec = walk - .files - .iter() - .filter_map(|p| p.strip_prefix(&root).ok()) - .map(|p| p.to_string_lossy().replace('\\', "/")) - .collect(); + // Write per-file stamps for walked files, including those later rejected + // as binary-by-content. A file that raced past max-file-size is different: + // withholding its stamp leaves it eligible for a later retry. + let raced_too_large = raced_too_large.into_inner().unwrap(); + if !raced_too_large.is_empty() { + eprintln!( + "Skipped {} files that grew past max-file-size during extraction", + raced_too_large.len() + ); + } + let all_walked = walked_paths_for_stamps(&root, &walk.files, &raced_too_large); let stamps = meta::collect_filestamps(&root, &all_walked); meta::write_filestamps(&stamps, &index_dir)?; @@ -572,11 +799,12 @@ pub fn build_index_for_files( for range in batch_ranges(&charges, INDEX_BUILD_BATCH_BYTES) { let batch = &files[range.clone()]; let batch_sizes = &sizes[range]; + let owned_budget = OwnedReadBudget::new(INDEX_BUILD_BATCH_BYTES); let batch_data: Vec> = batch .par_iter() .zip(batch_sizes.par_iter()) .filter_map(|(path, &size)| { - let data = match read_for_index(path, size) { + let data = match read_for_index(path, size, &owned_budget, None) { Ok(data) => data, Err(error) => { eprintln!("tgrep: skipping {}: {error}", path.display()); @@ -1296,6 +1524,19 @@ mod tests { sizes.iter().copied().map(batch_charge).collect() } + #[test] + fn raced_oversized_files_are_excluded_from_published_stamps() { + let root = std::path::PathBuf::from("repo"); + let kept = root.join("kept.rs"); + let skipped = root.join("grew.rs"); + let excluded = std::collections::HashSet::from([skipped.clone()]); + + assert_eq!( + walked_paths_for_stamps(&root, &[kept, skipped], &excluded), + vec!["kept.rs"] + ); + } + #[test] fn batches_are_bounded_by_cumulative_heap_bytes() { // Files below the mapping threshold are read, so their whole length is @@ -1369,6 +1610,189 @@ mod tests { ); } + #[test] + fn owned_read_rejects_files_beyond_the_batch_budget() { + let tmp = tempfile::tempdir().unwrap(); + let path = tmp.path().join("sparse.txt"); + std::fs::File::create(&path) + .unwrap() + .set_len(MAX_OWNED_FILE_BYTES + 1) + .unwrap(); + + let error = read_owned_for_index(&path, MAX_OWNED_FILE_BYTES + 1, MAX_OWNED_FILE_BYTES) + .unwrap_err(); + assert_eq!(error.kind(), std::io::ErrorKind::InvalidData); + } + + #[test] + fn configured_limit_is_checked_before_mmap() { + let tmp = tempfile::tempdir().unwrap(); + let path = tmp.path().join("too-large.txt"); + std::fs::File::create(&path) + .unwrap() + .set_len(MMAP_MIN_BYTES) + .unwrap(); + let budget = OwnedReadBudget::new(INDEX_BUILD_BATCH_BYTES); + + let error = match read_for_index(&path, MMAP_MIN_BYTES, &budget, Some(MMAP_MIN_BYTES - 1)) { + Ok(_) => panic!("configured max-file-size must be checked before mmap"), + Err(error) => error, + }; + assert_eq!(error.kind(), std::io::ErrorKind::InvalidData); + } + + #[test] + fn owned_read_reports_growth_beyond_its_reservation() { + let tmp = tempfile::tempdir().unwrap(); + let path = tmp.path().join("growing.txt"); + std::fs::write(&path, b"four").unwrap(); + + let error = read_owned_for_index(&path, 2, 2).unwrap_err(); + assert_eq!(error.kind(), std::io::ErrorKind::WouldBlock); + } + + #[test] + fn indexed_read_retries_growth_with_the_full_batch_reservation() { + let tmp = tempfile::tempdir().unwrap(); + let path = tmp.path().join("growing.txt"); + std::fs::write(&path, b"four").unwrap(); + let budget = OwnedReadBudget::new(INDEX_BUILD_BATCH_BYTES); + + let bytes = read_for_index(&path, 2, &budget, None).unwrap(); + assert_eq!(&*bytes, b"four"); + } + + #[test] + fn owned_read_permit_releases_batch_capacity() { + let budget = OwnedReadBudget::new(10); + let permit = budget.acquire(7); + assert_eq!(*budget.available.lock().unwrap(), 3); + drop(permit); + assert_eq!(*budget.available.lock().unwrap(), 10); + } + + #[test] + fn oversized_owned_read_claims_the_budget_exclusively() { + let budget = OwnedReadBudget::new(10); + let permit = budget.acquire(100); + assert_eq!(*budget.available.lock().unwrap(), 0); + drop(permit); + assert_eq!(*budget.available.lock().unwrap(), 10); + } + + #[test] + fn owned_read_budget_serializes_fallbacks_that_exceed_the_remaining_capacity() { + let budget = OwnedReadBudget::new(10); + let first = budget.acquire(100); + let (started_tx, started_rx) = std::sync::mpsc::channel(); + let (acquired_tx, acquired_rx) = std::sync::mpsc::channel(); + let waiting_budget = std::sync::Arc::clone(&budget); + let waiter = std::thread::spawn(move || { + started_tx.send(()).unwrap(); + let _second = waiting_budget.acquire(100); + acquired_tx.send(()).unwrap(); + }); + + started_rx.recv().unwrap(); + assert!( + acquired_rx + .recv_timeout(std::time::Duration::from_millis(50)) + .is_err(), + "a second fallback must wait rather than exceed the shared budget" + ); + drop(first); + acquired_rx + .recv_timeout(std::time::Duration::from_secs(1)) + .unwrap(); + waiter.join().unwrap(); + } + + #[test] + fn file_version_distinguishes_subsecond_modified_times() { + let stamp = meta::FileStamp { mtime: 1, size: 10 }; + let base = std::time::UNIX_EPOCH + std::time::Duration::from_secs(1); + let first = FileVersion { + stamp: stamp.clone(), + modified: Some(base + std::time::Duration::from_nanos(100)), + created: Some(base), + #[cfg(unix)] + device: 1, + #[cfg(unix)] + inode: 2, + #[cfg(unix)] + change_seconds: 1, + #[cfg(unix)] + change_nanos: 0, + }; + let second = FileVersion { + modified: Some(base + std::time::Duration::from_nanos(200)), + ..first.clone() + }; + + assert_ne!(first, second); + assert_eq!(first.stamp, second.stamp); + } + + fn write_test_git_index(root: &Path, tracked: &[&str]) { + let git = root.join(".git"); + std::fs::create_dir_all(&git).unwrap(); + std::fs::write(git.join("config"), "[core]\n\tignorecase = true\n").unwrap(); + let mut index = Vec::new(); + index.extend_from_slice(b"DIRC"); + index.extend_from_slice(&2u32.to_be_bytes()); + index.extend_from_slice(&(tracked.len() as u32).to_be_bytes()); + for path in tracked { + let start = index.len(); + index.extend_from_slice(&[0u8; 60]); + index.extend_from_slice(&(path.len() as u16).to_be_bytes()); + index.extend_from_slice(path.as_bytes()); + index.push(0); + while (index.len() - start) % 8 != 0 { + index.push(0); + } + } + std::fs::write(git.join("index"), index).unwrap(); + } + + #[test] + fn default_builder_constructs_snapshot_but_explicit_none_is_preserved() { + let repo = tempfile::tempdir().unwrap(); + let root = repo.path(); + std::fs::create_dir(root.join("ignored")).unwrap(); + std::fs::write(root.join(".gitignore"), "IGNORED/\n").unwrap(); + std::fs::write(root.join("ignored/tracked.rs"), "fn tracked() {}\n").unwrap(); + std::fs::write(root.join("ignored/untracked.rs"), "fn untracked() {}\n").unwrap(); + write_test_git_index(root, &[".gitignore", "ignored/tracked.rs"]); + + let default_index = tempfile::tempdir().unwrap(); + build_index_with_options(root, Some(default_index.path()), &BuildOptions::default()) + .unwrap(); + let default_reader = IndexReader::open(default_index.path()).unwrap(); + assert!( + default_reader.contains_path("ignored/tracked.rs"), + "the default builder must construct the tracked-file exemption" + ); + assert!( + !default_reader.contains_path("ignored/untracked.rs"), + "the default builder must apply case-insensitive root rules" + ); + + let none_index = tempfile::tempdir().unwrap(); + build_index_with_options_and_ignorecase( + root, + Some(none_index.path()), + &BuildOptions::default(), + None, + ) + .unwrap(); + assert!( + IndexReader::open(none_index.path()) + .unwrap() + .contains_path("ignored/untracked.rs"), + "an explicit None must not reconstruct the exemption" + ); + } + #[test] fn batches_cover_every_file_exactly_once() { let sizes: Vec = (0..500).map(|i| (i as u64 % 7) * 9 * MB).collect(); diff --git a/tgrep-core/src/git_index.rs b/tgrep-core/src/git_index.rs index 2b01e6e..e65307c 100644 --- a/tgrep-core/src/git_index.rs +++ b/tgrep-core/src/git_index.rs @@ -72,6 +72,25 @@ impl TrackedFiles { // matched — a handful per walk, not one per entry. self.paths.iter().any(|p| p.starts_with(prefix.as_str())) } + + pub(crate) fn fingerprint_matching( + &self, + mut include: impl FnMut(&str) -> bool, + ) -> (usize, u64, u64) { + let mut count = 0; + let mut xor = 0; + let mut sum = 0u64; + for path in &self.paths { + if !include(path) { + continue; + } + let hash = membership_hash(path); + count += 1; + xor ^= hash; + sum = sum.wrapping_add(hash); + } + (count, xor, sum) + } } fn normalise(path: &str) -> String { @@ -80,6 +99,14 @@ fn normalise(path: &str) -> String { .to_ascii_lowercase() } +fn membership_hash(path: &str) -> u64 { + path.as_bytes() + .iter() + .fold(0xcbf29ce484222325, |hash, byte| { + (hash ^ u64::from(*byte)).wrapping_mul(0x100000001b3) + }) +} + /// Locate the directory holding a repository's `.git` metadata. /// /// `.git` is usually a directory, but is a file holding `gitdir: ` in a @@ -103,41 +130,91 @@ pub(crate) fn git_dir(repo_root: &Path) -> Option { }) } -/// Whether the repository compares paths without regard to case. -/// -/// git writes `core.ignorecase = true` into the repository's own config when it -/// detects a case-insensitive filesystem, so the per-repository config is the -/// authoritative place to read it and needs no config-precedence handling. -pub fn ignores_case(repo_root: &Path) -> bool { - let Some(git_dir) = git_dir(repo_root) else { - return false; - }; - let Ok(config) = std::fs::read_to_string(git_dir.join("config")) else { - return false; +/// The repository metadata directory shared by all linked worktrees. +pub(crate) fn common_git_dir(git_dir: &Path) -> PathBuf { + let Ok(target) = std::fs::read_to_string(git_dir.join("commondir")) else { + return git_dir.to_path_buf(); }; - let mut in_core = false; + let target = target.trim(); + if target.is_empty() { + return git_dir.to_path_buf(); + } + let path = Path::new(target); + if path.is_absolute() { + path.to_path_buf() + } else { + git_dir.join(path) + } +} + +fn config_bool(config: &str, section: &str, key: &str) -> Option { + let mut in_section = false; + let mut result = None; for line in config.lines() { let line = line.trim(); - if let Some(section) = line.strip_prefix('[').and_then(|l| l.strip_suffix(']')) { - in_core = section.trim().eq_ignore_ascii_case("core"); + if line.is_empty() || line.starts_with('#') || line.starts_with(';') { continue; } - if !in_core { + if let Some(header) = line + .strip_prefix('[') + .and_then(|line| line.strip_suffix(']')) + { + in_section = header.trim().eq_ignore_ascii_case(section); continue; } - let Some((key, value)) = line.split_once('=') else { + if !in_section { + continue; + } + let (candidate, value) = line + .split_once('=') + .map_or((line, None), |(candidate, value)| (candidate, Some(value))); + if !candidate.trim().eq_ignore_ascii_case(key) { continue; - }; - if key.trim().eq_ignore_ascii_case("ignorecase") { - // git spells boolean true several ways. - let value = value.trim(); - return matches!( - value.to_ascii_lowercase().as_str(), - "true" | "yes" | "on" | "1" - ); } + result = match value.map(str::trim) { + None => Some(true), + Some(value) + if matches!( + value.to_ascii_lowercase().as_str(), + "true" | "yes" | "on" | "1" + ) => + { + Some(true) + } + Some(value) + if matches!( + value.to_ascii_lowercase().as_str(), + "false" | "no" | "off" | "0" | "" + ) => + { + Some(false) + } + Some(_) => None, + }; + } + result +} + +/// Whether the repository compares paths without regard to case. +/// +/// A repository may opt into per-worktree config. In that layout Git reads the +/// common config first, then lets `$GIT_DIR/config.worktree` override it. +pub fn ignores_case(repo_root: &Path) -> bool { + let Some(git_dir) = git_dir(repo_root) else { + return false; + }; + let common = common_git_dir(&git_dir); + let Ok(config) = std::fs::read_to_string(common.join("config")) else { + return false; + }; + let common_ignorecase = config_bool(&config, "core", "ignorecase").unwrap_or(false); + if !config_bool(&config, "extensions", "worktreeConfig").unwrap_or(false) { + return common_ignorecase; } - false + std::fs::read_to_string(git_dir.join("config.worktree")) + .ok() + .and_then(|config| config_bool(&config, "core", "ignorecase")) + .unwrap_or(common_ignorecase) } /// Read the tracked paths from `.git/index`. @@ -453,6 +530,63 @@ mod tests { assert!(tracked.contains("worktree/file.rs")); } + #[test] + fn linked_worktree_uses_the_common_config_and_its_own_index() { + let tmp = tempfile::tempdir().expect("tempdir"); + let common = tmp.path().join("repo.git"); + let worktree_git = common.join("worktrees").join("topic"); + std::fs::create_dir_all(&worktree_git).expect("mkdir"); + std::fs::write(worktree_git.join("commondir"), "../..\n").expect("write"); + std::fs::write(worktree_git.join("index"), v2_index(&["worktree/only.rs"])).expect("write"); + + let worktree = tmp.path().join("worktree"); + std::fs::create_dir_all(&worktree).expect("mkdir"); + std::fs::write( + worktree.join(".git"), + format!("gitdir: {}\n", worktree_git.display()), + ) + .expect("write"); + + std::fs::write( + common.join("config"), + "[core]\n\tignorecase = true\n[extensions]\n\tworktreeConfig\n", + ) + .expect("write"); + std::fs::write( + worktree_git.join("config.worktree"), + "[core]\n\tignorecase = false\n", + ) + .expect("write"); + assert!( + !ignores_case(&worktree), + "worktree false must override common true" + ); + + std::fs::write( + common.join("config"), + "[core]\n\tignorecase = false\n[extensions]\n\tworktreeConfig = true\n", + ) + .expect("write"); + std::fs::write( + worktree_git.join("config.worktree"), + "[core]\n\tignorecase = true\n", + ) + .expect("write"); + assert!(ignores_case(&worktree)); + + std::fs::write(common.join("config"), "[core]\n\tignorecase = false\n").expect("write"); + assert!( + !ignores_case(&worktree), + "config.worktree is inactive until the extension is enabled" + ); + assert_eq!(index_path(&worktree), Some(worktree_git.join("index"))); + assert!( + load_tracked(&worktree) + .expect("loads worktree index") + .contains("worktree/only.rs") + ); + } + #[test] fn a_directory_without_git_reports_nothing() { let tmp = tempfile::tempdir().expect("tempdir"); diff --git a/tgrep-core/src/gitignore.rs b/tgrep-core/src/gitignore.rs index 173dea4..2901281 100644 --- a/tgrep-core/src/gitignore.rs +++ b/tgrep-core/src/gitignore.rs @@ -106,7 +106,7 @@ pub struct IgnoreMatcher { /// enlistment it left the watcher subscribing to, and indexing, a 13.4 GiB /// build artifact the walk had already excluded — and every event under it /// re-added a file the next stale check then evicted. - ignorecase: Option, + ignorecase: Option>, } impl IgnoreMatcher { @@ -141,7 +141,7 @@ impl IgnoreMatcher { ancestors: Vec<(String, IgnoreKind, Gitignore)>, repo_exclude: Option<(String, Gitignore)>, global: Gitignore, - ignorecase: Option, + ignorecase: Option>, ) -> Option { let mut nested: Vec = nested .into_iter() @@ -308,6 +308,28 @@ impl IgnoreMatcher { .as_ref() .is_some_and(|ignorecase| ignorecase.excludes(&self.root.join(rel), is_dir)) } + + /// Fingerprint of the tracked paths behind the tracked-file exemption. + /// + /// `None` means this matcher has no case-insensitive exemption, so callers + /// need not poll anything. Index metadata is used internally to avoid + /// reparsing an unchanged index, but only path membership contributes to + /// this value. + pub fn tracked_membership_fingerprint(&self) -> Option { + self.ignorecase + .as_ref() + .map(|ignorecase| ignorecase.tracked_membership_fingerprint()) + } + + /// Fingerprint of the tracked exemption set in the current Git index. + /// + /// Unlike [`Self::tracked_membership_fingerprint`], this probes the current + /// index without changing the immutable set used by this matcher. + pub fn current_tracked_membership_fingerprint(&self) -> Option { + self.ignorecase + .as_ref() + .map(|ignorecase| ignorecase.current_tracked_membership_fingerprint()) + } } /// Return `rel` relative to `dir`, or `None` when `rel` is not beneath it. @@ -444,39 +466,69 @@ pub fn build_p4ignore_matcher(root: &Path) -> Option { pub struct CaseInsensitiveIgnore { matcher: Gitignore, repo_root: std::path::PathBuf, - /// Loaded on the first path this matcher actually claims, which for most - /// repositories is never. Reading it costs 163 ms and ~30 MB on a - /// 299k-file repository, and nothing at all if no rule ever matches. - /// - /// Reloaded when the index it was read from changes. A walk builds this - /// matcher, uses it and drops it, so a snapshot would do; the file watcher - /// holds one for the life of the server, and there a `git add -f` or a - /// `git rm --cached` rewrites only `.git/index` — which is hidden, so no - /// ignore source changes and nothing republishes the matcher. Frozen, the - /// exemption would keep answering from the tracked set as it stood at - /// startup, and the watcher would disagree with a fresh walk about which - /// files exist until the hourly reconcile. - tracked: std::sync::RwLock, + tracked: TrackedMembership, + /// Separate metadata-keyed cache for polling the current semantic + /// membership of a frozen matcher. It never participates in decisions. + current: std::sync::RwLock, } -/// The tracked-file set, together with the identity of the index it came from. -#[derive(Default)] -struct TrackedCache { - /// `None` until something is loaded; `Some` even when the load failed, so - /// an unreadable index is not retried on every path. - loaded_from: Option>, +enum TrackedMembership { + /// Ordinary walks load one immutable snapshot only if a matching ignore + /// rule actually needs the tracked-file exemption. + Lazy(std::sync::OnceLock), + /// Reconciliation behavior: one immutable set shared by a walk and the + /// point-query matcher published from that walk. + Frozen(TrackedSnapshot), +} + +struct TrackedSnapshot { tracked: Option, + fingerprint: TrackedMembershipFingerprint, +} + +/// A metadata-keyed semantic fingerprint that does not retain parsed paths. +#[derive(Default)] +struct TrackedFingerprintCache { + /// `None` until something is loaded; `Some` even when the probe failed, so + /// an unreadable index is not retried on every poll. + loaded_from: Option>, + fingerprint: TrackedMembershipFingerprint, +} + +/// Compact, order-independent identity of the tracked exemption set. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct TrackedMembershipFingerprint(Option<(usize, u64, u64)>); + +#[derive(Clone, Debug, PartialEq, Eq)] +struct IndexIdentity { + modified: std::time::SystemTime, + created: Option, + len: u64, + #[cfg(unix)] + device: u64, + #[cfg(unix)] + inode: u64, } -/// Modification time and length of the index, which is what identifies it. +/// Cheap metadata identity of the index file generation. /// /// git replaces the index by renaming `index.lock` over it, so any rewrite -/// lands as a new mtime — the same pair git's own racy-index handling relies -/// on. Two rewrites inside one filesystem mtime tick that leave the length -/// unchanged are the gap, and the periodic reconcile is what closes it. -fn index_identity(repo_root: &Path) -> Option<(std::time::SystemTime, u64)> { +/// normally lands as a new file instance as well as a new mtime. Creation time +/// covers that replacement on Windows, and device/inode covers it on Unix, +/// including an A→B→A rewrite inside one filesystem mtime tick. +fn index_identity(repo_root: &Path) -> Option { let meta = std::fs::metadata(crate::git_index::index_path(repo_root)?).ok()?; - Some((meta.modified().ok()?, meta.len())) + #[cfg(unix)] + use std::os::unix::fs::MetadataExt; + Some(IndexIdentity { + modified: meta.modified().ok()?, + created: meta.created().ok(), + len: meta.len(), + #[cfg(unix)] + device: meta.dev(), + #[cfg(unix)] + inode: meta.ino(), + }) } impl CaseInsensitiveIgnore { @@ -490,11 +542,40 @@ impl CaseInsensitiveIgnore { /// `--no-ignore-vcs` the case-sensitive pass lets everything through, which /// would leave this the only thing excluding — the exact opposite of what /// the flag asks for. + /// + /// The tracked-file exemption is loaded lazily on the first matching rule + /// and remains immutable for the lifetime of this value. pub fn new( root: &Path, use_gitignore: bool, use_exclude: bool, use_parents: bool, + ) -> Option { + Self::build(root, use_gitignore, use_exclude, use_parents, false) + } + + /// Build an immutable tracked-membership snapshot for a coordinated walk + /// and matcher publication. + /// + /// Ordinary point-query callers should use [`Self::new`], which defers the + /// Git-index read until a matching rule needs it. Server reconciliation + /// shares this eagerly initialized value through an `Arc` so one pass + /// cannot mix different tracked sets and can poll for later changes. + pub fn frozen_snapshot( + root: &Path, + use_gitignore: bool, + use_exclude: bool, + use_parents: bool, + ) -> Option { + Self::build(root, use_gitignore, use_exclude, use_parents, true) + } + + fn build( + root: &Path, + use_gitignore: bool, + use_exclude: bool, + use_parents: bool, + frozen: bool, ) -> Option { use ignore::gitignore::GitignoreBuilder; @@ -527,10 +608,16 @@ impl CaseInsensitiveIgnore { if matcher.is_empty() { return None; } + let tracked = if frozen { + TrackedMembership::Frozen(Self::load_snapshot(&repo_root)) + } else { + TrackedMembership::Lazy(std::sync::OnceLock::new()) + }; Some(Self { matcher, repo_root, - tracked: std::sync::RwLock::new(TrackedCache::default()), + tracked, + current: std::sync::RwLock::new(TrackedFingerprintCache::default()), }) } @@ -546,30 +633,66 @@ impl CaseInsensitiveIgnore { { return false; } - // Only now is the index worth reading. let relative = relative.to_string_lossy(); - self.tracked_hides(&relative, is_dir) + match &self.tracked { + TrackedMembership::Lazy(snapshot) => Self::hides( + snapshot + .get_or_init(|| Self::load_snapshot(&self.repo_root)) + .tracked + .as_ref(), + &relative, + is_dir, + ), + TrackedMembership::Frozen(snapshot) => { + Self::hides(snapshot.tracked.as_ref(), &relative, is_dir) + } + } } - /// Whether the tracked-file set leaves `relative` hidden. - /// - /// `false` when the index cannot be read: with no way to tell tracked from - /// untracked, excluding could hide real source, so it declines instead. - fn tracked_hides(&self, relative: &str, is_dir: bool) -> bool { + fn tracked_membership_fingerprint(&self) -> TrackedMembershipFingerprint { + match &self.tracked { + TrackedMembership::Lazy(snapshot) => snapshot + .get_or_init(|| Self::load_snapshot(&self.repo_root)) + .fingerprint + .clone(), + TrackedMembership::Frozen(snapshot) => snapshot.fingerprint.clone(), + } + } + + fn current_tracked_membership_fingerprint(&self) -> TrackedMembershipFingerprint { let identity = index_identity(&self.repo_root); { - let cache = self.tracked.read().unwrap(); + let cache = self.current.read().unwrap(); if cache.loaded_from.as_ref() == Some(&identity) { - return Self::hides(cache.tracked.as_ref(), relative, is_dir); + return cache.fingerprint.clone(); } } - let mut cache = self.tracked.write().unwrap(); - // Another thread may have reloaded it while this one waited. - if cache.loaded_from.as_ref() != Some(&identity) { - cache.tracked = crate::git_index::load_tracked(&self.repo_root); - cache.loaded_from = Some(identity); + let mut cache = self.current.write().unwrap(); + if cache.loaded_from.as_ref() == Some(&identity) { + return cache.fingerprint.clone(); + } + let tracked = crate::git_index::load_tracked(&self.repo_root); + cache.fingerprint = Self::fingerprint(tracked.as_ref()); + cache.loaded_from = Some(identity); + cache.fingerprint.clone() + } + + fn load_snapshot(repo_root: &Path) -> TrackedSnapshot { + let tracked = crate::git_index::load_tracked(repo_root); + let fingerprint = Self::fingerprint(tracked.as_ref()); + TrackedSnapshot { + tracked, + fingerprint, } - Self::hides(cache.tracked.as_ref(), relative, is_dir) + } + + fn fingerprint( + tracked: Option<&crate::git_index::TrackedFiles>, + ) -> TrackedMembershipFingerprint { + // Directory visibility depends on every tracked descendant, including + // paths whose final file match is whitelisted. Fingerprint the full set + // so changes that make an ignored ancestor walkable are observable. + TrackedMembershipFingerprint(tracked.map(|tracked| tracked.fingerprint_matching(|_| true))) } fn hides( @@ -626,20 +749,7 @@ fn git_repo_root(root: &Path) -> Option<&Path> { /// the next stale check evicts. pub fn repo_exclude_path(root: &Path) -> Option { let git_dir = crate::git_index::git_dir(git_repo_root(root)?)?; - let common = match std::fs::read_to_string(git_dir.join("commondir")) { - Ok(target) => { - let target = target.trim(); - let path = Path::new(target); - if target.is_empty() { - git_dir - } else if path.is_absolute() { - path.to_path_buf() - } else { - git_dir.join(path) - } - } - Err(_) => git_dir, - }; + let common = crate::git_index::common_git_dir(&git_dir); let path = common.join("info").join("exclude"); path.is_file().then_some(path) } @@ -692,6 +802,24 @@ pub fn matcher_from_ignore_paths_with_options( gitignore_files: &[std::path::PathBuf], ignore_files: &[std::path::PathBuf], no_require_git: bool, +) -> Option { + matcher_from_ignore_paths_with_options_and_ignorecase( + root, + gitignore_files, + ignore_files, + no_require_git, + CaseInsensitiveIgnore::new(root, true, true, true).map(std::sync::Arc::new), + ) +} + +/// Build a matcher using an immutable case-insensitive tracked-file snapshot +/// shared with the walk that discovered `gitignore_files`. +pub fn matcher_from_ignore_paths_with_options_and_ignorecase( + root: &Path, + gitignore_files: &[std::path::PathBuf], + ignore_files: &[std::path::PathBuf], + no_require_git: bool, + ignorecase: Option>, ) -> Option { use ignore::gitignore::GitignoreBuilder; @@ -780,12 +908,6 @@ pub fn matcher_from_ignore_paths_with_options( } else { GitignoreBuilder::new(root).build().ok()? }; - // The same narrowing `walker::walk_dir` and `walk_file_metadata` apply as a - // `filter_entry`. Serving takes no `--no-ignore-vcs` / `--no-ignore-exclude` - // / `--no-ignore-parent`, so the flags the walk was built with are the - // defaults; `no_ignore` is handled by the caller, which does not build a - // matcher at all in that case. - let ignorecase = CaseInsensitiveIgnore::new(root, true, true, true); IgnoreMatcher::with_all_sources( local, true, diff --git a/tgrep-core/src/hybrid.rs b/tgrep-core/src/hybrid.rs index bf96237..5a60168 100644 --- a/tgrep-core/src/hybrid.rs +++ b/tgrep-core/src/hybrid.rs @@ -293,6 +293,16 @@ impl HybridIndex { self.reader().all_paths().iter().cloned().collect() } + /// Whether the active on-disk reader contains `path`. + pub fn reader_has_path(&self, path: &str) -> bool { + self.reader().contains_path(path) + } + + /// Whether the active on-disk reader contains a path below `directory`. + pub fn reader_has_descendant_path(&self, directory: &str) -> bool { + self.reader().has_descendant_path(directory) + } + /// The reader paths `keep` accepts. /// /// For callers that want a few of them — everything under one directory, diff --git a/tgrep-core/src/live.rs b/tgrep-core/src/live.rs index 804e914..0f5ca2e 100644 --- a/tgrep-core/src/live.rs +++ b/tgrep-core/src/live.rs @@ -196,6 +196,17 @@ impl LiveIndex { self.path_to_id.contains_key(path) } + /// Whether the live overlay contains a path strictly below `directory`. + pub fn has_descendant_path(&self, directory: &str) -> bool { + if directory.is_empty() { + return !self.path_to_id.is_empty(); + } + self.path_to_id.keys().any(|path| { + path.strip_prefix(directory) + .is_some_and(|suffix| suffix.starts_with('/')) + }) + } + /// Number of files in the overlay. pub fn num_files(&self) -> usize { self.file_paths.len() @@ -499,6 +510,18 @@ impl LiveIndex { mod tests { use super::*; + #[test] + fn descendant_lookup_is_directory_boundary_aware() { + let mut live = LiveIndex::new(); + live.upsert_file_with_trigrams("src/deep/a.rs", Vec::new()); + live.upsert_file_with_trigrams("src2/not-a-child.rs", Vec::new()); + + assert!(live.has_descendant_path("src")); + assert!(live.has_descendant_path("src/deep")); + assert!(!live.has_descendant_path("sr")); + assert!(!live.has_descendant_path("src/deep/a.rs")); + } + #[test] fn clearing_reconciled_paths_removes_entries_and_tombstones_only_for_them() { let mut live = LiveIndex::new(); diff --git a/tgrep-core/src/meta.rs b/tgrep-core/src/meta.rs index 6bb40ae..98a1340 100644 --- a/tgrep-core/src/meta.rs +++ b/tgrep-core/src/meta.rs @@ -68,6 +68,21 @@ pub struct FileStamp { pub size: u64, } +/// Convert filesystem metadata into the persisted stamp used by change +/// detection. +pub fn file_stamp(metadata: &std::fs::Metadata) -> FileStamp { + let mtime = metadata + .modified() + .ok() + .and_then(|t| t.duration_since(SystemTime::UNIX_EPOCH).ok()) + .map(|d| d.as_secs()) + .unwrap_or(0); + FileStamp { + mtime, + size: metadata.len(), + } +} + /// Write per-file stamps to `filestamps.json` in the index directory. pub fn write_filestamps(stamps: &HashMap, index_dir: &Path) -> Result<()> { let path = index_dir.join(FILESTAMPS_FILENAME); @@ -95,21 +110,9 @@ pub fn collect_filestamps(root: &Path, paths: &[String]) -> HashMap, postings: Option, file_paths: Vec, + /// File IDs sorted by path, for exact membership checks without duplicating + /// every path string in a second collection. + path_order: Vec, num_entries: usize, } @@ -120,11 +123,14 @@ impl IndexReader { for (id, path) in file_entries { file_paths[id as usize] = path; } + let mut path_order: Vec = (0..file_paths.len()).collect(); + path_order.sort_unstable_by(|&a, &b| file_paths[a].cmp(&file_paths[b])); Ok(Self { lookup, postings, file_paths, + path_order, num_entries, }) } @@ -137,6 +143,7 @@ impl IndexReader { lookup: None, postings: None, file_paths: Vec::new(), + path_order: Vec::new(), num_entries: 0, } } @@ -146,6 +153,7 @@ impl IndexReader { self.lookup = None; self.postings = None; self.file_paths.clear(); + self.path_order.clear(); self.num_entries = 0; } @@ -179,6 +187,37 @@ impl IndexReader { self.file_paths.get(file_id as usize).map(|s| s.as_str()) } + pub fn contains_path(&self, path: &str) -> bool { + self.path_order + .binary_search_by(|&id| self.file_paths[id].as_str().cmp(path)) + .is_ok() + } + + /// Whether the reader contains a path strictly below `directory`. + pub fn has_descendant_path(&self, directory: &str) -> bool { + if directory.is_empty() { + return !self.path_order.is_empty(); + } + let prefix = directory + .as_bytes() + .iter() + .copied() + .chain(std::iter::once(b'/')); + let lower = self.path_order.partition_point(|&id| { + self.file_paths[id] + .as_bytes() + .iter() + .copied() + .cmp(prefix.clone()) + .is_lt() + }); + self.path_order.get(lower).is_some_and(|&id| { + self.file_paths[id] + .strip_prefix(directory) + .is_some_and(|suffix| suffix.starts_with('/')) + }) + } + /// Total number of indexed files. pub fn num_files(&self) -> usize { self.file_paths.len() @@ -518,6 +557,21 @@ mod tests { assert_eq!(reader.file_paths[2], "c.rs"); } + #[test] + fn descendant_lookup_is_sorted_and_directory_boundary_aware() { + let tmp = TempDir::new().unwrap(); + let mut files = ondisk::encode_file_entry(0, "elsewhere.rs").unwrap(); + files.extend_from_slice(&ondisk::encode_file_entry(1, "src/deep/a.rs").unwrap()); + files.extend_from_slice(&ondisk::encode_file_entry(2, "src2/not-a-child.rs").unwrap()); + write_index(tmp.path(), &[], &[], &files); + let reader = IndexReader::open(tmp.path()).unwrap(); + + assert!(reader.has_descendant_path("src")); + assert!(reader.has_descendant_path("src/deep")); + assert!(!reader.has_descendant_path("sr")); + assert!(!reader.has_descendant_path("src2/not-a-child.rs")); + } + #[test] fn is_degenerate_detects_mmap_counter_disagreement() { let tmp = TempDir::new().unwrap(); @@ -555,6 +609,7 @@ mod tests { lookup: opened.lookup, postings: opened.postings, file_paths: opened.file_paths, + path_order: opened.path_order, num_entries: 0, }; assert!( diff --git a/tgrep-core/src/walker.rs b/tgrep-core/src/walker.rs index cc5be0b..a8b536e 100644 --- a/tgrep-core/src/walker.rs +++ b/tgrep-core/src/walker.rs @@ -216,7 +216,7 @@ fn git_ignorecase_filter( no_ignore_vcs: bool, no_ignore_exclude: bool, no_ignore_parent: bool, -) -> Option { +) -> Option> { (!no_ignore) .then(|| { crate::gitignore::CaseInsensitiveIgnore::new( @@ -227,6 +227,7 @@ fn git_ignorecase_filter( ) }) .flatten() + .map(std::sync::Arc::new) } /// Walk a directory tree, respecting .gitignore rules (unless disabled). @@ -236,6 +237,25 @@ fn git_ignorecase_filter( /// detection is deferred to the caller (which reads the file anyway), /// avoiding an extra 8KB read per file during the walk. pub fn walk_dir(root: &Path, opts: &WalkOptions) -> WalkResult { + let ignorecase = git_ignorecase_filter( + root, + opts.no_ignore, + opts.no_ignore_vcs, + opts.no_ignore_exclude, + opts.no_ignore_parent, + ); + walk_dir_with_ignorecase(root, opts, ignorecase) +} + +/// Walk files using a supplied immutable tracked-file exemption snapshot. +/// +/// Callers that build a point-query matcher from the result can share this +/// value so the walk and publication make every decision from one snapshot. +pub fn walk_dir_with_ignorecase( + root: &Path, + opts: &WalkOptions, + ignorecase: Option>, +) -> WalkResult { let files = std::sync::Mutex::new(Vec::new()); let gitignore_files = std::sync::Mutex::new(Vec::new()); let ignore_files = std::sync::Mutex::new(Vec::new()); @@ -254,14 +274,6 @@ pub fn walk_dir(root: &Path, opts: &WalkOptions) -> WalkResult { .flatten() .map(std::sync::Arc::new); let p4ignore_root = root.clone(); - let ignorecase = git_ignorecase_filter( - &root, - opts.no_ignore, - opts.no_ignore_vcs, - opts.no_ignore_exclude, - opts.no_ignore_parent, - ); - let mut builder = WalkBuilder::new(&root); builder .hidden(!include_hidden) @@ -410,6 +422,24 @@ pub fn build_gitignore_matcher_from_files( ) } +/// Build a point-query matcher sharing the immutable tracked-file exemption +/// used by the walk that discovered these ignore files. +pub fn build_gitignore_matcher_from_files_with_ignorecase( + root: &Path, + gitignore_files: &[PathBuf], + ignore_files: &[PathBuf], + no_require_git: bool, + ignorecase: Option>, +) -> Option { + crate::gitignore::matcher_from_ignore_paths_with_options_and_ignorecase( + root, + gitignore_files, + ignore_files, + no_require_git, + ignorecase, + ) +} + /// Filesystem metadata for a single file (no content read). pub struct FileMeta { pub relative_path: String, @@ -479,6 +509,17 @@ impl Default for MetaWalkOptions { /// explicitly via [`crate::gitignore::ignore_files_in`], which also catches /// ignore files that their own rules would have filtered out of the walk. pub fn walk_file_metadata(root: &Path, opts: &MetaWalkOptions) -> MetaWalkResult { + let ignorecase = git_ignorecase_filter(root, opts.no_ignore, false, false, false); + walk_file_metadata_with_ignorecase(root, opts, ignorecase) +} + +/// Walk metadata using an immutable case-insensitive tracked-file exemption +/// that can also be shared with the resulting point-query matcher. +pub fn walk_file_metadata_with_ignorecase( + root: &Path, + opts: &MetaWalkOptions, + ignorecase: Option>, +) -> MetaWalkResult { let no_ignore = opts.no_ignore; let max_file_size = opts.max_file_size; let results = std::sync::Mutex::new(Vec::new()); @@ -491,11 +532,6 @@ pub fn walk_file_metadata(root: &Path, opts: &MetaWalkOptions) -> MetaWalkResult .flatten() .map(std::sync::Arc::new); let match_root = root.to_path_buf(); - // `MetaWalkOptions` carries no finer ignore flags, and the builder below - // enables gitignore and exclude together on `!no_ignore`. Passing `false` - // for both mirrors that exactly, which is what keeps this walk and - // `walk_dir` agreeing about the tree under `serve`. - let ignorecase = git_ignorecase_filter(root, no_ignore, false, false, false); let root = root.to_path_buf(); let walker = WalkBuilder::new(&root) @@ -580,16 +616,11 @@ pub fn walk_file_metadata(root: &Path, opts: &MetaWalkOptions) -> MetaWalkResult if max_file_size.is_some_and(|limit| meta.len() > limit) { return ignore::WalkState::Continue; } - let mtime = meta - .modified() - .ok() - .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()) - .map(|d| d.as_secs()) - .unwrap_or(0); + let stamp = crate::meta::file_stamp(&meta); results.lock().unwrap().push(FileMeta { relative_path: rel_path, - mtime, - size: meta.len(), + mtime: stamp.mtime, + size: stamp.size, }); ignore::WalkState::Continue @@ -1472,14 +1503,8 @@ mod tests { assert!(!matcher.is_ignored(Path::new("src/Gone.TXT"), false)); } - /// The exemption is answered from a cached read of `.git/index`. A walk - /// builds this matcher and drops it, so a snapshot would do — but the file - /// watcher holds one for the life of the server, and `git add -f` rewrites - /// only the index, which is hidden. No ignore source changes, nothing - /// republishes the matcher, and a frozen cache would keep hiding a file - /// that git now tracks until the hourly reconcile. #[test] - fn the_tracked_exemption_reloads_when_the_git_index_changes() { + fn public_matcher_lazily_freezes_tracked_exemptions_on_first_match() { let dir = ignorecase_fixture(true, &["src/main.rs", "src/Kept.TXT"]); let root = dir.path(); let matcher = crate::gitignore::matcher_from_ignore_paths( @@ -1489,21 +1514,81 @@ mod tests { ) .expect("the fixture has rules"); - // Untracked, and `*.txt` matches it once case is ignored. - assert!(matcher.is_ignored(Path::new("src/Gone.TXT"), false)); - + // Construction alone must not read the index. The first matching query + // observes this update and freezes that membership for later queries. fake_git_repo(root, true, &["src/main.rs", "src/Kept.TXT", "src/Gone.TXT"]); - assert!( !matcher.is_ignored(Path::new("src/Gone.TXT"), false), - "a file git now tracks must stop being hidden without rebuilding \ - the matcher" + "the first matching query must initialize from the current index" ); - // And back: `git rm --cached` is the same problem in reverse, where a - // stale cache keeps indexing a file the walk has started hiding. fake_git_repo(root, true, &["src/main.rs"]); - assert!(matcher.is_ignored(Path::new("src/Kept.TXT"), false)); + assert!( + !matcher.is_ignored(Path::new("src/Kept.TXT"), false), + "the lazy snapshot must remain immutable after initialization" + ); + } + + /// Reconciliation deliberately opts into the opposite behavior: the walk + /// and matcher publication must not change answers halfway through a pass. + #[test] + fn frozen_tracked_exemption_is_immutable_and_detects_changes() { + let dir = ignorecase_fixture(true, &["src/main.rs", "src/Kept.TXT"]); + let root = dir.path(); + let ignorecase = + crate::gitignore::CaseInsensitiveIgnore::frozen_snapshot(root, true, true, true) + .map(std::sync::Arc::new); + let matcher = crate::gitignore::matcher_from_ignore_paths_with_options_and_ignorecase( + root, + std::slice::from_ref(&root.join(".gitignore")), + &[], + false, + ignorecase, + ) + .expect("the fixture has rules"); + + assert!(matcher.is_ignored(Path::new("src/Gone.TXT"), false)); + fake_git_repo(root, true, &["src/main.rs", "src/Kept.TXT", "src/Gone.TXT"]); + + assert!( + matcher.is_ignored(Path::new("src/Gone.TXT"), false), + "a frozen matcher must preserve the pass snapshot" + ); + assert_ne!( + matcher.tracked_membership_fingerprint(), + matcher.current_tracked_membership_fingerprint(), + "semantic polling must detect changes without mutating decisions" + ); + } + + #[test] + fn tracked_fingerprint_includes_whitelisted_files_under_ignored_directories() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path(); + fake_git_repo(root, true, &["qlogs/kept.rs"]); + fs::write(root.join(".gitignore"), "QLogs/\n!qlogs/kept.rs\n").unwrap(); + fs::create_dir_all(root.join("qlogs")).unwrap(); + fs::write(root.join("qlogs/kept.rs"), "tracked").unwrap(); + + let ignorecase = + crate::gitignore::CaseInsensitiveIgnore::frozen_snapshot(root, true, true, true) + .map(std::sync::Arc::new); + let matcher = crate::gitignore::matcher_from_ignore_paths_with_options_and_ignorecase( + root, + std::slice::from_ref(&root.join(".gitignore")), + &[], + false, + ignorecase, + ) + .expect("the fixture has rules"); + let baseline = matcher.tracked_membership_fingerprint(); + + fake_git_repo(root, true, &[]); + assert_ne!( + baseline, + matcher.current_tracked_membership_fingerprint(), + "polling must notice when an ignored ancestor loses its tracked descendant" + ); } #[test]