diff --git a/crates/uffs-cli/src/main.rs b/crates/uffs-cli/src/main.rs index 48a93aaa1..105cc202e 100644 --- a/crates/uffs-cli/src/main.rs +++ b/crates/uffs-cli/src/main.rs @@ -55,6 +55,7 @@ use assert_cmd as _; pub mod args; pub mod commands; mod dispatch; +mod search_retry; /// Run the CLI and return a result. fn run() -> Result<()> { @@ -313,8 +314,7 @@ pub(crate) fn run_search(args: &[String]) -> Result<()> { let args_owned: Vec = commands::search::args::inject_no_output_for_null_stdout( commands::search::args::resolve_out_path(args), ); - let raw_response = client - .search_cli_raw(&args_owned) + let raw_response = search_retry::search_cli_with_warm_retry(&mut client, &args_owned) .with_context(|| "Daemon search_cli failed")?; let ipc_ms = t_search.elapsed().as_millis(); diff --git a/crates/uffs-cli/src/search_retry.rs b/crates/uffs-cli/src/search_retry.rs new file mode 100644 index 000000000..c4857fc4b --- /dev/null +++ b/crates/uffs-cli/src/search_retry.rs @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright (c) 2025-2026 SKY, LLC. + +//! Client-side resilience for a search that races a parked-drive re-warm. +//! +//! When the daemon has idled long enough to **park** drives (bodies evicted, +//! only the bloom + path trie resident), the first search re-warms them. On +//! Windows that re-warm reads MFT extents over the broker's overlapped handle, +//! and a pending read can be cancelled with `ERROR_OPERATION_ABORTED` (os error +//! 995) when the worker thread that issued it is reaped. The daemon's own read +//! primitive ([`uffs-mft`'s `read_handle_at`]) already retries that transient; +//! this is the client-side belt-and-suspenders so a user never sees a raw I/O +//! error for a recoverable warm-up hiccup — and gets a "warming" note instead. + +// Each helper is used exactly once on the search path; this is cohesion, not a +// smell — the `single_call_fn` restriction lint is relaxed for the module. +#![expect( + clippy::single_call_fn, + reason = "search-retry helpers, each called once on the search hot path" +)] + +use uffs_client::connect_sync::UffsClientSync; +use uffs_client::error::ClientError; + +/// Max client-side retries when a search aborts mid-index-warm. +const WARM_RETRY_MAX: u32 = 5; +/// Backoff between warm-retry attempts. +const WARM_RETRY_BACKOFF: core::time::Duration = core::time::Duration::from_millis(400); + +/// Run `search_cli_raw`, transparently retrying the one transient a search can +/// hit while the daemon re-warms parked drives: Windows +/// `ERROR_OPERATION_ABORTED` (os error 995). +/// +/// Bounded + back-off so a genuine, persistent failure still fails fast. +pub(crate) fn search_cli_with_warm_retry( + client: &mut UffsClientSync, + args: &[String], +) -> Result { + let mut attempt = 0_u32; + loop { + match client.search_cli_raw(args) { + Ok(response) => return Ok(response), + Err(err) if attempt < WARM_RETRY_MAX && is_index_warming_abort(&err) => { + attempt += 1; + report_index_warming(attempt); + std::thread::sleep(WARM_RETRY_BACKOFF); + } + Err(err) => return Err(err), + } + } +} + +/// `true` if `err` is the transient `ERROR_OPERATION_ABORTED` (os error 995) a +/// search hits when it races a parked-drive re-warm. Matched on the stable OS +/// error code in the rendered message — the daemon's typed I/O error is +/// flattened to a string across the JSON-RPC boundary, so the code is the only +/// portable signal left. +fn is_index_warming_abort(err: &ClientError) -> bool { + let message = err.to_string(); + message.contains("os error 995") || message.contains("operation has been aborted") +} + +/// Tell the user (on stderr, so it never pollutes stdout/CSV) that the index is +/// warming and the search is being retried. +#[expect(clippy::print_stderr, reason = "user-facing progress note, off stdout")] +fn report_index_warming(attempt: u32) { + eprintln!("UFFS index is warming up — retrying search ({attempt}/{WARM_RETRY_MAX})…"); +} + +#[cfg(test)] +mod tests { + #![expect( + clippy::default_numeric_fallback, + reason = "test module — relaxed linting" + )] + + use uffs_client::error::ClientError; + + use super::is_index_warming_abort; + + /// Only a transient `ERROR_OPERATION_ABORTED` (os error 995) — what a + /// search hits while racing a parked-drive re-warm — should be retried; + /// real errors must surface immediately. + #[test] + fn warming_abort_matches_only_995() { + assert!(is_index_warming_abort(&ClientError::Io( + "The I/O operation has been aborted because of either a thread exit \ + or an application request. (os error 995)" + .to_owned() + ))); + assert!(is_index_warming_abort(&ClientError::DaemonError { + code: -32000, + message: "I/O error: ... (os error 995)".to_owned(), + })); + // Real failures are NOT retried. + assert!(!is_index_warming_abort(&ClientError::Io( + "permission denied (os error 5)".to_owned() + ))); + assert!(!is_index_warming_abort(&ClientError::ConnectionClosed)); + } +} diff --git a/crates/uffs-mft/src/platform/volume.rs b/crates/uffs-mft/src/platform/volume.rs index 57aeb71a0..eae13dce3 100644 --- a/crates/uffs-mft/src/platform/volume.rs +++ b/crates/uffs-mft/src/platform/volume.rs @@ -173,8 +173,48 @@ pub(crate) fn try_adopt_broker_handle(drive: super::DriveLetter) -> Result Result<()> { + // `ERROR_OPERATION_ABORTED` (995) is transient: the OS cancels a *pending* + // overlapped read when the thread that issued it is reaped — e.g. a tokio + // `spawn_blocking` / rayon worker recycling during a parked-drive re-warm + // (the USN-refresh reads MFT extents over the broker's overlapped handle). + // The read is valid, so reissue it; bounded so a genuinely wedged handle + // still fails fast instead of spinning forever. + const MAX_ABORT_RETRIES: u32 = 8; + + let mut attempt = 0_u32; + loop { + match read_handle_at_once(handle, offset, buf) { + Ok(()) => return Ok(()), + Err(err) if attempt < MAX_ABORT_RETRIES && is_operation_aborted(&err) => { + attempt += 1; + // Yield so the recycled thread / overlapped slot can settle + // before the read is reissued. + std::thread::yield_now(); + } + Err(err) => return Err(err), + } + } +} + +/// `true` when `err` is a Windows `ERROR_OPERATION_ABORTED` (995) I/O failure — +/// the transient cancellation [`read_handle_at`] retries. +#[cfg(windows)] +fn is_operation_aborted(err: &MftError) -> bool { + let aborted = i32::try_from(ERROR_OPERATION_ABORTED_CODE).ok(); + matches!(err, MftError::Io(io) if io.raw_os_error() == aborted) +} + +/// One overlapped read attempt; the [`read_handle_at`] wrapper adds the +/// transient-abort (995) retry. +/// +/// # Errors +/// +/// [`MftError::Io`] if `ReadFile` / `GetOverlappedResult` fail, or +/// [`MftError::InvalidData`] on a short read. +#[cfg(windows)] +#[expect(unsafe_code, reason = "FFI: overlapped ReadFile + GetOverlappedResult")] +fn read_handle_at_once(handle: HANDLE, offset: u64, buf: &mut [u8]) -> Result<()> { use windows::Win32::Foundation::ERROR_IO_PENDING; use windows::Win32::Storage::FileSystem::ReadFile; use windows::Win32::System::IO::{GetOverlappedResult, OVERLAPPED};