diff --git a/crates/repo/src/bucket_transport.rs b/crates/repo/src/bucket_transport.rs index 431e0d8e..11db2d02 100644 --- a/crates/repo/src/bucket_transport.rs +++ b/crates/repo/src/bucket_transport.rs @@ -675,6 +675,23 @@ impl crate::reachable::ObjectSource for BucketSource<'_> { } } +/// Attempts `update_ref` makes at the manifest CAS before it gives up (the +/// documented "rerun the push" outcome). +const MAX_CAS_RETRIES: u32 = 16; + +/// Pause before CAS attempt number `attempt` (1-based after the first try): +/// "full jitter" exponential backoff, uniform in `[0, min(2ms << attempt, 250ms))`. +/// Every lost CAS means a rival landed, so a fleet that retries in lockstep +/// keeps colliding and an unlucky writer burns its whole budget while all its +/// rivals succeed (#271). Randomness comes from `RandomState`'s per-instance +/// keys: no RNG dependency, and the writers' draws differ. +fn cas_backoff(attempt: u32) -> std::time::Duration { + use std::hash::BuildHasher; + let ceiling_us = (2_000u64 << attempt.min(16)).min(250_000); + let draw = std::collections::hash_map::RandomState::new().hash_one(attempt); + std::time::Duration::from_micros(draw % ceiling_us) +} + impl Transport for BucketTransport { fn list_refs(&self) -> Result> { self.refresh()?; @@ -804,8 +821,10 @@ impl Transport for BucketTransport { // the bucket regardless (content-addressed, inert orphan garbage if // never referenced — correct per spec; `compact` sweeps it later, ADR-0057). let pending: Vec = std::mem::take(&mut *self.pending_packs.borrow_mut()); - const MAX_CAS_RETRIES: u32 = 16; - for _ in 0..MAX_CAS_RETRIES { + for attempt in 0..MAX_CAS_RETRIES { + if attempt > 0 { + std::thread::sleep(cas_backoff(attempt)); + } self.refresh()?; let (current, prev_tag, head_seq, checkpoint_seq, head_branch) = { let view = self.view.borrow(); @@ -885,9 +904,9 @@ impl Transport for BucketTransport { // under the new parent — `pending` (this call's local copy) // still lists our packs, so nothing is re-uploaded. } - Err(Error::Remote( - "manifest cas contention: gave up after 16 attempts".into(), - )) + Err(Error::Remote(format!( + "manifest cas contention: gave up after {MAX_CAS_RETRIES} attempts" + ))) } } @@ -1333,7 +1352,17 @@ mod tests { (obj.id(), obj.encode()) }; let gate = std::sync::Barrier::new(2); + let finished = std::sync::atomic::AtomicBool::new(false); + let status = crate::test_support::Status::default(); let results: Vec> = std::thread::scope(|s| { + // A racer that panics before the gate would leave its peer waiting. + let _finish = crate::test_support::Stop(&finished, true); + crate::test_support::watchdog( + s, + &finished, + &status, + std::time::Duration::from_secs(120), + ); let handles: Vec<_> = [b"racer-a".as_slice(), b"racer-b".as_slice()] .into_iter() .map(|tag| { @@ -1351,7 +1380,8 @@ mod tests { }) }) .collect(); - handles.into_iter().map(|h| h.join().unwrap()).collect() + let joined: Vec<_> = handles.into_iter().map(|h| h.join()).collect(); + joined.into_iter().map(|r| r.unwrap()).collect() }); let wins = results.iter().filter(|r| r.is_ok()).count(); let nffs = results @@ -1367,48 +1397,174 @@ mod tests { assert!(!broot.exists()); } - /// P36a acceptance proof #2: 8 threads push to 8 *distinct* branches - /// concurrently with no external coordinator. Contention is at the - /// manifest-CAS level only (every push races the same `manifest` key even - /// though the branches don't conflict), so every push must eventually - /// land via the retry loop. - #[test] - fn fleet_hammer_distinct_branches_all_land_without_coordinator() { - let pid = std::process::id(); - let broot = std::env::temp_dir().join(format!("scl-bt-fleet-{pid}")); + /// `n` threads push to `n` distinct branches at the same instant with no + /// external coordinator. Contention is at the manifest-CAS level only + /// (every push races the same `manifest` key even though the branches + /// don't conflict), so every push must land via the retry loop. A + /// watchdog turns a worker stuck at the gate (a peer panicked before it) + /// into a diagnosed abort. + fn fleet_lands(tag: &str, n: usize) { + use crate::test_support::{note, watchdog, Status, Stop}; + use std::sync::atomic::AtomicBool; + let broot = std::env::temp_dir().join(format!("scl-bt-{tag}-{}", std::process::id())); let _ = std::fs::remove_dir_all(&broot); - const N: usize = 8; - // A barrier forces all N threads to call `update_ref` at essentially + // A barrier forces all threads to call `update_ref` at essentially // the same instant — without it, short-lived threads could finish // one at a time with the OS never actually overlapping them, and - // "all N land" would hold even against a broken retry loop that was + // "all land" would hold even against a broken retry loop that was // never exercised under real contention. - let gate = std::sync::Barrier::new(N); + let gate = std::sync::Barrier::new(n); + let finished = AtomicBool::new(false); + let status = Status::default(); std::thread::scope(|s| { - for i in 0..N { - let broot = broot.clone(); - let gate = &gate; - s.spawn(move || { - let t = - BucketTransport::from_bucket(Box::new(DirBucket::open(&broot).unwrap())) - .unwrap(); - let obj = Object::blob(format!("agent-{i}").into_bytes()); - t.put_object(&obj.id(), &obj.encode()).unwrap(); - gate.wait(); - // distinct branches: contention is manifest-level only, so - // every one must eventually land via CAS retry. - t.update_ref(&format!("work-{i}"), &obj.id(), None).unwrap(); - }); + let _finish = Stop(&finished, true); + watchdog(s, &finished, &status, std::time::Duration::from_secs(120)); + let workers: Vec<_> = (0..n) + .map(|i| { + let (broot, gate, status) = (broot.clone(), &gate, &status); + s.spawn(move || { + note(status, format!("writer-{i:03}"), "setup".into()); + let t = BucketTransport::from_bucket(Box::new( + DirBucket::open(&broot).unwrap(), + )) + .unwrap(); + let obj = Object::blob(format!("agent-{i}").into_bytes()); + t.put_object(&obj.id(), &obj.encode()).unwrap(); + note(status, format!("writer-{i:03}"), "at the gate".into()); + gate.wait(); + note(status, format!("writer-{i:03}"), "update_ref".into()); + t.update_ref(&format!("work-{i}"), &obj.id(), None) + .unwrap_or_else(|e| panic!("writer {i} of {n}: {e}")); + note(status, format!("writer-{i:03}"), "landed".into()); + }) + }) + .collect(); + // Join every worker before `_finish` drops: the watchdog must keep + // running until the last one is done, or a worker stuck at the gate + // (its peer panicked) hangs the scope with no diagnostic. + let results: Vec<_> = workers.into_iter().map(|h| h.join()).collect(); + for r in results { + r.unwrap(); } }); let t = BucketTransport::from_bucket(Box::new(DirBucket::open(&broot).unwrap())).unwrap(); let refs = t.list_refs().unwrap(); - assert_eq!(refs.len(), N, "all {N} agent branches present: {refs:?}"); + assert_eq!(refs.len(), n, "all {n} agent branches present: {refs:?}"); drop(t); std::fs::remove_dir_all(&broot).unwrap(); assert!(!broot.exists()); } + /// P36a acceptance proof #2: 8 threads, 8 distinct branches. + #[test] + fn fleet_hammer_distinct_branches_all_land_without_coordinator() { + fleet_lands("fleet", 8); + } + + /// The #271 measurement: 48 simultaneous pushers, more than the 16-attempt + /// budget. Passes on an idle machine; heavy load can still exhaust the + /// budget (documented in ADR-0046), so it is not part of the default run. + /// `cargo test -p scl-repo wide_fleet -- --ignored` + #[test] + #[ignore = "stress: load-dependent by design"] + fn wide_fleet_of_48_pushers_lands_on_an_idle_machine() { + fleet_lands("fleet-wide", 48); + } + + /// A bucket whose manifest CAS loses the first `lose` attempts (as if a + /// rival landed each time) and then behaves like the real thing. + struct LoseCas { + inner: DirBucket, + lose: std::sync::atomic::AtomicU32, + lost: std::sync::atomic::AtomicU32, + } + impl Bucket for LoseCas { + fn get(&self, key: &str, cached: Option<&str>) -> scl_objio::Result { + self.inner.get(key, cached) + } + fn put_new(&self, key: &str, bytes: &[u8]) -> scl_objio::Result { + self.inner.put_new(key, bytes) + } + fn put_if_tag( + &self, + key: &str, + bytes: &[u8], + expected: Option<&str>, + ) -> scl_objio::Result> { + use std::sync::atomic::Ordering::SeqCst; + if key == "manifest" && self.lost.load(SeqCst) < self.lose.load(SeqCst) { + self.lost.fetch_add(1, SeqCst); + return Ok(None); + } + self.inner.put_if_tag(key, bytes, expected) + } + fn list(&self, prefix: &str) -> scl_objio::Result> { + self.inner.list(prefix) + } + fn list_meta(&self, prefix: &str) -> scl_objio::Result> { + self.inner.list_meta(prefix) + } + fn exists(&self, key: &str) -> scl_objio::Result { + self.inner.exists(key) + } + fn delete(&self, key: &str) -> scl_objio::Result<()> { + self.inner.delete(key) + } + } + + /// The CAS retry budget is exact and bounded (#271): losing 15 manifest + /// CASes in a row still lands on the 16th attempt, and losing all 16 + /// gives the documented "rerun the push" error, never a further try. + /// Deterministic: the losses are injected, so machine load cannot move + /// the outcome (a real fleet's losses depend on scheduling). + #[test] + fn the_cas_retry_budget_is_exactly_sixteen_attempts() { + use std::sync::atomic::AtomicU32; + for (lose, lands) in [(15u32, true), (16, false)] { + let broot = + std::env::temp_dir().join(format!("scl-bt-budget-{lose}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&broot); + let t = BucketTransport::from_bucket(Box::new(LoseCas { + inner: DirBucket::open(&broot).unwrap(), + lose: AtomicU32::new(lose), + lost: AtomicU32::new(0), + })) + .unwrap(); + let obj = Object::blob(b"budget".to_vec()); + t.put_object(&obj.id(), &obj.encode()).unwrap(); + let res = t.update_ref("main", &obj.id(), None); + if lands { + res.unwrap(); + assert_eq!(t.list_refs().unwrap().len(), 1, "lose {lose}"); + } else { + let e = res.unwrap_err().to_string(); + assert!(e.contains("gave up after 16 attempts"), "lose {lose}: {e}"); + } + drop(t); + std::fs::remove_dir_all(&broot).unwrap(); + } + } + + /// The pause between CAS attempts is jittered, grows with the attempt, and + /// stays under its cap, so total backoff for one push is bounded. + #[test] + fn cas_backoff_is_bounded_and_grows_to_its_cap() { + let max_us = |attempt| { + (0..200) + .map(|_| cas_backoff(attempt).as_micros()) + .max() + .unwrap() + }; + assert!(max_us(1) < 4_000); + assert!(max_us(8) > 4_000, "the ceiling must grow with the attempt"); + assert!(max_us(3) < 16_000); + assert!(max_us(15) < 250_000); + let total: std::time::Duration = (1..MAX_CAS_RETRIES).map(|_| cas_backoff(15)).sum(); + assert!(total < std::time::Duration::from_secs(4), "{total:?}"); + let draws: std::collections::HashSet<_> = (0..50).map(|_| cas_backoff(8)).collect(); + assert!(draws.len() > 1, "backoff must be jittered"); + } + /// P36a acceptance proof #3: a pusher that died after writing its pack + /// log entry, but before the manifest CAS, leaves debris that must be /// invisible to every reader (the manifest never points at it) — and a diff --git a/crates/repo/src/bucket_transport/compact/tests.rs b/crates/repo/src/bucket_transport/compact/tests.rs index fd2d584d..97cc032f 100644 --- a/crates/repo/src/bucket_transport/compact/tests.rs +++ b/crates/repo/src/bucket_transport/compact/tests.rs @@ -5,6 +5,7 @@ use super::super::CHECKPOINT_INTERVAL; use super::*; use crate::error::Error; +use crate::test_support::{note, watchdog, Status, Stop}; use crate::transport::Transport; use crate::walfmt::{idx_key, RefUpdate}; use scl_core::Object; @@ -485,45 +486,6 @@ fn a_failure_at_every_operation_boundary_leaves_a_readable_bucket() { assert!(n > 20, "the loop should cross many boundaries, got {n}"); } -/// Stops a concurrency test from hanging: a worker that panics raises the flag -/// (and the main thread always does, on exit), so one failure releases every -/// loop instead of leaving the scope waiting on workers that never see `done`. -struct Stop<'a>(&'a AtomicBool, bool); -impl Drop for Stop<'_> { - fn drop(&mut self) { - if self.1 || std::thread::panicking() { - self.0.store(true, Ordering::SeqCst); - } - } -} - -/// Aborts the process with each worker's last-known step if the test is still -/// running at `limit`: a worker stuck inside a call (a lock, a bucket op) can -/// not be unwound, so the hang becomes a diagnosed failure. Written straight to -/// stderr because the harness captures `eprintln!` until the test ends. -fn watchdog<'a>( - s: &'a std::thread::Scope<'a, '_>, - finished: &'a AtomicBool, - status: &'a Mutex>, - limit: Duration, -) { - s.spawn(move || { - let start = std::time::Instant::now(); - while !finished.load(Ordering::SeqCst) { - if start.elapsed() > limit { - use std::io::Write; - let _ = writeln!( - std::io::stderr(), - "WATCHDOG: test still running after {limit:?}; last steps: {:#?}", - status.lock().map(|m| m.clone()).unwrap_or_default() - ); - std::process::abort(); - } - std::thread::sleep(Duration::from_millis(200)); - } - }); -} - #[test] fn readers_and_writers_stay_consistent_while_compaction_runs() { let root = scratch("concurrent"); @@ -542,10 +504,8 @@ fn readers_and_writers_stay_consistent_while_compaction_runs() { let landed = Mutex::new(Vec::new()); let deleted = std::sync::atomic::AtomicUsize::new(0); let finished = AtomicBool::new(false); - let status = Mutex::new(BTreeMap::new()); - let note = |who: String, what: String| { - status.lock().unwrap().insert(who, what); - }; + let status = Status::default(); + let note = |who: String, what: String| note(&status, who, what); std::thread::scope(|s| { let _finish = Stop(&finished, true); let _stop = Stop(&done, false); diff --git a/crates/repo/src/fill.rs b/crates/repo/src/fill.rs index a452d0b0..91599f0d 100644 --- a/crates/repo/src/fill.rs +++ b/crates/repo/src/fill.rs @@ -108,7 +108,20 @@ impl Repo { ids: &[ObjectId], stats: &mut FillStats, ) -> Result<()> { - for chunk in ids.chunks(crate::wire::MAX_GET_OBJECTS_IDS) { + self.fetch_ids_chunked(transport, ids, crate::wire::MAX_GET_OBJECTS_IDS, stats) + } + + /// [`Self::fetch_ids_batched`] with the per-request id count as a + /// parameter, so a test can cross a chunk boundary without ingesting + /// thousands of loose objects (two fsyncs each). + fn fetch_ids_chunked( + &self, + transport: &dyn Transport, + ids: &[ObjectId], + max_ids: usize, + stats: &mut FillStats, + ) -> Result<()> { + for chunk in ids.chunks(max_ids) { self.fetch_ids_split(transport, chunk, stats)?; } Ok(()) @@ -972,11 +985,15 @@ mod tests { } } + /// Cost is deterministic: every ingested object is a durable loose write + /// (two fsyncs, F_FULLFSYNC on macOS), so this crosses the chunk boundary + /// with 20 ids at 8 per request instead of 2500 at 1024 (that took 25 s + /// unloaded and minutes under load, #271). #[test] fn a_wide_level_is_chunked_by_count_before_anything_is_split_by_size() { let fx = Fixture::new("chunks"); let dst = fx.clone(true); - let objs: std::collections::HashMap = (0..2500u32) + let objs: std::collections::HashMap = (0..20u32) .map(|i| { let o = Object::blob(i.to_le_bytes().to_vec()); (o.id(), o) @@ -988,14 +1005,14 @@ mod tests { sizes: Default::default(), }; let mut stats = FillStats::default(); - dst.fetch_ids_batched(&t, &ids, &mut stats).unwrap(); - // 2500 ids: two full 1024-id requests and one of 452, never split. - assert_eq!(*t.sizes.borrow(), vec![1024, 1024, 452]); + dst.fetch_ids_chunked(&t, &ids, 8, &mut stats).unwrap(); + // 20 ids: two full 8-id requests and one of 4, never split. + assert_eq!(*t.sizes.borrow(), vec![8, 8, 4]); assert_eq!( stats, FillStats { requests: 3, - objects: 2500 + objects: 20 } ); fx.cleanup(); diff --git a/crates/repo/src/lib.rs b/crates/repo/src/lib.rs index e3152403..cf976ab1 100644 --- a/crates/repo/src/lib.rs +++ b/crates/repo/src/lib.rs @@ -56,6 +56,8 @@ pub mod sparse; mod sparse_completion; pub mod stdio_transport; pub mod sync; +#[cfg(test)] +mod test_support; pub mod textdiff; pub mod tls_pins; pub mod transcripts; diff --git a/crates/repo/src/test_support.rs b/crates/repo/src/test_support.rs new file mode 100644 index 00000000..f6ae71f1 --- /dev/null +++ b/crates/repo/src/test_support.rs @@ -0,0 +1,57 @@ +//! Helpers for tests that run worker threads: a failure must end the test +//! with a diagnostic, never leave the scope or the harness waiting forever. + +use std::collections::BTreeMap; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Mutex; +use std::time::Duration; + +/// Stops a concurrency test from hanging: a worker that panics raises the flag +/// (and a guard built with `true` always does, on exit), so one failure +/// releases every loop instead of leaving the scope waiting on workers that +/// never see `done`. +pub(crate) struct Stop<'a>(pub &'a AtomicBool, pub bool); +impl Drop for Stop<'_> { + fn drop(&mut self) { + if self.1 || std::thread::panicking() { + self.0.store(true, Ordering::SeqCst); + } + } +} + +/// Last-known step per worker, shown by [`watchdog`] when it fires. +pub(crate) type Status = Mutex>; + +/// Record `what` as `who`'s current step. +pub(crate) fn note(status: &Status, who: String, what: String) { + status.lock().unwrap().insert(who, what); +} + +/// Aborts the process with each worker's last-known step if the test is still +/// running at `limit`: a worker stuck inside a call (a lock, a bucket op, a +/// barrier whose peer panicked) can not be unwound, so the hang becomes a +/// diagnosed failure. Written straight to stderr because the harness captures +/// `eprintln!` until the test ends. Raise `finished` (see [`Stop`]) to stand +/// the watchdog down. +pub(crate) fn watchdog<'a>( + s: &'a std::thread::Scope<'a, '_>, + finished: &'a AtomicBool, + status: &'a Status, + limit: Duration, +) { + s.spawn(move || { + let start = std::time::Instant::now(); + while !finished.load(Ordering::SeqCst) { + if start.elapsed() > limit { + use std::io::Write; + let _ = writeln!( + std::io::stderr(), + "WATCHDOG: test still running after {limit:?}; last steps: {:#?}", + status.lock().map(|m| m.clone()).unwrap_or_default() + ); + std::process::abort(); + } + std::thread::sleep(Duration::from_millis(200)); + } + }); +} diff --git a/docs/adr/0046-wal-bucket-remotes.md b/docs/adr/0046-wal-bucket-remotes.md index 71943842..07daaef0 100644 --- a/docs/adr/0046-wal-bucket-remotes.md +++ b/docs/adr/0046-wal-bucket-remotes.md @@ -243,3 +243,29 @@ ADR-0057. Effects on this ADR's contracts: objects and removes unreferenced ones. - The ADR-0046 `MAX_LIST` fold guard is unchanged. Compaction is what makes a skipped fold recoverable, on the pack axis. + +## As built (#271, 2026-10): bounded, jittered CAS retries + +`update_ref` still makes at most 16 attempts at the manifest CAS, and +exhausting them is still the documented outcome ("manifest cas contention: +gave up after 16 attempts"; the push is safe to rerun). Two things changed: + +- **Jittered exponential backoff between attempts.** Attempt `n > 0` first + sleeps a uniform random time in `[0, min(2 ms << n, 250 ms))`, the ceilings + are 4, 8, 16, 32, 64, 128 ms and then 250 ms, so one push sleeps about 2.5 s + in the worst case (about 1.25 s on average). A lost CAS always + means a rival landed, so a fleet retrying in lockstep keeps colliding and an + unlucky writer can burn all 16 attempts while every rival succeeds. Measured + with 48 simultaneous pushers to distinct branches on an idle machine: 0 of 20 + runs landed everyone without backoff, 40 of 40 with it. The budget stays + finite; there is no queue, so a large fleet on a heavily loaded host can + still exhaust it and must rerun the push (48 pushers at load average ~20: 5 + of 6 runs still hit the budget). The `--ignored` test + `wide_fleet_of_48_pushers_lands_on_an_idle_machine` reproduces the idle-host + measurement. +- **The `DirBucket` CAS lock is an OS advisory lock** (#264), not a + create-exclusive spin file. The spin lock gave up after 2000 tries with + "cas lock stuck" under load; that, not the retry budget, was what failed + `fleet_hammer_distinct_branches_all_land_without_coordinator` (11 of 60 runs + under load with the old lock, 0 of 100 with the new one). `S3Bucket` is + unaffected: its CAS is a server-side precondition.