From 863f1b6af5cdd860bc8f68a99759bcbe2693991d Mon Sep 17 00:00:00 2001 From: Toni Bergholm Date: Wed, 21 Oct 2026 04:06:34 +0300 Subject: [PATCH 1/2] fix(repo): jittered backoff between manifest CAS retries; deterministic load-proof tests (ref #271) - update_ref sleeps a jittered, capped exponential time between manifest CAS attempts. The budget stays 16 attempts. - fill: a_wide_level test crosses the chunk boundary with 20 ids at 8 per request (new fetch_ids_chunked) instead of 2500 loose objects, each two fsyncs. - test_support: shared Stop guard and watchdog (moved from the compaction test); the fleet and race tests use them so a panicking worker no longer leaves the scope waiting on a barrier. - Tests: exact 16-attempt budget with injected CAS losses, backoff bounds. - ADR-0046 addendum. --- crates/repo/src/bucket_transport.rs | 184 +++++++++++++++--- .../src/bucket_transport/compact/tests.rs | 46 +---- crates/repo/src/fill.rs | 29 ++- crates/repo/src/lib.rs | 2 + crates/repo/src/test_support.rs | 57 ++++++ docs/adr/0046-wal-bucket-remotes.md | 22 +++ scl-pins-relative-test-file | 0 scl-pins-relative-test-file.lock | 0 8 files changed, 266 insertions(+), 74 deletions(-) create mode 100644 crates/repo/src/test_support.rs create mode 100644 scl-pins-relative-test-file create mode 100644 scl-pins-relative-test-file.lock diff --git a/crates/repo/src/bucket_transport.rs b/crates/repo/src/bucket_transport.rs index 431e0d8e..5958c7a1 100644 --- a/crates/repo/src/bucket_transport.rs +++ b/crates/repo/src/bucket_transport.rs @@ -675,6 +675,23 @@ impl crate::reachable::ObjectSource for BucketSource<'_> { } } +/// Attempts `update_ref` makes at the manifest CAS before it gives up (the +/// documented "rerun the push" outcome). +const MAX_CAS_RETRIES: u32 = 16; + +/// Pause before CAS attempt number `attempt` (1-based after the first try): +/// "full jitter" exponential backoff, uniform in `[0, min(2ms << attempt, 250ms))`. +/// Every lost CAS means a rival landed, so a fleet that retries in lockstep +/// keeps colliding and an unlucky writer burns its whole budget while all its +/// rivals succeed (#271). Randomness comes from `RandomState`'s per-instance +/// keys: no RNG dependency, and the writers' draws differ. +fn cas_backoff(attempt: u32) -> std::time::Duration { + use std::hash::BuildHasher; + let ceiling_us = (2_000u64 << attempt.min(16)).min(250_000); + let draw = std::collections::hash_map::RandomState::new().hash_one(attempt); + std::time::Duration::from_micros(draw % ceiling_us) +} + impl Transport for BucketTransport { fn list_refs(&self) -> Result> { self.refresh()?; @@ -804,8 +821,10 @@ impl Transport for BucketTransport { // the bucket regardless (content-addressed, inert orphan garbage if // never referenced — correct per spec; `compact` sweeps it later, ADR-0057). let pending: Vec = std::mem::take(&mut *self.pending_packs.borrow_mut()); - const MAX_CAS_RETRIES: u32 = 16; - for _ in 0..MAX_CAS_RETRIES { + for attempt in 0..MAX_CAS_RETRIES { + if attempt > 0 { + std::thread::sleep(cas_backoff(attempt)); + } self.refresh()?; let (current, prev_tag, head_seq, checkpoint_seq, head_branch) = { let view = self.view.borrow(); @@ -885,9 +904,9 @@ impl Transport for BucketTransport { // under the new parent — `pending` (this call's local copy) // still lists our packs, so nothing is re-uploaded. } - Err(Error::Remote( - "manifest cas contention: gave up after 16 attempts".into(), - )) + Err(Error::Remote(format!( + "manifest cas contention: gave up after {MAX_CAS_RETRIES} attempts" + ))) } } @@ -1333,7 +1352,17 @@ mod tests { (obj.id(), obj.encode()) }; let gate = std::sync::Barrier::new(2); + let finished = std::sync::atomic::AtomicBool::new(false); + let status = crate::test_support::Status::default(); let results: Vec> = std::thread::scope(|s| { + // A racer that panics before the gate would leave its peer waiting. + let _finish = crate::test_support::Stop(&finished, true); + crate::test_support::watchdog( + s, + &finished, + &status, + std::time::Duration::from_secs(120), + ); let handles: Vec<_> = [b"racer-a".as_slice(), b"racer-b".as_slice()] .into_iter() .map(|tag| { @@ -1367,48 +1396,153 @@ mod tests { assert!(!broot.exists()); } - /// P36a acceptance proof #2: 8 threads push to 8 *distinct* branches - /// concurrently with no external coordinator. Contention is at the - /// manifest-CAS level only (every push races the same `manifest` key even - /// though the branches don't conflict), so every push must eventually - /// land via the retry loop. - #[test] - fn fleet_hammer_distinct_branches_all_land_without_coordinator() { - let pid = std::process::id(); - let broot = std::env::temp_dir().join(format!("scl-bt-fleet-{pid}")); + /// `n` threads push to `n` distinct branches at the same instant with no + /// external coordinator. Contention is at the manifest-CAS level only + /// (every push races the same `manifest` key even though the branches + /// don't conflict), so every push must land via the retry loop. A + /// watchdog turns a stuck worker into a diagnosed abort, and a panicking + /// worker releases the others. + fn fleet_lands(tag: &str, n: usize) { + use crate::test_support::{note, watchdog, Status, Stop}; + use std::sync::atomic::AtomicBool; + let broot = std::env::temp_dir().join(format!("scl-bt-{tag}-{}", std::process::id())); let _ = std::fs::remove_dir_all(&broot); - const N: usize = 8; - // A barrier forces all N threads to call `update_ref` at essentially + // A barrier forces all threads to call `update_ref` at essentially // the same instant — without it, short-lived threads could finish // one at a time with the OS never actually overlapping them, and - // "all N land" would hold even against a broken retry loop that was + // "all land" would hold even against a broken retry loop that was // never exercised under real contention. - let gate = std::sync::Barrier::new(N); + let gate = std::sync::Barrier::new(n); + let finished = AtomicBool::new(false); + let status = Status::default(); std::thread::scope(|s| { - for i in 0..N { - let broot = broot.clone(); - let gate = &gate; + let _finish = Stop(&finished, true); + watchdog(s, &finished, &status, std::time::Duration::from_secs(120)); + for i in 0..n { + let (broot, gate, status) = (broot.clone(), &gate, &status); s.spawn(move || { + note(status, format!("writer-{i:03}"), "setup".into()); let t = BucketTransport::from_bucket(Box::new(DirBucket::open(&broot).unwrap())) .unwrap(); let obj = Object::blob(format!("agent-{i}").into_bytes()); t.put_object(&obj.id(), &obj.encode()).unwrap(); + note(status, format!("writer-{i:03}"), "at the gate".into()); gate.wait(); - // distinct branches: contention is manifest-level only, so - // every one must eventually land via CAS retry. - t.update_ref(&format!("work-{i}"), &obj.id(), None).unwrap(); + note(status, format!("writer-{i:03}"), "update_ref".into()); + t.update_ref(&format!("work-{i}"), &obj.id(), None) + .unwrap_or_else(|e| panic!("writer {i} of {n}: {e}")); + note(status, format!("writer-{i:03}"), "landed".into()); }); } }); let t = BucketTransport::from_bucket(Box::new(DirBucket::open(&broot).unwrap())).unwrap(); let refs = t.list_refs().unwrap(); - assert_eq!(refs.len(), N, "all {N} agent branches present: {refs:?}"); + assert_eq!(refs.len(), n, "all {n} agent branches present: {refs:?}"); drop(t); std::fs::remove_dir_all(&broot).unwrap(); assert!(!broot.exists()); } + /// P36a acceptance proof #2: 8 threads, 8 distinct branches. + #[test] + fn fleet_hammer_distinct_branches_all_land_without_coordinator() { + fleet_lands("fleet", 8); + } + + /// A bucket whose manifest CAS loses the first `lose` attempts (as if a + /// rival landed each time) and then behaves like the real thing. + struct LoseCas { + inner: DirBucket, + lose: std::sync::atomic::AtomicU32, + lost: std::sync::atomic::AtomicU32, + } + impl Bucket for LoseCas { + fn get(&self, key: &str, cached: Option<&str>) -> scl_objio::Result { + self.inner.get(key, cached) + } + fn put_new(&self, key: &str, bytes: &[u8]) -> scl_objio::Result { + self.inner.put_new(key, bytes) + } + fn put_if_tag( + &self, + key: &str, + bytes: &[u8], + expected: Option<&str>, + ) -> scl_objio::Result> { + use std::sync::atomic::Ordering::SeqCst; + if key == "manifest" && self.lost.load(SeqCst) < self.lose.load(SeqCst) { + self.lost.fetch_add(1, SeqCst); + return Ok(None); + } + self.inner.put_if_tag(key, bytes, expected) + } + fn list(&self, prefix: &str) -> scl_objio::Result> { + self.inner.list(prefix) + } + fn list_meta(&self, prefix: &str) -> scl_objio::Result> { + self.inner.list_meta(prefix) + } + fn exists(&self, key: &str) -> scl_objio::Result { + self.inner.exists(key) + } + fn delete(&self, key: &str) -> scl_objio::Result<()> { + self.inner.delete(key) + } + } + + /// The CAS retry budget is exact and bounded (#271): losing 15 manifest + /// CASes in a row still lands on the 16th attempt, and losing all 16 + /// gives the documented "rerun the push" error, never a further try. + /// Deterministic: the losses are injected, so machine load cannot move + /// the outcome (a real fleet's losses depend on scheduling). + #[test] + fn the_cas_retry_budget_is_exactly_sixteen_attempts() { + use std::sync::atomic::AtomicU32; + for (lose, lands) in [(15u32, true), (16, false)] { + let broot = + std::env::temp_dir().join(format!("scl-bt-budget-{lose}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&broot); + let t = BucketTransport::from_bucket(Box::new(LoseCas { + inner: DirBucket::open(&broot).unwrap(), + lose: AtomicU32::new(lose), + lost: AtomicU32::new(0), + })) + .unwrap(); + let obj = Object::blob(b"budget".to_vec()); + t.put_object(&obj.id(), &obj.encode()).unwrap(); + let res = t.update_ref("main", &obj.id(), None); + if lands { + res.unwrap(); + assert_eq!(t.list_refs().unwrap().len(), 1, "lose {lose}"); + } else { + let e = res.unwrap_err().to_string(); + assert!(e.contains("gave up after 16 attempts"), "lose {lose}: {e}"); + } + drop(t); + std::fs::remove_dir_all(&broot).unwrap(); + } + } + + /// The pause between CAS attempts is jittered, grows with the attempt, and + /// stays under its cap, so total backoff for one push is bounded. + #[test] + fn cas_backoff_is_bounded_and_grows_to_its_cap() { + let max_us = |attempt| { + (0..200) + .map(|_| cas_backoff(attempt).as_micros()) + .max() + .unwrap() + }; + assert!(max_us(1) < 4_000); + assert!(max_us(3) < 16_000); + assert!(max_us(15) < 250_000); + let total: std::time::Duration = (1..MAX_CAS_RETRIES).map(|_| cas_backoff(15)).sum(); + assert!(total < std::time::Duration::from_secs(4), "{total:?}"); + let draws: std::collections::HashSet<_> = (0..50).map(|_| cas_backoff(8)).collect(); + assert!(draws.len() > 1, "backoff must be jittered"); + } + /// P36a acceptance proof #3: a pusher that died after writing its pack + /// log entry, but before the manifest CAS, leaves debris that must be /// invisible to every reader (the manifest never points at it) — and a diff --git a/crates/repo/src/bucket_transport/compact/tests.rs b/crates/repo/src/bucket_transport/compact/tests.rs index fd2d584d..97cc032f 100644 --- a/crates/repo/src/bucket_transport/compact/tests.rs +++ b/crates/repo/src/bucket_transport/compact/tests.rs @@ -5,6 +5,7 @@ use super::super::CHECKPOINT_INTERVAL; use super::*; use crate::error::Error; +use crate::test_support::{note, watchdog, Status, Stop}; use crate::transport::Transport; use crate::walfmt::{idx_key, RefUpdate}; use scl_core::Object; @@ -485,45 +486,6 @@ fn a_failure_at_every_operation_boundary_leaves_a_readable_bucket() { assert!(n > 20, "the loop should cross many boundaries, got {n}"); } -/// Stops a concurrency test from hanging: a worker that panics raises the flag -/// (and the main thread always does, on exit), so one failure releases every -/// loop instead of leaving the scope waiting on workers that never see `done`. -struct Stop<'a>(&'a AtomicBool, bool); -impl Drop for Stop<'_> { - fn drop(&mut self) { - if self.1 || std::thread::panicking() { - self.0.store(true, Ordering::SeqCst); - } - } -} - -/// Aborts the process with each worker's last-known step if the test is still -/// running at `limit`: a worker stuck inside a call (a lock, a bucket op) can -/// not be unwound, so the hang becomes a diagnosed failure. Written straight to -/// stderr because the harness captures `eprintln!` until the test ends. -fn watchdog<'a>( - s: &'a std::thread::Scope<'a, '_>, - finished: &'a AtomicBool, - status: &'a Mutex>, - limit: Duration, -) { - s.spawn(move || { - let start = std::time::Instant::now(); - while !finished.load(Ordering::SeqCst) { - if start.elapsed() > limit { - use std::io::Write; - let _ = writeln!( - std::io::stderr(), - "WATCHDOG: test still running after {limit:?}; last steps: {:#?}", - status.lock().map(|m| m.clone()).unwrap_or_default() - ); - std::process::abort(); - } - std::thread::sleep(Duration::from_millis(200)); - } - }); -} - #[test] fn readers_and_writers_stay_consistent_while_compaction_runs() { let root = scratch("concurrent"); @@ -542,10 +504,8 @@ fn readers_and_writers_stay_consistent_while_compaction_runs() { let landed = Mutex::new(Vec::new()); let deleted = std::sync::atomic::AtomicUsize::new(0); let finished = AtomicBool::new(false); - let status = Mutex::new(BTreeMap::new()); - let note = |who: String, what: String| { - status.lock().unwrap().insert(who, what); - }; + let status = Status::default(); + let note = |who: String, what: String| note(&status, who, what); std::thread::scope(|s| { let _finish = Stop(&finished, true); let _stop = Stop(&done, false); diff --git a/crates/repo/src/fill.rs b/crates/repo/src/fill.rs index a452d0b0..91599f0d 100644 --- a/crates/repo/src/fill.rs +++ b/crates/repo/src/fill.rs @@ -108,7 +108,20 @@ impl Repo { ids: &[ObjectId], stats: &mut FillStats, ) -> Result<()> { - for chunk in ids.chunks(crate::wire::MAX_GET_OBJECTS_IDS) { + self.fetch_ids_chunked(transport, ids, crate::wire::MAX_GET_OBJECTS_IDS, stats) + } + + /// [`Self::fetch_ids_batched`] with the per-request id count as a + /// parameter, so a test can cross a chunk boundary without ingesting + /// thousands of loose objects (two fsyncs each). + fn fetch_ids_chunked( + &self, + transport: &dyn Transport, + ids: &[ObjectId], + max_ids: usize, + stats: &mut FillStats, + ) -> Result<()> { + for chunk in ids.chunks(max_ids) { self.fetch_ids_split(transport, chunk, stats)?; } Ok(()) @@ -972,11 +985,15 @@ mod tests { } } + /// Cost is deterministic: every ingested object is a durable loose write + /// (two fsyncs, F_FULLFSYNC on macOS), so this crosses the chunk boundary + /// with 20 ids at 8 per request instead of 2500 at 1024 (that took 25 s + /// unloaded and minutes under load, #271). #[test] fn a_wide_level_is_chunked_by_count_before_anything_is_split_by_size() { let fx = Fixture::new("chunks"); let dst = fx.clone(true); - let objs: std::collections::HashMap = (0..2500u32) + let objs: std::collections::HashMap = (0..20u32) .map(|i| { let o = Object::blob(i.to_le_bytes().to_vec()); (o.id(), o) @@ -988,14 +1005,14 @@ mod tests { sizes: Default::default(), }; let mut stats = FillStats::default(); - dst.fetch_ids_batched(&t, &ids, &mut stats).unwrap(); - // 2500 ids: two full 1024-id requests and one of 452, never split. - assert_eq!(*t.sizes.borrow(), vec![1024, 1024, 452]); + dst.fetch_ids_chunked(&t, &ids, 8, &mut stats).unwrap(); + // 20 ids: two full 8-id requests and one of 4, never split. + assert_eq!(*t.sizes.borrow(), vec![8, 8, 4]); assert_eq!( stats, FillStats { requests: 3, - objects: 2500 + objects: 20 } ); fx.cleanup(); diff --git a/crates/repo/src/lib.rs b/crates/repo/src/lib.rs index e3152403..cf976ab1 100644 --- a/crates/repo/src/lib.rs +++ b/crates/repo/src/lib.rs @@ -56,6 +56,8 @@ pub mod sparse; mod sparse_completion; pub mod stdio_transport; pub mod sync; +#[cfg(test)] +mod test_support; pub mod textdiff; pub mod tls_pins; pub mod transcripts; diff --git a/crates/repo/src/test_support.rs b/crates/repo/src/test_support.rs new file mode 100644 index 00000000..f6ae71f1 --- /dev/null +++ b/crates/repo/src/test_support.rs @@ -0,0 +1,57 @@ +//! Helpers for tests that run worker threads: a failure must end the test +//! with a diagnostic, never leave the scope or the harness waiting forever. + +use std::collections::BTreeMap; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Mutex; +use std::time::Duration; + +/// Stops a concurrency test from hanging: a worker that panics raises the flag +/// (and a guard built with `true` always does, on exit), so one failure +/// releases every loop instead of leaving the scope waiting on workers that +/// never see `done`. +pub(crate) struct Stop<'a>(pub &'a AtomicBool, pub bool); +impl Drop for Stop<'_> { + fn drop(&mut self) { + if self.1 || std::thread::panicking() { + self.0.store(true, Ordering::SeqCst); + } + } +} + +/// Last-known step per worker, shown by [`watchdog`] when it fires. +pub(crate) type Status = Mutex>; + +/// Record `what` as `who`'s current step. +pub(crate) fn note(status: &Status, who: String, what: String) { + status.lock().unwrap().insert(who, what); +} + +/// Aborts the process with each worker's last-known step if the test is still +/// running at `limit`: a worker stuck inside a call (a lock, a bucket op, a +/// barrier whose peer panicked) can not be unwound, so the hang becomes a +/// diagnosed failure. Written straight to stderr because the harness captures +/// `eprintln!` until the test ends. Raise `finished` (see [`Stop`]) to stand +/// the watchdog down. +pub(crate) fn watchdog<'a>( + s: &'a std::thread::Scope<'a, '_>, + finished: &'a AtomicBool, + status: &'a Status, + limit: Duration, +) { + s.spawn(move || { + let start = std::time::Instant::now(); + while !finished.load(Ordering::SeqCst) { + if start.elapsed() > limit { + use std::io::Write; + let _ = writeln!( + std::io::stderr(), + "WATCHDOG: test still running after {limit:?}; last steps: {:#?}", + status.lock().map(|m| m.clone()).unwrap_or_default() + ); + std::process::abort(); + } + std::thread::sleep(Duration::from_millis(200)); + } + }); +} diff --git a/docs/adr/0046-wal-bucket-remotes.md b/docs/adr/0046-wal-bucket-remotes.md index 71943842..f437d4a7 100644 --- a/docs/adr/0046-wal-bucket-remotes.md +++ b/docs/adr/0046-wal-bucket-remotes.md @@ -243,3 +243,25 @@ ADR-0057. Effects on this ADR's contracts: objects and removes unreferenced ones. - The ADR-0046 `MAX_LIST` fold guard is unchanged. Compaction is what makes a skipped fold recoverable, on the pack axis. + +## As built (#271, 2026-10): bounded, jittered CAS retries + +`update_ref` still makes at most 16 attempts at the manifest CAS, and +exhausting them is still the documented outcome ("manifest cas contention: +gave up after 16 attempts"; the push is safe to rerun). Two things changed: + +- **Jittered exponential backoff between attempts.** Attempt `n > 0` first + sleeps a uniform random time in `[0, min(2 ms << n, 250 ms))`, so one push + waits at most about 3.75 s in total (average about half). A lost CAS always + means a rival landed, so a fleet retrying in lockstep keeps colliding and an + unlucky writer can burn all 16 attempts while every rival succeeds. Measured + with 48 simultaneous pushers to distinct branches on an idle machine: 0 of 20 + runs landed everyone without backoff, 40 of 40 with it. The budget stays + finite; there is no queue, so a very large fleet on a heavily loaded host can + still exhaust it and must rerun the push. +- **The `DirBucket` CAS lock is an OS advisory lock** (#264), not a + create-exclusive spin file. The spin lock gave up after 2000 tries with + "cas lock stuck" under load; that, not the retry budget, was what failed + `fleet_hammer_distinct_branches_all_land_without_coordinator` (11 of 60 runs + under load with the old lock, 0 of 100 with the new one). `S3Bucket` is + unaffected: its CAS is a server-side precondition. diff --git a/scl-pins-relative-test-file b/scl-pins-relative-test-file new file mode 100644 index 00000000..e69de29b diff --git a/scl-pins-relative-test-file.lock b/scl-pins-relative-test-file.lock new file mode 100644 index 00000000..e69de29b From 2120d131a320fd748bef7a591aa2dc1f45f55f93 Mon Sep 17 00:00:00 2001 From: Toni Bergholm Date: Wed, 21 Oct 2026 04:13:33 +0300 Subject: [PATCH 2/2] fix(repo): keep the fleet watchdog alive until every worker joins; drop stray files (ref #271) Review of #272: the scope body returned before joining, so the watchdog stood down before the workers raced. Join all handles inside the scope in the fleet and two-racer tests. Remove two empty files committed by mistake, correct the ADR backoff figures, add a growth check and an ignored 48-writer repro. --- crates/repo/src/bucket_transport.rs | 60 ++++++++++++++++++++--------- docs/adr/0046-wal-bucket-remotes.md | 12 ++++-- scl-pins-relative-test-file | 0 scl-pins-relative-test-file.lock | 0 4 files changed, 49 insertions(+), 23 deletions(-) delete mode 100644 scl-pins-relative-test-file delete mode 100644 scl-pins-relative-test-file.lock diff --git a/crates/repo/src/bucket_transport.rs b/crates/repo/src/bucket_transport.rs index 5958c7a1..11db2d02 100644 --- a/crates/repo/src/bucket_transport.rs +++ b/crates/repo/src/bucket_transport.rs @@ -1380,7 +1380,8 @@ mod tests { }) }) .collect(); - handles.into_iter().map(|h| h.join().unwrap()).collect() + let joined: Vec<_> = handles.into_iter().map(|h| h.join()).collect(); + joined.into_iter().map(|r| r.unwrap()).collect() }); let wins = results.iter().filter(|r| r.is_ok()).count(); let nffs = results @@ -1400,8 +1401,8 @@ mod tests { /// external coordinator. Contention is at the manifest-CAS level only /// (every push races the same `manifest` key even though the branches /// don't conflict), so every push must land via the retry loop. A - /// watchdog turns a stuck worker into a diagnosed abort, and a panicking - /// worker releases the others. + /// watchdog turns a worker stuck at the gate (a peer panicked before it) + /// into a diagnosed abort. fn fleet_lands(tag: &str, n: usize) { use crate::test_support::{note, watchdog, Status, Stop}; use std::sync::atomic::AtomicBool; @@ -1418,22 +1419,32 @@ mod tests { std::thread::scope(|s| { let _finish = Stop(&finished, true); watchdog(s, &finished, &status, std::time::Duration::from_secs(120)); - for i in 0..n { - let (broot, gate, status) = (broot.clone(), &gate, &status); - s.spawn(move || { - note(status, format!("writer-{i:03}"), "setup".into()); - let t = - BucketTransport::from_bucket(Box::new(DirBucket::open(&broot).unwrap())) - .unwrap(); - let obj = Object::blob(format!("agent-{i}").into_bytes()); - t.put_object(&obj.id(), &obj.encode()).unwrap(); - note(status, format!("writer-{i:03}"), "at the gate".into()); - gate.wait(); - note(status, format!("writer-{i:03}"), "update_ref".into()); - t.update_ref(&format!("work-{i}"), &obj.id(), None) - .unwrap_or_else(|e| panic!("writer {i} of {n}: {e}")); - note(status, format!("writer-{i:03}"), "landed".into()); - }); + let workers: Vec<_> = (0..n) + .map(|i| { + let (broot, gate, status) = (broot.clone(), &gate, &status); + s.spawn(move || { + note(status, format!("writer-{i:03}"), "setup".into()); + let t = BucketTransport::from_bucket(Box::new( + DirBucket::open(&broot).unwrap(), + )) + .unwrap(); + let obj = Object::blob(format!("agent-{i}").into_bytes()); + t.put_object(&obj.id(), &obj.encode()).unwrap(); + note(status, format!("writer-{i:03}"), "at the gate".into()); + gate.wait(); + note(status, format!("writer-{i:03}"), "update_ref".into()); + t.update_ref(&format!("work-{i}"), &obj.id(), None) + .unwrap_or_else(|e| panic!("writer {i} of {n}: {e}")); + note(status, format!("writer-{i:03}"), "landed".into()); + }) + }) + .collect(); + // Join every worker before `_finish` drops: the watchdog must keep + // running until the last one is done, or a worker stuck at the gate + // (its peer panicked) hangs the scope with no diagnostic. + let results: Vec<_> = workers.into_iter().map(|h| h.join()).collect(); + for r in results { + r.unwrap(); } }); let t = BucketTransport::from_bucket(Box::new(DirBucket::open(&broot).unwrap())).unwrap(); @@ -1450,6 +1461,16 @@ mod tests { fleet_lands("fleet", 8); } + /// The #271 measurement: 48 simultaneous pushers, more than the 16-attempt + /// budget. Passes on an idle machine; heavy load can still exhaust the + /// budget (documented in ADR-0046), so it is not part of the default run. + /// `cargo test -p scl-repo wide_fleet -- --ignored` + #[test] + #[ignore = "stress: load-dependent by design"] + fn wide_fleet_of_48_pushers_lands_on_an_idle_machine() { + fleet_lands("fleet-wide", 48); + } + /// A bucket whose manifest CAS loses the first `lose` attempts (as if a /// rival landed each time) and then behaves like the real thing. struct LoseCas { @@ -1535,6 +1556,7 @@ mod tests { .unwrap() }; assert!(max_us(1) < 4_000); + assert!(max_us(8) > 4_000, "the ceiling must grow with the attempt"); assert!(max_us(3) < 16_000); assert!(max_us(15) < 250_000); let total: std::time::Duration = (1..MAX_CAS_RETRIES).map(|_| cas_backoff(15)).sum(); diff --git a/docs/adr/0046-wal-bucket-remotes.md b/docs/adr/0046-wal-bucket-remotes.md index f437d4a7..07daaef0 100644 --- a/docs/adr/0046-wal-bucket-remotes.md +++ b/docs/adr/0046-wal-bucket-remotes.md @@ -251,14 +251,18 @@ exhausting them is still the documented outcome ("manifest cas contention: gave up after 16 attempts"; the push is safe to rerun). Two things changed: - **Jittered exponential backoff between attempts.** Attempt `n > 0` first - sleeps a uniform random time in `[0, min(2 ms << n, 250 ms))`, so one push - waits at most about 3.75 s in total (average about half). A lost CAS always + sleeps a uniform random time in `[0, min(2 ms << n, 250 ms))`, the ceilings + are 4, 8, 16, 32, 64, 128 ms and then 250 ms, so one push sleeps about 2.5 s + in the worst case (about 1.25 s on average). A lost CAS always means a rival landed, so a fleet retrying in lockstep keeps colliding and an unlucky writer can burn all 16 attempts while every rival succeeds. Measured with 48 simultaneous pushers to distinct branches on an idle machine: 0 of 20 runs landed everyone without backoff, 40 of 40 with it. The budget stays - finite; there is no queue, so a very large fleet on a heavily loaded host can - still exhaust it and must rerun the push. + finite; there is no queue, so a large fleet on a heavily loaded host can + still exhaust it and must rerun the push (48 pushers at load average ~20: 5 + of 6 runs still hit the budget). The `--ignored` test + `wide_fleet_of_48_pushers_lands_on_an_idle_machine` reproduces the idle-host + measurement. - **The `DirBucket` CAS lock is an OS advisory lock** (#264), not a create-exclusive spin file. The spin lock gave up after 2000 tries with "cas lock stuck" under load; that, not the retry budget, was what failed diff --git a/scl-pins-relative-test-file b/scl-pins-relative-test-file deleted file mode 100644 index e69de29b..00000000 diff --git a/scl-pins-relative-test-file.lock b/scl-pins-relative-test-file.lock deleted file mode 100644 index e69de29b..00000000