From 24fdeb1b46d5e1d1da3888be7723b9ee9ff57622 Mon Sep 17 00:00:00 2001 From: Toni Bergholm Date: Tue, 20 Oct 2026 23:20:40 +0300 Subject: [PATCH 1/4] fix(repo): sparse view change never aliases case variants; bucket compaction report and merge fixes (ref #241 #263) laid-out path as tracked. Both spellings come from one tree, so on a case-insensitive filesystem the newly visible path is the laid-out file itself. The alias decision is split out with the filesystem facts injected. between deletion batches. Consolidation drops merge parts built from fewer than two source packs, so two compressible packs that each exceed the decoded limit are no longer rewritten one by one. --- CLAUDE.md | 2 +- ROADMAP.md | 3 - crates/repo/src/bucket_transport/compact.rs | 68 +++++++--- .../src/bucket_transport/compact/tests.rs | 79 +++++++++++- crates/repo/src/repo.rs | 122 +++++++++++++++--- crates/repo/src/sparse.rs | 11 +- docs/adr/0034-sparse-checkouts.md | 13 +- docs/adr/0057-bucket-compaction.md | 8 ++ 8 files changed, 251 insertions(+), 55 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 2bbc91e9..975e1f9c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -344,7 +344,7 @@ truth — do not re-list items here); entries that name an issue mirror the open GitHub issues. Standing entries: `sc ws` workspace refresh (#215), a limit-aware `sc fetch` and an object-count clone limit (#217 remainder), the non-unix `.sc/local-key` ACL (#214), and the open -sparse guard and user-file safety limits (#241, ADR-0034 refinement 8). +user-file safety limits of the sparse guard (ADR-0034 refinement 8). ## Agent skills diff --git a/ROADMAP.md b/ROADMAP.md index e61abe13..63a1561d 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -782,9 +782,6 @@ P28 review follow-ons, the port de-flake, the clippy allow-list) are recorded in conflict, re-materialize under the recorded view) but not built: it needs a headless merge-into-tree entry point and a re-materialize that keeps files an agent may have open. Abandon and re-fork covers it meanwhile. -- **Sparse untracked-file guard edge cases.** #241: on a case-insensitive - filesystem the untracked-file guard can alias an old-view path with a newly - in-view one. - **User-file safety limits (ADR-0034 refinement 8, ADR-0044).** A file whose bytes equal the tracked or laid ones cannot be told from the stop's own write, so it is removed. `merge --abort` of a private-branch merge, which diff --git a/crates/repo/src/bucket_transport/compact.rs b/crates/repo/src/bucket_transport/compact.rs index 4771d31a..28f40dc9 100644 --- a/crates/repo/src/bucket_transport/compact.rs +++ b/crates/repo/src/bucket_transport/compact.rs @@ -91,6 +91,14 @@ fn hash_of(key: &str) -> Option<&str> { .or_else(|| name.strip_suffix(".idx")) } +/// Bytes of the `.pack` and `.idx` objects behind these packs. +fn pack_bytes(packs: &[(&String, &Vec)]) -> u64 { + packs + .iter() + .flat_map(|(_, ms)| ms.iter().map(|m| m.size)) + .sum() +} + /// Greedy size-tiered grouping: smallest candidates first, a group closes when /// the next pack would push it past `group_bytes`. Groups of one are dropped /// (there is nothing to merge). @@ -200,9 +208,13 @@ impl BucketTransport { return Ok(()); } let mut created = Vec::new(); + // The packs actually folded: a source that fills a part alone is + // left where it is (#263), so this can be a subset of `replaced`. + let mut merged: BTreeSet = BTreeSet::new(); for group in &groups { let mut objects: BTreeMap> = BTreeMap::new(); let mut decoded = 0u64; + let mut part: Vec<&String> = Vec::new(); for hash in group { let Fetched::New { bytes, .. } = self.bucket.get(&pack_key(hash), None)? else { // Either another compactor retired it (the manifest has @@ -223,19 +235,22 @@ impl BucketTransport { objects.insert(id, enc); Ok(()) })?; + part.push(hash); // Compressed sizes bound the group, not the decoded bytes held // here; spill to a pack of its own before RAM balloons. if decoded > opts.group_bytes.saturating_mul(DECODED_FACTOR) { - let part: Vec<(ObjectId, Vec)> = - std::mem::take(&mut objects).into_iter().collect(); - created.push(self.upload_pack(&part)?.0); + self.flush_part(&mut objects, &mut part, &mut created, &mut merged)?; decoded = 0; } } - if !objects.is_empty() { - let objects: Vec<(ObjectId, Vec)> = objects.into_iter().collect(); - created.push(self.upload_pack(&objects)?.0); - } + self.flush_part(&mut objects, &mut part, &mut created, &mut merged)?; + } + if created.is_empty() { + rep.notes.push( + "no merge would reduce the pack count: every candidate is large once decoded, nothing merged" + .into(), + ); + return Ok(()); } // Fence + fold in one CAS: the manifest tag we read is the state // the checkpoint describes, so a lost CAS just replans. @@ -258,7 +273,7 @@ impl BucketTransport { let mut seen = BTreeSet::new(); let mut kept: Vec = packs .into_iter() - .filter(|h| !replaced.contains(h) && seen.insert(h.clone())) + .filter(|h| !merged.contains(h) && seen.insert(h.clone())) .collect(); for h in &created { if seen.insert(h.clone()) { @@ -277,7 +292,7 @@ impl BucketTransport { } // Another compactor may have retired some of the packs we merged // already: replan rather than publish a second copy of the same data. - if !replaced.is_subset(&packs_now) { + if !merged.is_subset(&packs_now) { continue 'attempt; } let mut try_seq = head_seq + 1; @@ -320,7 +335,7 @@ impl BucketTransport { .is_some() { self.refresh()?; - rep.packs_merged = replaced.len(); + rep.packs_merged = merged.len(); rep.packs_created = created.len(); rep.checkpoint_seq = seq; return Ok(()); @@ -333,6 +348,28 @@ impl BucketTransport { Ok(()) } + /// Upload the objects of the source packs in `part` as one pack and mark + /// them merged, leaving `objects` and `part` empty. A part of fewer than two + /// sources is dropped: rewriting one pack into one gains nothing, and a + /// source whose decoded bytes alone pass the spill limit would otherwise + /// cost an upload per source (#263). + fn flush_part( + &self, + objects: &mut BTreeMap>, + part: &mut Vec<&String>, + created: &mut Vec, + merged: &mut BTreeSet, + ) -> Result<()> { + let objects: Vec<(ObjectId, Vec)> = std::mem::take(objects).into_iter().collect(); + let sources = std::mem::take(part); + if sources.len() < 2 { + return Ok(()); + } + created.push(self.upload_pack(&objects)?.0); + merged.extend(sources.into_iter().cloned()); + Ok(()) + } + /// Phase 2: delete what is behind the cutoff and what nothing names. fn sweep(&self, opts: &CompactOptions, now: SystemTime, rep: &mut CompactReport) -> Result<()> { // `modified <= old_before` means "at least prune_expire old". @@ -434,10 +471,7 @@ impl BucketTransport { } } rep.packs_deleted += del_packs.len(); - rep.bytes_deleted += del_packs - .iter() - .flat_map(|(_, ms)| ms.iter().map(|m| m.size)) - .sum::(); + rep.bytes_deleted += pack_bytes(&del_packs); if opts.dry_run { if rep.packs_created > 0 { rep.notes.push( @@ -458,10 +492,7 @@ impl BucketTransport { ); if !unmoved { rep.packs_deleted -= del_packs.len(); - rep.bytes_deleted -= del_packs - .iter() - .flat_map(|(_, ms)| ms.iter().map(|m| m.size)) - .sum::(); + rep.bytes_deleted -= pack_bytes(&del_packs); continue; } // The delete loop can run long (one request per object), so look at @@ -476,6 +507,7 @@ impl BucketTransport { { let left = del_packs.len() - i * PACK_DELETE_BATCH; rep.packs_deleted -= left; + rep.bytes_deleted -= pack_bytes(&del_packs[i * PACK_DELETE_BATCH..]); rep.notes.push(format!( "the manifest moved during pack deletion: {left} unreferenced pack(s) \ kept, rerun" diff --git a/crates/repo/src/bucket_transport/compact/tests.rs b/crates/repo/src/bucket_transport/compact/tests.rs index 621301a7..83e1d6a1 100644 --- a/crates/repo/src/bucket_transport/compact/tests.rs +++ b/crates/repo/src/bucket_transport/compact/tests.rs @@ -10,7 +10,7 @@ use crate::walfmt::{idx_key, RefUpdate}; use scl_core::Object; use scl_objio::{Bucket, DirBucket}; use std::path::PathBuf; -use std::sync::atomic::{AtomicBool, AtomicI64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; /// In-memory `Bucket` with the same semantics as `DirBucket`. @@ -853,6 +853,7 @@ fn a_manifest_that_moves_during_pack_deletion_stops_the_deletes() { // else: the sweep must notice at its next look and keep the rest. struct PushOnFirstPackDelete { inner: Shared, + freed: Arc, fired: Mutex, } impl Bucket for PushOnFirstPackDelete { @@ -880,6 +881,9 @@ fn a_manifest_that_moves_during_pack_deletion_stops_the_deletes() { self.inner.exists(k) } fn delete(&self, k: &str) -> scl_objio::Result<()> { + if let Fetched::New { bytes, .. } = self.inner.get(k, None)? { + self.freed.fetch_add(bytes.len() as u64, Ordering::SeqCst); + } self.inner.delete(k)?; let mut fired = self.fired.lock().unwrap(); if k.starts_with("packs/") && !*fired { @@ -892,9 +896,11 @@ fn a_manifest_that_moves_during_pack_deletion_stops_the_deletes() { } let mem = Arc::new(MemBucket::default()); let refs = push_commits(&open(&mem), "w", 70); + let freed = Arc::new(AtomicU64::new(0)); let t = BucketTransport::from_bucket(Box::new(PushOnFirstPackDelete { inner: Shared(mem.clone()), fired: Mutex::new(false), + freed: freed.clone(), })) .unwrap(); let rep = t.compact(&eager()).unwrap(); @@ -905,6 +911,9 @@ fn a_manifest_that_moves_during_pack_deletion_stops_the_deletes() { "{rep:?}" ); assert!(rep.packs_deleted < 70, "{rep:?}"); + // The packs kept after the manifest moved are not counted as reclaimed + // (#263): the report equals what really left the bucket. + assert_eq!(rep.bytes_deleted, freed.load(Ordering::SeqCst), "{rep:?}"); // some old packs were kept, the bucket is readable, and a rerun finishes assert!(keys(&mem, "packs/").len() > 4); let mut all = refs; @@ -914,3 +923,71 @@ fn a_manifest_that_moves_during_pack_deletion_stops_the_deletes() { assert_eq!(keys(&mem, "packs/").len(), 2); assert_readable(&mem, &all); } + +/// A branch whose tip carries one `len`-byte blob of `fill`: a pack that is a +/// few hundred bytes on disk but `len` decoded. +fn push_compressible(t: &BucketTransport, branch: &str, fill: u8, len: usize) { + let blob = Object::blob(vec![fill; len]); + let tree = Object::Tree(scl_core::Tree::new(vec![scl_core::TreeEntry { + name: "f".into(), + kind: scl_core::EntryKind::Blob, + id: blob.id(), + mode: scl_core::FileMode(0o644), + perms: 0, + }])); + let snap = Object::Snapshot(scl_core::Snapshot { + root: tree.id(), + parents: vec![], + author: "t".into(), + timestamp: 1, + message: branch.into(), + secrets: Default::default(), + protection: Default::default(), + }); + for o in [&blob, &tree, &snap] { + t.put_object(&o.id(), &o.encode()).unwrap(); + } + t.update_ref(branch, &snap.id(), None).unwrap(); +} + +#[test] +fn candidates_that_are_each_large_once_decoded_are_not_merged_one_by_one() { + let mem = Arc::new(MemBucket::default()); + let t = open(&mem); + // Tiny on disk, so both are candidates and share one group; each decodes + // past the spill limit (4 x group_bytes) on its own. + let opts = CompactOptions { + group_bytes: 64 << 10, + ..eager() + }; + push_compressible(&t, "big-0", 1, 1 << 20); + push_compressible(&t, "big-1", 2, 1 << 20); + let before = manifest(&mem); + let rep = t.compact(&opts).unwrap(); + assert_eq!((rep.packs_merged, rep.packs_created), (0, 0), "{rep:?}"); + assert!( + rep.notes + .iter() + .any(|n| n.contains("reduce the pack count")), + "{rep:?}" + ); + assert_eq!(manifest(&mem), before, "no empty log entry or checkpoint"); + assert_eq!(keys(&mem, "packs/").len(), 4); + + // Small packs beside them still merge, into fewer packs than they were. + push_commits(&t, "s", 2); + let rep = t.compact(&opts).unwrap(); + assert!(rep.packs_merged >= 2, "{rep:?}"); + assert!(rep.packs_created < rep.packs_merged, "{rep:?}"); + let t2 = open(&mem); + for b in ["big-0", "big-1", "s-0", "s-1"] { + let tip = t2 + .list_refs() + .unwrap() + .into_iter() + .find(|(n, _)| n == b) + .unwrap() + .1; + t2.get_object(&tip).unwrap(); + } +} diff --git a/crates/repo/src/repo.rs b/crates/repo/src/repo.rs index 5923944b..6b0d4303 100644 --- a/crates/repo/src/repo.rs +++ b/crates/repo/src/repo.rs @@ -2177,10 +2177,11 @@ impl Repo { /// there. Paths the old view matched are not checked: they are not written /// (a stopped op may have turned one into a directory, say). `already_laid` /// names paths that count as tracked because an earlier, interrupted run of - /// the same change wrote them. `alias_ok` lets a case alias of a laid-out - /// path count as tracked; a stopped-op widen passes false, because writing - /// a newly visible `SRC/a.txt` would overwrite the user's `src/a.txt`. Same rules as - /// [`Repo::refuse_untracked_overwrite_vs`]. + /// the same change wrote them. A case alias of a laid-out path never counts + /// as tracked here (#241): both spellings come from this one tree, so on a + /// case-insensitive filesystem a newly visible `SRC/a.txt` is the laid-out + /// `src/a.txt` itself, and writing it would replace that file's bytes. Same + /// rules as [`Repo::refuse_untracked_overwrite_vs`]. pub(crate) fn refuse_untracked_for_view_change( &self, op: &str, @@ -2188,7 +2189,6 @@ impl Repo { old_view: &crate::Sparse, new_view: &crate::Sparse, already_laid: &std::collections::BTreeSet, - alias_ok: bool, ) -> Result<()> { let partial = self.promisor()?.is_some(); let (mut laid_out, target) = { @@ -2211,7 +2211,7 @@ impl Repo { &laid_out, target.iter().filter(|p| !old_view.matches(p)), new_view, - alias_ok, + false, ) } @@ -2909,24 +2909,42 @@ fn is_tracked_case_alias( rel: &str, ) -> bool { use std::os::unix::fs::MetadataExt; - // Inode identity alone proves nothing: on a case-sensitive filesystem an - // untracked `Foo` can be a hard link to the tracked `foo`. Only a - // filesystem that answers to both spellings of a name that certainly - // exists (`.sc`) aliases case. - if root.join(".SC").symlink_metadata().is_err() { - return false; - } let Ok(here) = root.join(rel).symlink_metadata() else { return false; }; - tracked - .iter() - .filter(|t| t.as_str() != rel && t.eq_ignore_ascii_case(rel)) - .any(|t| { + case_alias_of_tracked( + // Inode identity alone proves nothing: on a case-sensitive filesystem + // an untracked `Foo` can be a hard link to the tracked `foo`. Only a + // filesystem that answers to both spellings of a name that certainly + // exists (`.sc`) aliases case. + || root.join(".SC").symlink_metadata().is_ok(), + tracked, + rel, + |t| { root.join(t) .symlink_metadata() .is_ok_and(|m| m.dev() == here.dev() && m.ino() == here.ino()) - }) + }, + ) +} + +/// The decision behind [`is_tracked_case_alias`], with the two filesystem +/// facts injected so it is testable without one: `case_insensitive` says the +/// filesystem folds case (asked at most once, and only when a tracked path +/// differing from `rel` only in case exists), `same_file(t)` that the tracked +/// path `t` is the very file at `rel`. +#[cfg(unix)] +fn case_alias_of_tracked( + case_insensitive: impl FnOnce() -> bool, + tracked: &std::collections::BTreeSet, + rel: &str, + same_file: impl Fn(&str) -> bool, +) -> bool { + let mut candidates = tracked + .iter() + .filter(|t| t.as_str() != rel && t.eq_ignore_ascii_case(rel)) + .peekable(); + candidates.peek().is_some() && case_insensitive() && candidates.any(|t| same_file(t)) } #[cfg(not(unix))] @@ -3081,6 +3099,74 @@ mod tests { std::fs::remove_dir_all(&root).unwrap(); } + /// The alias decision with the filesystem injected, so every platform runs + /// it: a case-sensitive filesystem never aliases, and a case-insensitive + /// one only for a case-variant that is the very same file. + #[cfg(unix)] + #[test] + fn the_case_alias_decision_needs_a_folding_filesystem_and_the_same_file() { + let tracked: std::collections::BTreeSet = + ["docs/a.txt".to_string(), "other".to_string()].into(); + let ask = + |ci: bool, rel: &str, same: bool| case_alias_of_tracked(|| ci, &tracked, rel, |_| same); + assert!(ask(true, "Docs/a.txt", true)); + // Case-sensitive: a hard link or a user's own `Docs/a.txt` is not it. + assert!(!ask(false, "Docs/a.txt", true)); + // Case-insensitive but another inode: the user's file, kept. + assert!(!ask(true, "Docs/a.txt", false)); + // The tracked spelling itself and a name with no case-variant never alias. + assert!(!ask(true, "docs/a.txt", true)); + assert!(!ask(true, "Nope", true)); + // The filesystem is probed only when a case-variant exists. + assert!(!case_alias_of_tracked( + || panic!("probed without a candidate"), + &tracked, + "Nope", + |_| true + )); + } + + /// A sparse view change over one tree holding `docs/a.txt` and `Docs/a.txt` + /// (#241). On a case-insensitive filesystem the laid-out `docs/a.txt` is + /// already what stands at `Docs/a.txt`, and writing it would replace that + /// file, so the guard must refuse; it must not count the alias as tracked. + /// Skipped on a case-sensitive filesystem, where nothing stands there. + #[cfg(unix)] + #[test] + fn a_view_change_refuses_a_newly_visible_case_alias_of_a_laid_out_file() { + let root = tmp_root("view-case-alias"); + let repo = Repo::init(&root).unwrap(); + if root.join(".SC").symlink_metadata().is_err() { + drop(repo); + std::fs::remove_dir_all(&root).unwrap(); + return; + } + std::fs::create_dir_all(root.join("docs")).unwrap(); + std::fs::write(root.join("docs/a.txt"), b"laid out\n").unwrap(); + let tree = { + let store_arc = repo.vfs.store(); + let mut store = store_arc.lock().unwrap(); + let mut entries = std::collections::BTreeMap::new(); + for (path, body) in [("docs/a.txt", "laid out\n"), ("Docs/a.txt", "other\n")] { + let id = store.put(scl_core::Object::blob(body)).unwrap(); + entries.insert(path.to_string(), (id, scl_core::FileMode(0o644), 0)); + } + let (tree, objs) = crate::private::build_tree_objects(&entries); + for o in objs { + store.put(o).unwrap(); + } + tree + }; + let old = crate::Sparse::new(vec!["docs/".into()]); + let new = crate::Sparse::new(vec!["docs/".into(), "Docs/".into()]); + let err = repo + .refuse_untracked_for_view_change("sparse set", tree, &old, &new, &Default::default()) + .unwrap_err(); + assert!(matches!(err, Error::InvalidArgument(_)), "got {err:?}"); + drop(repo); + std::fs::remove_dir_all(&root).unwrap(); + } + #[test] fn scignore_hides_untracked_from_status_and_commit_but_not_tracked() { let root = tmp_root("scignore"); diff --git a/crates/repo/src/sparse.rs b/crates/repo/src/sparse.rs index 655ebed4..31e23611 100644 --- a/crates/repo/src/sparse.rs +++ b/crates/repo/src/sparse.rs @@ -395,7 +395,6 @@ impl Repo { &old_spec, &spec, &Default::default(), - true, )?; } // Crash safety: record the spec being replaced before the new one is @@ -543,14 +542,7 @@ impl Repo { )? }; for root in roots { - self.refuse_untracked_for_view_change( - "sparse set", - root, - old, - new, - &already_laid, - false, - )?; + self.refuse_untracked_for_view_change("sparse set", root, old, new, &already_laid)?; } // Every guard passed, so each path below is the widen's to write: // record them (keeping an earlier interrupted run's) before the first. @@ -773,7 +765,6 @@ impl Repo { &self.sparse_spec()?, &Sparse::default(), &Default::default(), - true, )?; } clear(self.layout())?; diff --git a/docs/adr/0034-sparse-checkouts.md b/docs/adr/0034-sparse-checkouts.md index 5fac0879..26cd6d25 100644 --- a/docs/adr/0034-sparse-checkouts.md +++ b/docs/adr/0034-sparse-checkouts.md @@ -209,10 +209,15 @@ every claim below is checked against the shipped code, not the plan. laid; the marker is restored to its prior state once the spec is stored. The untracked-file guard checks only paths the old view did not match (those it laid out are not written, and a stopped op may have turned one - into a directory). Widening to the same spec is a quiet no-op. The guard does not count a case - alias of a laid-out path as tracked (`alias_ok = false`): on a case-insensitive - filesystem a decided `SRC/a.txt` would otherwise be written over the user's - resolution of `src/a.txt`. A crash between an atomic write's temp file + into a directory). Widening to the same spec is a quiet no-op. The guard never counts a case + alias of a laid-out path as tracked, for a stopped-op widen and for a plain + `sparse set`/`disable` alike (#241): both spellings come from one tree, so on a + case-insensitive filesystem a newly visible `SRC/a.txt` is the laid-out + `src/a.txt` itself, and writing it would replace that file's bytes (for a stopped + op, the user's resolution). The alias exemption stays for switch and the other + ops, where the old and new trees differ and a case-only rename is real; there it + needs a case-insensitive filesystem (probed with `.SC`, only when a case-variant + tracked path exists) and the same device and inode. A crash between an atomic write's temp file (`...tmp`) and its rename leaves that temp file beside the target; it is not recorded, delete it. The decided root is the right source: it is what completion carries an diff --git a/docs/adr/0057-bucket-compaction.md b/docs/adr/0057-bucket-compaction.md index 9c4b3daa..2ef32c1c 100644 --- a/docs/adr/0057-bucket-compaction.md +++ b/docs/adr/0057-bucket-compaction.md @@ -47,6 +47,14 @@ into groups of at most `group_bytes` (default 64 MiB); a group needs at least two members. Each group is read, its objects are deduplicated by id, and one new content-addressed pack plus idx is uploaded. This bounds peak RAM to a few times `group_bytes`, and packs above the threshold are never rewritten. +A group's decoded bytes are also bounded: past `4 x group_bytes` the objects read +so far are uploaded as a pack and reading continues. A part built from fewer +than two source packs is dropped, not uploaded, and its source stays where it +is (#263): rewriting one pack into one reduces nothing, and without this two +highly compressible packs that each decode past the limit cost one upload per +source. A run where no part reaches two sources merges nothing and writes no log +entry or checkpoint. `bytes_deleted` counts only objects the sweep actually +removed: packs kept because the manifest moved mid-sweep are subtracted. The result is committed with one CAS, which is both the fence and the fold: From 924c34a3431637e5bd919b080a81fceb82b5c71b Mon Sep 17 00:00:00 2001 From: Toni Bergholm Date: Tue, 20 Oct 2026 23:37:44 +0300 Subject: [PATCH 2/4] fix(repo): dry-run compaction splits like a real run; stricter case probe (ref #241 #263) Review follow-ups on #264: the dry run reads and splits candidates the same way, so its merge counts match the real run; the case-insensitivity probe requires .SC and .sc to be the same file; the new test asserts its cleanup. --- crates/repo/src/bucket_transport/compact.rs | 26 ++++++++++++------- .../src/bucket_transport/compact/tests.rs | 8 ++++++ crates/repo/src/repo.rs | 9 +++++-- 3 files changed, 32 insertions(+), 11 deletions(-) diff --git a/crates/repo/src/bucket_transport/compact.rs b/crates/repo/src/bucket_transport/compact.rs index 28f40dc9..c4075c32 100644 --- a/crates/repo/src/bucket_transport/compact.rs +++ b/crates/repo/src/bucket_transport/compact.rs @@ -202,11 +202,6 @@ impl BucketTransport { )); return Ok(()); } - if opts.dry_run { - rep.packs_merged = replaced.len(); - rep.packs_created = groups.len(); - return Ok(()); - } let mut created = Vec::new(); // The packs actually folded: a source that fills a part alone is // left where it is (#263), so this can be a subset of `replaced`. @@ -239,19 +234,25 @@ impl BucketTransport { // Compressed sizes bound the group, not the decoded bytes held // here; spill to a pack of its own before RAM balloons. if decoded > opts.group_bytes.saturating_mul(DECODED_FACTOR) { - self.flush_part(&mut objects, &mut part, &mut created, &mut merged)?; + self.flush_part(&mut objects, &mut part, &mut created, &mut merged, opts)?; decoded = 0; } } - self.flush_part(&mut objects, &mut part, &mut created, &mut merged)?; + self.flush_part(&mut objects, &mut part, &mut created, &mut merged, opts)?; } if created.is_empty() { rep.notes.push( - "no merge would reduce the pack count: every candidate is large once decoded, nothing merged" + "no merge would reduce the pack count: every candidate is large once decoded, \ + nothing merged" .into(), ); return Ok(()); } + if opts.dry_run { + rep.packs_merged = merged.len(); + rep.packs_created = created.len(); + return Ok(()); + } // Fence + fold in one CAS: the manifest tag we read is the state // the checkpoint describes, so a lost CAS just replans. self.refresh()?; @@ -359,13 +360,20 @@ impl BucketTransport { part: &mut Vec<&String>, created: &mut Vec, merged: &mut BTreeSet, + opts: &CompactOptions, ) -> Result<()> { let objects: Vec<(ObjectId, Vec)> = std::mem::take(objects).into_iter().collect(); let sources = std::mem::take(part); if sources.len() < 2 { return Ok(()); } - created.push(self.upload_pack(&objects)?.0); + // A dry run reads and splits the same way but uploads nothing, so its + // counts match what a real run would do. + created.push(if opts.dry_run { + String::new() + } else { + self.upload_pack(&objects)?.0 + }); merged.extend(sources.into_iter().cloned()); Ok(()) } diff --git a/crates/repo/src/bucket_transport/compact/tests.rs b/crates/repo/src/bucket_transport/compact/tests.rs index 83e1d6a1..20a708af 100644 --- a/crates/repo/src/bucket_transport/compact/tests.rs +++ b/crates/repo/src/bucket_transport/compact/tests.rs @@ -963,6 +963,14 @@ fn candidates_that_are_each_large_once_decoded_are_not_merged_one_by_one() { push_compressible(&t, "big-0", 1, 1 << 20); push_compressible(&t, "big-1", 2, 1 << 20); let before = manifest(&mem); + // The dry run reports what the real run will do: nothing. + let dry = t + .compact(&CompactOptions { + dry_run: true, + ..opts.clone() + }) + .unwrap(); + assert_eq!((dry.packs_merged, dry.packs_created), (0, 0), "{dry:?}"); let rep = t.compact(&opts).unwrap(); assert_eq!((rep.packs_merged, rep.packs_created), (0, 0), "{rep:?}"); assert!( diff --git a/crates/repo/src/repo.rs b/crates/repo/src/repo.rs index 6b0d4303..013efe18 100644 --- a/crates/repo/src/repo.rs +++ b/crates/repo/src/repo.rs @@ -2916,8 +2916,11 @@ fn is_tracked_case_alias( // Inode identity alone proves nothing: on a case-sensitive filesystem // an untracked `Foo` can be a hard link to the tracked `foo`. Only a // filesystem that answers to both spellings of a name that certainly - // exists (`.sc`) aliases case. - || root.join(".SC").symlink_metadata().is_ok(), + // exists (`.sc`) with the same file aliases case. + || { + let id = |n: &str| root.join(n).symlink_metadata().map(|m| (m.dev(), m.ino())); + matches!((id(".SC"), id(".sc")), (Ok(a), Ok(b)) if a == b) + }, tracked, rel, |t| { @@ -3139,6 +3142,7 @@ mod tests { if root.join(".SC").symlink_metadata().is_err() { drop(repo); std::fs::remove_dir_all(&root).unwrap(); + assert!(!root.exists()); return; } std::fs::create_dir_all(root.join("docs")).unwrap(); @@ -3165,6 +3169,7 @@ mod tests { assert!(matches!(err, Error::InvalidArgument(_)), "got {err:?}"); drop(repo); std::fs::remove_dir_all(&root).unwrap(); + assert!(!root.exists()); } #[test] From b9d92be170a14e7229ddd2b469c207c328120e00 Mon Sep 17 00:00:00 2001 From: Toni Bergholm Date: Wed, 21 Oct 2026 00:45:05 +0300 Subject: [PATCH 3/4] fix(repo): dry-run compaction checks the exact checkpoint size (ref #263) --- crates/repo/src/bucket_transport/compact.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/crates/repo/src/bucket_transport/compact.rs b/crates/repo/src/bucket_transport/compact.rs index c4075c32..44f65d07 100644 --- a/crates/repo/src/bucket_transport/compact.rs +++ b/crates/repo/src/bucket_transport/compact.rs @@ -249,6 +249,14 @@ impl BucketTransport { return Ok(()); } if opts.dry_run { + // Split parts can outnumber the planned groups: size the + // checkpoint the real run would build, as it does. + if super::fold_would_overflow(nrefs, live.len() - merged.len() + created.len()) { + rep.notes.push(format!( + "the merged checkpoint would exceed the cap of {MAX_LIST}: nothing merged" + )); + return Ok(()); + } rep.packs_merged = merged.len(); rep.packs_created = created.len(); return Ok(()); From 9be30ea8894e7ee7ee158c50c16a1f01390dd97b Mon Sep 17 00:00:00 2001 From: Toni Bergholm Date: Wed, 21 Oct 2026 02:19:33 +0300 Subject: [PATCH 4/4] fix(objio,repo): fair CAS lock for DirBucket; concurrency test fails fast instead of hanging (ref #263) readers_and_writers_stay_consistent_while_compaction_runs hung >30 min under load. A writer or compactor panic (DirBucket's create-exclusive spin lock timed out under contention: 'cas lock stuck') made the main thread unwind at a join before raising the done flag, so readers and compactors looped forever and the scope never ended. - DirBucket::lock takes an OS advisory file lock: queued, fair, released on holder death, no stale lock and no starved waiter. - The test raises done when any worker panics, caps writer retries, retries the documented 'manifest cas contention' outcome, paces readers, records each worker's last step, and a watchdog aborts with that state after 180s. --- crates/objio/src/dir.rs | 44 ++++----- .../src/bucket_transport/compact/tests.rs | 89 +++++++++++++++++-- 2 files changed, 97 insertions(+), 36 deletions(-) diff --git a/crates/objio/src/dir.rs b/crates/objio/src/dir.rs index 8957c64c..fd47c324 100644 --- a/crates/objio/src/dir.rs +++ b/crates/objio/src/dir.rs @@ -1,11 +1,11 @@ //! Local-directory `Bucket` — the test/demo backend behind `sc+wal://`. -use crate::{validate_key, Bucket, Error, Fetched, ObjectMeta, Result}; +use crate::{validate_key, Bucket, Fetched, ObjectMeta, Result}; use std::path::{Path, PathBuf}; /// A local filesystem bucket implementation using a directory as the backing /// store. Tags are blake3 hashes of content; compare-and-swap writes are -/// serialized via a spin-lock file to simulate S3 precondition semantics. +/// serialized via an advisory file lock to simulate S3 precondition semantics. pub struct DirBucket { root: PathBuf, } @@ -27,27 +27,19 @@ impl DirBucket { Ok(self.root.join(key)) } - /// Spin-acquire `/.cas-lock`; bounded so a crashed holder surfaces - /// as a loud error, not a hang. + /// Take the bucket's exclusive CAS lock: an OS advisory lock on + /// `/.cas-lock`. The OS queues waiters and drops the lock if its + /// holder dies, so there is no stale lock to detect and no waiter can be + /// starved by a rival that re-acquires in a loop (a create-exclusive spin + /// lock starved waiters for more than ten seconds under load). fn lock(&self) -> Result { - let path = self.root.join(".cas-lock"); - for _ in 0..2000 { - match std::fs::OpenOptions::new() - .write(true) - .create_new(true) - .open(&path) - { - Ok(_) => return Ok(CasLock { path }), - Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => { - std::thread::sleep(std::time::Duration::from_millis(1)); - } - Err(e) => return Err(e.into()), - } - } - Err(Error::Backend(format!( - "cas lock stuck (stale {} ?)", - path.display() - ))) + let file = std::fs::OpenOptions::new() + .write(true) + .create(true) + .truncate(false) + .open(self.root.join(".cas-lock"))?; + file.lock()?; + Ok(CasLock { _file: file }) } fn write_via_tmp(&self, path: &Path, bytes: &[u8]) -> Result<()> { @@ -61,13 +53,9 @@ impl DirBucket { } } +/// Holds the lock until dropped (closing the file releases it). struct CasLock { - path: PathBuf, -} -impl Drop for CasLock { - fn drop(&mut self) { - let _ = std::fs::remove_file(&self.path); - } + _file: std::fs::File, } impl Bucket for DirBucket { diff --git a/crates/repo/src/bucket_transport/compact/tests.rs b/crates/repo/src/bucket_transport/compact/tests.rs index 20a708af..fd2d584d 100644 --- a/crates/repo/src/bucket_transport/compact/tests.rs +++ b/crates/repo/src/bucket_transport/compact/tests.rs @@ -485,6 +485,45 @@ fn a_failure_at_every_operation_boundary_leaves_a_readable_bucket() { assert!(n > 20, "the loop should cross many boundaries, got {n}"); } +/// Stops a concurrency test from hanging: a worker that panics raises the flag +/// (and the main thread always does, on exit), so one failure releases every +/// loop instead of leaving the scope waiting on workers that never see `done`. +struct Stop<'a>(&'a AtomicBool, bool); +impl Drop for Stop<'_> { + fn drop(&mut self) { + if self.1 || std::thread::panicking() { + self.0.store(true, Ordering::SeqCst); + } + } +} + +/// Aborts the process with each worker's last-known step if the test is still +/// running at `limit`: a worker stuck inside a call (a lock, a bucket op) can +/// not be unwound, so the hang becomes a diagnosed failure. Written straight to +/// stderr because the harness captures `eprintln!` until the test ends. +fn watchdog<'a>( + s: &'a std::thread::Scope<'a, '_>, + finished: &'a AtomicBool, + status: &'a Mutex>, + limit: Duration, +) { + s.spawn(move || { + let start = std::time::Instant::now(); + while !finished.load(Ordering::SeqCst) { + if start.elapsed() > limit { + use std::io::Write; + let _ = writeln!( + std::io::stderr(), + "WATCHDOG: test still running after {limit:?}; last steps: {:#?}", + status.lock().map(|m| m.clone()).unwrap_or_default() + ); + std::process::abort(); + } + std::thread::sleep(Duration::from_millis(200)); + } + }); +} + #[test] fn readers_and_writers_stay_consistent_while_compaction_runs() { let root = scratch("concurrent"); @@ -502,24 +541,50 @@ fn readers_and_writers_stay_consistent_while_compaction_runs() { }; let landed = Mutex::new(Vec::new()); let deleted = std::sync::atomic::AtomicUsize::new(0); + let finished = AtomicBool::new(false); + let status = Mutex::new(BTreeMap::new()); + let note = |who: String, what: String| { + status.lock().unwrap().insert(who, what); + }; std::thread::scope(|s| { + let _finish = Stop(&finished, true); + let _stop = Stop(&done, false); + watchdog(s, &finished, &status, Duration::from_secs(180)); + let note = ¬e; let writers: Vec<_> = (0..WRITERS) .map(|w| { - let (root, landed) = (&root, &landed); + let (root, landed, done) = (&root, &landed, &done); s.spawn(move || { + let _stop = Stop(done, false); let t = open(root); for i in 0..PER_WRITER { let (tip, objects) = commit(&format!("writer-{w}-{i}")); let branch = format!("writer-{w}-{i}"); + let mut attempts = 0u32; loop { + attempts += 1; + note( + format!("writer-{w}"), + format!("push {i}, attempt {attempts}"), + ); + // A sweep only takes a pack older than GRACE, so a + // push that keeps losing its pack is not progressing. + assert!(attempts < 200, "writer {w}: push {i} never landed"); + assert!( + !done.load(Ordering::SeqCst), + "writer {w}: another worker failed" + ); for (id, bytes) in &objects { t.put_object(id, bytes).unwrap(); } match t.update_ref(&branch, &tip, None) { Ok(()) => break, - // The documented guard: a pack swept before - // commit fails the push; the retry re-uploads. - Err(Error::Remote(m)) if m.contains("removed from the bucket") => {} + // The documented outcomes a rerun fixes: a pack + // swept before commit (the retry re-uploads), and + // 16 lost manifest CASes against the compactors. + Err(Error::Remote(m)) + if m.contains("removed from the bucket") + || m.contains("manifest cas contention") => {} Err(e) => panic!("writer {w}: {e}"), } } @@ -533,11 +598,13 @@ fn readers_and_writers_stay_consistent_while_compaction_runs() { .map(|r| { let (root, done, seeded) = (&root, &done, &seeded); s.spawn(move || { + let _stop = Stop(done, false); let long_lived = open(root); let mut seen = seeded.len(); let mut rounds = 0u32; while !done.load(Ordering::SeqCst) || rounds < 3 { rounds += 1; + note(format!("reader-{r}"), format!("round {rounds}")); // alternate a long-lived (stale-view) reader and a cold start let cold; let t = if rounds.is_multiple_of(3) { @@ -558,18 +625,24 @@ fn readers_and_writers_stay_consistent_while_compaction_runs() { let tips: Vec = refs.iter().map(|(_, id)| *id).collect(); t.get_pack(&tips, &[], None, &mut Vec::new()) .unwrap_or_else(|e| panic!("reader {r}: get_pack: {e}")); + // Readers that never pause starve the writers' lock holders + // of filesystem time on a loaded machine. + std::thread::sleep(Duration::from_millis(5)); } }) }) .collect(); // Two compactors at once: the ADR promises that is safe (just wasteful). let compactors: Vec<_> = (0..2) - .map(|_| { - s.spawn(|| { - let t = open(&root); + .map(|c| { + let (root, done, opts, deleted) = (&root, &done, &opts, &deleted); + s.spawn(move || { + let _stop = Stop(done, false); + let t = open(root); let mut runs = 0u32; while !done.load(Ordering::SeqCst) { - match t.compact(&opts) { + note(format!("compactor-{c}"), format!("run {runs}")); + match t.compact(opts) { Ok(rep) => { deleted.fetch_add( rep.logs_deleted + rep.checkpoints_deleted + rep.packs_deleted,