diff --git a/CLAUDE.md b/CLAUDE.md index 290474d0..9a0289bb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -354,7 +354,7 @@ truth — do not re-list items here); entries that name an issue mirror the open GitHub issues. Standing entries: `sc ws` workspace refresh (#215), the non-unix `.sc/local-key` ACL (#214), and the open -sparse guard and user-file safety limits (#241, ADR-0034 refinement 8). +user-file safety limits of the sparse guard (ADR-0034 refinement 8). ## Agent skills diff --git a/ROADMAP.md b/ROADMAP.md index d31518d9..a2b88451 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -788,9 +788,6 @@ P28 review follow-ons, the port de-flake, the clippy allow-list) are recorded in conflict, re-materialize under the recorded view) but not built: it needs a headless merge-into-tree entry point and a re-materialize that keeps files an agent may have open. Abandon and re-fork covers it meanwhile. -- **Sparse untracked-file guard edge cases.** #241: on a case-insensitive - filesystem the untracked-file guard can alias an old-view path with a newly - in-view one. - **User-file safety limits (ADR-0034 refinement 8, ADR-0044).** A file whose bytes equal the tracked or laid ones cannot be told from the stop's own write, so it is removed. `merge --abort` of a private-branch merge, which diff --git a/crates/objio/src/dir.rs b/crates/objio/src/dir.rs index 8957c64c..fd47c324 100644 --- a/crates/objio/src/dir.rs +++ b/crates/objio/src/dir.rs @@ -1,11 +1,11 @@ //! Local-directory `Bucket` — the test/demo backend behind `sc+wal://`. -use crate::{validate_key, Bucket, Error, Fetched, ObjectMeta, Result}; +use crate::{validate_key, Bucket, Fetched, ObjectMeta, Result}; use std::path::{Path, PathBuf}; /// A local filesystem bucket implementation using a directory as the backing /// store. Tags are blake3 hashes of content; compare-and-swap writes are -/// serialized via a spin-lock file to simulate S3 precondition semantics. +/// serialized via an advisory file lock to simulate S3 precondition semantics. pub struct DirBucket { root: PathBuf, } @@ -27,27 +27,19 @@ impl DirBucket { Ok(self.root.join(key)) } - /// Spin-acquire `/.cas-lock`; bounded so a crashed holder surfaces - /// as a loud error, not a hang. + /// Take the bucket's exclusive CAS lock: an OS advisory lock on + /// `/.cas-lock`. The OS queues waiters and drops the lock if its + /// holder dies, so there is no stale lock to detect and no waiter can be + /// starved by a rival that re-acquires in a loop (a create-exclusive spin + /// lock starved waiters for more than ten seconds under load). fn lock(&self) -> Result { - let path = self.root.join(".cas-lock"); - for _ in 0..2000 { - match std::fs::OpenOptions::new() - .write(true) - .create_new(true) - .open(&path) - { - Ok(_) => return Ok(CasLock { path }), - Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => { - std::thread::sleep(std::time::Duration::from_millis(1)); - } - Err(e) => return Err(e.into()), - } - } - Err(Error::Backend(format!( - "cas lock stuck (stale {} ?)", - path.display() - ))) + let file = std::fs::OpenOptions::new() + .write(true) + .create(true) + .truncate(false) + .open(self.root.join(".cas-lock"))?; + file.lock()?; + Ok(CasLock { _file: file }) } fn write_via_tmp(&self, path: &Path, bytes: &[u8]) -> Result<()> { @@ -61,13 +53,9 @@ impl DirBucket { } } +/// Holds the lock until dropped (closing the file releases it). struct CasLock { - path: PathBuf, -} -impl Drop for CasLock { - fn drop(&mut self) { - let _ = std::fs::remove_file(&self.path); - } + _file: std::fs::File, } impl Bucket for DirBucket { diff --git a/crates/repo/src/bucket_transport/compact.rs b/crates/repo/src/bucket_transport/compact.rs index 4771d31a..44f65d07 100644 --- a/crates/repo/src/bucket_transport/compact.rs +++ b/crates/repo/src/bucket_transport/compact.rs @@ -91,6 +91,14 @@ fn hash_of(key: &str) -> Option<&str> { .or_else(|| name.strip_suffix(".idx")) } +/// Bytes of the `.pack` and `.idx` objects behind these packs. +fn pack_bytes(packs: &[(&String, &Vec)]) -> u64 { + packs + .iter() + .flat_map(|(_, ms)| ms.iter().map(|m| m.size)) + .sum() +} + /// Greedy size-tiered grouping: smallest candidates first, a group closes when /// the next pack would push it past `group_bytes`. Groups of one are dropped /// (there is nothing to merge). @@ -194,15 +202,14 @@ impl BucketTransport { )); return Ok(()); } - if opts.dry_run { - rep.packs_merged = replaced.len(); - rep.packs_created = groups.len(); - return Ok(()); - } let mut created = Vec::new(); + // The packs actually folded: a source that fills a part alone is + // left where it is (#263), so this can be a subset of `replaced`. + let mut merged: BTreeSet = BTreeSet::new(); for group in &groups { let mut objects: BTreeMap> = BTreeMap::new(); let mut decoded = 0u64; + let mut part: Vec<&String> = Vec::new(); for hash in group { let Fetched::New { bytes, .. } = self.bucket.get(&pack_key(hash), None)? else { // Either another compactor retired it (the manifest has @@ -223,19 +230,36 @@ impl BucketTransport { objects.insert(id, enc); Ok(()) })?; + part.push(hash); // Compressed sizes bound the group, not the decoded bytes held // here; spill to a pack of its own before RAM balloons. if decoded > opts.group_bytes.saturating_mul(DECODED_FACTOR) { - let part: Vec<(ObjectId, Vec)> = - std::mem::take(&mut objects).into_iter().collect(); - created.push(self.upload_pack(&part)?.0); + self.flush_part(&mut objects, &mut part, &mut created, &mut merged, opts)?; decoded = 0; } } - if !objects.is_empty() { - let objects: Vec<(ObjectId, Vec)> = objects.into_iter().collect(); - created.push(self.upload_pack(&objects)?.0); + self.flush_part(&mut objects, &mut part, &mut created, &mut merged, opts)?; + } + if created.is_empty() { + rep.notes.push( + "no merge would reduce the pack count: every candidate is large once decoded, \ + nothing merged" + .into(), + ); + return Ok(()); + } + if opts.dry_run { + // Split parts can outnumber the planned groups: size the + // checkpoint the real run would build, as it does. + if super::fold_would_overflow(nrefs, live.len() - merged.len() + created.len()) { + rep.notes.push(format!( + "the merged checkpoint would exceed the cap of {MAX_LIST}: nothing merged" + )); + return Ok(()); } + rep.packs_merged = merged.len(); + rep.packs_created = created.len(); + return Ok(()); } // Fence + fold in one CAS: the manifest tag we read is the state // the checkpoint describes, so a lost CAS just replans. @@ -258,7 +282,7 @@ impl BucketTransport { let mut seen = BTreeSet::new(); let mut kept: Vec = packs .into_iter() - .filter(|h| !replaced.contains(h) && seen.insert(h.clone())) + .filter(|h| !merged.contains(h) && seen.insert(h.clone())) .collect(); for h in &created { if seen.insert(h.clone()) { @@ -277,7 +301,7 @@ impl BucketTransport { } // Another compactor may have retired some of the packs we merged // already: replan rather than publish a second copy of the same data. - if !replaced.is_subset(&packs_now) { + if !merged.is_subset(&packs_now) { continue 'attempt; } let mut try_seq = head_seq + 1; @@ -320,7 +344,7 @@ impl BucketTransport { .is_some() { self.refresh()?; - rep.packs_merged = replaced.len(); + rep.packs_merged = merged.len(); rep.packs_created = created.len(); rep.checkpoint_seq = seq; return Ok(()); @@ -333,6 +357,35 @@ impl BucketTransport { Ok(()) } + /// Upload the objects of the source packs in `part` as one pack and mark + /// them merged, leaving `objects` and `part` empty. A part of fewer than two + /// sources is dropped: rewriting one pack into one gains nothing, and a + /// source whose decoded bytes alone pass the spill limit would otherwise + /// cost an upload per source (#263). + fn flush_part( + &self, + objects: &mut BTreeMap>, + part: &mut Vec<&String>, + created: &mut Vec, + merged: &mut BTreeSet, + opts: &CompactOptions, + ) -> Result<()> { + let objects: Vec<(ObjectId, Vec)> = std::mem::take(objects).into_iter().collect(); + let sources = std::mem::take(part); + if sources.len() < 2 { + return Ok(()); + } + // A dry run reads and splits the same way but uploads nothing, so its + // counts match what a real run would do. + created.push(if opts.dry_run { + String::new() + } else { + self.upload_pack(&objects)?.0 + }); + merged.extend(sources.into_iter().cloned()); + Ok(()) + } + /// Phase 2: delete what is behind the cutoff and what nothing names. fn sweep(&self, opts: &CompactOptions, now: SystemTime, rep: &mut CompactReport) -> Result<()> { // `modified <= old_before` means "at least prune_expire old". @@ -434,10 +487,7 @@ impl BucketTransport { } } rep.packs_deleted += del_packs.len(); - rep.bytes_deleted += del_packs - .iter() - .flat_map(|(_, ms)| ms.iter().map(|m| m.size)) - .sum::(); + rep.bytes_deleted += pack_bytes(&del_packs); if opts.dry_run { if rep.packs_created > 0 { rep.notes.push( @@ -458,10 +508,7 @@ impl BucketTransport { ); if !unmoved { rep.packs_deleted -= del_packs.len(); - rep.bytes_deleted -= del_packs - .iter() - .flat_map(|(_, ms)| ms.iter().map(|m| m.size)) - .sum::(); + rep.bytes_deleted -= pack_bytes(&del_packs); continue; } // The delete loop can run long (one request per object), so look at @@ -476,6 +523,7 @@ impl BucketTransport { { let left = del_packs.len() - i * PACK_DELETE_BATCH; rep.packs_deleted -= left; + rep.bytes_deleted -= pack_bytes(&del_packs[i * PACK_DELETE_BATCH..]); rep.notes.push(format!( "the manifest moved during pack deletion: {left} unreferenced pack(s) \ kept, rerun" diff --git a/crates/repo/src/bucket_transport/compact/tests.rs b/crates/repo/src/bucket_transport/compact/tests.rs index 621301a7..fd2d584d 100644 --- a/crates/repo/src/bucket_transport/compact/tests.rs +++ b/crates/repo/src/bucket_transport/compact/tests.rs @@ -10,7 +10,7 @@ use crate::walfmt::{idx_key, RefUpdate}; use scl_core::Object; use scl_objio::{Bucket, DirBucket}; use std::path::PathBuf; -use std::sync::atomic::{AtomicBool, AtomicI64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; /// In-memory `Bucket` with the same semantics as `DirBucket`. @@ -485,6 +485,45 @@ fn a_failure_at_every_operation_boundary_leaves_a_readable_bucket() { assert!(n > 20, "the loop should cross many boundaries, got {n}"); } +/// Stops a concurrency test from hanging: a worker that panics raises the flag +/// (and the main thread always does, on exit), so one failure releases every +/// loop instead of leaving the scope waiting on workers that never see `done`. +struct Stop<'a>(&'a AtomicBool, bool); +impl Drop for Stop<'_> { + fn drop(&mut self) { + if self.1 || std::thread::panicking() { + self.0.store(true, Ordering::SeqCst); + } + } +} + +/// Aborts the process with each worker's last-known step if the test is still +/// running at `limit`: a worker stuck inside a call (a lock, a bucket op) can +/// not be unwound, so the hang becomes a diagnosed failure. Written straight to +/// stderr because the harness captures `eprintln!` until the test ends. +fn watchdog<'a>( + s: &'a std::thread::Scope<'a, '_>, + finished: &'a AtomicBool, + status: &'a Mutex>, + limit: Duration, +) { + s.spawn(move || { + let start = std::time::Instant::now(); + while !finished.load(Ordering::SeqCst) { + if start.elapsed() > limit { + use std::io::Write; + let _ = writeln!( + std::io::stderr(), + "WATCHDOG: test still running after {limit:?}; last steps: {:#?}", + status.lock().map(|m| m.clone()).unwrap_or_default() + ); + std::process::abort(); + } + std::thread::sleep(Duration::from_millis(200)); + } + }); +} + #[test] fn readers_and_writers_stay_consistent_while_compaction_runs() { let root = scratch("concurrent"); @@ -502,24 +541,50 @@ fn readers_and_writers_stay_consistent_while_compaction_runs() { }; let landed = Mutex::new(Vec::new()); let deleted = std::sync::atomic::AtomicUsize::new(0); + let finished = AtomicBool::new(false); + let status = Mutex::new(BTreeMap::new()); + let note = |who: String, what: String| { + status.lock().unwrap().insert(who, what); + }; std::thread::scope(|s| { + let _finish = Stop(&finished, true); + let _stop = Stop(&done, false); + watchdog(s, &finished, &status, Duration::from_secs(180)); + let note = ¬e; let writers: Vec<_> = (0..WRITERS) .map(|w| { - let (root, landed) = (&root, &landed); + let (root, landed, done) = (&root, &landed, &done); s.spawn(move || { + let _stop = Stop(done, false); let t = open(root); for i in 0..PER_WRITER { let (tip, objects) = commit(&format!("writer-{w}-{i}")); let branch = format!("writer-{w}-{i}"); + let mut attempts = 0u32; loop { + attempts += 1; + note( + format!("writer-{w}"), + format!("push {i}, attempt {attempts}"), + ); + // A sweep only takes a pack older than GRACE, so a + // push that keeps losing its pack is not progressing. + assert!(attempts < 200, "writer {w}: push {i} never landed"); + assert!( + !done.load(Ordering::SeqCst), + "writer {w}: another worker failed" + ); for (id, bytes) in &objects { t.put_object(id, bytes).unwrap(); } match t.update_ref(&branch, &tip, None) { Ok(()) => break, - // The documented guard: a pack swept before - // commit fails the push; the retry re-uploads. - Err(Error::Remote(m)) if m.contains("removed from the bucket") => {} + // The documented outcomes a rerun fixes: a pack + // swept before commit (the retry re-uploads), and + // 16 lost manifest CASes against the compactors. + Err(Error::Remote(m)) + if m.contains("removed from the bucket") + || m.contains("manifest cas contention") => {} Err(e) => panic!("writer {w}: {e}"), } } @@ -533,11 +598,13 @@ fn readers_and_writers_stay_consistent_while_compaction_runs() { .map(|r| { let (root, done, seeded) = (&root, &done, &seeded); s.spawn(move || { + let _stop = Stop(done, false); let long_lived = open(root); let mut seen = seeded.len(); let mut rounds = 0u32; while !done.load(Ordering::SeqCst) || rounds < 3 { rounds += 1; + note(format!("reader-{r}"), format!("round {rounds}")); // alternate a long-lived (stale-view) reader and a cold start let cold; let t = if rounds.is_multiple_of(3) { @@ -558,18 +625,24 @@ fn readers_and_writers_stay_consistent_while_compaction_runs() { let tips: Vec = refs.iter().map(|(_, id)| *id).collect(); t.get_pack(&tips, &[], None, &mut Vec::new()) .unwrap_or_else(|e| panic!("reader {r}: get_pack: {e}")); + // Readers that never pause starve the writers' lock holders + // of filesystem time on a loaded machine. + std::thread::sleep(Duration::from_millis(5)); } }) }) .collect(); // Two compactors at once: the ADR promises that is safe (just wasteful). let compactors: Vec<_> = (0..2) - .map(|_| { - s.spawn(|| { - let t = open(&root); + .map(|c| { + let (root, done, opts, deleted) = (&root, &done, &opts, &deleted); + s.spawn(move || { + let _stop = Stop(done, false); + let t = open(root); let mut runs = 0u32; while !done.load(Ordering::SeqCst) { - match t.compact(&opts) { + note(format!("compactor-{c}"), format!("run {runs}")); + match t.compact(opts) { Ok(rep) => { deleted.fetch_add( rep.logs_deleted + rep.checkpoints_deleted + rep.packs_deleted, @@ -853,6 +926,7 @@ fn a_manifest_that_moves_during_pack_deletion_stops_the_deletes() { // else: the sweep must notice at its next look and keep the rest. struct PushOnFirstPackDelete { inner: Shared, + freed: Arc, fired: Mutex, } impl Bucket for PushOnFirstPackDelete { @@ -880,6 +954,9 @@ fn a_manifest_that_moves_during_pack_deletion_stops_the_deletes() { self.inner.exists(k) } fn delete(&self, k: &str) -> scl_objio::Result<()> { + if let Fetched::New { bytes, .. } = self.inner.get(k, None)? { + self.freed.fetch_add(bytes.len() as u64, Ordering::SeqCst); + } self.inner.delete(k)?; let mut fired = self.fired.lock().unwrap(); if k.starts_with("packs/") && !*fired { @@ -892,9 +969,11 @@ fn a_manifest_that_moves_during_pack_deletion_stops_the_deletes() { } let mem = Arc::new(MemBucket::default()); let refs = push_commits(&open(&mem), "w", 70); + let freed = Arc::new(AtomicU64::new(0)); let t = BucketTransport::from_bucket(Box::new(PushOnFirstPackDelete { inner: Shared(mem.clone()), fired: Mutex::new(false), + freed: freed.clone(), })) .unwrap(); let rep = t.compact(&eager()).unwrap(); @@ -905,6 +984,9 @@ fn a_manifest_that_moves_during_pack_deletion_stops_the_deletes() { "{rep:?}" ); assert!(rep.packs_deleted < 70, "{rep:?}"); + // The packs kept after the manifest moved are not counted as reclaimed + // (#263): the report equals what really left the bucket. + assert_eq!(rep.bytes_deleted, freed.load(Ordering::SeqCst), "{rep:?}"); // some old packs were kept, the bucket is readable, and a rerun finishes assert!(keys(&mem, "packs/").len() > 4); let mut all = refs; @@ -914,3 +996,79 @@ fn a_manifest_that_moves_during_pack_deletion_stops_the_deletes() { assert_eq!(keys(&mem, "packs/").len(), 2); assert_readable(&mem, &all); } + +/// A branch whose tip carries one `len`-byte blob of `fill`: a pack that is a +/// few hundred bytes on disk but `len` decoded. +fn push_compressible(t: &BucketTransport, branch: &str, fill: u8, len: usize) { + let blob = Object::blob(vec![fill; len]); + let tree = Object::Tree(scl_core::Tree::new(vec![scl_core::TreeEntry { + name: "f".into(), + kind: scl_core::EntryKind::Blob, + id: blob.id(), + mode: scl_core::FileMode(0o644), + perms: 0, + }])); + let snap = Object::Snapshot(scl_core::Snapshot { + root: tree.id(), + parents: vec![], + author: "t".into(), + timestamp: 1, + message: branch.into(), + secrets: Default::default(), + protection: Default::default(), + }); + for o in [&blob, &tree, &snap] { + t.put_object(&o.id(), &o.encode()).unwrap(); + } + t.update_ref(branch, &snap.id(), None).unwrap(); +} + +#[test] +fn candidates_that_are_each_large_once_decoded_are_not_merged_one_by_one() { + let mem = Arc::new(MemBucket::default()); + let t = open(&mem); + // Tiny on disk, so both are candidates and share one group; each decodes + // past the spill limit (4 x group_bytes) on its own. + let opts = CompactOptions { + group_bytes: 64 << 10, + ..eager() + }; + push_compressible(&t, "big-0", 1, 1 << 20); + push_compressible(&t, "big-1", 2, 1 << 20); + let before = manifest(&mem); + // The dry run reports what the real run will do: nothing. + let dry = t + .compact(&CompactOptions { + dry_run: true, + ..opts.clone() + }) + .unwrap(); + assert_eq!((dry.packs_merged, dry.packs_created), (0, 0), "{dry:?}"); + let rep = t.compact(&opts).unwrap(); + assert_eq!((rep.packs_merged, rep.packs_created), (0, 0), "{rep:?}"); + assert!( + rep.notes + .iter() + .any(|n| n.contains("reduce the pack count")), + "{rep:?}" + ); + assert_eq!(manifest(&mem), before, "no empty log entry or checkpoint"); + assert_eq!(keys(&mem, "packs/").len(), 4); + + // Small packs beside them still merge, into fewer packs than they were. + push_commits(&t, "s", 2); + let rep = t.compact(&opts).unwrap(); + assert!(rep.packs_merged >= 2, "{rep:?}"); + assert!(rep.packs_created < rep.packs_merged, "{rep:?}"); + let t2 = open(&mem); + for b in ["big-0", "big-1", "s-0", "s-1"] { + let tip = t2 + .list_refs() + .unwrap() + .into_iter() + .find(|(n, _)| n == b) + .unwrap() + .1; + t2.get_object(&tip).unwrap(); + } +} diff --git a/crates/repo/src/repo.rs b/crates/repo/src/repo.rs index ec5a4893..a8b43a10 100644 --- a/crates/repo/src/repo.rs +++ b/crates/repo/src/repo.rs @@ -2182,10 +2182,11 @@ impl Repo { /// there. Paths the old view matched are not checked: they are not written /// (a stopped op may have turned one into a directory, say). `already_laid` /// names paths that count as tracked because an earlier, interrupted run of - /// the same change wrote them. `alias_ok` lets a case alias of a laid-out - /// path count as tracked; a stopped-op widen passes false, because writing - /// a newly visible `SRC/a.txt` would overwrite the user's `src/a.txt`. Same rules as - /// [`Repo::refuse_untracked_overwrite_vs`]. + /// the same change wrote them. A case alias of a laid-out path never counts + /// as tracked here (#241): both spellings come from this one tree, so on a + /// case-insensitive filesystem a newly visible `SRC/a.txt` is the laid-out + /// `src/a.txt` itself, and writing it would replace that file's bytes. Same + /// rules as [`Repo::refuse_untracked_overwrite_vs`]. pub(crate) fn refuse_untracked_for_view_change( &self, op: &str, @@ -2193,7 +2194,6 @@ impl Repo { old_view: &crate::Sparse, new_view: &crate::Sparse, already_laid: &std::collections::BTreeSet, - alias_ok: bool, ) -> Result<()> { let partial = self.promisor()?.is_some(); let (mut laid_out, target) = { @@ -2216,7 +2216,7 @@ impl Repo { &laid_out, target.iter().filter(|p| !old_view.matches(p)), new_view, - alias_ok, + false, ) } @@ -2914,24 +2914,45 @@ fn is_tracked_case_alias( rel: &str, ) -> bool { use std::os::unix::fs::MetadataExt; - // Inode identity alone proves nothing: on a case-sensitive filesystem an - // untracked `Foo` can be a hard link to the tracked `foo`. Only a - // filesystem that answers to both spellings of a name that certainly - // exists (`.sc`) aliases case. - if root.join(".SC").symlink_metadata().is_err() { - return false; - } let Ok(here) = root.join(rel).symlink_metadata() else { return false; }; - tracked - .iter() - .filter(|t| t.as_str() != rel && t.eq_ignore_ascii_case(rel)) - .any(|t| { + case_alias_of_tracked( + // Inode identity alone proves nothing: on a case-sensitive filesystem + // an untracked `Foo` can be a hard link to the tracked `foo`. Only a + // filesystem that answers to both spellings of a name that certainly + // exists (`.sc`) with the same file aliases case. + || { + let id = |n: &str| root.join(n).symlink_metadata().map(|m| (m.dev(), m.ino())); + matches!((id(".SC"), id(".sc")), (Ok(a), Ok(b)) if a == b) + }, + tracked, + rel, + |t| { root.join(t) .symlink_metadata() .is_ok_and(|m| m.dev() == here.dev() && m.ino() == here.ino()) - }) + }, + ) +} + +/// The decision behind [`is_tracked_case_alias`], with the two filesystem +/// facts injected so it is testable without one: `case_insensitive` says the +/// filesystem folds case (asked at most once, and only when a tracked path +/// differing from `rel` only in case exists), `same_file(t)` that the tracked +/// path `t` is the very file at `rel`. +#[cfg(unix)] +fn case_alias_of_tracked( + case_insensitive: impl FnOnce() -> bool, + tracked: &std::collections::BTreeSet, + rel: &str, + same_file: impl Fn(&str) -> bool, +) -> bool { + let mut candidates = tracked + .iter() + .filter(|t| t.as_str() != rel && t.eq_ignore_ascii_case(rel)) + .peekable(); + candidates.peek().is_some() && case_insensitive() && candidates.any(|t| same_file(t)) } #[cfg(not(unix))] @@ -3086,6 +3107,76 @@ mod tests { std::fs::remove_dir_all(&root).unwrap(); } + /// The alias decision with the filesystem injected, so every platform runs + /// it: a case-sensitive filesystem never aliases, and a case-insensitive + /// one only for a case-variant that is the very same file. + #[cfg(unix)] + #[test] + fn the_case_alias_decision_needs_a_folding_filesystem_and_the_same_file() { + let tracked: std::collections::BTreeSet = + ["docs/a.txt".to_string(), "other".to_string()].into(); + let ask = + |ci: bool, rel: &str, same: bool| case_alias_of_tracked(|| ci, &tracked, rel, |_| same); + assert!(ask(true, "Docs/a.txt", true)); + // Case-sensitive: a hard link or a user's own `Docs/a.txt` is not it. + assert!(!ask(false, "Docs/a.txt", true)); + // Case-insensitive but another inode: the user's file, kept. + assert!(!ask(true, "Docs/a.txt", false)); + // The tracked spelling itself and a name with no case-variant never alias. + assert!(!ask(true, "docs/a.txt", true)); + assert!(!ask(true, "Nope", true)); + // The filesystem is probed only when a case-variant exists. + assert!(!case_alias_of_tracked( + || panic!("probed without a candidate"), + &tracked, + "Nope", + |_| true + )); + } + + /// A sparse view change over one tree holding `docs/a.txt` and `Docs/a.txt` + /// (#241). On a case-insensitive filesystem the laid-out `docs/a.txt` is + /// already what stands at `Docs/a.txt`, and writing it would replace that + /// file, so the guard must refuse; it must not count the alias as tracked. + /// Skipped on a case-sensitive filesystem, where nothing stands there. + #[cfg(unix)] + #[test] + fn a_view_change_refuses_a_newly_visible_case_alias_of_a_laid_out_file() { + let root = tmp_root("view-case-alias"); + let repo = Repo::init(&root).unwrap(); + if root.join(".SC").symlink_metadata().is_err() { + drop(repo); + std::fs::remove_dir_all(&root).unwrap(); + assert!(!root.exists()); + return; + } + std::fs::create_dir_all(root.join("docs")).unwrap(); + std::fs::write(root.join("docs/a.txt"), b"laid out\n").unwrap(); + let tree = { + let store_arc = repo.vfs.store(); + let mut store = store_arc.lock().unwrap(); + let mut entries = std::collections::BTreeMap::new(); + for (path, body) in [("docs/a.txt", "laid out\n"), ("Docs/a.txt", "other\n")] { + let id = store.put(scl_core::Object::blob(body)).unwrap(); + entries.insert(path.to_string(), (id, scl_core::FileMode(0o644), 0)); + } + let (tree, objs) = crate::private::build_tree_objects(&entries); + for o in objs { + store.put(o).unwrap(); + } + tree + }; + let old = crate::Sparse::new(vec!["docs/".into()]); + let new = crate::Sparse::new(vec!["docs/".into(), "Docs/".into()]); + let err = repo + .refuse_untracked_for_view_change("sparse set", tree, &old, &new, &Default::default()) + .unwrap_err(); + assert!(matches!(err, Error::InvalidArgument(_)), "got {err:?}"); + drop(repo); + std::fs::remove_dir_all(&root).unwrap(); + assert!(!root.exists()); + } + #[test] fn scignore_hides_untracked_from_status_and_commit_but_not_tracked() { let root = tmp_root("scignore"); diff --git a/crates/repo/src/sparse.rs b/crates/repo/src/sparse.rs index 655ebed4..31e23611 100644 --- a/crates/repo/src/sparse.rs +++ b/crates/repo/src/sparse.rs @@ -395,7 +395,6 @@ impl Repo { &old_spec, &spec, &Default::default(), - true, )?; } // Crash safety: record the spec being replaced before the new one is @@ -543,14 +542,7 @@ impl Repo { )? }; for root in roots { - self.refuse_untracked_for_view_change( - "sparse set", - root, - old, - new, - &already_laid, - false, - )?; + self.refuse_untracked_for_view_change("sparse set", root, old, new, &already_laid)?; } // Every guard passed, so each path below is the widen's to write: // record them (keeping an earlier interrupted run's) before the first. @@ -773,7 +765,6 @@ impl Repo { &self.sparse_spec()?, &Sparse::default(), &Default::default(), - true, )?; } clear(self.layout())?; diff --git a/docs/adr/0034-sparse-checkouts.md b/docs/adr/0034-sparse-checkouts.md index 5fac0879..26cd6d25 100644 --- a/docs/adr/0034-sparse-checkouts.md +++ b/docs/adr/0034-sparse-checkouts.md @@ -209,10 +209,15 @@ every claim below is checked against the shipped code, not the plan. laid; the marker is restored to its prior state once the spec is stored. The untracked-file guard checks only paths the old view did not match (those it laid out are not written, and a stopped op may have turned one - into a directory). Widening to the same spec is a quiet no-op. The guard does not count a case - alias of a laid-out path as tracked (`alias_ok = false`): on a case-insensitive - filesystem a decided `SRC/a.txt` would otherwise be written over the user's - resolution of `src/a.txt`. A crash between an atomic write's temp file + into a directory). Widening to the same spec is a quiet no-op. The guard never counts a case + alias of a laid-out path as tracked, for a stopped-op widen and for a plain + `sparse set`/`disable` alike (#241): both spellings come from one tree, so on a + case-insensitive filesystem a newly visible `SRC/a.txt` is the laid-out + `src/a.txt` itself, and writing it would replace that file's bytes (for a stopped + op, the user's resolution). The alias exemption stays for switch and the other + ops, where the old and new trees differ and a case-only rename is real; there it + needs a case-insensitive filesystem (probed with `.SC`, only when a case-variant + tracked path exists) and the same device and inode. A crash between an atomic write's temp file (`...tmp`) and its rename leaves that temp file beside the target; it is not recorded, delete it. The decided root is the right source: it is what completion carries an diff --git a/docs/adr/0057-bucket-compaction.md b/docs/adr/0057-bucket-compaction.md index 9c4b3daa..2ef32c1c 100644 --- a/docs/adr/0057-bucket-compaction.md +++ b/docs/adr/0057-bucket-compaction.md @@ -47,6 +47,14 @@ into groups of at most `group_bytes` (default 64 MiB); a group needs at least two members. Each group is read, its objects are deduplicated by id, and one new content-addressed pack plus idx is uploaded. This bounds peak RAM to a few times `group_bytes`, and packs above the threshold are never rewritten. +A group's decoded bytes are also bounded: past `4 x group_bytes` the objects read +so far are uploaded as a pack and reading continues. A part built from fewer +than two source packs is dropped, not uploaded, and its source stays where it +is (#263): rewriting one pack into one reduces nothing, and without this two +highly compressible packs that each decode past the limit cost one upload per +source. A run where no part reaches two sources merges nothing and writes no log +entry or checkpoint. `bytes_deleted` counts only objects the sweep actually +removed: packs kept because the manifest moved mid-sweep are subtracted. The result is committed with one CAS, which is both the fence and the fold: