Repository navigation
fix: sparse case-alias guard (#241) and compaction report/merge follow-ups (#263) #264
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
24fdeb1
924c34a
b9d92be
3715443
acd5cfd
9be30ea
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -91,6 +91,14 @@ fn hash_of(key: &str) -> Option<&str> { | |
| .or_else(|| name.strip_suffix(".idx")) | ||
| } | ||
|
|
||
| /// Bytes of the `.pack` and `.idx` objects behind these packs. | ||
| fn pack_bytes(packs: &[(&String, &Vec<ObjectMeta>)]) -> u64 { | ||
| packs | ||
| .iter() | ||
| .flat_map(|(_, ms)| ms.iter().map(|m| m.size)) | ||
| .sum() | ||
| } | ||
|
|
||
| /// Greedy size-tiered grouping: smallest candidates first, a group closes when | ||
| /// the next pack would push it past `group_bytes`. Groups of one are dropped | ||
| /// (there is nothing to merge). | ||
|
|
@@ -194,15 +202,14 @@ impl BucketTransport { | |
| )); | ||
| return Ok(()); | ||
| } | ||
| if opts.dry_run { | ||
| rep.packs_merged = replaced.len(); | ||
| rep.packs_created = groups.len(); | ||
| return Ok(()); | ||
| } | ||
| let mut created = Vec::new(); | ||
| // The packs actually folded: a source that fills a part alone is | ||
| // left where it is (#263), so this can be a subset of `replaced`. | ||
| let mut merged: BTreeSet<String> = BTreeSet::new(); | ||
| for group in &groups { | ||
| let mut objects: BTreeMap<ObjectId, Vec<u8>> = BTreeMap::new(); | ||
| let mut decoded = 0u64; | ||
| let mut part: Vec<&String> = Vec::new(); | ||
| for hash in group { | ||
| let Fetched::New { bytes, .. } = self.bucket.get(&pack_key(hash), None)? else { | ||
| // Either another compactor retired it (the manifest has | ||
|
|
@@ -223,19 +230,36 @@ impl BucketTransport { | |
| objects.insert(id, enc); | ||
| Ok(()) | ||
| })?; | ||
| part.push(hash); | ||
| // Compressed sizes bound the group, not the decoded bytes held | ||
| // here; spill to a pack of its own before RAM balloons. | ||
| if decoded > opts.group_bytes.saturating_mul(DECODED_FACTOR) { | ||
| let part: Vec<(ObjectId, Vec<u8>)> = | ||
| std::mem::take(&mut objects).into_iter().collect(); | ||
| created.push(self.upload_pack(&part)?.0); | ||
| self.flush_part(&mut objects, &mut part, &mut created, &mut merged, opts)?; | ||
| decoded = 0; | ||
| } | ||
| } | ||
| if !objects.is_empty() { | ||
| let objects: Vec<(ObjectId, Vec<u8>)> = objects.into_iter().collect(); | ||
| created.push(self.upload_pack(&objects)?.0); | ||
| self.flush_part(&mut objects, &mut part, &mut created, &mut merged, opts)?; | ||
| } | ||
| if created.is_empty() { | ||
| rep.notes.push( | ||
| "no merge would reduce the pack count: every candidate is large once decoded, \ | ||
| nothing merged" | ||
| .into(), | ||
| ); | ||
| return Ok(()); | ||
| } | ||
| if opts.dry_run { | ||
| // Split parts can outnumber the planned groups: size the | ||
| // checkpoint the real run would build, as it does. | ||
| if super::fold_would_overflow(nrefs, live.len() - merged.len() + created.len()) { | ||
| rep.notes.push(format!( | ||
| "the merged checkpoint would exceed the cap of {MAX_LIST}: nothing merged" | ||
| )); | ||
| return Ok(()); | ||
| } | ||
| rep.packs_merged = merged.len(); | ||
| rep.packs_created = created.len(); | ||
| return Ok(()); | ||
|
Comment on lines
+251
to
+262
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🎯 Functional Correctness | 🟡 Minor | ⚡ Quick win 🔎 Supported by static analysis🏁 Script executed: sed -n '1,130p' crates/repo/src/bucket_transport/compact.rs
sed -n '190,365p' crates/repo/src/bucket_transport/compact.rs
rg -n -C 4 'fold_would_overflow|created|merged|put_if_tag|pack_name|hash' crates/repo/src/bucket_transport/compact.rs crates/repo/src/bucket_transport
git diff --unified=100 38beb7b2922b4f68b2c128f451f45102e4249939 b9d92be170a14e7229ddd2b469c207c328120e00 -- crates/repo/src/bucket_transport/compact.rsRepository: git-agentic/src-control Length of output: 42083 🏁 Script executed: rg -n -F -- 'fn upload_pack' crates/repo/src crates/core/src
rg -n -F -- 'upload_pack(' crates/repo/src/bucket_transport
rg -n -F -- 'pack_key(' crates/repo/src crates/core/src
rg -n -e 'ObjectId|hash|digest|Sha|blake|pack' crates/core/src/pack crates/repo/src/bucket_transport/compact.rs | head -240
sed -n '340,430p' crates/repo/src/bucket_transport/compact.rs
sed -n '900,1040p' crates/repo/src/bucket_transport/compact_tests.rs 2>/dev/null || true
rg -n -F -- 'dry_run' crates/repo/src/bucket_transportRepository: git-agentic/src-control Length of output: 18361 🏁 Script executed: sed -n '500,590p' crates/repo/src/bucket_transport.rs
rg -n -F -- 'write_pack' crates/core crates/repo
rg -n -e 'PackWriter|pack_hash|hash.*pack|Sha256|blake3|ObjectId' crates/core crates/repo/src | head -260
rg --files crates/core | head -120
sed -n '280,330p' crates/repo/src/walfmt.rs
sed -n '280,340p' crates/repo/src/bucket_transport/compact/tests.rs
sed -n '930,1010p' crates/repo/src/bucket_transport/compact/tests.rsRepository: git-agentic/src-control Length of output: 34484 🏁 Script executed: sed -n '390,555p' crates/repo/src/bucket_transport.rs
sed -n '555,625p' crates/repo/src/bucket_transport.rs
rg -n -F -- 'fn put_object' crates/repo/src/bucket_transport.rs
rg -n -F -- 'fn put_pack' crates/repo/src/bucket_transport.rs
rg -n -F -- 'staged' crates/repo/src/bucket_transport.rs
sed -n '1,230p' crates/repo/src/bucket_transport/compact/tests.rs
git diff --unified=20 38beb7b2922b4f68b2c128f451f45102e4249939 b9d92be170a14e7229ddd2b469c207c328120e00 -- crates/repo/src/bucket_transport/compact.rs | sed -n '1,180p'Repository: git-agentic/src-control Length of output: 28065 🏁 Script executed: sed -n '730,815p' crates/repo/src/bucket_transport.rs
sed -n '1140,1235p' crates/repo/src/bucket_transport.rs
rg -n -F -- '.put_object(' crates/repo/src crates/cli/src
rg -n -F -- 'put_object(' crates/repo/src crates/cli/src | head -180
sed -n '590,635p' crates/repo/src/bucket_transport.rs
rg -n -F -- 'duplicate' crates/repo/src/bucket_transport crates/repo/src | head -120
rg -n -F -- 'fold_would_overflow' crates/repo/src/bucket_transport.rs crates/repo/src/bucket_transport/compact.rsRepository: git-agentic/src-control Length of output: 22635 Deduplicate simulated pack hashes before applying the checkpoint cap.
Suggested fix- if super::fold_would_overflow(nrefs, live.len() - merged.len() + created.len()) {
+ let mut kept = BTreeSet::new();
+ for hash in &live {
+ if !merged.contains(hash) {
+ kept.insert(hash.clone());
+ }
+ }
+ kept.extend(created.iter().cloned());
+ if super::fold_would_overflow(nrefs, kept.len()) {
@@
- created.push(if opts.dry_run {
- String::new()
- } else {
- self.upload_pack(&objects)?.0
- });
+ created.push(if opts.dry_run {
+ let (pack, _) = scl_core::pack::build_pack(&objects)?;
+ hex::encode(blake3::hash(&pack).as_bytes())
+ } else {
+ self.upload_pack(&objects)?.0
+ });🤖 Prompt for AI Agents |
||
| } | ||
| // Fence + fold in one CAS: the manifest tag we read is the state | ||
| // the checkpoint describes, so a lost CAS just replans. | ||
|
|
@@ -258,7 +282,7 @@ impl BucketTransport { | |
| let mut seen = BTreeSet::new(); | ||
| let mut kept: Vec<String> = packs | ||
| .into_iter() | ||
| .filter(|h| !replaced.contains(h) && seen.insert(h.clone())) | ||
| .filter(|h| !merged.contains(h) && seen.insert(h.clone())) | ||
| .collect(); | ||
| for h in &created { | ||
| if seen.insert(h.clone()) { | ||
|
|
@@ -277,7 +301,7 @@ impl BucketTransport { | |
| } | ||
| // Another compactor may have retired some of the packs we merged | ||
| // already: replan rather than publish a second copy of the same data. | ||
| if !replaced.is_subset(&packs_now) { | ||
| if !merged.is_subset(&packs_now) { | ||
| continue 'attempt; | ||
| } | ||
| let mut try_seq = head_seq + 1; | ||
|
|
@@ -320,7 +344,7 @@ impl BucketTransport { | |
| .is_some() | ||
| { | ||
| self.refresh()?; | ||
| rep.packs_merged = replaced.len(); | ||
| rep.packs_merged = merged.len(); | ||
| rep.packs_created = created.len(); | ||
| rep.checkpoint_seq = seq; | ||
| return Ok(()); | ||
|
|
@@ -333,6 +357,35 @@ impl BucketTransport { | |
| Ok(()) | ||
| } | ||
|
|
||
| /// Upload the objects of the source packs in `part` as one pack and mark | ||
| /// them merged, leaving `objects` and `part` empty. A part of fewer than two | ||
| /// sources is dropped: rewriting one pack into one gains nothing, and a | ||
| /// source whose decoded bytes alone pass the spill limit would otherwise | ||
| /// cost an upload per source (#263). | ||
| fn flush_part( | ||
| &self, | ||
| objects: &mut BTreeMap<ObjectId, Vec<u8>>, | ||
| part: &mut Vec<&String>, | ||
| created: &mut Vec<String>, | ||
| merged: &mut BTreeSet<String>, | ||
| opts: &CompactOptions, | ||
| ) -> Result<()> { | ||
| let objects: Vec<(ObjectId, Vec<u8>)> = std::mem::take(objects).into_iter().collect(); | ||
| let sources = std::mem::take(part); | ||
| if sources.len() < 2 { | ||
| return Ok(()); | ||
|
coderabbitai[bot] marked this conversation as resolved.
Resolved
|
||
| } | ||
| // A dry run reads and splits the same way but uploads nothing, so its | ||
| // counts match what a real run would do. | ||
| created.push(if opts.dry_run { | ||
| String::new() | ||
| } else { | ||
| self.upload_pack(&objects)?.0 | ||
| }); | ||
| merged.extend(sources.into_iter().cloned()); | ||
| Ok(()) | ||
| } | ||
|
|
||
| /// Phase 2: delete what is behind the cutoff and what nothing names. | ||
| fn sweep(&self, opts: &CompactOptions, now: SystemTime, rep: &mut CompactReport) -> Result<()> { | ||
| // `modified <= old_before` means "at least prune_expire old". | ||
|
|
@@ -434,10 +487,7 @@ impl BucketTransport { | |
| } | ||
| } | ||
| rep.packs_deleted += del_packs.len(); | ||
| rep.bytes_deleted += del_packs | ||
| .iter() | ||
| .flat_map(|(_, ms)| ms.iter().map(|m| m.size)) | ||
| .sum::<u64>(); | ||
| rep.bytes_deleted += pack_bytes(&del_packs); | ||
| if opts.dry_run { | ||
| if rep.packs_created > 0 { | ||
| rep.notes.push( | ||
|
|
@@ -458,10 +508,7 @@ impl BucketTransport { | |
| ); | ||
| if !unmoved { | ||
| rep.packs_deleted -= del_packs.len(); | ||
| rep.bytes_deleted -= del_packs | ||
| .iter() | ||
| .flat_map(|(_, ms)| ms.iter().map(|m| m.size)) | ||
| .sum::<u64>(); | ||
| rep.bytes_deleted -= pack_bytes(&del_packs); | ||
| continue; | ||
| } | ||
| // The delete loop can run long (one request per object), so look at | ||
|
|
@@ -476,6 +523,7 @@ impl BucketTransport { | |
| { | ||
| let left = del_packs.len() - i * PACK_DELETE_BATCH; | ||
| rep.packs_deleted -= left; | ||
| rep.bytes_deleted -= pack_bytes(&del_packs[i * PACK_DELETE_BATCH..]); | ||
| rep.notes.push(format!( | ||
| "the manifest moved during pack deletion: {left} unreferenced pack(s) \ | ||
| kept, rerun" | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
When the remote is near
MAX_LISTand decoded splitting leaves only a subset of the planned sources inmerged, the earlier capacity check is optimistic because it useslive.len() - replaced.len() + groups.len(). This dry-run return therefore reports a merge, while the corresponding real run buildskept, failsfold_would_overflow, and reports that nothing merged. The new partial-part behavior makes this possible even without concurrent pushes; apply the exactlive - merged + created/deduplicated checkpoint check before returning the dry-run report.Useful? React with 👍 / 👎.