From b27ad2b365374c82e7fd6791da8a57cd5859c2dd Mon Sep 17 00:00:00 2001 From: Brandon Black Date: Fri, 2 Oct 2026 10:26:42 -0700 Subject: [PATCH 01/10] confirm: collect spend abs once after fill The write phase looked up each spend abs in fill, again in ensure_spend_abs_layouts, and a third time inside structural. One spend_abs_jobs walk after fill is the Corrupt gate before tx.head insert, and structural reuses those jobs. --- crates/rbitcoin-consensus/src/block/mod.rs | 51 +++++++++--- .../src/block/structure_rule_tests.rs | 3 + .../rbitcoin-consensus/src/confirm_run/mod.rs | 2 +- .../src/confirm_run/phases.rs | 7 ++ .../rbitcoin-consensus/src/confirm_run/pin.rs | 30 +++++++ .../src/confirm_run/write.rs | 9 +-- .../src/confirm_run/write_idempotent_tests.rs | 81 +++++++++++++++++++ docs/invariants.md | 3 +- 8 files changed, 168 insertions(+), 18 deletions(-) diff --git a/crates/rbitcoin-consensus/src/block/mod.rs b/crates/rbitcoin-consensus/src/block/mod.rs index d13fb7f22..c91b9f6db 100644 --- a/crates/rbitcoin-consensus/src/block/mod.rs +++ b/crates/rbitcoin-consensus/src/block/mod.rs @@ -1541,6 +1541,7 @@ pub(crate) fn structural_validate_spends( mtp_cache: &mut U32Map, run_create_height: &FkMap, scratch: &mut StructuralScratch, + precomputed_abs: Option<&[StructuralAbsJob]>, ) -> Result { use std::time::Instant; @@ -1550,7 +1551,14 @@ pub(crate) fn structural_validate_spends( let t_spent = Instant::now(); reject_bip30_unspent_overwrite(query, block, ctx)?; let t_abs = Instant::now(); - structural_abs_heights(query, spends, batch_parents, run_create_height, scratch)?; + structural_abs_heights( + query, + spends, + batch_parents, + run_create_height, + scratch, + precomputed_abs, + )?; let tip = query.tip_height().map(|h| h.0); let mut spent_strong_ns = 0u64; let mut multi_list_ns = 0u64; @@ -1592,7 +1600,7 @@ pub(crate) fn structural_validate_spends( }) } -type StructuralAbsJob = (u64, u32, u64, rbitcoin_primitives::Fk, u32); +pub(crate) type StructuralAbsJob = (u64, u32, u64, rbitcoin_primitives::Fk, u32); type DurableSpentSet = std::collections::HashSet<(u64, u32), BuildHasherDefault>; type OverlayMetaSkip = std::collections::HashMap< @@ -1651,16 +1659,37 @@ fn structural_abs_heights( batch_parents: &rbitcoin_query::BatchParents, run_create_height: &FkMap, scratch: &mut StructuralScratch, + precomputed_abs: Option<&[StructuralAbsJob]>, ) -> Result<(), ConsensusError> { - batch_parents - .spend_abs_jobs_into( - spends - .iter() - .map(|&(_, vout, sfk, cfk, vin)| (cfk, vout, sfk, vin)), - &mut scratch.abs_jobs, - &mut scratch.abs_seen, - ) - .map_err(ConsensusError::from)?; + if let Some(jobs) = precomputed_abs { + // Tests recompute so a stale list cannot skip a missing abs. + #[cfg(test)] + { + let fresh = batch_parents + .spend_abs_jobs( + spends + .iter() + .map(|&(_, vout, sfk, cfk, vin)| (cfk, vout, sfk, vin)), + ) + .map_err(ConsensusError::from)?; + if fresh.as_slice() != jobs { + return Err(ConsensusError::Store(rbitcoin_store::StoreError::Corrupt( + "invariant: precomputed spend abs diverged from pin", + ))); + } + } + scratch.abs_jobs.extend_from_slice(jobs); + } else { + batch_parents + .spend_abs_jobs_into( + spends + .iter() + .map(|&(_, vout, sfk, cfk, vin)| (cfk, vout, sfk, vin)), + &mut scratch.abs_jobs, + &mut scratch.abs_seen, + ) + .map_err(ConsensusError::from)?; + } scratch.unique_fks.extend( scratch .abs_jobs diff --git a/crates/rbitcoin-consensus/src/block/structure_rule_tests.rs b/crates/rbitcoin-consensus/src/block/structure_rule_tests.rs index b51944ac6..0e8adeda5 100644 --- a/crates/rbitcoin-consensus/src/block/structure_rule_tests.rs +++ b/crates/rbitcoin-consensus/src/block/structure_rule_tests.rs @@ -369,6 +369,7 @@ fn rejects_unspent_overwrite(q: &rbitcoin_query::Query, first: &Transaction) { &mut U32Map::default(), &FkMap::default(), &mut crate::block::StructuralScratch::default(), + None, ) .expect_err("an unspent overwrite must trip BIP30"); let msg = format!("{err}"); @@ -454,6 +455,7 @@ fn bip30_message_at_mainnet_above_bip34( &mut U32Map::default(), &FkMap::default(), &mut crate::block::StructuralScratch::default(), + None, ) .map(|_| ()) } @@ -2099,6 +2101,7 @@ fn already_archived_schema13_pin_identity_tip_follow() { &mut mtp, &FkMap::default(), &mut crate::block::StructuralScratch::default(), + None, ) .expect_err("missing denserels abs must hard-fail"); let msg = format!("{err}"); diff --git a/crates/rbitcoin-consensus/src/confirm_run/mod.rs b/crates/rbitcoin-consensus/src/confirm_run/mod.rs index 3418a4ccc..6115a187c 100644 --- a/crates/rbitcoin-consensus/src/confirm_run/mod.rs +++ b/crates/rbitcoin-consensus/src/confirm_run/mod.rs @@ -69,7 +69,7 @@ pub use lookup::{ use phases::{assemble_run, Assembled}; #[cfg(test)] use phases::{check_bip34, expected_bits_extending, post_commit}; -use pin::{ensure_spend_abs_layouts, pin_for_wire_batch}; +use pin::{collect_spend_abs_after_fill, ensure_spend_abs_layouts, pin_for_wire_batch}; pub use scripts::{confirm_scripts_phase, drive_script_waves_with}; pub(crate) use write::finish_post_commit; pub use write::{confirm_write_phase, finish_post_commit_hashes, replay_spend_annotations}; diff --git a/crates/rbitcoin-consensus/src/confirm_run/phases.rs b/crates/rbitcoin-consensus/src/confirm_run/phases.rs index 40c2ba09a..c2694505c 100644 --- a/crates/rbitcoin-consensus/src/confirm_run/phases.rs +++ b/crates/rbitcoin-consensus/src/confirm_run/phases.rs @@ -238,6 +238,7 @@ pub(super) fn structural_run( prepared: &[Prepared], wire_blocks: &[Arc], batch_parents: &rbitcoin_query::BatchParents, + abs_jobs: &[Vec], ) -> Result<(crate::block::StructuralPhaseNs, crate::block::AnnotateSlots), ConsensusError> { use crate::block::{StructuralPhaseNs, StructuralScratch}; let t0 = Instant::now(); @@ -249,6 +250,11 @@ pub(super) fn structural_run( mtp_cache.insert(p.height.0 - 1, p.prev_mtp); } } + if abs_jobs.len() != prepared.len() { + return Err(ConsensusError::Store(rbitcoin_store::StoreError::Corrupt( + "invariant: spend abs jobs length", + ))); + } let mut tot = StructuralPhaseNs::default(); let mut run_create_height: FkMap = FkMap::default(); for p in prepared { @@ -270,6 +276,7 @@ pub(super) fn structural_run( &mut mtp_cache, &run_create_height, &mut scratch, + Some(&abs_jobs[i]), )?; tot.spent_ns = tot.spent_ns.saturating_add(ph.spent_ns); tot.spent_abs_ns = tot.spent_abs_ns.saturating_add(ph.spent_abs_ns); diff --git a/crates/rbitcoin-consensus/src/confirm_run/pin.rs b/crates/rbitcoin-consensus/src/confirm_run/pin.rs index 9ee751eaf..65d63e394 100644 --- a/crates/rbitcoin-consensus/src/confirm_run/pin.rs +++ b/crates/rbitcoin-consensus/src/confirm_run/pin.rs @@ -441,11 +441,41 @@ pub(super) fn pin_for_wire_batch( Ok((batch_parents, spend_edges)) } +/// One `spend_abs_jobs` walk per prepared block, after fill has stamped ranges. +/// +/// Missing in-range abs is the ensure `Corrupt`. The write phase returns that +/// before `tx.head` insert and passes the jobs to structural, which does not +/// look them up again. Null create fk is skipped; duplicate abs is one job. +pub(super) fn collect_spend_abs_after_fill( + batch_parents: &rbitcoin_query::BatchParents, + prepared: &[Prepared], +) -> Result>, ConsensusError> { + let mut out = Vec::with_capacity(prepared.len()); + for p in prepared { + let jobs = batch_parents + .spend_abs_jobs( + p.spends + .iter() + .map(|&(_txid, vout, sfk, cfk, vin)| (cfk, vout, sfk, vin)), + ) + .map_err(|_| { + ConsensusError::Store(StoreError::Corrupt( + "invariant: ensure denserels/abs incomplete for spend edge", + )) + })?; + out.push(jobs); + } + Ok(out) +} + /// Ensure spend abs for every spend edge on the write batch. /// /// Lookup stamps archived-parent spent ranges; load copies them onto /// the pin. Same-batch abs comes from append RAM in write fill. Missing /// abs is `Corrupt`. Never `put_spend*` and never preads `create.loc`. +/// +/// Direct tests keep this walk, including its skip of a null spend fk. +/// [`collect_spend_abs_after_fill`] is the write-phase post-condition. pub(super) fn ensure_spend_abs_layouts( batch_parents: &rbitcoin_query::BatchParents, prepared: &[Prepared], diff --git a/crates/rbitcoin-consensus/src/confirm_run/write.rs b/crates/rbitcoin-consensus/src/confirm_run/write.rs index f8b5c9987..691e361f2 100644 --- a/crates/rbitcoin-consensus/src/confirm_run/write.rs +++ b/crates/rbitcoin-consensus/src/confirm_run/write.rs @@ -162,11 +162,9 @@ pub fn confirm_write_phase( plan_take_ns, create_map_ns, } = apply_archive_plan(query, &mut batch)?; - { - let t_ens = Instant::now(); - ensure_spend_abs_layouts(&batch.batch_parents, &batch.prepared)?; - ensure_ns = ensure_ns.saturating_add(t_ens.elapsed().as_nanos() as u64); - } + let t_ens = Instant::now(); + let abs_jobs = super::collect_spend_abs_after_fill(&batch.batch_parents, &batch.prepared)?; + ensure_ns = ensure_ns.saturating_add(t_ens.elapsed().as_nanos() as u64); if class_a_ns > 0 { rbitcoin_query::note_confirm(&query.confirm_stats().class_a_ns, class_a_ns); } @@ -200,6 +198,7 @@ pub fn confirm_write_phase( &batch.prepared, &batch.wire_blocks, &batch.batch_parents, + &abs_jobs, )?; let structural_ns = t_struct.elapsed().as_nanos() as u64; diff --git a/crates/rbitcoin-consensus/src/confirm_run/write_idempotent_tests.rs b/crates/rbitcoin-consensus/src/confirm_run/write_idempotent_tests.rs index 545265360..7a8ec6682 100644 --- a/crates/rbitcoin-consensus/src/confirm_run/write_idempotent_tests.rs +++ b/crates/rbitcoin-consensus/src/confirm_run/write_idempotent_tests.rs @@ -1460,6 +1460,7 @@ fn structural_same_batch_overlay_skips_meta_pread() { &mut mtp, &run, &mut scratch, + None, ) .expect("overlay spend is not durable-spent before tip"); let meta_n = q @@ -1606,6 +1607,7 @@ fn structural_scratch_second_block_does_not_replay_first_slots() { &mut mtp, &run, &mut scratch, + None, ) .unwrap(); } @@ -3103,6 +3105,7 @@ fn structural_pinned_without_abs_is_invariant_error() { &mut mtp, &rbitcoin_query::FkMap::default(), &mut crate::block::StructuralScratch::default(), + None, ) .expect_err("pinned without abs must be invariant"); let msg = format!("{err}"); @@ -3215,3 +3218,81 @@ fn one_shot_load_matches_stamp_then_load_from_plan() { let _ = std::fs::remove_dir_all(&path_a); let _ = std::fs::remove_dir_all(&path_b); } + +/// Post-fill abs jobs come from one `spend_abs_jobs` walk. A vout outside the +/// spent range is the ensure `Corrupt` (before head insert). Duplicate abs +/// collapses to one job. A null create fk is skipped. +#[test] +fn collect_spend_abs_after_fill_is_one_walk() { + use super::{collect_spend_abs_after_fill, Prepared}; + use rbitcoin_primitives::{Fk, Height}; + use rbitcoin_query::BatchParents; + use rbitcoin_store::OutputRecord; + + let mut bp = BatchParents::new(); + bp.insert_owned( + Fk(1), + rec_tx(0x11, 2), + vec![ + (0, OutputRecord::unspent(1, vec![0x51])), + (1, OutputRecord::unspent(1, vec![0x51])), + ], + vec![0, 1], + Some(false), + None, + Vec::new(), + ); + // One spent slot: vout 0 is in range, vout 1 is not. + bp.set_spent_range_only(Fk(1), (1000, OutputRecord::SPENT_SLOT_LEN as u64)); + + let bits = bitcoin::CompactTarget::from_consensus(0x207f_ffff); + let prepared_ok = [Prepared { + height: Height(1), + header_fk: Fk(1), + tx_fks: vec![Fk(9)], + jobs: vec![], + spends: vec![ + ([0x11; 32], 0, Fk(9), Fk(1), 0), + ([0x11; 32], 0, Fk(9), Fk(1), 1), + ([0; 32], 0, Fk(9), Fk::NULL, 0), + ], + fees: 0, + check_scripts: false, + time: 1, + bits, + hash: [1u8; 32], + prev_mtp: 0, + }]; + let jobs = collect_spend_abs_after_fill(&bp, &prepared_ok).expect("in-range vout 0"); + assert_eq!( + jobs, + vec![vec![( + 1u64, + 0u32, + rbitcoin_store::spent_abs(1000, 0), + Fk(9), + 0u32, + )]] + ); + + let prepared_miss = [Prepared { + height: Height(1), + header_fk: Fk(1), + tx_fks: vec![Fk(9)], + jobs: vec![], + spends: vec![([0x11; 32], 1, Fk(9), Fk(1), 0)], + fees: 0, + check_scripts: false, + time: 1, + bits, + hash: [1u8; 32], + prev_mtp: 0, + }]; + let err = + collect_spend_abs_after_fill(&bp, &prepared_miss).expect_err("vout past the spent range"); + let msg = format!("{err}"); + assert!( + msg.contains("invariant: ensure denserels/abs incomplete for spend edge"), + "got {msg}" + ); +} diff --git a/docs/invariants.md b/docs/invariants.md index 5122c2863..6bbb2af85 100644 --- a/docs/invariants.md +++ b/docs/invariants.md @@ -152,7 +152,7 @@ packs at/above the leaving **pack** height **before** the next bind. ## Related code -- Confirm write annotate / ensure: `rbitcoin-consensus` `confirm_run::{post_commit,ensure_spend_abs_layouts,pin_for_wire_batch}` +- Confirm write annotate / ensure: `rbitcoin-consensus` `confirm_run::{post_commit,collect_spend_abs_after_fill,ensure_spend_abs_layouts,pin_for_wire_batch}` - Structural: `rbitcoin-consensus` `block::structural_validate_spends` - Pin / denserels: `rbitcoin-query` `confirm_load`, `BatchParents`, `pin_for_wire_batch` (cold range / adopt) - Abs annotate: `rbitcoin-store` `put_spend_batch_by_abs_meta` @@ -165,6 +165,7 @@ packs at/above the leaving **pack** height **before** the next bind. | `pin_for_wire_incomplete_outs_is_invariant_error` | `pin_for_wire_batch` incomplete outs → cold miss | | `post_commit_missing_denserels_is_invariant_error` | `post_commit` abs-only annotate | | `ensure_spend_abs_incomplete_is_invariant_error` | `ensure_spend_abs_layouts` post-condition | +| `collect_spend_abs_after_fill_is_one_walk` | one `spend_abs_jobs` after fill; vout past the range is ensure Corrupt; duplicate abs is one job | | `write_ensure_stamps_spent_range_after_load_pin` / `pin_and_ensure_journey` / `fill_same_batch_abs_from_append_loc_ram` / `fill_just_written_survives_until_last_started_write` / `fill_stamp_spent_hole_from_write_tls` / `pin_and_ensure_from_pin_loc_without_tls` | load pin copies lookup spent range; missing stamp is Corrupt; same-batch abs from append RAM; just-written loc until write of last started height; stamp spent hole filled by write TLS; pin loc after TLS prune (no write loc pread) | | `fill_missing_parent_ranges_stamps_spent_idx_for_archived` / `inflight_hit_adopts_skeleton_loc` / `inflight_hit_skeleton_miss_leaves_spent_unset_despite_disk_loc` / `plan_inflight_skeleton_miss_leaves_spent_unset_despite_disk_loc` / `inflight_hit_uses_pin_loc_without_disk` | leftover TipOnly still loc-fills archived spent; InFlight takes skeleton loc; IBD skeleton miss leaves spent unset (no loc-by-fk); pin loc binds without disk | | `spend_abs_jobs_unique_and_missing_is_corrupt` | pin arithmetic abs list; missing → Corrupt | From 02ab9fb920b7ff854677c7f9c86a2adafc37f3b2 Mon Sep 17 00:00:00 2001 From: Brandon Black Date: Fri, 2 Oct 2026 10:44:51 -0700 Subject: [PATCH 02/10] confirm: reuse the write-thread structural scratch Each write batch allocated a new spentness scratch, pending-spent set, and create-height map. The write thread keeps one of each and clears slots, the pending set, and heights at the start of the batch so annotate and double-spend state cannot cross batches. The pending check is one insert. --- crates/rbitcoin-consensus/src/block/mod.rs | 13 +- .../src/confirm_run/phases.rs | 39 +++-- .../src/confirm_run/write.rs | 27 ++-- .../src/confirm_run/write_idempotent_tests.rs | 148 ++++++++++++++++++ 4 files changed, 199 insertions(+), 28 deletions(-) diff --git a/crates/rbitcoin-consensus/src/block/mod.rs b/crates/rbitcoin-consensus/src/block/mod.rs index c91b9f6db..673c2f355 100644 --- a/crates/rbitcoin-consensus/src/block/mod.rs +++ b/crates/rbitcoin-consensus/src/block/mod.rs @@ -1499,6 +1499,13 @@ impl Default for StructuralScratch { } impl StructuralScratch { + /// Drop annotate slots from the previous write batch. Per-block buffers + /// stay for [`Self::begin_block`]; slots accumulate inside one batch. + pub(crate) fn begin_batch(&mut self) { + self.slots.abs_edges.clear(); + self.slots.known.clear(); + } + fn begin_block(&mut self) { self.abs_jobs.clear(); self.abs_seen.clear(); @@ -1865,9 +1872,6 @@ fn structural_mark_pending( ) -> Result<(), ConsensusError> { for &(prev_txid, vout, _spend_fk, create_fk, _vin) in spends { let key = (prev_txid, vout); - if pending_spent.contains(&key) { - return Err(ConsensusError::PrevoutSpent); - } let spent = if create_fk.is_null() { false } else if let Some(id) = create_fk.get() { @@ -1875,10 +1879,9 @@ fn structural_mark_pending( } else { false }; - if spent { + if spent || !pending_spent.insert(key) { return Err(ConsensusError::PrevoutSpent); } - pending_spent.insert(key); } Ok(()) } diff --git a/crates/rbitcoin-consensus/src/confirm_run/phases.rs b/crates/rbitcoin-consensus/src/confirm_run/phases.rs index c2694505c..a5fb67f20 100644 --- a/crates/rbitcoin-consensus/src/confirm_run/phases.rs +++ b/crates/rbitcoin-consensus/src/confirm_run/phases.rs @@ -230,6 +230,14 @@ pub(super) fn assemble_run( }) } +/// Scratch reused by the confirm write thread. Cleared at each batch. +#[derive(Default)] +pub(super) struct StructuralReuse { + pub scratch: crate::block::StructuralScratch, + pub pending: rbitcoin_query::OutPointSet, + pub heights: FkMap, +} + /// Durable spentness + maturity + subsidy after scripts (height order). pub(super) fn structural_run( query: &Query, @@ -239,27 +247,30 @@ pub(super) fn structural_run( wire_blocks: &[Arc], batch_parents: &rbitcoin_query::BatchParents, abs_jobs: &[Vec], + reuse: &mut StructuralReuse, ) -> Result<(crate::block::StructuralPhaseNs, crate::block::AnnotateSlots), ConsensusError> { - use crate::block::{StructuralPhaseNs, StructuralScratch}; + use crate::block::StructuralPhaseNs; + if abs_jobs.len() != prepared.len() { + return Err(ConsensusError::Store(rbitcoin_store::StoreError::Corrupt( + "invariant: spend abs jobs length", + ))); + } let t0 = Instant::now(); - let mut scratch = StructuralScratch::default(); - let mut pending_spent: rbitcoin_query::OutPointSet = Default::default(); + // Reused across write batches on this thread. Slots, the pack-local + // double-spend set, and create heights are this batch only. + reuse.scratch.begin_batch(); + reuse.pending.clear(); + reuse.heights.clear(); let mut mtp_cache: U32Map = U32Map::default(); for p in prepared { if p.height.0 > 0 { mtp_cache.insert(p.height.0 - 1, p.prev_mtp); } } - if abs_jobs.len() != prepared.len() { - return Err(ConsensusError::Store(rbitcoin_store::StoreError::Corrupt( - "invariant: spend abs jobs length", - ))); - } let mut tot = StructuralPhaseNs::default(); - let mut run_create_height: FkMap = FkMap::default(); for p in prepared { for fk in &p.tx_fks { - run_create_height.insert(*fk, p.height.0); + reuse.heights.insert(*fk, p.height.0); } } for (i, p) in prepared.iter().enumerate() { @@ -271,11 +282,11 @@ pub(super) fn structural_run( Some(&p.tx_fks), &p.spends, p.fees, - &mut pending_spent, + &mut reuse.pending, batch_parents, &mut mtp_cache, - &run_create_height, - &mut scratch, + &reuse.heights, + &mut reuse.scratch, Some(&abs_jobs[i]), )?; tot.spent_ns = tot.spent_ns.saturating_add(ph.spent_ns); @@ -313,7 +324,7 @@ pub(super) fn structural_run( tot.create_h_ns, ); rbitcoin_query::note_confirm(&query.confirm_stats().structural_bip68_ns, tot.bip68_ns); - Ok((tot, scratch.slots)) + Ok((tot, std::mem::take(&mut reuse.scratch.slots))) } pub(super) fn class_c_commit( diff --git a/crates/rbitcoin-consensus/src/confirm_run/write.rs b/crates/rbitcoin-consensus/src/confirm_run/write.rs index 691e361f2..a59fa7342 100644 --- a/crates/rbitcoin-consensus/src/confirm_run/write.rs +++ b/crates/rbitcoin-consensus/src/confirm_run/write.rs @@ -2,6 +2,12 @@ use super::phases::{class_c_commit, post_commit, structural_run}; use super::*; +use std::cell::RefCell; + +thread_local! { + static WRITE_REUSE: RefCell = + RefCell::new(super::phases::StructuralReuse::default()); +} pub(super) fn write_height_needed(tip: Option, height: u32) -> bool { match tip { @@ -191,15 +197,18 @@ pub fn confirm_write_phase( let overlap = (|| -> Result<_, ConsensusError> { // Local Instant totals (not atomic deltas) — sample_and_reset races mid-batch. let t_struct = Instant::now(); - let (struct_ph, slots) = structural_run( - query, - params, - milestone, - &batch.prepared, - &batch.wire_blocks, - &batch.batch_parents, - &abs_jobs, - )?; + let (struct_ph, slots) = WRITE_REUSE.with(|reuse| { + structural_run( + query, + params, + milestone, + &batch.prepared, + &batch.wire_blocks, + &batch.batch_parents, + &abs_jobs, + &mut reuse.borrow_mut(), + ) + })?; let structural_ns = t_struct.elapsed().as_nanos() as u64; let n_blocks = batch.prepared.len(); diff --git a/crates/rbitcoin-consensus/src/confirm_run/write_idempotent_tests.rs b/crates/rbitcoin-consensus/src/confirm_run/write_idempotent_tests.rs index 7a8ec6682..c444d2021 100644 --- a/crates/rbitcoin-consensus/src/confirm_run/write_idempotent_tests.rs +++ b/crates/rbitcoin-consensus/src/confirm_run/write_idempotent_tests.rs @@ -3296,3 +3296,151 @@ fn collect_spend_abs_after_fill_is_one_walk() { "got {msg}" ); } + +/// A scratch reused for the next write batch must not keep the previous +/// batch's annotate slots or pack-local double-spend set. +#[test] +fn structural_run_clears_reused_scratch_between_batches() { + use super::phases::{structural_run, StructuralReuse}; + use super::{collect_spend_abs_after_fill, Prepared}; + use crate::milestone::Milestone; + use crate::params::ChainParams; + use bitcoin::absolute::LockTime; + use bitcoin::block::{Header, Version}; + use bitcoin::hashes::Hash; + use bitcoin::script::ScriptBuf; + use bitcoin::transaction::Version as TxVersion; + use bitcoin::{ + Amount, Block, BlockHash, CompactTarget, OutPoint, Sequence, Transaction, TxIn, TxOut, + Witness, + }; + use rbitcoin_primitives::{Fk, Height}; + use rbitcoin_query::BatchParents; + use rbitcoin_store::{InputRecord, OutputRecord}; + use std::sync::Arc; + + let (path, q) = tiny_query(); + let parent_pin = rbitcoin_query::CreatePinInner::records( + rec_tx(0x41, 1), + vec![OutputRecord::unspent(1, vec![0x51])], + ); + let (fks, loc) = q + .store() + .put_tx_full_batch_from_pins( + &[( + std::sync::Arc::clone(&parent_pin), + vec![InputRecord::coinbase(u32::MAX, vec![0x01], vec![])], + )], + false, + &[], + ) + .unwrap(); + let mut bp = BatchParents::new(); + bp.insert_create_pin( + fks[0], + parent_pin, + vec![0], + Some(false), + Some(loc[0].txout), + Vec::new(), + ); + bp.set_spent_range_only(fks[0], loc[0].spent); + q.store().header_txs.put_range(Fk(100), fks[0], 1).unwrap(); + q.store().confirmed.set(Height(0), Fk(100)).unwrap(); + q.store().rebuild_height_fence().unwrap(); + + let coinbase = Transaction { + version: TxVersion::ONE, + lock_time: LockTime::ZERO, + input: vec![TxIn { + previous_output: OutPoint::null(), + script_sig: ScriptBuf::from_bytes(vec![0x00, 0x01]), + sequence: Sequence::MAX, + witness: Witness::new(), + }], + output: vec![TxOut { + value: Amount::from_sat(50_0000_0000), + script_pubkey: ScriptBuf::from_bytes(vec![0x51]), + }], + }; + let mut block = Block { + header: Header { + version: Version::from_consensus(4), + prev_blockhash: BlockHash::from_byte_array([0u8; 32]), + merkle_root: bitcoin::TxMerkleNode::from_byte_array([0u8; 32]), + time: 1_300_000_000, + bits: CompactTarget::from_consensus(0x207f_ffff), + nonce: 0, + }, + txdata: vec![ + coinbase, + Transaction { + version: TxVersion::ONE, + lock_time: LockTime::ZERO, + input: vec![TxIn { + previous_output: OutPoint { + txid: bitcoin::Txid::from_byte_array([0x41; 32]), + vout: 0, + }, + script_sig: ScriptBuf::new(), + sequence: Sequence::MAX, + witness: Witness::new(), + }], + output: vec![TxOut { + value: Amount::from_sat(1), + script_pubkey: ScriptBuf::from_bytes(vec![0x51]), + }], + }, + ], + }; + block.header.merkle_root = block.compute_merkle_root().unwrap(); + let params = ChainParams::regtest(); + let bits = CompactTarget::from_consensus(0x207f_ffff); + let prepared = [Prepared { + height: Height(1), + header_fk: Fk(100), + tx_fks: vec![Fk(9)], + jobs: vec![], + spends: vec![([0x41; 32], 0, Fk(9), fks[0], 0)], + fees: 0, + check_scripts: false, + time: 1_300_000_000, + bits, + hash: [2u8; 32], + prev_mtp: 1_300_000_000, + }]; + let jobs = collect_spend_abs_after_fill(&bp, &prepared).expect("parent abs"); + let wire = vec![Arc::new(block)]; + + let mut reuse = StructuralReuse::default(); + reuse + .scratch + .slots + .push((999, Fk(1), 0, Fk(8), 0), (Fk::NULL, 0, 0)); + reuse.pending.insert(([0x41; 32], 0u32)); + + let run = |reuse: &mut StructuralReuse| { + structural_run( + &q, + ¶ms, + Milestone::NONE, + &prepared, + &wire, + &bp, + &jobs, + reuse, + ) + .expect("reused scratch still connects") + }; + let (_ph, slots) = run(&mut reuse); + assert!( + slots.abs_edges.iter().all(|e| e.0 != 999), + "previous batch slot leaked: {:?}", + slots.abs_edges + ); + assert_eq!(slots.abs_edges.len(), 1); + let (_ph, slots2) = run(&mut reuse); + assert_eq!(slots2.abs_edges.len(), 1); + assert_ne!(slots2.abs_edges[0].0, 999); + let _ = std::fs::remove_dir_all(&path); +} From 5cfa7bcfb411cc36ff8ec5577b4c4bc7e53885d8 Mon Sep 17 00:00:00 2001 From: Brandon Black Date: Fri, 2 Oct 2026 10:58:56 -0700 Subject: [PATCH 03/10] confirm: look up batch create heights by fk span IBD create fks are one contiguous range per block. Binary search over those spans replaces a HashMap insert per transaction on the write thread. A gap, a null fk, or an overlap keeps the map so a missing fk does not inherit a neighbor height. --- crates/rbitcoin-consensus/src/block/mod.rs | 133 ++++++++++++++++-- .../src/block/structure_rule_tests.rs | 21 +-- .../src/confirm_run/phases.rs | 15 +- .../src/confirm_run/write_idempotent_tests.rs | 20 +-- 4 files changed, 151 insertions(+), 38 deletions(-) diff --git a/crates/rbitcoin-consensus/src/block/mod.rs b/crates/rbitcoin-consensus/src/block/mod.rs index 673c2f355..f6296ed0b 100644 --- a/crates/rbitcoin-consensus/src/block/mod.rs +++ b/crates/rbitcoin-consensus/src/block/mod.rs @@ -1546,7 +1546,7 @@ pub(crate) fn structural_validate_spends( pending_spent: &mut rbitcoin_query::OutPointSet, batch_parents: &rbitcoin_query::BatchParents, mtp_cache: &mut U32Map, - run_create_height: &FkMap, + run_create_height: &RunCreateHeight, scratch: &mut StructuralScratch, precomputed_abs: Option<&[StructuralAbsJob]>, ) -> Result { @@ -1608,6 +1608,81 @@ pub(crate) fn structural_validate_spends( } pub(crate) type StructuralAbsJob = (u64, u32, u64, rbitcoin_primitives::Fk, u32); + +/// Create heights for one write batch. +/// +/// Contiguous per-block fk spans are the IBD shape (`first` + count). A gap +/// or overlap uses the map so a fk between spans does not inherit a height. +pub(crate) enum RunCreateHeight { + Spans(Vec<(u64, u64, u32)>), + Map(FkMap), +} + +impl RunCreateHeight { + pub(crate) fn from_blocks<'a>( + blocks: impl IntoIterator, + ) -> Self { + let blocks: Vec<(u32, &'a [rbitcoin_primitives::Fk])> = blocks.into_iter().collect(); + if let Some(spans) = contiguous_create_spans(&blocks) { + Self::Spans(spans) + } else { + let mut map = FkMap::default(); + for (height, fks) in blocks { + for fk in fks { + map.insert(*fk, height); + } + } + Self::Map(map) + } + } + + pub(crate) fn get(&self, fk: rbitcoin_primitives::Fk) -> Option { + match self { + Self::Map(map) => map.get(&fk).copied(), + Self::Spans(spans) => { + let id = fk.get()?; + let i = spans.partition_point(|span| span.0 <= id); + let (_, end, height) = spans.get(i.checked_sub(1)?)?; + (*end > id).then_some(*height) + } + } + } +} + +/// `Some` when every non-empty block is `[first, first+n)` and spans do not overlap. +/// +/// A null fk, a gap, or an id that would wrap is `None` so the caller uses the map. +/// A wrapped end must not cover fks that are not in the block. +fn contiguous_create_spans( + blocks: &[(u32, &[rbitcoin_primitives::Fk])], +) -> Option> { + let mut spans = Vec::with_capacity(blocks.len()); + for &(height, fks) in blocks { + if fks.is_empty() { + continue; + } + let mut ids = fks.iter().map(|fk| fk.get()); + let first = ids.next().flatten()?; + let mut prev = first; + for id in ids { + let id = id?; + let next = prev.checked_add(1)?; + if id != next { + return None; + } + prev = id; + } + let end = prev.checked_add(1)?; + spans.push((first, end, height)); + } + spans.sort_unstable_by_key(|span| span.0); + for pair in spans.windows(2) { + if pair[0].1 > pair[1].0 { + return None; + } + } + Some(spans) +} type DurableSpentSet = std::collections::HashSet<(u64, u32), BuildHasherDefault>; type OverlayMetaSkip = std::collections::HashMap< @@ -1624,11 +1699,11 @@ fn fill_overlay_skip( rbitcoin_primitives::Fk, u32, )], - run_create_height: &FkMap, + run_create_height: &RunCreateHeight, scratch: &mut StructuralScratch, ) { for &(_, vout, sfk, cfk, vin) in spends { - if !run_create_height.contains_key(&cfk) { + if run_create_height.get(cfk).is_none() { continue; } let Some(id) = cfk.get() else { @@ -1664,7 +1739,7 @@ fn structural_abs_heights( u32, )], batch_parents: &rbitcoin_query::BatchParents, - run_create_height: &FkMap, + run_create_height: &RunCreateHeight, scratch: &mut StructuralScratch, precomputed_abs: Option<&[StructuralAbsJob]>, ) -> Result<(), ConsensusError> { @@ -1713,7 +1788,7 @@ fn structural_abs_heights( let Some(id) = fk.get() else { continue; }; - let Some(h) = h.or_else(|| run_create_height.get(fk).copied()) else { + let Some(h) = h.or_else(|| run_create_height.get(*fk)) else { continue; }; scratch.height_by_id.insert(id, h); @@ -2347,8 +2422,9 @@ mod overlay_meta_skip_tests { #[test] fn overlay_meta_skip_omits_matching_abs() { - let mut run = FkMap::default(); - run.insert(Fk(10), 5); + let mut map = FkMap::default(); + map.insert(Fk(10), 5); + let run = RunCreateHeight::Map(map); let spends = spends(&[(0, Fk(11), Fk(10), 0)]); let mut scratch = StructuralScratch::default(); fill_overlay_skip(&spends, &run, &mut scratch); @@ -2360,8 +2436,9 @@ mod overlay_meta_skip_tests { #[test] fn overlay_meta_skip_keeps_conflicting_spender_on_disk_list() { - let mut run = FkMap::default(); - run.insert(Fk(10), 5); + let mut map = FkMap::default(); + map.insert(Fk(10), 5); + let run = RunCreateHeight::Map(map); let spends = spends(&[(0, Fk(11), Fk(10), 0), (0, Fk(12), Fk(10), 0)]); let mut scratch = StructuralScratch::default(); fill_overlay_skip(&spends, &run, &mut scratch); @@ -2371,7 +2448,7 @@ mod overlay_meta_skip_tests { #[test] fn overlay_meta_skip_historical_create_stays_on_disk() { - let run = FkMap::default(); + let run = RunCreateHeight::Spans(Vec::new()); let spends = spends(&[(0, Fk(11), Fk(10), 0)]); let mut scratch = StructuralScratch::default(); fill_overlay_skip(&spends, &run, &mut scratch); @@ -2379,6 +2456,42 @@ mod overlay_meta_skip_tests { } } +#[cfg(test)] +mod run_create_height_tests { + use super::RunCreateHeight; + use rbitcoin_primitives::Fk; + + fn fk(id: u64) -> Fk { + Fk(id) + } + + #[test] + fn run_create_height_span_misses_gaps() { + let one = [fk(10), fk(11)]; + let idx = RunCreateHeight::from_blocks([(7u32, one.as_slice())]); + assert!(matches!(idx, RunCreateHeight::Spans(_))); + assert_eq!(idx.get(fk(10)), Some(7)); + assert_eq!(idx.get(fk(11)), Some(7)); + assert_eq!(idx.get(fk(9)), None); + assert_eq!(idx.get(fk(12)), None); + + let low = [fk(10), fk(11)]; + let high = [fk(20), fk(21)]; + let idx = RunCreateHeight::from_blocks([(1u32, low.as_slice()), (2, high.as_slice())]); + assert_eq!(idx.get(fk(15)), None); + assert_eq!(idx.get(fk(10)), Some(1)); + assert_eq!(idx.get(fk(21)), Some(2)); + + let gapped = [fk(1), fk(2), fk(4)]; + let idx = RunCreateHeight::from_blocks([(3u32, gapped.as_slice())]); + assert!(matches!(idx, RunCreateHeight::Map(_))); + assert_eq!(idx.get(fk(3)), None); + assert_eq!(idx.get(fk(1)), Some(3)); + assert_eq!(idx.get(fk(2)), Some(3)); + assert_eq!(idx.get(fk(4)), Some(3)); + } +} + #[cfg(test)] mod bip34_tests; #[cfg(test)] diff --git a/crates/rbitcoin-consensus/src/block/structure_rule_tests.rs b/crates/rbitcoin-consensus/src/block/structure_rule_tests.rs index 0e8adeda5..e6888f8dc 100644 --- a/crates/rbitcoin-consensus/src/block/structure_rule_tests.rs +++ b/crates/rbitcoin-consensus/src/block/structure_rule_tests.rs @@ -345,9 +345,9 @@ fn padded_spend(data_len: usize) -> Transaction { /// 91842/91880 grandfather). Signet activates BIP34 at height 1, and Core's /// empty BIP34 hash still enforces BIP30 on every signet block. fn rejects_unspent_overwrite(q: &rbitcoin_query::Query, first: &Transaction) { - use crate::block::structural_validate_spends; + use crate::block::{structural_validate_spends, RunCreateHeight}; use rbitcoin_primitives::Fk; - use rbitcoin_query::{BatchParents, FkMap, OutPointSet, U32Map}; + use rbitcoin_query::{BatchParents, OutPointSet, U32Map}; let dup = block_with(vec![first.clone()]); let signet = Box::leak(Box::new(ChainParams::signet())); for (ctx, net) in [ @@ -357,6 +357,7 @@ fn rejects_unspent_overwrite(q: &rbitcoin_query::Query, first: &Transaction) { "signet", ), ] { + let heights = RunCreateHeight::Spans(Vec::new()); let err = structural_validate_spends( q, &dup, @@ -367,7 +368,7 @@ fn rejects_unspent_overwrite(q: &rbitcoin_query::Query, first: &Transaction) { &mut OutPointSet::default(), &BatchParents::new(), &mut U32Map::default(), - &FkMap::default(), + &heights, &mut crate::block::StructuralScratch::default(), None, ) @@ -439,10 +440,11 @@ fn bip30_message_at_mainnet_above_bip34( q: &rbitcoin_query::Query, block: &Block, ) -> Result<(), ConsensusError> { - use crate::block::structural_validate_spends; - use rbitcoin_query::{BatchParents, FkMap, OutPointSet, U32Map}; + use crate::block::{structural_validate_spends, RunCreateHeight}; + use rbitcoin_query::{BatchParents, OutPointSet, U32Map}; let main = Box::leak(Box::new(ChainParams::mainnet())); let ctx = ValidationContext::at(main, Height(main.btc.bip34_height + 1), Milestone::NONE); + let heights = RunCreateHeight::Spans(Vec::new()); structural_validate_spends( q, block, @@ -453,7 +455,7 @@ fn bip30_message_at_mainnet_above_bip34( &mut OutPointSet::default(), &BatchParents::new(), &mut U32Map::default(), - &FkMap::default(), + &heights, &mut crate::block::StructuralScratch::default(), None, ) @@ -2080,15 +2082,16 @@ fn already_archived_schema13_pin_identity_tip_follow() { // Structural without denserels/abs is invariant — not soft PrevoutSpent recovery. { - use super::structural_validate_spends; + use super::{structural_validate_spends, RunCreateHeight}; use rbitcoin_primitives::Fk; - use rbitcoin_query::{BatchParents, FkMap, OutPointSet, U32Map}; + use rbitcoin_query::{BatchParents, OutPointSet, U32Map}; let c2_fk = q.tx_fk_by_txid(c2_txid.as_byte_array()).unwrap().unwrap(); let spends = vec![(c2_txid.to_byte_array(), 0u32, Fk(9_000_001), c2_fk, 0)]; let parents = BatchParents::new(); let ctx = ValidationContext::at(Box::leak(Box::new(params.clone())), Height(h_n1), ms); let mut pending = OutPointSet::default(); let mut mtp = U32Map::default(); + let heights = RunCreateHeight::Spans(Vec::new()); let err = structural_validate_spends( &q, &b_n1, @@ -2099,7 +2102,7 @@ fn already_archived_schema13_pin_identity_tip_follow() { &mut pending, &parents, &mut mtp, - &FkMap::default(), + &heights, &mut crate::block::StructuralScratch::default(), None, ) diff --git a/crates/rbitcoin-consensus/src/confirm_run/phases.rs b/crates/rbitcoin-consensus/src/confirm_run/phases.rs index a5fb67f20..b2dcf1b4e 100644 --- a/crates/rbitcoin-consensus/src/confirm_run/phases.rs +++ b/crates/rbitcoin-consensus/src/confirm_run/phases.rs @@ -235,7 +235,6 @@ pub(super) fn assemble_run( pub(super) struct StructuralReuse { pub scratch: crate::block::StructuralScratch, pub pending: rbitcoin_query::OutPointSet, - pub heights: FkMap, } /// Durable spentness + maturity + subsidy after scripts (height order). @@ -256,11 +255,12 @@ pub(super) fn structural_run( ))); } let t0 = Instant::now(); - // Reused across write batches on this thread. Slots, the pack-local - // double-spend set, and create heights are this batch only. + // Slots and the pack-local double-spend set are this batch only. reuse.scratch.begin_batch(); reuse.pending.clear(); - reuse.heights.clear(); + let index = crate::block::RunCreateHeight::from_blocks( + prepared.iter().map(|p| (p.height.0, p.tx_fks.as_slice())), + ); let mut mtp_cache: U32Map = U32Map::default(); for p in prepared { if p.height.0 > 0 { @@ -268,11 +268,6 @@ pub(super) fn structural_run( } } let mut tot = StructuralPhaseNs::default(); - for p in prepared { - for fk in &p.tx_fks { - reuse.heights.insert(*fk, p.height.0); - } - } for (i, p) in prepared.iter().enumerate() { let ctx = ValidationContext::at(params, p.height, milestone); let ph = structural_validate_spends( @@ -285,7 +280,7 @@ pub(super) fn structural_run( &mut reuse.pending, batch_parents, &mut mtp_cache, - &reuse.heights, + &index, &mut reuse.scratch, Some(&abs_jobs[i]), )?; diff --git a/crates/rbitcoin-consensus/src/confirm_run/write_idempotent_tests.rs b/crates/rbitcoin-consensus/src/confirm_run/write_idempotent_tests.rs index c444d2021..74f8f9489 100644 --- a/crates/rbitcoin-consensus/src/confirm_run/write_idempotent_tests.rs +++ b/crates/rbitcoin-consensus/src/confirm_run/write_idempotent_tests.rs @@ -1326,7 +1326,7 @@ fn fill_same_batch_abs_from_append_loc_ram() { /// Overlay slot already in Class A: structural must not meta-pread it. #[test] fn structural_same_batch_overlay_skips_meta_pread() { - use crate::block::structural_validate_spends; + use crate::block::{structural_validate_spends, RunCreateHeight}; use crate::milestone::Milestone; use crate::params::ChainParams; use bitcoin::absolute::LockTime; @@ -1434,9 +1434,10 @@ fn structural_same_batch_overlay_skips_meta_pread() { bp.set_spent_range_only(fks[0], loc[0].spent); let spends = vec![([0x32u8; 32], 0u32, fks[1], fks[0], 0)]; - let mut run = FkMap::default(); - run.insert(fks[0], 1); - run.insert(fks[1], 1); + let mut map = FkMap::default(); + map.insert(fks[0], 1); + map.insert(fks[1], 1); + let run = RunCreateHeight::Map(map); let params = ChainParams::regtest(); let ctx = crate::block::ValidationContext::at(¶ms, Height(1), Milestone::NONE); let mut pending = OutPointSet::default(); @@ -1493,7 +1494,7 @@ fn structural_same_batch_overlay_skips_meta_pread() { #[test] fn structural_scratch_second_block_does_not_replay_first_slots() { - use crate::block::{structural_validate_spends, StructuralScratch}; + use crate::block::{structural_validate_spends, RunCreateHeight, StructuralScratch}; use crate::milestone::Milestone; use crate::params::ChainParams; use bitcoin::absolute::LockTime; @@ -1506,7 +1507,7 @@ fn structural_scratch_second_block_does_not_replay_first_slots() { Witness, }; use rbitcoin_primitives::{Fk, Height}; - use rbitcoin_query::{BatchParents, FkMap, OutPointSet}; + use rbitcoin_query::{BatchParents, OutPointSet}; use rbitcoin_store::{InputRecord, OutputRecord}; let (path, q) = tiny_query(); @@ -1592,7 +1593,7 @@ fn structural_scratch_second_block_does_not_replay_first_slots() { let mut mtp = rbitcoin_query::U32Map::::default(); mtp.insert(0, 1_300_000_000); let mut scratch = StructuralScratch::default(); - let run = FkMap::default(); + let run = RunCreateHeight::Spans(Vec::new()); for vout in [0u32, 1] { let spends = vec![([0x41u8; 32], vout, Fk(9), fks[0], 0)]; structural_validate_spends( @@ -3019,7 +3020,7 @@ fn wire_lookup_empty_and_noncontiguous() { /// Pin-covered parent without denserels/abs fails structural (no body-range cold). #[test] fn structural_pinned_without_abs_is_invariant_error() { - use crate::block::structural_validate_spends; + use crate::block::{structural_validate_spends, RunCreateHeight}; use crate::milestone::Milestone; use crate::params::ChainParams; use bitcoin::absolute::LockTime; @@ -3093,6 +3094,7 @@ fn structural_pinned_without_abs_is_invariant_error() { let ctx = crate::block::ValidationContext::at(¶ms, Height(1), Milestone::NONE); let mut pending = OutPointSet::default(); let mut mtp = rbitcoin_query::U32Map::::default(); + let heights = RunCreateHeight::Spans(Vec::new()); let err = structural_validate_spends( &q, &block, @@ -3103,7 +3105,7 @@ fn structural_pinned_without_abs_is_invariant_error() { &mut pending, &bp, &mut mtp, - &rbitcoin_query::FkMap::default(), + &heights, &mut crate::block::StructuralScratch::default(), None, ) From b01abb35ef9feb695f2d77bbbff0616c5b742623 Mon Sep 17 00:00:00 2001 From: Brandon Black Date: Fri, 2 Oct 2026 11:13:11 -0700 Subject: [PATCH 04/10] confirm: send the validated wire header on the tip event The write thread re-read header.body for each connected block while holding the confirmed-hash lock. Copy the wire headers before the batch is consumed and put those on TipEvent. A count that does not match the batch is Corrupt. Mempool relay stays gated on relay_enabled. ensure_spend_abs_layouts is test-only. The non-test library was carrying that walker after the write path stopped calling it. --- .../rbitcoin-consensus/src/confirm_run/mod.rs | 14 +++- .../rbitcoin-consensus/src/confirm_run/pin.rs | 3 +- crates/rbitcoin-net/src/chain.rs | 69 ++++++++++++++++--- crates/rbitcoin-net/src/ibd/archive.rs | 5 +- crates/rbitcoin-net/src/ibd/confirm/mod.rs | 12 +++- 5 files changed, 88 insertions(+), 15 deletions(-) diff --git a/crates/rbitcoin-consensus/src/confirm_run/mod.rs b/crates/rbitcoin-consensus/src/confirm_run/mod.rs index 6115a187c..44b8839d4 100644 --- a/crates/rbitcoin-consensus/src/confirm_run/mod.rs +++ b/crates/rbitcoin-consensus/src/confirm_run/mod.rs @@ -69,7 +69,9 @@ pub use lookup::{ use phases::{assemble_run, Assembled}; #[cfg(test)] use phases::{check_bip34, expected_bits_extending, post_commit}; -use pin::{collect_spend_abs_after_fill, ensure_spend_abs_layouts, pin_for_wire_batch}; +#[cfg(test)] +use pin::ensure_spend_abs_layouts; +use pin::{collect_spend_abs_after_fill, pin_for_wire_batch}; pub use scripts::{confirm_scripts_phase, drive_script_waves_with}; pub(crate) use write::finish_post_commit; pub use write::{confirm_write_phase, finish_post_commit_hashes, replay_spend_annotations}; @@ -354,6 +356,16 @@ impl ScriptOkBatch { self.batch_parents.len() } + /// Wire headers in the same order as [`Self::heights_hashes`]. + pub fn wire_headers(&self) -> Result, ConsensusError> { + if self.wire_blocks.len() != self.prepared.len() { + return Err(ConsensusError::Store(rbitcoin_store::StoreError::Corrupt( + "invariant: wire headers length", + ))); + } + Ok(self.wire_blocks.iter().map(|b| b.header).collect()) + } + #[allow(clippy::result_large_err)] // public error enum /// Absorb another script-ok batch for write batch (FIFO drain). /// diff --git a/crates/rbitcoin-consensus/src/confirm_run/pin.rs b/crates/rbitcoin-consensus/src/confirm_run/pin.rs index 65d63e394..90cc971d3 100644 --- a/crates/rbitcoin-consensus/src/confirm_run/pin.rs +++ b/crates/rbitcoin-consensus/src/confirm_run/pin.rs @@ -311,7 +311,7 @@ fn denserels_by_stamped_range( /// Sources: plan/in-flight offline denserels → stamp-carried CreatePin → /// **txout body by range** from [`ParentPinStamp`] (lookup-stamped). Load never /// reads head / `tx.idx` / `txid.body`. Load **copies** lookup-stamped -/// `spent_range` onto pins. Write [`ensure_spend_abs_layouts`] is abs-or-Corrupt. +/// `spent_range` onto pins. Write [`collect_spend_abs_after_fill`] is abs-or-Corrupt. pub(super) fn pin_for_wire_batch( query: &Query, plan: Option<&rbitcoin_query::ArchiveWritePlan>, @@ -476,6 +476,7 @@ pub(super) fn collect_spend_abs_after_fill( /// /// Direct tests keep this walk, including its skip of a null spend fk. /// [`collect_spend_abs_after_fill`] is the write-phase post-condition. +#[cfg(test)] pub(super) fn ensure_spend_abs_layouts( batch_parents: &rbitcoin_query::BatchParents, prepared: &[Prepared], diff --git a/crates/rbitcoin-net/src/chain.rs b/crates/rbitcoin-net/src/chain.rs index adb668d39..40df68f64 100644 --- a/crates/rbitcoin-net/src/chain.rs +++ b/crates/rbitcoin-net/src/chain.rs @@ -1637,6 +1637,7 @@ impl ChainHub { /// WRITE stage: structural + Class C + spend annotate (ordered). pub fn confirm_write(&self, batch: ScriptOkBatch) -> Result, NetError> { + let headers = batch.wire_headers().map_err(NetError::from_consensus)?; let meta: Vec<(u32, BlockHash)> = batch .heights_hashes() .into_iter() @@ -1644,7 +1645,7 @@ impl ChainHub { .collect(); confirm_write_phase(&self.query, &self.params, self.milestone, batch) .map_err(NetError::from_consensus)?; - self.note_confirmed_tip(&meta)?; + self.note_confirmed_tip(&meta, &headers)?; Ok(meta .iter() .map(|&(height, _)| AcceptOutcome::Accepted { height }) @@ -1654,7 +1655,15 @@ impl ChainHub { pub(crate) fn note_confirmed_tip( &self, need_meta: &[(u32, BlockHash)], + headers: &[Header], ) -> Result<(), NetError> { + if headers.len() != need_meta.len() { + return Err(NetError::from_consensus( + rbitcoin_consensus::ConsensusError::Store(rbitcoin_store::StoreError::Corrupt( + "invariant: wire headers length", + )), + )); + } if let Some(mp) = self.mempool() { mp.clear_recent_rejects(); if mp.relay_enabled() { @@ -1680,16 +1689,14 @@ impl ChainHub { } } let mut confirmed = self.confirmed.write().unwrap(); - for &(height, hash) in need_meta { + for (&(height, hash), header) in need_meta.iter().zip(headers.iter()) { confirmed.insert(hash); - if let Ok(hdr) = self.query.wire_header_at_height(Height(height)) { - let _ = self.tip_tx.send(TipEvent { - height, - hash, - header: hdr, - reorg_branch_len: 0, - }); - } + let _ = self.tip_tx.send(TipEvent { + height, + hash, + header: *header, + reorg_branch_len: 0, + }); } drop(confirmed); self.notify.notify_waiters(); @@ -4665,6 +4672,48 @@ mod tests { let _ = std::fs::remove_dir_all(dir); } + #[test] + fn note_confirmed_tip_sends_the_wire_header() { + let (dir, hub) = tmp_hub(); + hub.ensure_genesis().unwrap(); + let block = hub + .assemble_block_to_script(ScriptBuf::from_bytes(vec![0x51]), vec![]) + .expect("assemble"); + let wire = block.header; + let hash = block.block_hash(); + match hub.accept_block(block).expect("connect") { + AcceptOutcome::Accepted { height } => assert_eq!(height, 1), + other => panic!("expected Accepted, got {other:?}"), + } + let stored = hub + .query + .wire_header_at_height(Height(1)) + .expect("stored header"); + assert_eq!(stored.nonce, wire.nonce); + let mut claimed = wire; + claimed.nonce = wire.nonce.wrapping_add(1); + let mut tip_rx = hub.subscribe_tips(); + hub.note_confirmed_tip(&[(1, hash)], &[claimed]) + .expect("note"); + let ev = tip_rx.try_recv().expect("tip event"); + assert_eq!(ev.header, claimed); + assert_eq!( + hub.query + .wire_header_at_height(Height(1)) + .expect("store") + .nonce, + stored.nonce + ); + let err = hub + .note_confirmed_tip(&[(1, hash)], &[]) + .expect_err("header count"); + assert!( + err.to_string().contains("invariant: wire headers length"), + "{err}" + ); + let _ = std::fs::remove_dir_all(dir); + } + #[test] fn exclusive_sh_handoff_mempool_to_pending() { use rbitcoin_store::script_hash; diff --git a/crates/rbitcoin-net/src/ibd/archive.rs b/crates/rbitcoin-net/src/ibd/archive.rs index 95dc9e878..8c4026c5e 100644 --- a/crates/rbitcoin-net/src/ibd/archive.rs +++ b/crates/rbitcoin-net/src/ibd/archive.rs @@ -461,10 +461,12 @@ mod class_a_rehydrate_tests { let time = 1_300_000_000u32; let mut prev = gen; let mut hashes = Vec::new(); + let mut headers = Vec::new(); for h in 1u32..=4 { let b = mine(prev, time + h * 600, h); hub.ensure_header(&b.header).unwrap(); hashes.push(b.block_hash()); + headers.push(b.header); prev = b.block_hash(); } @@ -486,7 +488,8 @@ mod class_a_rehydrate_tests { hub.query .block_queue_offer(2, hashes[1].to_byte_array(), 7, b"wire2") .unwrap(); - hub.note_confirmed_tip(&[(3, hashes[2])]).unwrap(); + hub.note_confirmed_tip(&[(3, hashes[2])], &[headers[2]]) + .unwrap(); assert!( hub.has_block(&hashes[2]), "stale confirmed-set must not dequeue above-tip wire" diff --git a/crates/rbitcoin-net/src/ibd/confirm/mod.rs b/crates/rbitcoin-net/src/ibd/confirm/mod.rs index 9fd6463eb..2b3875a93 100644 --- a/crates/rbitcoin-net/src/ibd/confirm/mod.rs +++ b/crates/rbitcoin-net/src/ibd/confirm/mod.rs @@ -1466,6 +1466,14 @@ pub(crate) fn spawn_confirm_engine( feed_wb.finish(heights_hashes.iter().map(|(h, _)| *h)); continue; } + let headers = match batch.wire_headers() { + Ok(h) => h, + Err(e) => { + warn!("ibd: confirm write {e}"); + feed_wb.finish(heights_hashes.iter().map(|(h, _)| *h)); + break; + } + }; let meta: Vec<(u32, BlockHash)> = heights_hashes .iter() .map(|&(h, raw)| (h, BlockHash::from_byte_array(raw))) @@ -1477,7 +1485,7 @@ pub(crate) fn spawn_confirm_engine( batch, ) { Ok(_fks) => { - if let Err(e) = hub_wb.note_confirmed_tip(&meta) { + if let Err(e) = hub_wb.note_confirmed_tip(&meta, &headers) { warn!("ibd: confirm write note tip: {e}"); } let t_deq = Instant::now(); @@ -1551,7 +1559,7 @@ pub(crate) fn spawn_confirm_engine( &heights_hashes, ) { Ok(()) => { - if let Err(e) = hub_wb.note_confirmed_tip(&meta) { + if let Err(e) = hub_wb.note_confirmed_tip(&meta, &headers) { warn!("ibd: confirm write note tip: {e}"); } for (h, raw) in &heights_hashes { From 5089f4ed750cafd4a7c79919d9c5bd693afa540c Mon Sep 17 00:00:00 2001 From: Brandon Black Date: Fri, 2 Oct 2026 11:14:43 -0700 Subject: [PATCH 05/10] confirm: allow structural_run's unbundled arguments Clippy's argument cap is 7. The write-thread entry keeps its call-site arguments unbundled, matching the other confirm helpers. --- crates/rbitcoin-consensus/src/confirm_run/phases.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/rbitcoin-consensus/src/confirm_run/phases.rs b/crates/rbitcoin-consensus/src/confirm_run/phases.rs index b2dcf1b4e..738dcc545 100644 --- a/crates/rbitcoin-consensus/src/confirm_run/phases.rs +++ b/crates/rbitcoin-consensus/src/confirm_run/phases.rs @@ -238,6 +238,7 @@ pub(super) struct StructuralReuse { } /// Durable spentness + maturity + subsidy after scripts (height order). +#[allow(clippy::too_many_arguments)] // call-site args stay unbundled pub(super) fn structural_run( query: &Query, params: &ChainParams, From 684227ce8cd61b8b3de31c2d7e5479f54b46d9c6 Mon Sep 17 00:00:00 2001 From: Brandon Black Date: Fri, 2 Oct 2026 11:17:43 -0700 Subject: [PATCH 06/10] confirm: allow the write-thread closure's complexity Copying the wire headers before commit adds one arm to the write loop, which was already at clippy's complexity cap. Same allowance as the load thread. --- crates/rbitcoin-net/src/ibd/confirm/mod.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/rbitcoin-net/src/ibd/confirm/mod.rs b/crates/rbitcoin-net/src/ibd/confirm/mod.rs index 2b3875a93..c09ef9900 100644 --- a/crates/rbitcoin-net/src/ibd/confirm/mod.rs +++ b/crates/rbitcoin-net/src/ibd/confirm/mod.rs @@ -1419,7 +1419,9 @@ pub(crate) fn spawn_confirm_engine( let load_ahead_reset_wb = Arc::clone(&load_ahead_reset); let write_thr = std::thread::Builder::new() .name("ibd-confirm-write".into()) - .spawn(move || { + .spawn( + #[allow(clippy::cognitive_complexity)] // confirm write OS pipeline + move || { info!("ibd: confirm write on dedicated OS thread"); let stats = hub_wb.query.confirm_stats_arc(); // Non-contig leftover already note_write_recv'd; write it next iter. From 20d061a606085bc2e4d0f53e02cd998a1bbfd01f Mon Sep 17 00:00:00 2001 From: Brandon Black Date: Fri, 2 Oct 2026 13:23:36 -0700 Subject: [PATCH 07/10] test: find the new tip in fee history, not the file size A restart preload rewrites the snapshot and truncates the journal to its header. The next connect appends one record. When the previous run already journaled a record of that size, the two files stay the same length while the new height is stored. The cross-surface check now reads that height. --- crates/rbitcoin-test/tests/cross_surface.rs | 72 ++++++++++++++++----- 1 file changed, 55 insertions(+), 17 deletions(-) diff --git a/crates/rbitcoin-test/tests/cross_surface.rs b/crates/rbitcoin-test/tests/cross_surface.rs index 369ec7ce3..e97096853 100644 --- a/crates/rbitcoin-test/tests/cross_surface.rs +++ b/crates/rbitcoin-test/tests/cross_surface.rs @@ -1089,6 +1089,49 @@ async fn electrum_rpc(stream: &mut TcpStream, id: u64, method: &str, params: Val serde_json::from_str(&resp_line).unwrap() } +/// True when `height` is a snapshot row, a snapshot hash, or a journal record. +/// +/// Preload rewrites `fee_history` and truncates `fee_history.log` to its +/// header. A later connect appends one 52-byte record. When the previous run +/// already journaled a record of that size, the sum of the two files stays +/// put while the height moves. +fn fee_history_has_height(dir: &std::path::Path, height: u32) -> bool { + fn u32_at(b: &[u8], at: usize) -> Option { + let end = at.checked_add(4)?; + Some(u32::from_le_bytes(b.get(at..end)?.try_into().ok()?)) + } + let snap = std::fs::read(dir.join("fee_history")).unwrap_or_default(); + if snap.len() >= 16 && &snap[..4] == b"RBFH" { + if let (Some(count), Some(n_hashes)) = (u32_at(&snap, 8), u32_at(&snap, 12)) { + let rows = (count as usize).min(snap.len().saturating_sub(16) / 16); + let row_bytes = rows * 16; + for i in 0..rows { + if u32_at(&snap, 16 + i * 16) == Some(height) { + return true; + } + } + let hash_base = 16 + row_bytes; + let hashes = (n_hashes as usize).min(snap.len().saturating_sub(hash_base) / 36); + for i in 0..hashes { + if u32_at(&snap, hash_base + i * 36) == Some(height) { + return true; + } + } + } + } + let journal = std::fs::read(dir.join("fee_history.log")).unwrap_or_default(); + if journal.len() >= 16 && &journal[..4] == b"RBFJ" { + let mut at = 16; + while at + 52 <= journal.len() { + if u32_at(&journal, at) == Some(height) { + return true; + } + at += 52; + } + } + false +} + /// A node that leaves IBD with relay on preloads fee history from the chain /// and keeps it in the mempool dir (snapshot plus per-connect journal), and a /// restart preloads again on top of that file. With flow cold and too little history for any target, @@ -1107,13 +1150,6 @@ async fn fee_history_backfills_from_the_chain_when_relay_starts() { } std::fs::write(td.path().join("rpc.token"), "pass").unwrap(); let mempool_dir = td.path().join("mempool"); - let file_len = || { - ["fee_history", "fee_history.log"] - .iter() - .map(|f| std::fs::metadata(mempool_dir.join(f)).map_or(0, |m| m.len())) - .sum::() - }; - let mut history_len = 0; for run in ["first start", "restart"] { let rpc_addr = ephemeral_addr(); let mut cfg = NodeConfig::default() @@ -1129,22 +1165,24 @@ async fn fee_history_backfills_from_the_chain_when_relay_starts() { wait_listeners(&[rpc_addr]).await; // A fresh tip leaves IBD and turns relay on (a restart is already - // out of IBD); the new height lands in the snapshot or the journal. + // out of IBD). The new height is in the snapshot or the journal. let mined = jsonrpc(rpc_addr, "generate", json!([1])).await; assert!(mined["result"].is_array(), "{run}: {mined}"); + let count = jsonrpc(rpc_addr, "getblockcount", json!([])).await; + let tip = u32::try_from( + count["result"] + .as_u64() + .unwrap_or_else(|| panic!("{run}: {count}")), + ) + .unwrap_or_else(|_| panic!("{run}: {count}")); let deadline = Instant::now() + Duration::from_secs(10); - let written = loop { - let len = file_len(); - if len > history_len || Instant::now() > deadline { - break len; + let seen = loop { + if fee_history_has_height(&mempool_dir, tip) || Instant::now() > deadline { + break fee_history_has_height(&mempool_dir, tip); } tokio::time::sleep(Duration::from_millis(50)).await; }; - assert!( - written > history_len, - "{run}: history file {written} B, was {history_len} B" - ); - history_len = written; + assert!(seen, "{run}: fee history missing height {tip}"); let fee = jsonrpc(rpc_addr, "estimatesmartfee", json!([2])).await; assert!(fee["result"].get("feerate").is_none(), "{run}: {fee}"); From b63aae7468b51b480f2e3c18af62151f6d2c045a Mon Sep 17 00:00:00 2001 From: Brandon Black Date: Fri, 2 Oct 2026 13:45:21 -0700 Subject: [PATCH 08/10] test: give the lagged tip burst 45s to catch up Under a full workspace run the follower was still mid-burst when the 10s wait ended (55 of 80). Forty-five seconds stays inside the 60s sync_blocks interval. --- crates/rbitcoin-net/src/peer_tests.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/crates/rbitcoin-net/src/peer_tests.rs b/crates/rbitcoin-net/src/peer_tests.rs index 964349756..f463296fc 100644 --- a/crates/rbitcoin-net/src/peer_tests.rs +++ b/crates/rbitcoin-net/src/peer_tests.rs @@ -2491,19 +2491,17 @@ async fn tip_burst_past_broadcast_capacity_still_syncs_peer() { assert!(want >= BURST, "miner tip {want}"); let mut peer_tip = nb.tip_height().unwrap_or(0); - for _ in 0..200 { - peer_tip = nb.tip_height().unwrap_or(0); - if peer_tip >= want { - break; - } + let deadline = std::time::Instant::now() + Duration::from_secs(45); + while peer_tip < want && std::time::Instant::now() < deadline { tokio::time::sleep(Duration::from_millis(50)).await; + peer_tip = nb.tip_height().unwrap_or(0); } na.shutdown().await; nb.shutdown().await; let _ = std::fs::remove_dir_all(&dir); assert_eq!( peer_tip, want, - "peer must catch tip after Lagged burst within ~10s (not headers_poll 120s)" + "peer must catch tip after Lagged burst within 45s" ); } /// Core `disconnect_nodes` waits ≤5s for the far side's `getpeerinfo` to drop us. From aa1d2ed48c898e19bc9b9b9da5343cc31d63d54d Mon Sep 17 00:00:00 2001 From: Brandon Black Date: Fri, 2 Oct 2026 13:58:08 -0700 Subject: [PATCH 09/10] test: give compact-block orphan follow 60s The workspace run hit the 30s wall around the getblocktxn and orphan steps while other tests were running. Neighboring multinode tests already use a 60s wall. --- crates/rbitcoin-test/tests/integration_multinode.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/rbitcoin-test/tests/integration_multinode.rs b/crates/rbitcoin-test/tests/integration_multinode.rs index 2914c3d2e..5c7ef55a3 100644 --- a/crates/rbitcoin-test/tests/integration_multinode.rs +++ b/crates/rbitcoin-test/tests/integration_multinode.rs @@ -1390,7 +1390,7 @@ async fn p2p_compact_hb_getblocktxn_and_orphan() { seed.shutdown().await; peer.shutdown().await; }; - tokio::time::timeout(llvm_cov_wall(30, 90), fut) + tokio::time::timeout(llvm_cov_wall(60, 90), fut) .await .expect("p2p_compact_hb_getblocktxn_and_orphan wall timeout"); } From 125029cc5ad588a9f02a017338b18a2f4f2ecd14 Mon Sep 17 00:00:00 2001 From: Brandon Black Date: Fri, 2 Oct 2026 15:12:22 -0700 Subject: [PATCH 10/10] confirm: record the write-thread CPU cuts changelog.d fragment for the single spend-abs walk, the reused structural scratch, foreign-key span heights, and the validated wire header on the tip event. --- changelog.d/write-thread-cpu.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 changelog.d/write-thread-cpu.md diff --git a/changelog.d/write-thread-cpu.md b/changelog.d/write-thread-cpu.md new file mode 100644 index 000000000..e2ffc40ef --- /dev/null +++ b/changelog.d/write-thread-cpu.md @@ -0,0 +1,7 @@ +Changed + +- Confirm write collects each block's spend absolute offsets once, reuses + the structural scratch and the in-batch double-spend set, and looks up + create heights by foreign-key span when every block in the batch is + contiguous. The tip event carries the wire header already validated on + the write path.