diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5073f5e..b9f6755 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,13 +68,13 @@ jobs: # The portable configurations the README advertises, so they cannot rot. portable: - name: no_std + wasm32 + name: no_std + wasm32 + bare metal runs-on: ubuntu-latest steps: - uses: actions/checkout@v5 - uses: dtolnay/rust-toolchain@stable with: - targets: wasm32-unknown-unknown + targets: wasm32-unknown-unknown, thumbv7em-none-eabihf, riscv32imac-unknown-none-elf - uses: Swatinem/rust-cache@v2 - name: no_std + alloc run: cargo check -p rusty_zstd --no-default-features --features alloc @@ -84,6 +84,19 @@ jobs: run: cargo check -p rusty_zstd --target wasm32-unknown-unknown --no-default-features --features alloc - name: profile feature builds run: cargo check -p rusty_zstd --features profile + # BARE METAL. A `no-std` category on the registry is not a claim; a target + # that compiles is. Both of these failed with 204 errors until the census + # counters stopped requiring 64-bit atomics -- and every one of those errors + # was a counter, not a line of codec. `census64` makes the counter a + # zero-sized stub where `AtomicU64` does not exist, so the instrument costs + # the crate a diagnostic on those parts instead of costing it the parts. + # + # Cortex-M4F and RV32IMAC are what rusty_RTOS (Kairos) consumes the codec on, + # for on-chip OTA payloads and trace capture. Neither has 64-bit atomics. + - name: bare metal (Cortex-M4F, no 64-bit atomics) + run: cargo check -p rusty_zstd --target thumbv7em-none-eabihf --no-default-features --features alloc + - name: bare metal (RV32IMAC, no 64-bit atomics) + run: cargo check -p rusty_zstd --target riscv32imac-unknown-none-elf --no-default-features --features alloc lint: runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index 23f5e00..faba3b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,51 @@ based on [Keep a Changelog](https://keepachangelog.com/); this project uses ## [Unreleased] +### Fixed -- the census counters cost the crate BARE METAL; they no longer do + +`cargo check -p rusty_zstd --no-default-features --features alloc` failed with +**204 errors** on `thumbv7em-none-eabihf` (Cortex-M4F) and the same on +`riscv32imac-unknown-none-elf`. Every one of them was +`cannot find AtomicU64 in atomic`, and every one was an INSTRUMENT: reload +counts, decode band histograms, kernel reach, walk exits, copy bytes. Not one +was a line of codec. Neither part has 64-bit atomics. + +This was found by `rusty_RTOS`'s house gate, which consumes the crate for +on-chip OTA payloads and trace capture, and it is the reason its plan says a +`no-std` category on the registry is not a claim: **the target that compiles is +the claim.** The plan estimated eight statics from a capped rustc report; the +real count is 427 fully-qualified uses plus six imports across sixteen files. + +`census64` is now the one seam. On every target that HAS 64-bit atomics it is +`pub use core::sync::atomic::AtomicU64` -- the same type, not a wrapper -- so +the public statics keep their published type and the emitted code is unchanged. +**Proved, not asserted:** the board over every shipping kernel, fill and finder +is identical to the pre-change assembly in all thirty-two columns, and the +identity gates still read GOLD `2F6594F7EEDBD12B` / 59,680,638 and LDM +`57BE83EA4E1199E8` / 57,796,847. + +On a part without them it is a zero-sized stub: the statics leave BSS entirely +and every `fetch_add` folds away. + +**Why a stub rather than `portable-atomic`.** That crate would keep the census +READABLE there, and it is the other honest answer. But on a core with no 64-bit +atomic instruction it needs a critical-section implementation, and a library +that turns that feature on conscripts every downstream firmware's interrupt +policy so a diagnostic counter can increment. The seam is one type wide, so a +firmware that does want the count can supply `portable_atomic::AtomicU64`. + +**A zero there means "not measurable on this target", never "measured zero"**, +which is the same trap this crate's own rules warn about -- so it is published, +not buried: `census64::CENSUS_LIVE` is `false` on such a build. The reverse +mistake would be far worse (a stub selected on a HOSTED target would make every +count-based verdict fiction while every gate still passed), so it is gated two +ways: a compile-time size check, and a unit test that asserts on BEHAVIOUR +rather than on the constant -- it counts to 42 and reads it back. + +CI gains the rung, in the job that already guards the portable configurations: +`--target thumbv7em-none-eabihf` and `--target riscv32imac-unknown-none-elf`, +both `--no-default-features --features alloc`, on every push. + ### Measured -- what the encode campaign bought: 1.08-1.27x at L3-L12 (2026-09-09) Every verdict in the sections below is an INSTRUCTION COUNT, because this box diff --git a/README.md b/README.md index 974045c..d094e87 100644 --- a/README.md +++ b/README.md @@ -347,8 +347,13 @@ implementation cannot leak into the dependency graph. - **Byte-identity where it is owed.** The XXH64 checksum is gated against the published XXH64 vectors and a GOLD oracle; every SIMD kernel is gated against its scalar twin; the Huffman and FSE table decoders are gated against C's. -- **`no_std + alloc` and `wasm32-unknown-unknown` build in CI**, so the portable - configuration cannot rot. +- **`no_std + alloc`, `wasm32-unknown-unknown` and two bare-metal targets build + in CI**, so the portable configuration cannot rot. The bare-metal rungs are + Cortex-M4F and RV32IMAC, neither of which has 64-bit atomics: a `no-std` + category is not a claim, a target that compiles is. Note that the measurement + counters need `AtomicU64`, so on a part without it they compile to a + zero-sized stub and `census64::CENSUS_LIVE` reads `false` -- a zero from a + counter there means "not measurable on this target", never "measured zero". Fetch the oracle with `pwsh scripts/fetch-oracle.ps1`, or point `RUSTY_ZSTD_ORACLE` at any `zstd` binary whose `--version` says 1.5.7. Details: @@ -363,6 +368,8 @@ Fetch the oracle with `pwsh scripts/fetch-oracle.ps1`, or point | macOS (x86-64 / aarch64) | ✅ builds + tests | | `wasm32-unknown-unknown` | ✅ builds (`std` and `alloc`) | | `no_std + alloc` | ✅ builds | +| `thumbv7em-none-eabihf` (Cortex-M4F) | ✅ builds (`no_std + alloc`) | +| `riscv32imac-unknown-none-elf` | ✅ builds (`no_std + alloc`) | AVX2 and NEON kernels are selected at **runtime**. On a CPU without them, the scalar twins run and the output is identical. diff --git a/crates/rusty_zstd/src/bit.rs b/crates/rusty_zstd/src/bit.rs index 49ba0f5..594a8b5 100644 --- a/crates/rusty_zstd/src/bit.rs +++ b/crates/rusty_zstd/src/bit.rs @@ -16,12 +16,12 @@ pub(crate) struct BitRev<'a> { /// Executed `BitRev::reload` calls. See the note inside `reload`. #[cfg(feature = "profile")] -pub static RELOAD_CALLS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static RELOAD_CALLS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Executed refills -- reloads that reached the container load rather than /// taking one of the four early-outs. See the note inside `reload`. #[cfg(feature = "profile")] -pub static RELOAD_REFILLS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static RELOAD_REFILLS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear the refill counter. #[cfg(feature = "profile")] diff --git a/crates/rusty_zstd/src/census64.rs b/crates/rusty_zstd/src/census64.rs new file mode 100644 index 0000000..9e13e67 --- /dev/null +++ b/crates/rusty_zstd/src/census64.rs @@ -0,0 +1,183 @@ +//! The census counter's 64-bit atomic, and what it becomes on a part that has +//! no 64-bit atomics. +//! +//! WHY THIS MODULE EXISTS. Every measurement instrument in this crate counts +//! into a `static AtomicU64`: reload calls, decode band histograms, kernel +//! reach, walk exits, copy bytes. That is deliberate and the codec skills +//! depend on it -- a count is the only instrument that separates "this kernel +//! does not help" from "this arm is wired to nothing". +//! +//! It also cost the crate two targets. `core::sync::atomic::AtomicU64` does +//! not exist on a Cortex-M4F (`thumbv7em-none-eabihf`) or on RV32 +//! (`riscv32imac-unknown-none-elf`), so `--no-default-features --features +//! alloc` failed with 204 errors on both -- every one of them a counter, not a +//! line of codec. A `no-std` label on the registry is not a claim; a target +//! that compiles is. +//! +//! WHAT IT DOES. On every target that HAS 64-bit atomics this is exactly +//! `core::sync::atomic::AtomicU64`, re-exported. Same type, same ABI, no +//! wrapper, so the public statics keep their published type and the emitted +//! code is byte-for-byte what it was. +//! +//! On a target that does not, it is a zero-sized stub whose operations do +//! nothing and whose loads read 0. The statics stop occupying BSS, the +//! `fetch_add`s vanish, and the codec keeps its targets. +//! +//! WHY A STUB AND NOT `portable-atomic`. That crate is the other honest +//! answer, and it would keep the census READABLE on those parts. But on a +//! core with no 64-bit atomic instruction it needs a critical-section +//! implementation, and a library that turns that feature on forces the choice +//! onto every downstream firmware. A codec should not conscript the +//! application's interrupt policy so that a diagnostic counter can increment. +//! Anyone who does want the count on such a part can supply +//! `portable_atomic::AtomicU64` here; the seam is one type wide. +//! +//! HONESTY. A counter that silently reads zero is exactly the "stale +//! instrument" this crate's own rules warn about, so the fact is published +//! rather than buried: [`CENSUS_LIVE`] is `false` on such a target, and any +//! gate or report that asserts on a census must consult it before believing a +//! zero. **Zero here means "not measurable on this target", never "measured +//! zero".** +//! +//! The reverse mistake would be worse and is gated below: if this stub were +//! ever selected on a HOSTED target, every counter in the crate would read 0 +//! and every census-based verdict would silently become fiction. The unit test +//! fails the build if that happens. + +/// Whether the census counters in this build can actually count. +/// +/// `true` wherever `AtomicU64` exists (every hosted target this crate ships +/// on). `false` on a part without 64-bit atomics, where every counter is a +/// stub and every reader returns 0. Check this before believing a zero. +pub const CENSUS_LIVE: bool = cfg!(target_has_atomic = "64"); + +#[cfg(target_has_atomic = "64")] +pub use core::sync::atomic::AtomicU64; + +#[cfg(not(target_has_atomic = "64"))] +pub use stub::AtomicU64; + +#[cfg(not(target_has_atomic = "64"))] +mod stub { + use core::sync::atomic::Ordering; + + /// A do-nothing stand-in for `core::sync::atomic::AtomicU64` on a target + /// that has no 64-bit atomics. Zero-sized: a `static` of this type costs + /// no BSS and every operation folds away. + /// + /// It is `Sync` for the same reason `()` is: there is no state to race + /// on. Loads return 0, and [`super::CENSUS_LIVE`] is `false` so a reader + /// can tell that apart from a real zero. + #[derive(Debug, Default)] + pub struct AtomicU64(()); + + impl AtomicU64 { + /// The value is discarded: this build cannot hold one. + #[inline(always)] + pub const fn new(_v: u64) -> Self { + Self(()) + } + + /// Always 0. See [`super::CENSUS_LIVE`]. + #[inline(always)] + pub fn load(&self, _order: Ordering) -> u64 { + 0 + } + + #[inline(always)] + pub fn store(&self, _v: u64, _order: Ordering) {} + + /// Always 0 (the previous value this build never held). + #[inline(always)] + pub fn swap(&self, _v: u64, _order: Ordering) -> u64 { + 0 + } + + #[inline(always)] + pub fn fetch_add(&self, _v: u64, _order: Ordering) -> u64 { + 0 + } + + #[inline(always)] + pub fn fetch_sub(&self, _v: u64, _order: Ordering) -> u64 { + 0 + } + + #[inline(always)] + pub fn fetch_max(&self, _v: u64, _order: Ordering) -> u64 { + 0 + } + + #[inline(always)] + pub fn fetch_min(&self, _v: u64, _order: Ordering) -> u64 { + 0 + } + + #[inline(always)] + pub fn fetch_or(&self, _v: u64, _order: Ordering) -> u64 { + 0 + } + + #[inline(always)] + pub fn fetch_and(&self, _v: u64, _order: Ordering) -> u64 { + 0 + } + + /// Always reports the stored value as 0, so a compare against + /// anything else fails -- the same shape a real CAS has when it loses. + #[inline(always)] + pub fn compare_exchange( + &self, + current: u64, + _new: u64, + _success: Ordering, + _failure: Ordering, + ) -> Result { + if current == 0 { + Ok(0) + } else { + Err(0) + } + } + + #[inline(always)] + pub fn compare_exchange_weak( + &self, + current: u64, + new: u64, + success: Ordering, + failure: Ordering, + ) -> Result { + self.compare_exchange(current, new, success, failure) + } + } +} + +/// The selected type must be a REAL 64-bit atomic wherever one exists. If a cfg +/// edit ever let the stub through here, this fails the build rather than the +/// census. (The stub is zero-sized; the real one is eight bytes.) +#[cfg(target_has_atomic = "64")] +const _: () = [(); 1][(core::mem::size_of::() != 8) as usize]; + +#[cfg(test)] +mod tests { + /// The census must actually COUNT anywhere tests run. Asserted on behaviour, + /// not on the `CENSUS_LIVE` constant: if a cfg edit ever selected the stub on + /// a hosted target, every counter in the crate would read 0, `kreach_gate` + /// would see a perfectly-routed 0/0 everywhere, and the whole instrument + /// would report success while measuring nothing. + #[test] + fn census_counts_wherever_tests_run() { + assert!(super::CENSUS_LIVE == cfg!(target_has_atomic = "64")); + let c = super::AtomicU64::new(0); + c.fetch_add(7, core::sync::atomic::Ordering::Relaxed); + c.fetch_add(35, core::sync::atomic::Ordering::Relaxed); + assert_eq!( + c.swap(0, core::sync::atomic::Ordering::Relaxed), + 42, + "the AtomicU64 stub was selected on a target that HAS 64-bit atomics: \ + every census counter in this crate now reads 0 and every count-based \ + verdict from it is fiction" + ); + } +} diff --git a/crates/rusty_zstd/src/compressed.rs b/crates/rusty_zstd/src/compressed.rs index 03916e8..0899696 100644 --- a/crates/rusty_zstd/src/compressed.rs +++ b/crates/rusty_zstd/src/compressed.rs @@ -15,44 +15,44 @@ use crate::reader::Reader; /// Only >=32-byte ops can benefit from AVX2 -- a 16-byte copy is already one /// `movups`. #[cfg(feature = "profile")] -pub static DEC_LIT32: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static DEC_LIT32: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static DEC_MATCH32: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static DEC_MATCH32: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static DEC_LIT16: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static DEC_LIT16: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Literal copies served by the 64-byte tier (BRICK 80's missing third rung). #[cfg(feature = "profile")] -pub static DEC_LIT64: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static DEC_LIT64: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static DEC_MATCH16: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static DEC_MATCH16: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// T4 band census for `copy_match`: which route does each match actually take? /// 0 = offset 1 splat, 1 = 32-byte tier, 2 = 16-byte tier, /// 3 = extend_from_within (offset >= len, runtime-length memcpy CALL), /// 4 = overlapping loop (offset < len). #[cfg(feature = "profile")] -pub static DEC_BAND: [core::sync::atomic::AtomicU64; 8] = [ - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), +pub static DEC_BAND: [crate::census64::AtomicU64; 8] = [ + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), ]; /// Bytes moved by each route, so a rare-but-long band cannot hide. #[cfg(feature = "profile")] -pub static DEC_BAND_B: [core::sync::atomic::AtomicU64; 8] = [ - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), +pub static DEC_BAND_B: [crate::census64::AtomicU64; 8] = [ + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), ]; /// Length histogram for the UN-TIERED bands (3 = `extend_from_within`, @@ -61,8 +61,8 @@ pub static DEC_BAND_B: [core::sync::atomic::AtomicU64; 8] = [ /// the high half -- a band's mean hides its distribution, and the tier width has /// to be chosen from the distribution. #[cfg(feature = "profile")] -pub static DEC_UNTIERED: [core::sync::atomic::AtomicU64; 16] = - [const { core::sync::atomic::AtomicU64::new(0) }; 16]; +pub static DEC_UNTIERED: [crate::census64::AtomicU64; 16] = + [const { crate::census64::AtomicU64::new(0) }; 16]; #[cfg(feature = "profile")] pub fn take_dec_untiered() -> [u64; 16] { @@ -2109,8 +2109,8 @@ static MATCHCOPY_ARM: core::sync::atomic::AtomicU8 = core::sync::atomic::AtomicU /// as out-of-range. Deterministic -- this is what SIZES D9, because a prefetch /// only helps the sequences it is actually issued for. #[cfg(feature = "pfcensus")] -pub static PF_CENSUS: [core::sync::atomic::AtomicU64; 3] = - [const { core::sync::atomic::AtomicU64::new(0) }; 3]; +pub static PF_CENSUS: [crate::census64::AtomicU64; 3] = + [const { crate::census64::AtomicU64::new(0) }; 3]; /// Read and clear `(issued, skipped_rep, skipped_oob)`. #[cfg(feature = "pfcensus")] pub fn take_pf_census() -> (u64, u64, u64) { @@ -2721,7 +2721,7 @@ fn copy_match_nodict( /// Each is a full `from_norm` -- heap alloc + serial spread + finalize -- for a /// value fixed by RFC 8878 and identical for the life of the process. #[cfg(feature = "profile")] -pub static N21_PREDEF: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static N21_PREDEF: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear the N21 probe. #[cfg(feature = "profile")] pub fn take_n21_predef() -> u64 { @@ -2731,7 +2731,7 @@ pub fn take_n21_predef() -> u64 { /// D3 probe: `extend_from_within` calls made by the overlapping (band 4) loop. /// D3 assumes "a memcpy call per period"; this counts what actually happens. #[cfg(feature = "profile")] -pub static D3_ITERS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static D3_ITERS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear the D3 probe. #[cfg(feature = "profile")] pub fn take_d3_iters() -> u64 { @@ -2741,10 +2741,10 @@ pub fn take_d3_iters() -> u64 { /// D4 coverage census: `[frame_only, dict_only, dict_CROSSING]` calls. /// A brick on the crossing path is unverified until index 2 is non-zero. #[cfg(feature = "profile")] -pub static D4_PATHS: [core::sync::atomic::AtomicU64; 3] = [ - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), +pub static D4_PATHS: [crate::census64::AtomicU64; 3] = [ + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), ]; /// Read and clear the D4 coverage census. #[cfg(feature = "profile")] diff --git a/crates/rusty_zstd/src/copies.rs b/crates/rusty_zstd/src/copies.rs index c194a69..6ec4e95 100644 --- a/crates/rusty_zstd/src/copies.rs +++ b/crates/rusty_zstd/src/copies.rs @@ -160,8 +160,9 @@ mod imp { #[cfg(feature = "profile")] mod imp { use super::N_COPY_SLOTS; + use crate::census64::AtomicU64; use core::cell::Cell; - use core::sync::atomic::{AtomicU64, Ordering::Relaxed}; + use core::sync::atomic::Ordering::Relaxed; pub(super) static G_BYTES: [AtomicU64; N_COPY_SLOTS] = [const { AtomicU64::new(0) }; N_COPY_SLOTS]; diff --git a/crates/rusty_zstd/src/encode.rs b/crates/rusty_zstd/src/encode.rs index be598f8..c199d06 100644 --- a/crates/rusty_zstd/src/encode.rs +++ b/crates/rusty_zstd/src/encode.rs @@ -1850,19 +1850,19 @@ pub fn set_prime_stride_arm(n: usize) { /// Deterministic work counter for the priming loop: positions inserted. /// Accumulated LOCALLY and published once per call -- an atomic inside the loop /// is the bricks 49/64/77 defect this campaign keeps finding. -pub static PRIME_ITERS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static PRIME_ITERS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// E4 ceiling probe: `[calls, positions hashed]` in the post-match fill helpers. /// E4 proposes batching these into a vector tile; a tile needs positions. /// N9 probe: rebuilds of the RFC-constant default FSE ctable in `select_seq_table`. -pub static N9_BASIC: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static N9_BASIC: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// ALLOC-8 probe: `[speculative EntropyState saves, saves actually ROLLED BACK]`. /// The save clones 3 FSE tables + a Huffman table per block; if the rollback /// almost never fires, the clone is almost pure waste. -pub static ENT_SAVE: [core::sync::atomic::AtomicU64; 2] = [ - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), +pub static ENT_SAVE: [crate::census64::AtomicU64; 2] = [ + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), ]; /// Read and clear the ALLOC-8 probe. pub fn take_ent_save() -> [u64; 2] { @@ -2203,10 +2203,10 @@ fn adaptive_block_max( base } -pub static G5_RPREV: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static G5_RPREV_N: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static G5_DRIFTSUM: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static G5_DRIFT_N: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static G5_RPREV: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static G5_RPREV_N: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static G5_DRIFTSUM: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static G5_DRIFT_N: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// GATE 5 inputs, as block means: `(mean r_prev, mean drift)`. pub fn take_g5_inputs() -> (f64, f64) { @@ -2221,9 +2221,9 @@ pub fn take_g5_inputs() -> (f64, f64) { ) } -pub static G5_CALLS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static G5_HIT_RATIO: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static G5_HIT_DRIFT: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static G5_CALLS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static G5_HIT_RATIO: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static G5_HIT_DRIFT: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// GATE 5 coverage: `(calls, raw-escape fires, drift fires)`. pub fn take_g5() -> (u64, u64, u64) { @@ -3303,10 +3303,10 @@ fn raw_probe_period() -> u32 { /// 1 = `early_raw_skip` (needs Fast + tlen 1..7) /// 2 = payload did not beat `raw_limit` #[cfg(feature = "profile")] -pub static RAW_EXIT: [core::sync::atomic::AtomicU64; 3] = [ - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), +pub static RAW_EXIT: [crate::census64::AtomicU64; 3] = [ + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), ]; /// Read and clear the three raw-exit counts. @@ -3322,15 +3322,15 @@ pub fn take_raw_exits() -> [u64; 3] { /// GATE 16 study: how far past `raw_limit` did blocks that went RAW land? #[cfg(feature = "profile")] -pub static RAW_MARGIN_SUM: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static RAW_MARGIN_SUM: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static RAW_MARGIN_N: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static RAW_MARGIN_N: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static RAW_MARGIN_HIST: [core::sync::atomic::AtomicU64; 4] = [ - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), +pub static RAW_MARGIN_HIST: [crate::census64::AtomicU64; 4] = [ + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), ]; /// Read and clear `(sum_permille, n, [<=1.01, <=1.05, <=1.20, >1.20])`. @@ -6405,9 +6405,9 @@ fn bt_fill_stride() -> usize { 1 } -pub static LF_FILLS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static LF_NONEMPTY: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static LF_INSERTS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static LF_FILLS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static LF_NONEMPTY: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static LF_INSERTS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// `(fill_sites_reached, sites_with_at_least_one_insert, total_inserts)` pub fn take_lazy_fill() -> (u64, u64, u64) { @@ -6489,19 +6489,19 @@ fn row_fill_stride() -> usize { /// Set the lazy back-fill stride in-process. /// GATE 6 next-long probe outcomes, for GATE 14's dispatch study. -pub static NL_PROBES_G: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static NL_HITS_G: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static NL_PROBES_G: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static NL_HITS_G: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Total match-length GAIN the next-long probe bought, across its hits. -pub static NL_GAIN_G: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static NL_GAIN_G: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Hits in the RAISED band (`best_ml >= 8`) -- what a higher cut newly enables. -pub static NL_BAND_HITS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static NL_BAND_GAIN: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static NL_BAND_OLD: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static NL_BAND_HITS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static NL_BAND_GAIN: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static NL_BAND_OLD: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Offsets the raised-band hits take, and the offsets they replace. -pub static NL_OFF_NEW: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static NL_OFF_OLD: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static NL_OFF_NEW: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static NL_OFF_OLD: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Raised-band hits whose new offset is LARGER than the one they replaced. -pub static NL_OFF_WORSE: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static NL_OFF_WORSE: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear `(off_new_sum, off_old_sum, hits_with_worse_offset)`. pub fn take_nl_off() -> (u64, u64, u64) { @@ -6769,7 +6769,7 @@ fn dfast_fill_stride() -> usize { /// fills (short and long counted separately). The sparse arm's saving is paid /// in this unit; §4.39 priced only the main-loop positions it costs and so /// called the arm "dominated" while ignoring the larger term. -pub static DF_ENDFILL: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static DF_ENDFILL: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear the per-match end-fill write count. pub fn take_dfast_endfill() -> u64 { @@ -6837,7 +6837,7 @@ fn dfast_fill_anchor_c() -> bool { } /// Interior back-fill positions inserted by GATE 12 @ L3's stride arm. -pub static DF_FILL: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static DF_FILL: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Bench hook: interior DFast back-fill inserts since the last call. pub fn take_dfast_fill() -> u64 { @@ -7131,11 +7131,11 @@ fn step_probe_on() -> bool { /// GATE 18 study: the measured step-2 forfeit, per mille x10. #[cfg(feature = "profile")] -pub static STEP_FORFEIT_SUM: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static STEP_FORFEIT_SUM: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static STEP_FORFEIT_N: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static STEP_FORFEIT_N: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static STEP_SEQ_SUM: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static STEP_SEQ_SUM: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear `(sum_x10000, n)`. #[cfg(feature = "profile")] @@ -7423,8 +7423,8 @@ fn hash4_tag_from(v: u64, hash_shift: u32, smask: u64) -> (usize, u8) { /// speculation serve? `MM_MISS / MM_TOTAL` is the share of positions that reach /// the miss-advance, i.e. where a speculated next-position load is CONSUMED. /// Gate 7 audit: tag rejections that `fast_probe` would have ACCEPTED. -pub static TAG_FALSE_REJECT: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static TAG_REJECT_TOTAL: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static TAG_FALSE_REJECT: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static TAG_REJECT_TOTAL: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear `(false_rejects, total_rejects)`. pub fn take_tag_rejects() -> (u64, u64) { @@ -7436,12 +7436,12 @@ pub fn take_tag_rejects() -> (u64, u64) { } /// GATE 2 candidate signal: rep match BYTES per rep PROBE. -pub static REP_PROBES: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static REP_BYTES: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static REP_PROBES: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static REP_BYTES: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); -pub static REP_HITS_G: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static ALL_MATCH_BYTES: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static ALL_SEQS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static REP_HITS_G: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static ALL_MATCH_BYTES: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static ALL_SEQS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// `(rep_probes, rep_bytes, rep_hits, all_match_bytes, all_seqs)` pub fn take_rep_rate() -> (u64, u64, u64, u64, u64) { @@ -7455,8 +7455,8 @@ pub fn take_rep_rate() -> (u64, u64, u64, u64, u64) { ) } -pub static MM_TOTAL: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static MM_MISS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static MM_TOTAL: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static MM_MISS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear `(main_loop_positions, positions_reaching_the_advance)`. pub fn take_mm() -> (u64, u64) { @@ -7467,11 +7467,11 @@ pub fn take_mm() -> (u64, u64) { /// ffanat: dispatch-arm census. 0 = specialised (false,false,pipe), /// 1 = tag arm (generic), 2 = rep arms (generic), 3 = rest. #[cfg(feature = "profile")] -pub static FF_ARM: [core::sync::atomic::AtomicU64; 4] = [ - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), +pub static FF_ARM: [crate::census64::AtomicU64; 4] = [ + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), ]; /// Read and clear the dispatch-arm census. @@ -7484,9 +7484,9 @@ pub fn take_ff_arms() -> [u64; 4] { o } -pub static FF_PIPE_BLOCKS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static FF_SPEC_MADE: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static FF_SPEC_USED: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static FF_PIPE_BLOCKS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static FF_SPEC_MADE: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static FF_SPEC_USED: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear `(pipelined_blocks, speculations_made, speculations_used)`. pub fn take_ff_pipe() -> (u64, u64, u64) { @@ -7810,9 +7810,9 @@ fn lit_short_min() -> f32 { } /// Deterministic instrument: guard evaluations that FAILED, i.e. wasted work. -pub static LP_GUARD_FAIL: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static LP_GUARD_FAIL: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Guard evaluations SKIPPED by the Gate 13 dispatch. -pub static LP_GUARD_SKIP: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static LP_GUARD_SKIP: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear the Gate 13 guard instruments. pub fn take_lp_guard() -> (u64, u64) { @@ -8017,13 +8017,13 @@ fn push_literals_tiers( } /// Literal run-length histogram: 0-4, 5-8, 9-16, 17-32, 33-64, 65+. -pub static LP_HIST: [core::sync::atomic::AtomicU64; 6] = [ - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), +pub static LP_HIST: [crate::census64::AtomicU64; 6] = [ + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), ]; /// Read and clear the literal run-length histogram. @@ -8037,8 +8037,8 @@ pub fn take_lit_hist() -> [u64; 6] { } /// Literal appends served by tier 2 (32 bytes) and tier 3 (64 bytes). -pub static LP_FAST2: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static LP_FAST3: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static LP_FAST2: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static LP_FAST3: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear the tier-2 and tier-3 counts. pub fn take_lit_tiers() -> (u64, u64) { @@ -8062,8 +8062,8 @@ pub fn take_lp_stats() -> ([u64; 6], u64, u64) { ) } -pub static LP_FAST: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static LP_SLOW: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static LP_FAST: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static LP_SLOW: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear `(fixed_width, fallback)` literal-append counts. pub fn take_lit_push() -> (u64, u64) { @@ -9966,14 +9966,14 @@ fn dfast_step_forced() -> usize { 1 } -pub static DFAST_MATCH_BYTES: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static DFAST_SEQS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static DFAST_BLOCK_BYTES: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static DFAST_MATCH_BYTES: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static DFAST_SEQS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static DFAST_BLOCK_BYTES: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); -pub static DFAST_BLOCKS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static DFAST_REP_BLOCKS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static DFAST_BLOCKS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static DFAST_REP_BLOCKS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Positions over which `try_rep1` is live -- the work a rep dispatch removes. -pub static DFAST_REP_POS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static DFAST_REP_POS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// `(blocks, rep_blocks, rep_positions)` pub fn take_dfast_rep_blocks() -> (u64, u64, u64) { @@ -9985,8 +9985,8 @@ pub fn take_dfast_rep_blocks() -> (u64, u64, u64) { ) } -pub static DFAST_REP_BYTES: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static DFAST_REP_HITS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static DFAST_REP_BYTES: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static DFAST_REP_HITS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// `(match_bytes, seqs, block_bytes, rep_bytes, rep_hits)` for DFast. pub fn take_dfast_match_stats() -> (u64, u64, u64, u64, u64) { @@ -10049,8 +10049,8 @@ static DFAST_PIPE_ARM: core::sync::atomic::AtomicU8 = core::sync::atomic::Atomic /// A/B the DFast 2-way software pipeline in-process -- both shapes, one binary, /// so the comparison is immune to cross-binary drift. -pub static DFAST_SPEC_MADE: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static DFAST_SPEC_USED: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static DFAST_SPEC_MADE: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static DFAST_SPEC_USED: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear `(speculations_made, speculations_consumed)`. pub fn take_dfast_spec() -> (u64, u64) { @@ -12552,9 +12552,9 @@ fn bt_depth_steps() -> u32 { 1 } -pub static BT_WALKS2: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static BT_ITERS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static BT_FULL: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static BT_WALKS2: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static BT_ITERS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static BT_FULL: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// `(walks, total_iterations, walks_that_used_ALL attempts)` pub fn take_bt_iters() -> (u64, u64, u64) { @@ -13347,14 +13347,14 @@ fn opt_rep_enabled() -> bool { /// GATE 10 @ L19: what the DP's repcode candidate earns. `try_rep1` runs at /// every position of every opt block, unconditionally. -pub static OPT_REP_PROBES: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static OPT_REP_HITS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static OPT_REP_BYTES: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static OPT_REP_PROBES: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static OPT_REP_HITS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static OPT_REP_BYTES: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); -pub static OPT_POS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static OPT_SKIP_INF: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static OPT_SKIP_JUMP: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static OPT_SKIP_JUMPS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static OPT_POS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static OPT_SKIP_INF: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static OPT_SKIP_JUMP: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static OPT_SKIP_JUMPS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// `(positions, skipped_price_inf, bytes_jumped, jumps)` pub fn take_opt_skips() -> (u64, u64, u64, u64) { @@ -13367,10 +13367,10 @@ pub fn take_opt_skips() -> (u64, u64, u64, u64) { ) } -pub static OPT_BT_CALLS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static OPT_BT_DRY: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static OPT_BT_LEN: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static OPT_SEQS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static OPT_BT_CALLS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static OPT_BT_DRY: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static OPT_BT_LEN: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static OPT_SEQS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// `(bt_calls, bt_calls_returning_nothing, total_match_len, emitted_seqs)` pub fn take_opt_bt() -> (u64, u64, u64, u64) { @@ -13475,7 +13475,7 @@ pub fn set_opt_fill_max_arm(v: usize) { } /// Positions inserted by the opt back-fill -- the work GATE 12 controls at L19. -pub static OPT_FILL_INS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static OPT_FILL_INS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear the opt back-fill insert count. pub fn take_opt_fill_ins() -> u64 { @@ -14380,9 +14380,9 @@ fn fused_ml(x: u64, src: &[u8], m: usize, ip: usize, block_end: usize) -> usize } #[cfg(feature = "profile")] -static FUSED_SHORT: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +static FUSED_SHORT: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -static FUSED_LONG: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +static FUSED_LONG: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// (short, long) fused-head resolutions since the last call -- BRICK 11's verdict. #[cfg(feature = "profile")] @@ -14669,8 +14669,8 @@ fn row_auto_ok(params: CompressionParameters, src_len: Option) -> bool { /// WIDE-CHAIN LATCH census: `[events, positions_rescanned]`. The latch does a /// full O(window) chain rebuild when it fires; this is what that costs. #[cfg(feature = "profile")] -pub static WIDE_LATCH: [core::sync::atomic::AtomicU64; 2] = - [const { core::sync::atomic::AtomicU64::new(0) }; 2]; +pub static WIDE_LATCH: [crate::census64::AtomicU64; 2] = + [const { crate::census64::AtomicU64::new(0) }; 2]; /// Read and clear `(latch_events, positions_rescanned)`. #[cfg(feature = "profile")] @@ -14697,8 +14697,8 @@ pub fn take_wide_latch() -> (u64, u64) { /// smaller `chain_mask` aliases many positions onto one slot, so the link a /// walk reads may belong to some other position entirely. #[cfg(feature = "profile")] -pub static WALK_EXIT: [core::sync::atomic::AtomicU64; 8] = - [const { core::sync::atomic::AtomicU64::new(0) }; 8]; +pub static WALK_EXIT: [crate::census64::AtomicU64; 8] = + [const { crate::census64::AtomicU64::new(0) }; 8]; /// Read and clear the walk-exit census. #[cfg(feature = "profile")] @@ -14716,7 +14716,7 @@ pub fn take_walk_exit() -> [u64; 8] { /// the same candidates with FEWER DEPENDENT LOADS, so both numbers are needed: /// `ROW_EXAM` is candidates, `ROW_LOADS` is rows touched (one load each). #[cfg(feature = "profile")] -pub static ROW_EXAM: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static ROW_EXAM: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] /// SECTION 14.9 census: the 16-to-1 bucket-sharing cost. /// `[examined, same_bucket, mls_eq_pass, gtag0_probes, probes]`. @@ -14725,12 +14725,12 @@ pub static ROW_EXAM: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU /// ends in a random `src` load inside `mls_eq` and (for mls >= 4) cannot /// possibly succeed. #[cfg(feature = "profile")] -pub static ROW_BUCKET: [core::sync::atomic::AtomicU64; 5] = [ - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), +pub static ROW_BUCKET: [crate::census64::AtomicU64; 5] = [ + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), ]; /// Read and clear the bucket-sharing census. #[cfg(feature = "profile")] @@ -14781,12 +14781,11 @@ pub(crate) fn dfast_bext_enabled() -> bool { /// backward walk WOULD recover at the commit point, and how many matches could /// move at all. Measurement only; the walk is not applied. #[cfg(feature = "profile")] -pub static DFAST_BEXT_BYTES: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static DFAST_BEXT_BYTES: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static DFAST_BEXT_MATCHES: core::sync::atomic::AtomicU64 = - core::sync::atomic::AtomicU64::new(0); +pub static DFAST_BEXT_MATCHES: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static DFAST_BEXT_SEQS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static DFAST_BEXT_SEQS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear `(bytes_recoverable, matches_that_could_move, matches_seen)`. #[cfg(feature = "profile")] @@ -14800,7 +14799,7 @@ pub fn take_dfast_bext() -> (u64, u64, u64) { } #[cfg(feature = "profile")] -pub static ROW_LOADS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static ROW_LOADS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear `(candidates_examined, rows_loaded)`. #[cfg(feature = "profile")] pub fn take_row_census() -> (u64, u64) { @@ -15061,9 +15060,9 @@ fn chain_null_tag(src: &[u8], mls: usize) -> u8 { /// FALSE-skip re-probe -- a skipped candidate whose bytes would have matched /// must never exist. #[cfg(feature = "profile")] -pub static LINK_SKIPS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static LINK_SKIPS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static LINK_FALSE: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static LINK_FALSE: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] pub fn take_link_tag() -> (u64, u64) { use core::sync::atomic::Ordering::Relaxed; @@ -15138,11 +15137,11 @@ pub static WALK_SIG_REP: core::sync::atomic::AtomicU32 = core::sync::atomic::Ato #[cfg(feature = "profile")] pub static WALK_SIG_SPB: core::sync::atomic::AtomicU32 = core::sync::atomic::AtomicU32::new(0); #[cfg(feature = "profile")] -pub static WALK_SIG_MB: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static WALK_SIG_MB: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static WALK_SIG_NS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static WALK_SIG_NS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static WALK_SIG_OB: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static WALK_SIG_OB: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] pub fn take_walk_signals() -> (f32, f32, f32, u64, u64, u64) { use core::sync::atomic::Ordering::Relaxed; @@ -15159,13 +15158,13 @@ pub fn take_walk_signals() -> (f32, f32, f32, u64, u64, u64) { /// Back-extension census for the SIMD question: (extensions > 0, total /// bytes, extensions >= 8 -- the class a u64 backward step would win). #[cfg(feature = "profile")] -pub static BEXT_N: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static BEXT_N: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static BEXT_BYTES: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static BEXT_BYTES: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static BEXT_GE8: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static BEXT_GE8: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static BEXT_MATCHES: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static BEXT_MATCHES: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] pub fn take_bext() -> (u64, u64, u64, u64) { use core::sync::atomic::Ordering::Relaxed; @@ -15192,15 +15191,15 @@ fn note_bext(ext: u64) { /// Chain-walk census: (candidates examined, byte-mismatch steps). #[cfg(feature = "profile")] -pub static WALK_EXAM: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static WALK_EXAM: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static WALK_BYTEMISS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static WALK_BYTEMISS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Walk-continue accept classes: (first-find past a collision -- legacy would /// have emitted a literal; upgrade past a collision -- legacy had a match). #[cfg(feature = "profile")] -pub static WALK_CONT_FIRST: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static WALK_CONT_FIRST: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static WALK_CONT_UPGRADE: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static WALK_CONT_UPGRADE: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] pub fn take_walk_classes() -> (u64, u64) { use core::sync::atomic::Ordering::Relaxed; @@ -15222,9 +15221,9 @@ pub fn take_walk_census() -> (u64, u64) { /// accept count is what a representation with an unambiguous null would /// change. #[cfg(feature = "profile")] -pub static WALK_M0: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static WALK_M0: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static WALK_M0_ACCEPT: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static WALK_M0_ACCEPT: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] pub fn take_walk_phantom() -> (u64, u64) { use core::sync::atomic::Ordering::Relaxed; @@ -15728,9 +15727,8 @@ pub fn set_fast_spec_arm(on: bool) { /// Which `find_dfast` body actually executed. Probe counts and output bytes are /// IDENTICAL between the two, by design -- they examine the same candidates in /// the same order -- so neither can show which one ran. These can. -pub static DFAST_SPEC_CALLS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static DFAST_RUNTIME_CALLS: core::sync::atomic::AtomicU64 = - core::sync::atomic::AtomicU64::new(0); +pub static DFAST_SPEC_CALLS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static DFAST_RUNTIME_CALLS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear both call counters. pub fn take_dfast_calls() -> (u64, u64) { @@ -15742,9 +15740,9 @@ pub fn take_dfast_calls() -> (u64, u64) { } /// Calls into `find_fast`'s Gate-4 dispatcher, for reachability proofs. -pub static FAST_CALLS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static FAST_CALLS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Calls into `find_opt` (L16+). -pub static OPT_CALLS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static OPT_CALLS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear the finder reachability counters: `(find_fast, find_opt)`. pub fn take_finder_calls() -> (u64, u64) { @@ -15756,8 +15754,8 @@ pub fn take_finder_calls() -> (u64, u64) { } /// Which `bt_find_best` body ran: `(specialised, runtime_fallback)`. -pub static BT_SPEC_CALLS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static BT_RUNTIME_CALLS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static BT_SPEC_CALLS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static BT_RUNTIME_CALLS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear the bt-path body counters. pub fn take_bt_calls() -> (u64, u64) { @@ -15954,21 +15952,21 @@ static PAIR_T_CACHE: core::sync::atomic::AtomicU32 = core::sync::atomic::AtomicU /// Each of these accessors calls `std::env::var` with no cache -- a /// GetEnvironmentVariableW plus a String allocation for a process constant. #[cfg(feature = "profile")] -pub static ENVHIT: [core::sync::atomic::AtomicU64; 14] = [ - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), +pub static ENVHIT: [crate::census64::AtomicU64; 14] = [ + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), ]; /// Read and clear the fitted-constant read counts. @@ -15984,18 +15982,18 @@ pub fn take_envhits() -> [u64; 14] { /// Diagnostic counters for Gate 6 candidate variables: how often the pair probe /// fires, how often it HITS, and how many bytes those hits cover. Activity vs /// outcome -- the campaign's law says the signal must predict the outcome. -pub static PAIR_PROBES: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static PAIR_HITS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static PAIR_BYTES: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static PAIR_PROBES: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static PAIR_HITS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static PAIR_BYTES: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Split the pair probe by the MAIN probe's slot state -- `m0 == 0` means that /// hash bucket has never been written, which is free information already in a /// register at the probe site. -pub static PAIR_M0_EMPTY: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static PAIR_M0_LIVE: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static PAIR_HIT_EMPTY: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static PAIR_HIT_LIVE: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static PAIR_BYTES_EMPTY: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static PAIR_BYTES_LIVE: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static PAIR_M0_EMPTY: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static PAIR_M0_LIVE: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static PAIR_HIT_EMPTY: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static PAIR_HIT_LIVE: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static PAIR_BYTES_EMPTY: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static PAIR_BYTES_LIVE: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// `(probes_empty, probes_live, hits_empty, hits_live, bytes_empty, bytes_live)` pub fn take_pair_split() -> (u64, u64, u64, u64, u64, u64) { @@ -16010,25 +16008,25 @@ pub fn take_pair_split() -> (u64, u64, u64, u64, u64, u64) { ) } -pub static MAIN_BYTES: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static MAIN_BYTES: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear: `(probes, hits, pair_match_bytes, all_match_bytes)`. -static ROUTE_HIST: [core::sync::atomic::AtomicU64; 3] = [ - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), +static ROUTE_HIST: [crate::census64::AtomicU64; 3] = [ + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), ]; -static ROUTE_GAIN: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -static ROUTE_REP: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -static ROUTE_N: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); - -static SIG_GAIN: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -static SIG_REP: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -static SIG_N: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -static SIG_TAG: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -static SIG_REPLEN: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -static SIG_NSEQ: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -static SIG_OPTREP: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +static ROUTE_GAIN: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +static ROUTE_REP: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +static ROUTE_N: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); + +static SIG_GAIN: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +static SIG_REP: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +static SIG_N: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +static SIG_TAG: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +static SIG_REPLEN: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +static SIG_NSEQ: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +static SIG_OPTREP: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// EVERY per-block content signal the encoder already maintains, as block means: /// `(pair_gain, rep_yield, tag_yield, rep_len_ratio, last_nseq, opt_rep_rate)`. @@ -16156,9 +16154,9 @@ fn cand_yield((f, t): (u64, u64)) -> f32 { /// L19-native accounting: tree probes, those too SHORT to use, and those that /// could not IMPROVE on the best so far. -pub static BT_PROBE: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static BT_SHORT: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static BT_NOGAIN: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static BT_PROBE: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static BT_SHORT: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static BT_NOGAIN: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// `(probes, too_short, no_gain)` pub fn take_bt_probe_stats() -> (u64, u64, u64) { @@ -16314,11 +16312,11 @@ fn long_tag_enabled() -> bool { /// 1a ledger: (nonempty long probes, rejections, FALSE rejections). Three /// counters with one meaning each -- see the tag audit's instrument trap. #[cfg(feature = "profile")] -pub static LTAG_NONEMPTY: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static LTAG_NONEMPTY: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static LTAG_REJECT: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static LTAG_REJECT: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static LTAG_FALSE: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static LTAG_FALSE: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear the LONG-table tag audit: `[nonempty, rejected, FALSE]`. /// A FALSE rejection is one the byte compare would have ACCEPTED -- i.e. a @@ -16348,18 +16346,18 @@ pub fn take_long_tag() -> (u64, u64, u64) { /// at the MAIN long consume site. The fail share is the ceiling on what a /// stronger tag could still remove. #[cfg(feature = "profile")] -pub static LTAG_SURV_FAIL: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static LTAG_SURV_FAIL: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static LTAG_SURV_WFAIL: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static LTAG_SURV_WFAIL: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static LTAG_SURV_ACC: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static LTAG_SURV_ACC: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// SHORT-table consume-site residual, mirror of the long table's. #[cfg(feature = "profile")] -pub static STAG_SURV_FAIL: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static STAG_SURV_FAIL: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static STAG_SURV_WFAIL: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static STAG_SURV_WFAIL: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static STAG_SURV_ACC: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static STAG_SURV_ACC: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// `(bytes_fail, window_fail, accepted)` for the SHORT consume site. #[cfg(feature = "profile")] pub fn take_short_tag_residual() -> (u64, u64, u64) { @@ -16386,9 +16384,9 @@ pub fn take_long_tag_residual() -> (u64, u64, u64) { /// `TAGARR_READS` is a load from a SECOND random cache line; `PACKED_TAG_READS` /// reads the byte that arrived with the position. #[cfg(feature = "profile")] -pub static TAGARR_READS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static TAGARR_READS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static PACKED_TAG_READS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static PACKED_TAG_READS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear `(tag-array reads, packed reads)`. #[cfg(feature = "profile")] @@ -16423,13 +16421,13 @@ fn fast_pack_enabled() -> bool { /// every such candidate costs a random `src[m]` load, a compare, and a /// `count_match` that dies below `mls`. #[cfg(feature = "profile")] -pub static FF_LAZY_FIRES: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static FF_LAZY_FIRES: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static FF_LATCH: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static FF_LATCH: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static FF_CAND4: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static FF_CAND4: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static FF_ACCEPT: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static FF_ACCEPT: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear `(four-byte passes, accepted matches)`. #[cfg(feature = "profile")] diff --git a/crates/rusty_zstd/src/fse.rs b/crates/rusty_zstd/src/fse.rs index 9bee1cc..4659ee4 100644 --- a/crates/rusty_zstd/src/fse.rs +++ b/crates/rusty_zstd/src/fse.rs @@ -35,10 +35,10 @@ pub(crate) struct FseView<'a> { /// applies only when `high_threshold == table_size - 1`. This counts how often /// that is true, before anything is built. #[cfg(feature = "profile")] -pub static D6_SPREAD: [core::sync::atomic::AtomicU64; 3] = [ - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), +pub static D6_SPREAD: [crate::census64::AtomicU64; 3] = [ + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), ]; /// Read and clear the D6 probe. #[cfg(feature = "profile")] diff --git a/crates/rusty_zstd/src/huffman.rs b/crates/rusty_zstd/src/huffman.rs index b6538fc..3d1d2fa 100644 --- a/crates/rusty_zstd/src/huffman.rs +++ b/crates/rusty_zstd/src/huffman.rs @@ -27,10 +27,10 @@ pub(crate) struct HuffmanTable { /// The tree-merge loop is (n-1) iterations of an adaptive sort plus two /// `Vec::remove(0)` memmoves, so n^2 is the work proxy. #[cfg(feature = "profile")] -pub static N13_STATS: [core::sync::atomic::AtomicU64; 3] = [ - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), +pub static N13_STATS: [crate::census64::AtomicU64; 3] = [ + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), ]; /// Read and clear the N13 probe. #[cfg(feature = "profile")] @@ -53,9 +53,9 @@ pub fn take_n13_stats() -> [u64; 3] { #[cfg(feature = "profile")] /// `fast_4x2` outcomes: [bailed, succeeded]. #[cfg(feature = "profile")] -pub static F4X2_ARM: [core::sync::atomic::AtomicU64; 2] = [ - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), +pub static F4X2_ARM: [crate::census64::AtomicU64; 2] = [ + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), ]; /// Read and clear the `fast_4x2` outcome census. @@ -70,7 +70,7 @@ pub fn take_f4x2_arm() -> (u64, u64) { /// Sections that took `decode_4x`'s X1 arm. #[cfg(feature = "profile")] -pub static X4_X1_CALLS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static X4_X1_CALLS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Sections that took `decode_4x`'s X2 arm. /// @@ -86,7 +86,7 @@ pub static X4_X1_CALLS: core::sync::atomic::AtomicU64 = core::sync::atomic::Atom /// "OUTLINED, on a census" note rests on, and a conflated counter cannot /// support it in either direction. #[cfg(feature = "profile")] -pub static X4_X2_CALLS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static X4_X2_CALLS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear the X1-arm count. #[cfg(feature = "profile")] @@ -106,9 +106,9 @@ pub fn take_x4_arms() -> (u64, u64) { } #[cfg(feature = "profile")] -pub static X2_STATS: [core::sync::atomic::AtomicU64; 2] = [ - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), +pub static X2_STATS: [crate::census64::AtomicU64; 2] = [ + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), ]; /// Read and clear the N2 instrument: `(builds, uses)`. #[cfg(feature = "profile")] @@ -1456,9 +1456,9 @@ pub(crate) enum HuffUpdate { /// E11 census: `(literal bytes the old O(n) `covers` walk would have read, /// calls)`. `profile`-gated; the shipping build carries nothing. #[cfg(feature = "profile")] -pub static E11_WALKED: (core::sync::atomic::AtomicU64, core::sync::atomic::AtomicU64) = ( - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), +pub static E11_WALKED: (crate::census64::AtomicU64, crate::census64::AtomicU64) = ( + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), ); /// Read and clear the E11 census. @@ -2010,10 +2010,10 @@ fn huffman_nbits(freq: &[u32; 256]) -> Result<[u8; 256], Error> { /// E12 ceiling probe: `(calls, total inner-scan element visits, adjustment steps)`. #[cfg(feature = "profile")] -pub static E12_SCAN: [core::sync::atomic::AtomicU64; 3] = [ - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), +pub static E12_SCAN: [crate::census64::AtomicU64; 3] = [ + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), ]; /// Read and clear the E12 ceiling probe. #[cfg(feature = "profile")] diff --git a/crates/rusty_zstd/src/kreach.rs b/crates/rusty_zstd/src/kreach.rs index 38fc3b6..2536f43 100644 --- a/crates/rusty_zstd/src/kreach.rs +++ b/crates/rusty_zstd/src/kreach.rs @@ -91,8 +91,9 @@ mod imp { #[cfg(feature = "profile")] mod imp { use super::N_SLOTS; + use crate::census64::AtomicU64; use core::cell::Cell; - use core::sync::atomic::{AtomicU64, Ordering::Relaxed}; + use core::sync::atomic::Ordering::Relaxed; pub(super) static G_HIT: [AtomicU64; N_SLOTS] = [const { AtomicU64::new(0) }; N_SLOTS]; pub(super) static G_MISS: [AtomicU64; N_SLOTS] = [const { AtomicU64::new(0) }; N_SLOTS]; diff --git a/crates/rusty_zstd/src/ldm.rs b/crates/rusty_zstd/src/ldm.rs index 2e31e62..f6acabd 100644 --- a/crates/rusty_zstd/src/ldm.rs +++ b/crates/rusty_zstd/src/ldm.rs @@ -257,9 +257,9 @@ fn ldm_hash(src: &[u8], ip: usize, hash_log: u32) -> usize { /// population the old `memcmp` ran on) and those the 8-byte head let through /// to `count_eq`. #[cfg(feature = "profile")] -pub static LDM_CANDS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static LDM_CANDS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static LDM_COUNTS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static LDM_COUNTS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// `(candidates, counted)` since the last call. #[cfg(feature = "profile")] diff --git a/crates/rusty_zstd/src/lib.rs b/crates/rusty_zstd/src/lib.rs index 8592d4d..3e4945a 100644 --- a/crates/rusty_zstd/src/lib.rs +++ b/crates/rusty_zstd/src/lib.rs @@ -101,7 +101,7 @@ pub(crate) fn env_knob_is1(name: &str) -> bool { /// Count of `env_knob` reads -- see the note there. #[cfg(feature = "profile")] -pub static ENV_READS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static ENV_READS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear the env-read counter. #[cfg(feature = "profile")] @@ -119,6 +119,10 @@ pub(crate) fn env_knob(_name: &str) -> Result { mod bit; mod block; +/// The census counters' 64-bit atomic, and the zero-sized stub it becomes +/// on a part without 64-bit atomics. See the module docs: a zero read there +/// means "not measurable on this target", never "measured zero". +pub mod census64; mod compressed; /// Copy census: how many times does the encoder move each input byte? pub mod copies; diff --git a/crates/rusty_zstd/src/prof.rs b/crates/rusty_zstd/src/prof.rs index a15e7bc..d50c582 100644 --- a/crates/rusty_zstd/src/prof.rs +++ b/crates/rusty_zstd/src/prof.rs @@ -183,9 +183,10 @@ pub struct EncodeCounts { #[cfg(feature = "profile")] mod on { use super::{BlockTap, EncodeCounts, Stage, NAMES, N_STAGES}; + use crate::census64::AtomicU64; use crate::profclock as cpuclock; use core::cell::RefCell; - use core::sync::atomic::{AtomicU64, Ordering}; + use core::sync::atomic::Ordering; thread_local! { static BLOCK_TAPS: RefCell> = const { RefCell::new(alloc::vec::Vec::new()) }; diff --git a/crates/rusty_zstd/src/profclock.rs b/crates/rusty_zstd/src/profclock.rs index 583ca3b..92314c0 100644 --- a/crates/rusty_zstd/src/profclock.rs +++ b/crates/rusty_zstd/src/profclock.rs @@ -46,7 +46,8 @@ mod imp { // This module is FFI by definition: the whole point is to reach a // clock the standard library does not expose. #![allow(unsafe_code)] - use core::sync::atomic::{AtomicU64, Ordering}; + use crate::census64::AtomicU64; + use core::sync::atomic::Ordering; #[link(name = "kernel32")] extern "system" { diff --git a/crates/rusty_zstd/src/rowfind.rs b/crates/rusty_zstd/src/rowfind.rs index 5c3eb69..42626de 100644 --- a/crates/rusty_zstd/src/rowfind.rs +++ b/crates/rusty_zstd/src/rowfind.rs @@ -50,10 +50,10 @@ pub(crate) const ROW: usize = 16; /// models are evaluated from the SAME mask in one run, so the comparison needs /// no A/B build and carries no clock. #[cfg(feature = "profile")] -pub static ROW_WALK: [core::sync::atomic::AtomicU64; 3] = [ - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), +pub static ROW_WALK: [crate::census64::AtomicU64; 3] = [ + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), ]; /// Read and clear the row-walk census. #[cfg(feature = "profile")] diff --git a/crates/rusty_zstd/src/scratch.rs b/crates/rusty_zstd/src/scratch.rs index eff6f1f..6e1b943 100644 --- a/crates/rusty_zstd/src/scratch.rs +++ b/crates/rusty_zstd/src/scratch.rs @@ -270,20 +270,20 @@ pub(crate) fn note_pool(hit: bool) { /// Pool hit/miss census. A miss is an allocation. #[cfg(feature = "profile")] -pub static POOL_HIT: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static POOL_HIT: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Pool misses -- each one is a fresh allocation. #[cfg(feature = "profile")] -pub static POOL_MISS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static POOL_MISS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Buffers handed back with real capacity. #[cfg(feature = "profile")] -pub static POOL_GIVE: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static POOL_GIVE: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Buffers handed back with ZERO capacity -- never pooled, so the matching /// take must allocate. A take/give imbalance shows up here first. #[cfg(feature = "profile")] -pub static POOL_GIVE_EMPTY: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static POOL_GIVE_EMPTY: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Buffers DROPPED because the free list was full. #[cfg(feature = "profile")] -pub static POOL_DROP: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static POOL_DROP: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear the pool census: `(hits, misses, drops, gives, give_empty)`. #[cfg(feature = "profile")] diff --git a/crates/rusty_zstd/src/simd.rs b/crates/rusty_zstd/src/simd.rs index 4d06ed5..a9fd659 100644 --- a/crates/rusty_zstd/src/simd.rs +++ b/crates/rusty_zstd/src/simd.rs @@ -180,8 +180,9 @@ fn eqlen_arm() -> u8 { /// benches compress single-threaded, so the receipt is exact for them. #[cfg(feature = "profile")] mod counters { + use crate::census64::AtomicU64; use core::cell::Cell; - use core::sync::atomic::{AtomicU64, Ordering::Relaxed}; + use core::sync::atomic::Ordering::Relaxed; pub(super) static G_CALLS: AtomicU64 = AtomicU64::new(0); pub(super) static G_WIDE: AtomicU64 = AtomicU64::new(0); diff --git a/crates/rusty_zstd/src/stream.rs b/crates/rusty_zstd/src/stream.rs index d392913..dae0101 100644 --- a/crates/rusty_zstd/src/stream.rs +++ b/crates/rusty_zstd/src/stream.rs @@ -17,9 +17,9 @@ use crate::DecompressOptions; /// The memmove per `decoded.drain(..drop)` is `len - drop` -- the retained /// window -- and section 18's dig found it running once per `stream()` CALL. #[cfg(feature = "profile")] -pub static DEC_COMPACT: [core::sync::atomic::AtomicU64; 2] = [ - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), +pub static DEC_COMPACT: [crate::census64::AtomicU64; 2] = [ + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), ]; /// Read and clear the decoder-compaction census. #[cfg(feature = "profile")] @@ -35,9 +35,9 @@ pub fn take_dec_compact() -> [u64; 2] { /// Each slide memmoves the retained window, zeroes six match tables and /// re-primes the whole window -- section 19's decoder finding, mirrored. #[cfg(feature = "profile")] -pub static ENC_SLIDE: [core::sync::atomic::AtomicU64; 2] = [ - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), +pub static ENC_SLIDE: [crate::census64::AtomicU64; 2] = [ + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), ]; /// History multiplier at which the encoder slides its window. /// diff --git a/crates/rusty_zstd/src/xxh64.rs b/crates/rusty_zstd/src/xxh64.rs index 9521840..a301623 100644 --- a/crates/rusty_zstd/src/xxh64.rs +++ b/crates/rusty_zstd/src/xxh64.rs @@ -421,7 +421,8 @@ fn finish_tail(mut tail: &[u8], mut acc: u64, total: u64) -> u64 { /// `profile`-gated so the shipping path carries nothing. #[cfg(feature = "profile")] pub mod census { - use core::sync::atomic::{AtomicU64, Ordering}; + use crate::census64::AtomicU64; + use core::sync::atomic::Ordering; pub static HYBRID_BYTES: AtomicU64 = AtomicU64::new(0); pub static SCALAR_BYTES: AtomicU64 = AtomicU64::new(0); pub static HYBRID_CALLS: AtomicU64 = AtomicU64::new(0);