From a1f3809ffd4ff944a346ae7512e9c2e8501e0ac5 Mon Sep 17 00:00:00 2001 From: Tim Date: Wed, 9 Sep 2026 16:15:02 -0700 Subject: [PATCH] fix(no_std): the census counters no longer cost the crate bare metal `cargo check -p rusty_zstd --no-default-features --features alloc` failed with 204 errors on thumbv7em-none-eabihf (Cortex-M4F) and the same on riscv32imac-unknown-none-elf. Every one was `cannot find AtomicU64 in atomic`, and every one was an INSTRUMENT -- reload counts, decode band histograms, kernel reach, walk exits, copy bytes. Not one was a line of codec. Neither part has 64-bit atomics. Found by rusty_RTOS's house gate, which wants the codec on those parts for on-chip OTA payloads and trace capture. Its plan estimated eight statics from a capped rustc report; the real count is 427 fully-qualified uses plus six imports across sixteen files, which is why this is one seam rather than a hundred `#[cfg]`s. `census64` is that seam: - Where 64-bit atomics EXIST it is `pub use core::sync::atomic::AtomicU64` -- the same type, not a wrapper -- so the public statics keep their published type and codegen cannot change. Proved rather than asserted: the asm board over every shipping kernel, fill and finder is identical in all thirty-two columns to the pre-change build, and the identity gates still read GOLD 2F6594F7EEDBD12B / 59,680,638 and LDM 57BE83EA4E1199E8 / 57,796,847. - Where they do not, it is a zero-sized stub: the statics leave BSS and every fetch_add folds away. Not `portable-atomic`, deliberately. It would keep the census readable there and it is the other honest answer, but on a core with no 64-bit atomic instruction it needs a critical-section implementation, and a library that enables that conscripts every downstream firmware's interrupt policy so a diagnostic counter can increment. The seam is one type wide, so a firmware that wants the count can supply portable_atomic::AtomicU64 itself. A zero there means "not measurable on this target", never "measured zero" -- the exact trap this crate's own rules warn about -- so it is published, not buried: census64::CENSUS_LIVE is false on such a build. The REVERSE mistake would be far worse, because a stub selected on a hosted target would make every count-based verdict fiction while every gate still passed. Two guards: a compile-time size check, and a unit test that asserts on BEHAVIOUR rather than on the constant (it counts to 42 and reads it back). CI gains both rungs in the job that already guards the portable configurations, and the README's platform table gains both rows. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/ci.yml | 17 +- CHANGELOG.md | 45 ++++ README.md | 11 +- crates/rusty_zstd/src/bit.rs | 4 +- crates/rusty_zstd/src/census64.rs | 183 +++++++++++++ crates/rusty_zstd/src/compressed.rs | 66 ++--- crates/rusty_zstd/src/copies.rs | 3 +- crates/rusty_zstd/src/encode.rs | 384 ++++++++++++++-------------- crates/rusty_zstd/src/fse.rs | 8 +- crates/rusty_zstd/src/huffman.rs | 38 +-- crates/rusty_zstd/src/kreach.rs | 3 +- crates/rusty_zstd/src/ldm.rs | 4 +- crates/rusty_zstd/src/lib.rs | 6 +- crates/rusty_zstd/src/prof.rs | 3 +- crates/rusty_zstd/src/profclock.rs | 3 +- crates/rusty_zstd/src/rowfind.rs | 8 +- crates/rusty_zstd/src/scratch.rs | 10 +- crates/rusty_zstd/src/simd.rs | 3 +- crates/rusty_zstd/src/stream.rs | 12 +- crates/rusty_zstd/src/xxh64.rs | 3 +- 20 files changed, 535 insertions(+), 279 deletions(-) create mode 100644 crates/rusty_zstd/src/census64.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5073f5e..b9f6755 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,13 +68,13 @@ jobs: # The portable configurations the README advertises, so they cannot rot. portable: - name: no_std + wasm32 + name: no_std + wasm32 + bare metal runs-on: ubuntu-latest steps: - uses: actions/checkout@v5 - uses: dtolnay/rust-toolchain@stable with: - targets: wasm32-unknown-unknown + targets: wasm32-unknown-unknown, thumbv7em-none-eabihf, riscv32imac-unknown-none-elf - uses: Swatinem/rust-cache@v2 - name: no_std + alloc run: cargo check -p rusty_zstd --no-default-features --features alloc @@ -84,6 +84,19 @@ jobs: run: cargo check -p rusty_zstd --target wasm32-unknown-unknown --no-default-features --features alloc - name: profile feature builds run: cargo check -p rusty_zstd --features profile + # BARE METAL. A `no-std` category on the registry is not a claim; a target + # that compiles is. Both of these failed with 204 errors until the census + # counters stopped requiring 64-bit atomics -- and every one of those errors + # was a counter, not a line of codec. `census64` makes the counter a + # zero-sized stub where `AtomicU64` does not exist, so the instrument costs + # the crate a diagnostic on those parts instead of costing it the parts. + # + # Cortex-M4F and RV32IMAC are what rusty_RTOS (Kairos) consumes the codec on, + # for on-chip OTA payloads and trace capture. Neither has 64-bit atomics. + - name: bare metal (Cortex-M4F, no 64-bit atomics) + run: cargo check -p rusty_zstd --target thumbv7em-none-eabihf --no-default-features --features alloc + - name: bare metal (RV32IMAC, no 64-bit atomics) + run: cargo check -p rusty_zstd --target riscv32imac-unknown-none-elf --no-default-features --features alloc lint: runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index 23f5e00..faba3b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,51 @@ based on [Keep a Changelog](https://keepachangelog.com/); this project uses ## [Unreleased] +### Fixed -- the census counters cost the crate BARE METAL; they no longer do + +`cargo check -p rusty_zstd --no-default-features --features alloc` failed with +**204 errors** on `thumbv7em-none-eabihf` (Cortex-M4F) and the same on +`riscv32imac-unknown-none-elf`. Every one of them was +`cannot find AtomicU64 in atomic`, and every one was an INSTRUMENT: reload +counts, decode band histograms, kernel reach, walk exits, copy bytes. Not one +was a line of codec. Neither part has 64-bit atomics. + +This was found by `rusty_RTOS`'s house gate, which consumes the crate for +on-chip OTA payloads and trace capture, and it is the reason its plan says a +`no-std` category on the registry is not a claim: **the target that compiles is +the claim.** The plan estimated eight statics from a capped rustc report; the +real count is 427 fully-qualified uses plus six imports across sixteen files. + +`census64` is now the one seam. On every target that HAS 64-bit atomics it is +`pub use core::sync::atomic::AtomicU64` -- the same type, not a wrapper -- so +the public statics keep their published type and the emitted code is unchanged. +**Proved, not asserted:** the board over every shipping kernel, fill and finder +is identical to the pre-change assembly in all thirty-two columns, and the +identity gates still read GOLD `2F6594F7EEDBD12B` / 59,680,638 and LDM +`57BE83EA4E1199E8` / 57,796,847. + +On a part without them it is a zero-sized stub: the statics leave BSS entirely +and every `fetch_add` folds away. + +**Why a stub rather than `portable-atomic`.** That crate would keep the census +READABLE there, and it is the other honest answer. But on a core with no 64-bit +atomic instruction it needs a critical-section implementation, and a library +that turns that feature on conscripts every downstream firmware's interrupt +policy so a diagnostic counter can increment. The seam is one type wide, so a +firmware that does want the count can supply `portable_atomic::AtomicU64`. + +**A zero there means "not measurable on this target", never "measured zero"**, +which is the same trap this crate's own rules warn about -- so it is published, +not buried: `census64::CENSUS_LIVE` is `false` on such a build. The reverse +mistake would be far worse (a stub selected on a HOSTED target would make every +count-based verdict fiction while every gate still passed), so it is gated two +ways: a compile-time size check, and a unit test that asserts on BEHAVIOUR +rather than on the constant -- it counts to 42 and reads it back. + +CI gains the rung, in the job that already guards the portable configurations: +`--target thumbv7em-none-eabihf` and `--target riscv32imac-unknown-none-elf`, +both `--no-default-features --features alloc`, on every push. + ### Measured -- what the encode campaign bought: 1.08-1.27x at L3-L12 (2026-09-09) Every verdict in the sections below is an INSTRUCTION COUNT, because this box diff --git a/README.md b/README.md index 974045c..d094e87 100644 --- a/README.md +++ b/README.md @@ -347,8 +347,13 @@ implementation cannot leak into the dependency graph. - **Byte-identity where it is owed.** The XXH64 checksum is gated against the published XXH64 vectors and a GOLD oracle; every SIMD kernel is gated against its scalar twin; the Huffman and FSE table decoders are gated against C's. -- **`no_std + alloc` and `wasm32-unknown-unknown` build in CI**, so the portable - configuration cannot rot. +- **`no_std + alloc`, `wasm32-unknown-unknown` and two bare-metal targets build + in CI**, so the portable configuration cannot rot. The bare-metal rungs are + Cortex-M4F and RV32IMAC, neither of which has 64-bit atomics: a `no-std` + category is not a claim, a target that compiles is. Note that the measurement + counters need `AtomicU64`, so on a part without it they compile to a + zero-sized stub and `census64::CENSUS_LIVE` reads `false` -- a zero from a + counter there means "not measurable on this target", never "measured zero". Fetch the oracle with `pwsh scripts/fetch-oracle.ps1`, or point `RUSTY_ZSTD_ORACLE` at any `zstd` binary whose `--version` says 1.5.7. Details: @@ -363,6 +368,8 @@ Fetch the oracle with `pwsh scripts/fetch-oracle.ps1`, or point | macOS (x86-64 / aarch64) | ✅ builds + tests | | `wasm32-unknown-unknown` | ✅ builds (`std` and `alloc`) | | `no_std + alloc` | ✅ builds | +| `thumbv7em-none-eabihf` (Cortex-M4F) | ✅ builds (`no_std + alloc`) | +| `riscv32imac-unknown-none-elf` | ✅ builds (`no_std + alloc`) | AVX2 and NEON kernels are selected at **runtime**. On a CPU without them, the scalar twins run and the output is identical. diff --git a/crates/rusty_zstd/src/bit.rs b/crates/rusty_zstd/src/bit.rs index 49ba0f5..594a8b5 100644 --- a/crates/rusty_zstd/src/bit.rs +++ b/crates/rusty_zstd/src/bit.rs @@ -16,12 +16,12 @@ pub(crate) struct BitRev<'a> { /// Executed `BitRev::reload` calls. See the note inside `reload`. #[cfg(feature = "profile")] -pub static RELOAD_CALLS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static RELOAD_CALLS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Executed refills -- reloads that reached the container load rather than /// taking one of the four early-outs. See the note inside `reload`. #[cfg(feature = "profile")] -pub static RELOAD_REFILLS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static RELOAD_REFILLS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear the refill counter. #[cfg(feature = "profile")] diff --git a/crates/rusty_zstd/src/census64.rs b/crates/rusty_zstd/src/census64.rs new file mode 100644 index 0000000..9e13e67 --- /dev/null +++ b/crates/rusty_zstd/src/census64.rs @@ -0,0 +1,183 @@ +//! The census counter's 64-bit atomic, and what it becomes on a part that has +//! no 64-bit atomics. +//! +//! WHY THIS MODULE EXISTS. Every measurement instrument in this crate counts +//! into a `static AtomicU64`: reload calls, decode band histograms, kernel +//! reach, walk exits, copy bytes. That is deliberate and the codec skills +//! depend on it -- a count is the only instrument that separates "this kernel +//! does not help" from "this arm is wired to nothing". +//! +//! It also cost the crate two targets. `core::sync::atomic::AtomicU64` does +//! not exist on a Cortex-M4F (`thumbv7em-none-eabihf`) or on RV32 +//! (`riscv32imac-unknown-none-elf`), so `--no-default-features --features +//! alloc` failed with 204 errors on both -- every one of them a counter, not a +//! line of codec. A `no-std` label on the registry is not a claim; a target +//! that compiles is. +//! +//! WHAT IT DOES. On every target that HAS 64-bit atomics this is exactly +//! `core::sync::atomic::AtomicU64`, re-exported. Same type, same ABI, no +//! wrapper, so the public statics keep their published type and the emitted +//! code is byte-for-byte what it was. +//! +//! On a target that does not, it is a zero-sized stub whose operations do +//! nothing and whose loads read 0. The statics stop occupying BSS, the +//! `fetch_add`s vanish, and the codec keeps its targets. +//! +//! WHY A STUB AND NOT `portable-atomic`. That crate is the other honest +//! answer, and it would keep the census READABLE on those parts. But on a +//! core with no 64-bit atomic instruction it needs a critical-section +//! implementation, and a library that turns that feature on forces the choice +//! onto every downstream firmware. A codec should not conscript the +//! application's interrupt policy so that a diagnostic counter can increment. +//! Anyone who does want the count on such a part can supply +//! `portable_atomic::AtomicU64` here; the seam is one type wide. +//! +//! HONESTY. A counter that silently reads zero is exactly the "stale +//! instrument" this crate's own rules warn about, so the fact is published +//! rather than buried: [`CENSUS_LIVE`] is `false` on such a target, and any +//! gate or report that asserts on a census must consult it before believing a +//! zero. **Zero here means "not measurable on this target", never "measured +//! zero".** +//! +//! The reverse mistake would be worse and is gated below: if this stub were +//! ever selected on a HOSTED target, every counter in the crate would read 0 +//! and every census-based verdict would silently become fiction. The unit test +//! fails the build if that happens. + +/// Whether the census counters in this build can actually count. +/// +/// `true` wherever `AtomicU64` exists (every hosted target this crate ships +/// on). `false` on a part without 64-bit atomics, where every counter is a +/// stub and every reader returns 0. Check this before believing a zero. +pub const CENSUS_LIVE: bool = cfg!(target_has_atomic = "64"); + +#[cfg(target_has_atomic = "64")] +pub use core::sync::atomic::AtomicU64; + +#[cfg(not(target_has_atomic = "64"))] +pub use stub::AtomicU64; + +#[cfg(not(target_has_atomic = "64"))] +mod stub { + use core::sync::atomic::Ordering; + + /// A do-nothing stand-in for `core::sync::atomic::AtomicU64` on a target + /// that has no 64-bit atomics. Zero-sized: a `static` of this type costs + /// no BSS and every operation folds away. + /// + /// It is `Sync` for the same reason `()` is: there is no state to race + /// on. Loads return 0, and [`super::CENSUS_LIVE`] is `false` so a reader + /// can tell that apart from a real zero. + #[derive(Debug, Default)] + pub struct AtomicU64(()); + + impl AtomicU64 { + /// The value is discarded: this build cannot hold one. + #[inline(always)] + pub const fn new(_v: u64) -> Self { + Self(()) + } + + /// Always 0. See [`super::CENSUS_LIVE`]. + #[inline(always)] + pub fn load(&self, _order: Ordering) -> u64 { + 0 + } + + #[inline(always)] + pub fn store(&self, _v: u64, _order: Ordering) {} + + /// Always 0 (the previous value this build never held). + #[inline(always)] + pub fn swap(&self, _v: u64, _order: Ordering) -> u64 { + 0 + } + + #[inline(always)] + pub fn fetch_add(&self, _v: u64, _order: Ordering) -> u64 { + 0 + } + + #[inline(always)] + pub fn fetch_sub(&self, _v: u64, _order: Ordering) -> u64 { + 0 + } + + #[inline(always)] + pub fn fetch_max(&self, _v: u64, _order: Ordering) -> u64 { + 0 + } + + #[inline(always)] + pub fn fetch_min(&self, _v: u64, _order: Ordering) -> u64 { + 0 + } + + #[inline(always)] + pub fn fetch_or(&self, _v: u64, _order: Ordering) -> u64 { + 0 + } + + #[inline(always)] + pub fn fetch_and(&self, _v: u64, _order: Ordering) -> u64 { + 0 + } + + /// Always reports the stored value as 0, so a compare against + /// anything else fails -- the same shape a real CAS has when it loses. + #[inline(always)] + pub fn compare_exchange( + &self, + current: u64, + _new: u64, + _success: Ordering, + _failure: Ordering, + ) -> Result { + if current == 0 { + Ok(0) + } else { + Err(0) + } + } + + #[inline(always)] + pub fn compare_exchange_weak( + &self, + current: u64, + new: u64, + success: Ordering, + failure: Ordering, + ) -> Result { + self.compare_exchange(current, new, success, failure) + } + } +} + +/// The selected type must be a REAL 64-bit atomic wherever one exists. If a cfg +/// edit ever let the stub through here, this fails the build rather than the +/// census. (The stub is zero-sized; the real one is eight bytes.) +#[cfg(target_has_atomic = "64")] +const _: () = [(); 1][(core::mem::size_of::() != 8) as usize]; + +#[cfg(test)] +mod tests { + /// The census must actually COUNT anywhere tests run. Asserted on behaviour, + /// not on the `CENSUS_LIVE` constant: if a cfg edit ever selected the stub on + /// a hosted target, every counter in the crate would read 0, `kreach_gate` + /// would see a perfectly-routed 0/0 everywhere, and the whole instrument + /// would report success while measuring nothing. + #[test] + fn census_counts_wherever_tests_run() { + assert!(super::CENSUS_LIVE == cfg!(target_has_atomic = "64")); + let c = super::AtomicU64::new(0); + c.fetch_add(7, core::sync::atomic::Ordering::Relaxed); + c.fetch_add(35, core::sync::atomic::Ordering::Relaxed); + assert_eq!( + c.swap(0, core::sync::atomic::Ordering::Relaxed), + 42, + "the AtomicU64 stub was selected on a target that HAS 64-bit atomics: \ + every census counter in this crate now reads 0 and every count-based \ + verdict from it is fiction" + ); + } +} diff --git a/crates/rusty_zstd/src/compressed.rs b/crates/rusty_zstd/src/compressed.rs index 03916e8..0899696 100644 --- a/crates/rusty_zstd/src/compressed.rs +++ b/crates/rusty_zstd/src/compressed.rs @@ -15,44 +15,44 @@ use crate::reader::Reader; /// Only >=32-byte ops can benefit from AVX2 -- a 16-byte copy is already one /// `movups`. #[cfg(feature = "profile")] -pub static DEC_LIT32: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static DEC_LIT32: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static DEC_MATCH32: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static DEC_MATCH32: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static DEC_LIT16: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static DEC_LIT16: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Literal copies served by the 64-byte tier (BRICK 80's missing third rung). #[cfg(feature = "profile")] -pub static DEC_LIT64: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static DEC_LIT64: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static DEC_MATCH16: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static DEC_MATCH16: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// T4 band census for `copy_match`: which route does each match actually take? /// 0 = offset 1 splat, 1 = 32-byte tier, 2 = 16-byte tier, /// 3 = extend_from_within (offset >= len, runtime-length memcpy CALL), /// 4 = overlapping loop (offset < len). #[cfg(feature = "profile")] -pub static DEC_BAND: [core::sync::atomic::AtomicU64; 8] = [ - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), +pub static DEC_BAND: [crate::census64::AtomicU64; 8] = [ + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), ]; /// Bytes moved by each route, so a rare-but-long band cannot hide. #[cfg(feature = "profile")] -pub static DEC_BAND_B: [core::sync::atomic::AtomicU64; 8] = [ - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), +pub static DEC_BAND_B: [crate::census64::AtomicU64; 8] = [ + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), ]; /// Length histogram for the UN-TIERED bands (3 = `extend_from_within`, @@ -61,8 +61,8 @@ pub static DEC_BAND_B: [core::sync::atomic::AtomicU64; 8] = [ /// the high half -- a band's mean hides its distribution, and the tier width has /// to be chosen from the distribution. #[cfg(feature = "profile")] -pub static DEC_UNTIERED: [core::sync::atomic::AtomicU64; 16] = - [const { core::sync::atomic::AtomicU64::new(0) }; 16]; +pub static DEC_UNTIERED: [crate::census64::AtomicU64; 16] = + [const { crate::census64::AtomicU64::new(0) }; 16]; #[cfg(feature = "profile")] pub fn take_dec_untiered() -> [u64; 16] { @@ -2109,8 +2109,8 @@ static MATCHCOPY_ARM: core::sync::atomic::AtomicU8 = core::sync::atomic::AtomicU /// as out-of-range. Deterministic -- this is what SIZES D9, because a prefetch /// only helps the sequences it is actually issued for. #[cfg(feature = "pfcensus")] -pub static PF_CENSUS: [core::sync::atomic::AtomicU64; 3] = - [const { core::sync::atomic::AtomicU64::new(0) }; 3]; +pub static PF_CENSUS: [crate::census64::AtomicU64; 3] = + [const { crate::census64::AtomicU64::new(0) }; 3]; /// Read and clear `(issued, skipped_rep, skipped_oob)`. #[cfg(feature = "pfcensus")] pub fn take_pf_census() -> (u64, u64, u64) { @@ -2721,7 +2721,7 @@ fn copy_match_nodict( /// Each is a full `from_norm` -- heap alloc + serial spread + finalize -- for a /// value fixed by RFC 8878 and identical for the life of the process. #[cfg(feature = "profile")] -pub static N21_PREDEF: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static N21_PREDEF: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear the N21 probe. #[cfg(feature = "profile")] pub fn take_n21_predef() -> u64 { @@ -2731,7 +2731,7 @@ pub fn take_n21_predef() -> u64 { /// D3 probe: `extend_from_within` calls made by the overlapping (band 4) loop. /// D3 assumes "a memcpy call per period"; this counts what actually happens. #[cfg(feature = "profile")] -pub static D3_ITERS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static D3_ITERS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear the D3 probe. #[cfg(feature = "profile")] pub fn take_d3_iters() -> u64 { @@ -2741,10 +2741,10 @@ pub fn take_d3_iters() -> u64 { /// D4 coverage census: `[frame_only, dict_only, dict_CROSSING]` calls. /// A brick on the crossing path is unverified until index 2 is non-zero. #[cfg(feature = "profile")] -pub static D4_PATHS: [core::sync::atomic::AtomicU64; 3] = [ - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), +pub static D4_PATHS: [crate::census64::AtomicU64; 3] = [ + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), ]; /// Read and clear the D4 coverage census. #[cfg(feature = "profile")] diff --git a/crates/rusty_zstd/src/copies.rs b/crates/rusty_zstd/src/copies.rs index c194a69..6ec4e95 100644 --- a/crates/rusty_zstd/src/copies.rs +++ b/crates/rusty_zstd/src/copies.rs @@ -160,8 +160,9 @@ mod imp { #[cfg(feature = "profile")] mod imp { use super::N_COPY_SLOTS; + use crate::census64::AtomicU64; use core::cell::Cell; - use core::sync::atomic::{AtomicU64, Ordering::Relaxed}; + use core::sync::atomic::Ordering::Relaxed; pub(super) static G_BYTES: [AtomicU64; N_COPY_SLOTS] = [const { AtomicU64::new(0) }; N_COPY_SLOTS]; diff --git a/crates/rusty_zstd/src/encode.rs b/crates/rusty_zstd/src/encode.rs index be598f8..c199d06 100644 --- a/crates/rusty_zstd/src/encode.rs +++ b/crates/rusty_zstd/src/encode.rs @@ -1850,19 +1850,19 @@ pub fn set_prime_stride_arm(n: usize) { /// Deterministic work counter for the priming loop: positions inserted. /// Accumulated LOCALLY and published once per call -- an atomic inside the loop /// is the bricks 49/64/77 defect this campaign keeps finding. -pub static PRIME_ITERS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static PRIME_ITERS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// E4 ceiling probe: `[calls, positions hashed]` in the post-match fill helpers. /// E4 proposes batching these into a vector tile; a tile needs positions. /// N9 probe: rebuilds of the RFC-constant default FSE ctable in `select_seq_table`. -pub static N9_BASIC: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static N9_BASIC: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// ALLOC-8 probe: `[speculative EntropyState saves, saves actually ROLLED BACK]`. /// The save clones 3 FSE tables + a Huffman table per block; if the rollback /// almost never fires, the clone is almost pure waste. -pub static ENT_SAVE: [core::sync::atomic::AtomicU64; 2] = [ - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), +pub static ENT_SAVE: [crate::census64::AtomicU64; 2] = [ + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), ]; /// Read and clear the ALLOC-8 probe. pub fn take_ent_save() -> [u64; 2] { @@ -2203,10 +2203,10 @@ fn adaptive_block_max( base } -pub static G5_RPREV: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static G5_RPREV_N: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static G5_DRIFTSUM: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static G5_DRIFT_N: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static G5_RPREV: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static G5_RPREV_N: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static G5_DRIFTSUM: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static G5_DRIFT_N: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// GATE 5 inputs, as block means: `(mean r_prev, mean drift)`. pub fn take_g5_inputs() -> (f64, f64) { @@ -2221,9 +2221,9 @@ pub fn take_g5_inputs() -> (f64, f64) { ) } -pub static G5_CALLS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static G5_HIT_RATIO: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static G5_HIT_DRIFT: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static G5_CALLS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static G5_HIT_RATIO: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static G5_HIT_DRIFT: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// GATE 5 coverage: `(calls, raw-escape fires, drift fires)`. pub fn take_g5() -> (u64, u64, u64) { @@ -3303,10 +3303,10 @@ fn raw_probe_period() -> u32 { /// 1 = `early_raw_skip` (needs Fast + tlen 1..7) /// 2 = payload did not beat `raw_limit` #[cfg(feature = "profile")] -pub static RAW_EXIT: [core::sync::atomic::AtomicU64; 3] = [ - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), +pub static RAW_EXIT: [crate::census64::AtomicU64; 3] = [ + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), ]; /// Read and clear the three raw-exit counts. @@ -3322,15 +3322,15 @@ pub fn take_raw_exits() -> [u64; 3] { /// GATE 16 study: how far past `raw_limit` did blocks that went RAW land? #[cfg(feature = "profile")] -pub static RAW_MARGIN_SUM: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static RAW_MARGIN_SUM: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static RAW_MARGIN_N: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static RAW_MARGIN_N: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static RAW_MARGIN_HIST: [core::sync::atomic::AtomicU64; 4] = [ - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), +pub static RAW_MARGIN_HIST: [crate::census64::AtomicU64; 4] = [ + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), ]; /// Read and clear `(sum_permille, n, [<=1.01, <=1.05, <=1.20, >1.20])`. @@ -6405,9 +6405,9 @@ fn bt_fill_stride() -> usize { 1 } -pub static LF_FILLS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static LF_NONEMPTY: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static LF_INSERTS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static LF_FILLS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static LF_NONEMPTY: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static LF_INSERTS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// `(fill_sites_reached, sites_with_at_least_one_insert, total_inserts)` pub fn take_lazy_fill() -> (u64, u64, u64) { @@ -6489,19 +6489,19 @@ fn row_fill_stride() -> usize { /// Set the lazy back-fill stride in-process. /// GATE 6 next-long probe outcomes, for GATE 14's dispatch study. -pub static NL_PROBES_G: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static NL_HITS_G: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static NL_PROBES_G: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static NL_HITS_G: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Total match-length GAIN the next-long probe bought, across its hits. -pub static NL_GAIN_G: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static NL_GAIN_G: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Hits in the RAISED band (`best_ml >= 8`) -- what a higher cut newly enables. -pub static NL_BAND_HITS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static NL_BAND_GAIN: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static NL_BAND_OLD: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static NL_BAND_HITS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static NL_BAND_GAIN: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static NL_BAND_OLD: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Offsets the raised-band hits take, and the offsets they replace. -pub static NL_OFF_NEW: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static NL_OFF_OLD: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static NL_OFF_NEW: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static NL_OFF_OLD: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Raised-band hits whose new offset is LARGER than the one they replaced. -pub static NL_OFF_WORSE: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static NL_OFF_WORSE: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear `(off_new_sum, off_old_sum, hits_with_worse_offset)`. pub fn take_nl_off() -> (u64, u64, u64) { @@ -6769,7 +6769,7 @@ fn dfast_fill_stride() -> usize { /// fills (short and long counted separately). The sparse arm's saving is paid /// in this unit; §4.39 priced only the main-loop positions it costs and so /// called the arm "dominated" while ignoring the larger term. -pub static DF_ENDFILL: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static DF_ENDFILL: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear the per-match end-fill write count. pub fn take_dfast_endfill() -> u64 { @@ -6837,7 +6837,7 @@ fn dfast_fill_anchor_c() -> bool { } /// Interior back-fill positions inserted by GATE 12 @ L3's stride arm. -pub static DF_FILL: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static DF_FILL: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Bench hook: interior DFast back-fill inserts since the last call. pub fn take_dfast_fill() -> u64 { @@ -7131,11 +7131,11 @@ fn step_probe_on() -> bool { /// GATE 18 study: the measured step-2 forfeit, per mille x10. #[cfg(feature = "profile")] -pub static STEP_FORFEIT_SUM: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static STEP_FORFEIT_SUM: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static STEP_FORFEIT_N: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static STEP_FORFEIT_N: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static STEP_SEQ_SUM: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static STEP_SEQ_SUM: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear `(sum_x10000, n)`. #[cfg(feature = "profile")] @@ -7423,8 +7423,8 @@ fn hash4_tag_from(v: u64, hash_shift: u32, smask: u64) -> (usize, u8) { /// speculation serve? `MM_MISS / MM_TOTAL` is the share of positions that reach /// the miss-advance, i.e. where a speculated next-position load is CONSUMED. /// Gate 7 audit: tag rejections that `fast_probe` would have ACCEPTED. -pub static TAG_FALSE_REJECT: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static TAG_REJECT_TOTAL: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static TAG_FALSE_REJECT: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static TAG_REJECT_TOTAL: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear `(false_rejects, total_rejects)`. pub fn take_tag_rejects() -> (u64, u64) { @@ -7436,12 +7436,12 @@ pub fn take_tag_rejects() -> (u64, u64) { } /// GATE 2 candidate signal: rep match BYTES per rep PROBE. -pub static REP_PROBES: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static REP_BYTES: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static REP_PROBES: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static REP_BYTES: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); -pub static REP_HITS_G: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static ALL_MATCH_BYTES: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static ALL_SEQS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static REP_HITS_G: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static ALL_MATCH_BYTES: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static ALL_SEQS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// `(rep_probes, rep_bytes, rep_hits, all_match_bytes, all_seqs)` pub fn take_rep_rate() -> (u64, u64, u64, u64, u64) { @@ -7455,8 +7455,8 @@ pub fn take_rep_rate() -> (u64, u64, u64, u64, u64) { ) } -pub static MM_TOTAL: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static MM_MISS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static MM_TOTAL: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static MM_MISS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear `(main_loop_positions, positions_reaching_the_advance)`. pub fn take_mm() -> (u64, u64) { @@ -7467,11 +7467,11 @@ pub fn take_mm() -> (u64, u64) { /// ffanat: dispatch-arm census. 0 = specialised (false,false,pipe), /// 1 = tag arm (generic), 2 = rep arms (generic), 3 = rest. #[cfg(feature = "profile")] -pub static FF_ARM: [core::sync::atomic::AtomicU64; 4] = [ - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), +pub static FF_ARM: [crate::census64::AtomicU64; 4] = [ + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), ]; /// Read and clear the dispatch-arm census. @@ -7484,9 +7484,9 @@ pub fn take_ff_arms() -> [u64; 4] { o } -pub static FF_PIPE_BLOCKS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static FF_SPEC_MADE: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static FF_SPEC_USED: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static FF_PIPE_BLOCKS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static FF_SPEC_MADE: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static FF_SPEC_USED: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear `(pipelined_blocks, speculations_made, speculations_used)`. pub fn take_ff_pipe() -> (u64, u64, u64) { @@ -7810,9 +7810,9 @@ fn lit_short_min() -> f32 { } /// Deterministic instrument: guard evaluations that FAILED, i.e. wasted work. -pub static LP_GUARD_FAIL: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static LP_GUARD_FAIL: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Guard evaluations SKIPPED by the Gate 13 dispatch. -pub static LP_GUARD_SKIP: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static LP_GUARD_SKIP: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear the Gate 13 guard instruments. pub fn take_lp_guard() -> (u64, u64) { @@ -8017,13 +8017,13 @@ fn push_literals_tiers( } /// Literal run-length histogram: 0-4, 5-8, 9-16, 17-32, 33-64, 65+. -pub static LP_HIST: [core::sync::atomic::AtomicU64; 6] = [ - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), +pub static LP_HIST: [crate::census64::AtomicU64; 6] = [ + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), ]; /// Read and clear the literal run-length histogram. @@ -8037,8 +8037,8 @@ pub fn take_lit_hist() -> [u64; 6] { } /// Literal appends served by tier 2 (32 bytes) and tier 3 (64 bytes). -pub static LP_FAST2: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static LP_FAST3: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static LP_FAST2: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static LP_FAST3: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear the tier-2 and tier-3 counts. pub fn take_lit_tiers() -> (u64, u64) { @@ -8062,8 +8062,8 @@ pub fn take_lp_stats() -> ([u64; 6], u64, u64) { ) } -pub static LP_FAST: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static LP_SLOW: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static LP_FAST: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static LP_SLOW: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear `(fixed_width, fallback)` literal-append counts. pub fn take_lit_push() -> (u64, u64) { @@ -9966,14 +9966,14 @@ fn dfast_step_forced() -> usize { 1 } -pub static DFAST_MATCH_BYTES: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static DFAST_SEQS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static DFAST_BLOCK_BYTES: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static DFAST_MATCH_BYTES: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static DFAST_SEQS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static DFAST_BLOCK_BYTES: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); -pub static DFAST_BLOCKS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static DFAST_REP_BLOCKS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static DFAST_BLOCKS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static DFAST_REP_BLOCKS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Positions over which `try_rep1` is live -- the work a rep dispatch removes. -pub static DFAST_REP_POS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static DFAST_REP_POS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// `(blocks, rep_blocks, rep_positions)` pub fn take_dfast_rep_blocks() -> (u64, u64, u64) { @@ -9985,8 +9985,8 @@ pub fn take_dfast_rep_blocks() -> (u64, u64, u64) { ) } -pub static DFAST_REP_BYTES: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static DFAST_REP_HITS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static DFAST_REP_BYTES: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static DFAST_REP_HITS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// `(match_bytes, seqs, block_bytes, rep_bytes, rep_hits)` for DFast. pub fn take_dfast_match_stats() -> (u64, u64, u64, u64, u64) { @@ -10049,8 +10049,8 @@ static DFAST_PIPE_ARM: core::sync::atomic::AtomicU8 = core::sync::atomic::Atomic /// A/B the DFast 2-way software pipeline in-process -- both shapes, one binary, /// so the comparison is immune to cross-binary drift. -pub static DFAST_SPEC_MADE: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static DFAST_SPEC_USED: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static DFAST_SPEC_MADE: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static DFAST_SPEC_USED: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear `(speculations_made, speculations_consumed)`. pub fn take_dfast_spec() -> (u64, u64) { @@ -12552,9 +12552,9 @@ fn bt_depth_steps() -> u32 { 1 } -pub static BT_WALKS2: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static BT_ITERS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static BT_FULL: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static BT_WALKS2: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static BT_ITERS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static BT_FULL: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// `(walks, total_iterations, walks_that_used_ALL attempts)` pub fn take_bt_iters() -> (u64, u64, u64) { @@ -13347,14 +13347,14 @@ fn opt_rep_enabled() -> bool { /// GATE 10 @ L19: what the DP's repcode candidate earns. `try_rep1` runs at /// every position of every opt block, unconditionally. -pub static OPT_REP_PROBES: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static OPT_REP_HITS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static OPT_REP_BYTES: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static OPT_REP_PROBES: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static OPT_REP_HITS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static OPT_REP_BYTES: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); -pub static OPT_POS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static OPT_SKIP_INF: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static OPT_SKIP_JUMP: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static OPT_SKIP_JUMPS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static OPT_POS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static OPT_SKIP_INF: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static OPT_SKIP_JUMP: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static OPT_SKIP_JUMPS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// `(positions, skipped_price_inf, bytes_jumped, jumps)` pub fn take_opt_skips() -> (u64, u64, u64, u64) { @@ -13367,10 +13367,10 @@ pub fn take_opt_skips() -> (u64, u64, u64, u64) { ) } -pub static OPT_BT_CALLS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static OPT_BT_DRY: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static OPT_BT_LEN: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static OPT_SEQS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static OPT_BT_CALLS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static OPT_BT_DRY: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static OPT_BT_LEN: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static OPT_SEQS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// `(bt_calls, bt_calls_returning_nothing, total_match_len, emitted_seqs)` pub fn take_opt_bt() -> (u64, u64, u64, u64) { @@ -13475,7 +13475,7 @@ pub fn set_opt_fill_max_arm(v: usize) { } /// Positions inserted by the opt back-fill -- the work GATE 12 controls at L19. -pub static OPT_FILL_INS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static OPT_FILL_INS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear the opt back-fill insert count. pub fn take_opt_fill_ins() -> u64 { @@ -14380,9 +14380,9 @@ fn fused_ml(x: u64, src: &[u8], m: usize, ip: usize, block_end: usize) -> usize } #[cfg(feature = "profile")] -static FUSED_SHORT: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +static FUSED_SHORT: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -static FUSED_LONG: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +static FUSED_LONG: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// (short, long) fused-head resolutions since the last call -- BRICK 11's verdict. #[cfg(feature = "profile")] @@ -14669,8 +14669,8 @@ fn row_auto_ok(params: CompressionParameters, src_len: Option) -> bool { /// WIDE-CHAIN LATCH census: `[events, positions_rescanned]`. The latch does a /// full O(window) chain rebuild when it fires; this is what that costs. #[cfg(feature = "profile")] -pub static WIDE_LATCH: [core::sync::atomic::AtomicU64; 2] = - [const { core::sync::atomic::AtomicU64::new(0) }; 2]; +pub static WIDE_LATCH: [crate::census64::AtomicU64; 2] = + [const { crate::census64::AtomicU64::new(0) }; 2]; /// Read and clear `(latch_events, positions_rescanned)`. #[cfg(feature = "profile")] @@ -14697,8 +14697,8 @@ pub fn take_wide_latch() -> (u64, u64) { /// smaller `chain_mask` aliases many positions onto one slot, so the link a /// walk reads may belong to some other position entirely. #[cfg(feature = "profile")] -pub static WALK_EXIT: [core::sync::atomic::AtomicU64; 8] = - [const { core::sync::atomic::AtomicU64::new(0) }; 8]; +pub static WALK_EXIT: [crate::census64::AtomicU64; 8] = + [const { crate::census64::AtomicU64::new(0) }; 8]; /// Read and clear the walk-exit census. #[cfg(feature = "profile")] @@ -14716,7 +14716,7 @@ pub fn take_walk_exit() -> [u64; 8] { /// the same candidates with FEWER DEPENDENT LOADS, so both numbers are needed: /// `ROW_EXAM` is candidates, `ROW_LOADS` is rows touched (one load each). #[cfg(feature = "profile")] -pub static ROW_EXAM: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static ROW_EXAM: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] /// SECTION 14.9 census: the 16-to-1 bucket-sharing cost. /// `[examined, same_bucket, mls_eq_pass, gtag0_probes, probes]`. @@ -14725,12 +14725,12 @@ pub static ROW_EXAM: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU /// ends in a random `src` load inside `mls_eq` and (for mls >= 4) cannot /// possibly succeed. #[cfg(feature = "profile")] -pub static ROW_BUCKET: [core::sync::atomic::AtomicU64; 5] = [ - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), +pub static ROW_BUCKET: [crate::census64::AtomicU64; 5] = [ + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), ]; /// Read and clear the bucket-sharing census. #[cfg(feature = "profile")] @@ -14781,12 +14781,11 @@ pub(crate) fn dfast_bext_enabled() -> bool { /// backward walk WOULD recover at the commit point, and how many matches could /// move at all. Measurement only; the walk is not applied. #[cfg(feature = "profile")] -pub static DFAST_BEXT_BYTES: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static DFAST_BEXT_BYTES: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static DFAST_BEXT_MATCHES: core::sync::atomic::AtomicU64 = - core::sync::atomic::AtomicU64::new(0); +pub static DFAST_BEXT_MATCHES: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static DFAST_BEXT_SEQS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static DFAST_BEXT_SEQS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear `(bytes_recoverable, matches_that_could_move, matches_seen)`. #[cfg(feature = "profile")] @@ -14800,7 +14799,7 @@ pub fn take_dfast_bext() -> (u64, u64, u64) { } #[cfg(feature = "profile")] -pub static ROW_LOADS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static ROW_LOADS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear `(candidates_examined, rows_loaded)`. #[cfg(feature = "profile")] pub fn take_row_census() -> (u64, u64) { @@ -15061,9 +15060,9 @@ fn chain_null_tag(src: &[u8], mls: usize) -> u8 { /// FALSE-skip re-probe -- a skipped candidate whose bytes would have matched /// must never exist. #[cfg(feature = "profile")] -pub static LINK_SKIPS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static LINK_SKIPS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static LINK_FALSE: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static LINK_FALSE: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] pub fn take_link_tag() -> (u64, u64) { use core::sync::atomic::Ordering::Relaxed; @@ -15138,11 +15137,11 @@ pub static WALK_SIG_REP: core::sync::atomic::AtomicU32 = core::sync::atomic::Ato #[cfg(feature = "profile")] pub static WALK_SIG_SPB: core::sync::atomic::AtomicU32 = core::sync::atomic::AtomicU32::new(0); #[cfg(feature = "profile")] -pub static WALK_SIG_MB: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static WALK_SIG_MB: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static WALK_SIG_NS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static WALK_SIG_NS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static WALK_SIG_OB: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static WALK_SIG_OB: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] pub fn take_walk_signals() -> (f32, f32, f32, u64, u64, u64) { use core::sync::atomic::Ordering::Relaxed; @@ -15159,13 +15158,13 @@ pub fn take_walk_signals() -> (f32, f32, f32, u64, u64, u64) { /// Back-extension census for the SIMD question: (extensions > 0, total /// bytes, extensions >= 8 -- the class a u64 backward step would win). #[cfg(feature = "profile")] -pub static BEXT_N: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static BEXT_N: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static BEXT_BYTES: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static BEXT_BYTES: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static BEXT_GE8: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static BEXT_GE8: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static BEXT_MATCHES: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static BEXT_MATCHES: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] pub fn take_bext() -> (u64, u64, u64, u64) { use core::sync::atomic::Ordering::Relaxed; @@ -15192,15 +15191,15 @@ fn note_bext(ext: u64) { /// Chain-walk census: (candidates examined, byte-mismatch steps). #[cfg(feature = "profile")] -pub static WALK_EXAM: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static WALK_EXAM: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static WALK_BYTEMISS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static WALK_BYTEMISS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Walk-continue accept classes: (first-find past a collision -- legacy would /// have emitted a literal; upgrade past a collision -- legacy had a match). #[cfg(feature = "profile")] -pub static WALK_CONT_FIRST: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static WALK_CONT_FIRST: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static WALK_CONT_UPGRADE: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static WALK_CONT_UPGRADE: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] pub fn take_walk_classes() -> (u64, u64) { use core::sync::atomic::Ordering::Relaxed; @@ -15222,9 +15221,9 @@ pub fn take_walk_census() -> (u64, u64) { /// accept count is what a representation with an unambiguous null would /// change. #[cfg(feature = "profile")] -pub static WALK_M0: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static WALK_M0: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static WALK_M0_ACCEPT: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static WALK_M0_ACCEPT: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] pub fn take_walk_phantom() -> (u64, u64) { use core::sync::atomic::Ordering::Relaxed; @@ -15728,9 +15727,8 @@ pub fn set_fast_spec_arm(on: bool) { /// Which `find_dfast` body actually executed. Probe counts and output bytes are /// IDENTICAL between the two, by design -- they examine the same candidates in /// the same order -- so neither can show which one ran. These can. -pub static DFAST_SPEC_CALLS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static DFAST_RUNTIME_CALLS: core::sync::atomic::AtomicU64 = - core::sync::atomic::AtomicU64::new(0); +pub static DFAST_SPEC_CALLS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static DFAST_RUNTIME_CALLS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear both call counters. pub fn take_dfast_calls() -> (u64, u64) { @@ -15742,9 +15740,9 @@ pub fn take_dfast_calls() -> (u64, u64) { } /// Calls into `find_fast`'s Gate-4 dispatcher, for reachability proofs. -pub static FAST_CALLS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static FAST_CALLS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Calls into `find_opt` (L16+). -pub static OPT_CALLS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static OPT_CALLS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear the finder reachability counters: `(find_fast, find_opt)`. pub fn take_finder_calls() -> (u64, u64) { @@ -15756,8 +15754,8 @@ pub fn take_finder_calls() -> (u64, u64) { } /// Which `bt_find_best` body ran: `(specialised, runtime_fallback)`. -pub static BT_SPEC_CALLS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static BT_RUNTIME_CALLS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static BT_SPEC_CALLS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static BT_RUNTIME_CALLS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear the bt-path body counters. pub fn take_bt_calls() -> (u64, u64) { @@ -15954,21 +15952,21 @@ static PAIR_T_CACHE: core::sync::atomic::AtomicU32 = core::sync::atomic::AtomicU /// Each of these accessors calls `std::env::var` with no cache -- a /// GetEnvironmentVariableW plus a String allocation for a process constant. #[cfg(feature = "profile")] -pub static ENVHIT: [core::sync::atomic::AtomicU64; 14] = [ - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), +pub static ENVHIT: [crate::census64::AtomicU64; 14] = [ + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), ]; /// Read and clear the fitted-constant read counts. @@ -15984,18 +15982,18 @@ pub fn take_envhits() -> [u64; 14] { /// Diagnostic counters for Gate 6 candidate variables: how often the pair probe /// fires, how often it HITS, and how many bytes those hits cover. Activity vs /// outcome -- the campaign's law says the signal must predict the outcome. -pub static PAIR_PROBES: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static PAIR_HITS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static PAIR_BYTES: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static PAIR_PROBES: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static PAIR_HITS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static PAIR_BYTES: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Split the pair probe by the MAIN probe's slot state -- `m0 == 0` means that /// hash bucket has never been written, which is free information already in a /// register at the probe site. -pub static PAIR_M0_EMPTY: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static PAIR_M0_LIVE: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static PAIR_HIT_EMPTY: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static PAIR_HIT_LIVE: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static PAIR_BYTES_EMPTY: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static PAIR_BYTES_LIVE: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static PAIR_M0_EMPTY: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static PAIR_M0_LIVE: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static PAIR_HIT_EMPTY: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static PAIR_HIT_LIVE: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static PAIR_BYTES_EMPTY: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static PAIR_BYTES_LIVE: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// `(probes_empty, probes_live, hits_empty, hits_live, bytes_empty, bytes_live)` pub fn take_pair_split() -> (u64, u64, u64, u64, u64, u64) { @@ -16010,25 +16008,25 @@ pub fn take_pair_split() -> (u64, u64, u64, u64, u64, u64) { ) } -pub static MAIN_BYTES: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static MAIN_BYTES: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear: `(probes, hits, pair_match_bytes, all_match_bytes)`. -static ROUTE_HIST: [core::sync::atomic::AtomicU64; 3] = [ - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), +static ROUTE_HIST: [crate::census64::AtomicU64; 3] = [ + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), ]; -static ROUTE_GAIN: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -static ROUTE_REP: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -static ROUTE_N: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); - -static SIG_GAIN: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -static SIG_REP: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -static SIG_N: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -static SIG_TAG: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -static SIG_REPLEN: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -static SIG_NSEQ: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -static SIG_OPTREP: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +static ROUTE_GAIN: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +static ROUTE_REP: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +static ROUTE_N: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); + +static SIG_GAIN: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +static SIG_REP: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +static SIG_N: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +static SIG_TAG: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +static SIG_REPLEN: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +static SIG_NSEQ: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +static SIG_OPTREP: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// EVERY per-block content signal the encoder already maintains, as block means: /// `(pair_gain, rep_yield, tag_yield, rep_len_ratio, last_nseq, opt_rep_rate)`. @@ -16156,9 +16154,9 @@ fn cand_yield((f, t): (u64, u64)) -> f32 { /// L19-native accounting: tree probes, those too SHORT to use, and those that /// could not IMPROVE on the best so far. -pub static BT_PROBE: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static BT_SHORT: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub static BT_NOGAIN: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static BT_PROBE: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static BT_SHORT: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); +pub static BT_NOGAIN: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// `(probes, too_short, no_gain)` pub fn take_bt_probe_stats() -> (u64, u64, u64) { @@ -16314,11 +16312,11 @@ fn long_tag_enabled() -> bool { /// 1a ledger: (nonempty long probes, rejections, FALSE rejections). Three /// counters with one meaning each -- see the tag audit's instrument trap. #[cfg(feature = "profile")] -pub static LTAG_NONEMPTY: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static LTAG_NONEMPTY: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static LTAG_REJECT: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static LTAG_REJECT: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static LTAG_FALSE: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static LTAG_FALSE: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear the LONG-table tag audit: `[nonempty, rejected, FALSE]`. /// A FALSE rejection is one the byte compare would have ACCEPTED -- i.e. a @@ -16348,18 +16346,18 @@ pub fn take_long_tag() -> (u64, u64, u64) { /// at the MAIN long consume site. The fail share is the ceiling on what a /// stronger tag could still remove. #[cfg(feature = "profile")] -pub static LTAG_SURV_FAIL: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static LTAG_SURV_FAIL: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static LTAG_SURV_WFAIL: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static LTAG_SURV_WFAIL: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static LTAG_SURV_ACC: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static LTAG_SURV_ACC: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// SHORT-table consume-site residual, mirror of the long table's. #[cfg(feature = "profile")] -pub static STAG_SURV_FAIL: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static STAG_SURV_FAIL: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static STAG_SURV_WFAIL: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static STAG_SURV_WFAIL: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static STAG_SURV_ACC: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static STAG_SURV_ACC: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// `(bytes_fail, window_fail, accepted)` for the SHORT consume site. #[cfg(feature = "profile")] pub fn take_short_tag_residual() -> (u64, u64, u64) { @@ -16386,9 +16384,9 @@ pub fn take_long_tag_residual() -> (u64, u64, u64) { /// `TAGARR_READS` is a load from a SECOND random cache line; `PACKED_TAG_READS` /// reads the byte that arrived with the position. #[cfg(feature = "profile")] -pub static TAGARR_READS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static TAGARR_READS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static PACKED_TAG_READS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static PACKED_TAG_READS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear `(tag-array reads, packed reads)`. #[cfg(feature = "profile")] @@ -16423,13 +16421,13 @@ fn fast_pack_enabled() -> bool { /// every such candidate costs a random `src[m]` load, a compare, and a /// `count_match` that dies below `mls`. #[cfg(feature = "profile")] -pub static FF_LAZY_FIRES: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static FF_LAZY_FIRES: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static FF_LATCH: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static FF_LATCH: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static FF_CAND4: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static FF_CAND4: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static FF_ACCEPT: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static FF_ACCEPT: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear `(four-byte passes, accepted matches)`. #[cfg(feature = "profile")] diff --git a/crates/rusty_zstd/src/fse.rs b/crates/rusty_zstd/src/fse.rs index 9bee1cc..4659ee4 100644 --- a/crates/rusty_zstd/src/fse.rs +++ b/crates/rusty_zstd/src/fse.rs @@ -35,10 +35,10 @@ pub(crate) struct FseView<'a> { /// applies only when `high_threshold == table_size - 1`. This counts how often /// that is true, before anything is built. #[cfg(feature = "profile")] -pub static D6_SPREAD: [core::sync::atomic::AtomicU64; 3] = [ - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), +pub static D6_SPREAD: [crate::census64::AtomicU64; 3] = [ + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), ]; /// Read and clear the D6 probe. #[cfg(feature = "profile")] diff --git a/crates/rusty_zstd/src/huffman.rs b/crates/rusty_zstd/src/huffman.rs index b6538fc..3d1d2fa 100644 --- a/crates/rusty_zstd/src/huffman.rs +++ b/crates/rusty_zstd/src/huffman.rs @@ -27,10 +27,10 @@ pub(crate) struct HuffmanTable { /// The tree-merge loop is (n-1) iterations of an adaptive sort plus two /// `Vec::remove(0)` memmoves, so n^2 is the work proxy. #[cfg(feature = "profile")] -pub static N13_STATS: [core::sync::atomic::AtomicU64; 3] = [ - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), +pub static N13_STATS: [crate::census64::AtomicU64; 3] = [ + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), ]; /// Read and clear the N13 probe. #[cfg(feature = "profile")] @@ -53,9 +53,9 @@ pub fn take_n13_stats() -> [u64; 3] { #[cfg(feature = "profile")] /// `fast_4x2` outcomes: [bailed, succeeded]. #[cfg(feature = "profile")] -pub static F4X2_ARM: [core::sync::atomic::AtomicU64; 2] = [ - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), +pub static F4X2_ARM: [crate::census64::AtomicU64; 2] = [ + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), ]; /// Read and clear the `fast_4x2` outcome census. @@ -70,7 +70,7 @@ pub fn take_f4x2_arm() -> (u64, u64) { /// Sections that took `decode_4x`'s X1 arm. #[cfg(feature = "profile")] -pub static X4_X1_CALLS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static X4_X1_CALLS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Sections that took `decode_4x`'s X2 arm. /// @@ -86,7 +86,7 @@ pub static X4_X1_CALLS: core::sync::atomic::AtomicU64 = core::sync::atomic::Atom /// "OUTLINED, on a census" note rests on, and a conflated counter cannot /// support it in either direction. #[cfg(feature = "profile")] -pub static X4_X2_CALLS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static X4_X2_CALLS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear the X1-arm count. #[cfg(feature = "profile")] @@ -106,9 +106,9 @@ pub fn take_x4_arms() -> (u64, u64) { } #[cfg(feature = "profile")] -pub static X2_STATS: [core::sync::atomic::AtomicU64; 2] = [ - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), +pub static X2_STATS: [crate::census64::AtomicU64; 2] = [ + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), ]; /// Read and clear the N2 instrument: `(builds, uses)`. #[cfg(feature = "profile")] @@ -1456,9 +1456,9 @@ pub(crate) enum HuffUpdate { /// E11 census: `(literal bytes the old O(n) `covers` walk would have read, /// calls)`. `profile`-gated; the shipping build carries nothing. #[cfg(feature = "profile")] -pub static E11_WALKED: (core::sync::atomic::AtomicU64, core::sync::atomic::AtomicU64) = ( - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), +pub static E11_WALKED: (crate::census64::AtomicU64, crate::census64::AtomicU64) = ( + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), ); /// Read and clear the E11 census. @@ -2010,10 +2010,10 @@ fn huffman_nbits(freq: &[u32; 256]) -> Result<[u8; 256], Error> { /// E12 ceiling probe: `(calls, total inner-scan element visits, adjustment steps)`. #[cfg(feature = "profile")] -pub static E12_SCAN: [core::sync::atomic::AtomicU64; 3] = [ - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), +pub static E12_SCAN: [crate::census64::AtomicU64; 3] = [ + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), ]; /// Read and clear the E12 ceiling probe. #[cfg(feature = "profile")] diff --git a/crates/rusty_zstd/src/kreach.rs b/crates/rusty_zstd/src/kreach.rs index 38fc3b6..2536f43 100644 --- a/crates/rusty_zstd/src/kreach.rs +++ b/crates/rusty_zstd/src/kreach.rs @@ -91,8 +91,9 @@ mod imp { #[cfg(feature = "profile")] mod imp { use super::N_SLOTS; + use crate::census64::AtomicU64; use core::cell::Cell; - use core::sync::atomic::{AtomicU64, Ordering::Relaxed}; + use core::sync::atomic::Ordering::Relaxed; pub(super) static G_HIT: [AtomicU64; N_SLOTS] = [const { AtomicU64::new(0) }; N_SLOTS]; pub(super) static G_MISS: [AtomicU64; N_SLOTS] = [const { AtomicU64::new(0) }; N_SLOTS]; diff --git a/crates/rusty_zstd/src/ldm.rs b/crates/rusty_zstd/src/ldm.rs index 2e31e62..f6acabd 100644 --- a/crates/rusty_zstd/src/ldm.rs +++ b/crates/rusty_zstd/src/ldm.rs @@ -257,9 +257,9 @@ fn ldm_hash(src: &[u8], ip: usize, hash_log: u32) -> usize { /// population the old `memcmp` ran on) and those the 8-byte head let through /// to `count_eq`. #[cfg(feature = "profile")] -pub static LDM_CANDS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static LDM_CANDS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); #[cfg(feature = "profile")] -pub static LDM_COUNTS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static LDM_COUNTS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// `(candidates, counted)` since the last call. #[cfg(feature = "profile")] diff --git a/crates/rusty_zstd/src/lib.rs b/crates/rusty_zstd/src/lib.rs index 8592d4d..3e4945a 100644 --- a/crates/rusty_zstd/src/lib.rs +++ b/crates/rusty_zstd/src/lib.rs @@ -101,7 +101,7 @@ pub(crate) fn env_knob_is1(name: &str) -> bool { /// Count of `env_knob` reads -- see the note there. #[cfg(feature = "profile")] -pub static ENV_READS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static ENV_READS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear the env-read counter. #[cfg(feature = "profile")] @@ -119,6 +119,10 @@ pub(crate) fn env_knob(_name: &str) -> Result { mod bit; mod block; +/// The census counters' 64-bit atomic, and the zero-sized stub it becomes +/// on a part without 64-bit atomics. See the module docs: a zero read there +/// means "not measurable on this target", never "measured zero". +pub mod census64; mod compressed; /// Copy census: how many times does the encoder move each input byte? pub mod copies; diff --git a/crates/rusty_zstd/src/prof.rs b/crates/rusty_zstd/src/prof.rs index a15e7bc..d50c582 100644 --- a/crates/rusty_zstd/src/prof.rs +++ b/crates/rusty_zstd/src/prof.rs @@ -183,9 +183,10 @@ pub struct EncodeCounts { #[cfg(feature = "profile")] mod on { use super::{BlockTap, EncodeCounts, Stage, NAMES, N_STAGES}; + use crate::census64::AtomicU64; use crate::profclock as cpuclock; use core::cell::RefCell; - use core::sync::atomic::{AtomicU64, Ordering}; + use core::sync::atomic::Ordering; thread_local! { static BLOCK_TAPS: RefCell> = const { RefCell::new(alloc::vec::Vec::new()) }; diff --git a/crates/rusty_zstd/src/profclock.rs b/crates/rusty_zstd/src/profclock.rs index 583ca3b..92314c0 100644 --- a/crates/rusty_zstd/src/profclock.rs +++ b/crates/rusty_zstd/src/profclock.rs @@ -46,7 +46,8 @@ mod imp { // This module is FFI by definition: the whole point is to reach a // clock the standard library does not expose. #![allow(unsafe_code)] - use core::sync::atomic::{AtomicU64, Ordering}; + use crate::census64::AtomicU64; + use core::sync::atomic::Ordering; #[link(name = "kernel32")] extern "system" { diff --git a/crates/rusty_zstd/src/rowfind.rs b/crates/rusty_zstd/src/rowfind.rs index 5c3eb69..42626de 100644 --- a/crates/rusty_zstd/src/rowfind.rs +++ b/crates/rusty_zstd/src/rowfind.rs @@ -50,10 +50,10 @@ pub(crate) const ROW: usize = 16; /// models are evaluated from the SAME mask in one run, so the comparison needs /// no A/B build and carries no clock. #[cfg(feature = "profile")] -pub static ROW_WALK: [core::sync::atomic::AtomicU64; 3] = [ - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), +pub static ROW_WALK: [crate::census64::AtomicU64; 3] = [ + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), ]; /// Read and clear the row-walk census. #[cfg(feature = "profile")] diff --git a/crates/rusty_zstd/src/scratch.rs b/crates/rusty_zstd/src/scratch.rs index eff6f1f..6e1b943 100644 --- a/crates/rusty_zstd/src/scratch.rs +++ b/crates/rusty_zstd/src/scratch.rs @@ -270,20 +270,20 @@ pub(crate) fn note_pool(hit: bool) { /// Pool hit/miss census. A miss is an allocation. #[cfg(feature = "profile")] -pub static POOL_HIT: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static POOL_HIT: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Pool misses -- each one is a fresh allocation. #[cfg(feature = "profile")] -pub static POOL_MISS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static POOL_MISS: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Buffers handed back with real capacity. #[cfg(feature = "profile")] -pub static POOL_GIVE: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static POOL_GIVE: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Buffers handed back with ZERO capacity -- never pooled, so the matching /// take must allocate. A take/give imbalance shows up here first. #[cfg(feature = "profile")] -pub static POOL_GIVE_EMPTY: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static POOL_GIVE_EMPTY: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Buffers DROPPED because the free list was full. #[cfg(feature = "profile")] -pub static POOL_DROP: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub static POOL_DROP: crate::census64::AtomicU64 = crate::census64::AtomicU64::new(0); /// Read and clear the pool census: `(hits, misses, drops, gives, give_empty)`. #[cfg(feature = "profile")] diff --git a/crates/rusty_zstd/src/simd.rs b/crates/rusty_zstd/src/simd.rs index 4d06ed5..a9fd659 100644 --- a/crates/rusty_zstd/src/simd.rs +++ b/crates/rusty_zstd/src/simd.rs @@ -180,8 +180,9 @@ fn eqlen_arm() -> u8 { /// benches compress single-threaded, so the receipt is exact for them. #[cfg(feature = "profile")] mod counters { + use crate::census64::AtomicU64; use core::cell::Cell; - use core::sync::atomic::{AtomicU64, Ordering::Relaxed}; + use core::sync::atomic::Ordering::Relaxed; pub(super) static G_CALLS: AtomicU64 = AtomicU64::new(0); pub(super) static G_WIDE: AtomicU64 = AtomicU64::new(0); diff --git a/crates/rusty_zstd/src/stream.rs b/crates/rusty_zstd/src/stream.rs index d392913..dae0101 100644 --- a/crates/rusty_zstd/src/stream.rs +++ b/crates/rusty_zstd/src/stream.rs @@ -17,9 +17,9 @@ use crate::DecompressOptions; /// The memmove per `decoded.drain(..drop)` is `len - drop` -- the retained /// window -- and section 18's dig found it running once per `stream()` CALL. #[cfg(feature = "profile")] -pub static DEC_COMPACT: [core::sync::atomic::AtomicU64; 2] = [ - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), +pub static DEC_COMPACT: [crate::census64::AtomicU64; 2] = [ + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), ]; /// Read and clear the decoder-compaction census. #[cfg(feature = "profile")] @@ -35,9 +35,9 @@ pub fn take_dec_compact() -> [u64; 2] { /// Each slide memmoves the retained window, zeroes six match tables and /// re-primes the whole window -- section 19's decoder finding, mirrored. #[cfg(feature = "profile")] -pub static ENC_SLIDE: [core::sync::atomic::AtomicU64; 2] = [ - core::sync::atomic::AtomicU64::new(0), - core::sync::atomic::AtomicU64::new(0), +pub static ENC_SLIDE: [crate::census64::AtomicU64; 2] = [ + crate::census64::AtomicU64::new(0), + crate::census64::AtomicU64::new(0), ]; /// History multiplier at which the encoder slides its window. /// diff --git a/crates/rusty_zstd/src/xxh64.rs b/crates/rusty_zstd/src/xxh64.rs index 9521840..a301623 100644 --- a/crates/rusty_zstd/src/xxh64.rs +++ b/crates/rusty_zstd/src/xxh64.rs @@ -421,7 +421,8 @@ fn finish_tail(mut tail: &[u8], mut acc: u64, total: u64) -> u64 { /// `profile`-gated so the shipping path carries nothing. #[cfg(feature = "profile")] pub mod census { - use core::sync::atomic::{AtomicU64, Ordering}; + use crate::census64::AtomicU64; + use core::sync::atomic::Ordering; pub static HYBRID_BYTES: AtomicU64 = AtomicU64::new(0); pub static SCALAR_BYTES: AtomicU64 = AtomicU64::new(0); pub static HYBRID_CALLS: AtomicU64 = AtomicU64::new(0);