perf(encode): 1.08-1.27x faster matchfind at L3-L12, byte-identical; rusty_alloc 2.0.5

.github/workflows/ci.yml

@@ -34,6 +34,29 @@ jobs:
       - uses: Swatinem/rust-cache@v2
       - name: Test
         run: cargo test -p rusty_zstd -p rusty_zstd-cli -p rzstd-alloc
+      # KERNEL REACH. A twin that exists but is not CALLED passes every other
+      # gate in this file: byte-identity passes because the two paths agree by
+      # design, the round-trip passes, and an arm-toggle A/B reads FLAT --
+      # indistinguishable from "the kernel does not help". Exactly that shipped
+      # here: the xxh64 AVX2 kernel was reachable only from a test and a bench,
+      # and DECODE ran it on 0% of its bytes for months. This counts instead.
+      #
+      # Needs `profile` for the census taps (they compile to nothing without
+      # it, so the shipped build is unaffected). Slots whose ISA the runner
+      # lacks are skipped, not failed.
+      - name: Kernel reach gate
+        run: cargo test -p rusty_zstd --release --features profile --test kreach_gate -- --nocapture
+      # The gate must be able to FAIL. Forcing every arm scalar has to break
+      # it; if this step SUCCEEDS, the census is detached from the dispatch it
+      # names and the green run above meant nothing.
+      - name: Kernel reach gate self-check (must fail)
+        shell: bash
+        run: |
+          if RZSTD_KREACH_POISON=1 cargo test -p rusty_zstd --release --features profile --test kreach_gate; then
+            echo '::error::kreach gate passed with every arm forced scalar -- the census is not wired'
+            exit 1
+          fi
+          echo 'poison check ok: the gate fails when the arms are forced scalar'
       - name: Doc tests (the README's examples)
         run: cargo test -p rusty_zstd --doc
       - name: CLI version + aliases

bench/vsc.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+# Shipped-CLI vs shipped-zstd, same input, single-threaded both sides.
+#
+# Why CLI-vs-CLI and not the in-process bench: `rzstd-bench` installs
+# `rzstd-alloc` as its global allocator, and that allocator's background thread
+# is charged to our process. Pinned to ONE core it contends with the codec
+# thread -- `cores_busy` reads ~2.0 against the reference's 1.0, which is a
+# work-parity violation and voids the comparison. Our CLI uses the system
+# allocator (control measured: cores_busy 0.72-0.91), so CLI-vs-CLI is
+# like-for-like.
+#
+# Discipline: arms ABBA-alternated so drift cancels instead of landing on one
+# arm; min-of-N, because the floor is what survives a noisy box; a NULL arm
+# (ours against itself) to establish what this box can resolve; and work parity
+# asserted per row by decoding BOTH outputs and comparing byte counts.
+set -u
+Z=${Z:-./third_party/zstd/extracted/zstd-v1.5.7-win64/zstd.exe}
+US=${US:-./target/release/rzstd.exe}
+REPS=${REPS:-5}
+LEVELS=${LEVELS:-"1 3 9"}
+CORPORA=${CORPORA:-"dickens samba webster mozilla x-ray nci xml osdb"}
+
+t() { python -c "import time;print(repr(time.perf_counter()))"; }
+mn() { python -c "print(repr(min($1,$2)))"; }
+
+printf "%-9s %2s %9s %10s %10s %7s %9s %7s\n" \
+  corpus L src_MiB us_MB/s zstd_MB/s "zstd/us" size_us/c null
+for id in $CORPORA; do
+  f="corpora/data/silesia/$id"; [ -f "$f" ] || f="corpora/data/generated/$id"
+  [ -f "$f" ] || continue
+  src=$(stat -c%s "$f")
+  for L in $LEVELS; do
+    "$US" -"$L" -c "$f" > /tmp/vs_us.zst 2>/dev/null
+    "$Z"  -"$L" -T1 -c "$f" > /tmp/vs_c.zst 2>/dev/null
+    us_b=$(stat -c%s /tmp/vs_us.zst); c_b=$(stat -c%s /tmp/vs_c.zst)
+    d1=$("$US" -d -c /tmp/vs_us.zst 2>/dev/null | wc -c)
+    d2=$("$Z"  -d -c /tmp/vs_c.zst  2>/dev/null | wc -c)
+    if [ "$d1" != "$src" ] || [ "$d2" != "$src" ]; then
+      printf "%-9s %2s  VOID work parity: decoded %s / %s vs src %s\n" "$id" "$L" "$d1" "$d2" "$src"
+      continue
+    fi
+    bu=1e9; bc=1e9; bn=1e9
+    for i in $(seq 1 "$REPS"); do
+      if [ $((i % 2)) -eq 0 ]; then
+        a0=$(t); "$US" -"$L" -c "$f" >/dev/null 2>&1; a1=$(t)
+        b0=$(t); "$Z" -"$L" -T1 -c "$f" >/dev/null 2>&1; b1=$(t)
+      else
+        b0=$(t); "$Z" -"$L" -T1 -c "$f" >/dev/null 2>&1; b1=$(t)
+        a0=$(t); "$US" -"$L" -c "$f" >/dev/null 2>&1; a1=$(t)
+      fi
+      n0=$(t); "$US" -"$L" -c "$f" >/dev/null 2>&1; n1=$(t)
+      bu=$(mn "$bu" "$a1-$a0"); bc=$(mn "$bc" "$b1-$b0"); bn=$(mn "$bn" "$n1-$n0")
+    done
+    python - <<PY
+mib=$src/1048576.0; bu=$bu; bc=$bc; bn=$bn
+print(f"{'$id':<9} {'$L':>2} {mib:9.1f} {mib/bu:10.1f} {mib/bc:10.1f} "
+      f"{bu/bc:7.2f} {$us_b/$c_b:9.4f} {bn/bu:7.3f}")
+PY
+  done
+done
+echo
+echo "zstd/us = how many times faster the C reference is. size_us/c = our output"
+echo "over theirs. null = our arm against itself; a result no further from 1.0"
+echo "than the null is not a result. Work parity asserted per row."

crates/rusty_zstd-bench/examples/accel10.rs

@@ -0,0 +1,37 @@
+//! Per-corpus size impact of the incompressible-section acceleration, and a
+//! round-trip on every cell. An aggregate near zero can hide a big regression
+//! cancelled by a big gain; this checks.
+use rusty_zstd as rz;
+const IDS: &[&str] = &["jsonlog-16m","smallmsg-8m","versions-16m","mr","ooffice","osdb",
+    "reymont","sao","webster","dickens","mozilla","nci","samba","xml","x-ray",
+    "text-32m","incomp-32m","zeros-32m"];
+fn main() {
+    let sh: usize = std::env::args().nth(1).and_then(|s| s.parse().ok()).unwrap_or(10);
+    for cap in [1usize << 20, 4 << 20] {
+        println!("\n===== shift {sh}, cap {} MiB =====", cap >> 20);
+        println!("{:<14}{:>10}{:>10}{:>10}{:>10}", "corpus", "L5 d", "L7 d", "L9 d", "L12 d");
+        println!("{}", "-".repeat(54));
+        let mut tot = [0i64; 4];
+        for id in IDS {
+            let Ok(f) = std::fs::read(format!("corpora/data/generated/{id}"))
+                .or_else(|_| std::fs::read(format!("corpora/data/silesia/{id}"))) else { continue };
+            let s = &f[..f.len().min(cap)];
+            print!("{:<14}", id);
+            for (k, lvl) in [5i32, 7, 9, 12].iter().enumerate() {
+                let o = rz::CompressOptions { level: *lvl, checksum: false };
+                rz::set_lazy_accel_arm(0);
+                let a = rz::compress_with(s, o).unwrap().len() as i64;
+                rz::set_lazy_accel_arm(sh);
+                let z = rz::compress_with(s, o).unwrap();
+                assert_eq!(rz::decompress(&z).unwrap(), s, "{id} L{lvl} round-trip");
+                let d = z.len() as i64 - a;
+                tot[k] += d;
+                print!("{:>10}", d);
+            }
+            println!();
+        }
+        rz::set_lazy_accel_arm(0);
+        println!("{:<14}{:>10}{:>10}{:>10}{:>10}   <== TOTAL",
+                 "", tot[0], tot[1], tot[2], tot[3]);
+    }
+}

crates/rusty_zstd-bench/examples/accelsweep.rs

@@ -0,0 +1,50 @@
+//! Sweep C's incompressible-section acceleration on our chain ladder.
+//!
+//! Two questions, two currencies:
+//!  * SIZE across the board -- exact, load-immune.
+//!  * TIME on incompressible data -- measured ONLY as an arm-vs-arm ratio in
+//!    ONE process with a null, so a loaded box moves both arms together.
+use rusty_zstd as rz;
+use std::time::Instant;
+const IDS: &[&str] = &["jsonlog-16m","smallmsg-8m","mr","ooffice","osdb","reymont","sao",
+    "webster","dickens","mozilla","nci","samba","xml","x-ray","text-32m","incomp-32m"];
+fn main() {
+    let cap = 1usize << 20;
+    let srcs: Vec<(&str, Vec<u8>)> = IDS.iter().filter_map(|id| {
+        std::fs::read(format!("corpora/data/generated/{id}"))
+            .or_else(|_| std::fs::read(format!("corpora/data/silesia/{id}")))
+            .ok().map(|f| { let n = f.len().min(cap); (*id, f[..n].to_vec()) })
+    }).collect();
+    let inc: Vec<u8> = srcs.iter().find(|(i, _)| *i == "incomp-32m").unwrap().1.clone();
+    let go = |lvl: i32| -> u64 { srcs.iter().map(|(_, s)|
+        rz::compress_with(s, rz::CompressOptions { level: lvl, checksum: false })
+            .unwrap().len() as u64).sum() };
+    let t_inc = |lvl: i32| -> f64 {
+        let p = rz::compression_params(lvl, Some(inc.len() as u64)).unwrap();
+        let mut b = f64::MAX;
+        for _ in 0..9 {
+            let t = Instant::now();
+            let z = rz::compress_with_params(&inc, p, false).unwrap();
+            let e = t.elapsed().as_secs_f64();
+            std::hint::black_box(z.len());
+            if e < b { b = e }
+        }
+        b * 1000.0
+    };
+    for lvl in [5i32, 7, 9, 12] {
+        rz::set_lazy_accel_arm(0);
+        let base = go(lvl);
+        let tb = t_inc(lvl);
+        let tb2 = t_inc(lvl);                       // null arm: same setting twice
+        println!("\n=== L{lvl} ===  base {base} B | incomp {tb:.2} ms (null {:+.1}%)",
+                 (tb2 / tb - 1.0) * 100.0);
+        println!("  {:>6}{:>12}{:>10}{:>12}{:>10}", "shift", "bytes", "d size", "incomp ms", "speedup");
+        for sh in [4usize, 6, 7, 8, 9, 10, 12] {
+            rz::set_lazy_accel_arm(sh);
+            let n = go(lvl);
+            let t = t_inc(lvl);
+            println!("  {:>6}{:>12}{:>+10}{:>12.2}{:>9.2}x", sh, n, n as i64 - base as i64, t, tb / t);
+        }
+        rz::set_lazy_accel_arm(0);
+    }
+}