fix(encode): copy the dictionary where the reference copies it, index it the way it does, and stop re-entering the parser per literal

ffi-bench/tests/dictionary_ffi.rs

@@ -122,3 +122,84 @@ fn dict_frames_decode_with_c_across_levels_and_reuse() {
         }
     }
 }
+
+/// A dictionary frame on the optimal band must not come out LARGER than the
+/// reference's, on input the dictionary describes well.
+///
+/// The parser walks a binary tree it fills lazily: after a search it jumps its
+/// insert cursor to where the match it found ENDS, on the reasoning that the
+/// positions inside a match are covered. That end is the end of the match in
+/// the SOURCE (upstream `matchEndIdx = matchIndex + matchLength`,
+/// zstd_opt.c:747-748 and :794-795), and for a candidate drawn from the
+/// dictionary the source lies BEFORE the position being searched. Measuring the
+/// jump from the searched position instead skips the positions in between, they
+/// never enter the tree, and a later search finds an empty bucket where the
+/// reference finds a long match. It only shows with a dictionary attached,
+/// because only a dictionary candidate sits that far back.
+///
+/// Level 11 at 4 KiB is where it bites hardest: upstream resolves it to btopt
+/// with `searchLog` 3, so a search gets eight candidates and cannot afford to
+/// look in an empty bucket.
+#[test]
+fn dict_frames_on_the_optimal_band_are_no_larger_than_the_reference() {
+    use structured_zstd::decoding::Dictionary;
+    use structured_zstd::encoding::{CompressionLevel, FrameCompressor};
+
+    // The benchmark's `small-4k-log-lines` scenario, byte for byte: its lines
+    // carry a region field the shorter fixture above does not, and which long
+    // matches the dictionary offers is exactly what this pins.
+    const LINES: &[&str] = &[
+        "ts=2026-03-26T21:39:28Z level=INFO msg=\"flush memtable\" tenant=demo table=orders region=eu-west\n",
+        "ts=2026-03-26T21:39:29Z level=INFO msg=\"rotate segment\" tenant=demo table=orders region=eu-west\n",
+        "ts=2026-03-26T21:39:30Z level=INFO msg=\"compact level\" tenant=demo table=orders region=eu-west\n",
+        "ts=2026-03-26T21:39:31Z level=INFO msg=\"write block\" tenant=demo table=orders region=eu-west\n",
+    ];
+    let payload = {
+        let mut bytes = Vec::with_capacity(4 * 1024);
+        while bytes.len() < 4 * 1024 {
+            for line in LINES {
+                let remaining = (4 * 1024) - bytes.len();
+                if remaining == 0 {
+                    break;
+                }
+                bytes.extend_from_slice(&line.as_bytes()[..line.len().min(remaining)]);
+            }
+        }
+        bytes
+    };
+    // Trained the way the benchmark trains it: 256-byte samples, an eighth of
+    // the input as the size request.
+    let samples: Vec<&[u8]> = payload.chunks(256).collect();
+    let dict = zstd::dict::from_samples(&samples, payload.len() / 8)
+        .expect("dictionary should train from the log-line samples");
+
+    for level in [10i32, 11, 12, 13] {
+        let mut cctx: FrameCompressor = FrameCompressor::new(CompressionLevel::Level(level));
+        cctx.set_dictionary_id_flag(false);
+        cctx.set_dictionary(
+            Dictionary::from_serialized_or_raw_content(dict.as_slice()).expect("dictionary parses"),
+        )
+        .expect("attach dict");
+        let mut reference = zstd::bulk::Compressor::with_dictionary(level, dict.as_slice())
+            .expect("reference accepts the dictionary");
+        let theirs = reference
+            .compress(&payload)
+            .expect("reference compresses the payload");
+
+        // Three frames on the SAME compressor. A reused dictionary context
+        // advances the history base between frames, so anything the parser
+        // keeps in absolute coordinates has to carry that base: a value left
+        // relative to the live history reads correctly on the first frame and
+        // silently stops working on the second.
+        for frame in 0..3 {
+            cctx.set_source_size_hint(payload.len() as u64);
+            let ours = cctx.compress_independent_frame(&payload);
+            assert!(
+                ours.len() <= theirs.len(),
+                "level {level} frame {frame}: {} bytes against the reference's {}",
+                ours.len(),
+                theirs.len(),
+            );
+        }
+    }
+}

ffi-bench/tests/zz_cparams.rs

@@ -21,7 +21,13 @@ fn reference_cparams(level: i32, src: u64, dict: usize) -> (u32, u32, u32, u32,
 
 #[test]
 fn cparams_match_reference_over_grid() {
-    let levels = [-7, -5, -3, -1, 0, 1, 2, 3, 4, 5, 7, 10, 12, 15, 17, 19, 22];
+    // Every level, not a sample of them: the sampled list skipped 11 and 13,
+    // and a level row is exactly the kind of thing that can be wrong on its own
+    // while its neighbours are right.
+    let levels = [
+        -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17,
+        18, 19, 20, 21, 22,
+    ];
     let srcs: [u64; 9] = [0, 1, 100, 4096, 6806, 16_384, 100_000, 131_072, 5_000_000];
     let dicts: [usize; 5] = [0, 437, 2048, 65_536, 1_000_000];
     let mut mismatches = 0usize;

zstd/examples/encode_loop_dict.rs

@@ -82,8 +82,18 @@ fn main() {
     let iters: u32 = args.get(2).and_then(|s| s.parse().ok()).unwrap_or(20_000);
     let input_spec: &str = args.get(3).map(|s| s.as_str()).unwrap_or("logs4096");
     let dict_path: Option<&str> = args.get(4).map(|s| s.as_str());
+    // 6th arg `alt<N>`: alternate each frame between the input and its first N
+    // bytes. A dictionary resolves attach-vs-copy from the source size, so two
+    // sizes either side of that cutoff make the compressor switch modes every
+    // frame — the shape that shows what switching costs, and the one a caller
+    // with variable-sized records actually has.
+    let alt_len: Option<usize> = args
+        .get(5)
+        .and_then(|s| s.strip_prefix("alt"))
+        .and_then(|n| n.parse().ok());
 
     let src = resolve_input(input_spec);
+    let alt = alt_len.map(|n| src[..n.min(src.len())].to_vec());
 
     // One reused compressor: matcher tables + any dictionary parse happen
     // once here, mirroring a consumer that reuses a context across N
@@ -104,15 +114,31 @@ fn main() {
     // zero output allocation.
     let mut out: Vec<u8> = Vec::new();
     let mut sink: usize = 0;
-    for _ in 0..iters {
-        cctx.compress_independent_frame_into(&src, &mut out);
+    for i in 0..iters {
+        let frame = match (&alt, i % 2) {
+            (Some(short), 1) => short.as_slice(),
+            _ => src.as_slice(),
+        };
+        cctx.compress_independent_frame_into(frame, &mut out);
         sink = sink.wrapping_add(out.len());
         core::hint::black_box(&out);
     }
 
+    // Under `alt<N>` the frames are not all `src.len()` long, so the total is
+    // counted from the schedule rather than multiplied out. Odd iterations take
+    // the short frame, which is `iters / 2` of them. Counting it here instead of
+    // accumulating inside the loop keeps the timed body exactly as it is
+    // measured.
+    let short_frames = (iters / 2) as usize;
+    let long_frames = iters as usize - short_frames;
+    let input_bytes = match &alt {
+        Some(short) => long_frames * src.len() + short_frames * short.len(),
+        None => iters as usize * src.len(),
+    };
+
     eprintln!(
-        "encoded {} bytes × {} iters at level {} dict={}; last-out-sum={}",
-        src.len(),
+        "encoded {} input bytes in {} iters at level {} dict={}; last-out-sum={}",
+        input_bytes,
         iters,
         level,
         dict_path.unwrap_or("none"),

zstd/src/bin/structured-zstd/main.rs

@@ -176,6 +176,37 @@ fn parse_size(text: &str) -> Result<u64> {
         .ok_or_else(|| eyre!("size `{text}` does not fit in 64 bits"))
 }
 
+/// Read the leading unsigned number the way upstream's argument reader does
+/// (`readU32FromCharChecked`, zstdcli.c:350-376): a run of decimal digits,
+/// then an optional `K` or `M` multiplier which may be spelled `KiB` / `MB`.
+/// Reading STOPS there and the remainder is handed back — no sign is accepted,
+/// and an empty digit run reads as zero, which every caller treats as invalid.
+fn read_leading_u32(text: &str) -> Result<(u32, &str)> {
+    let bytes = text.as_bytes();
+    let mut at = 0;
+    let mut value: u32 = 0;
+    while at < bytes.len() && bytes[at].is_ascii_digit() {
+        value = value
+            .checked_mul(10)
+            .and_then(|v| v.checked_add(u32::from(bytes[at] - b'0')))
+            .ok_or_else(|| eyre!("numeric value `{text}` overflows 32-bit unsigned int"))?;
+        at += 1;
+    }
+    if at < bytes.len() && matches!(bytes[at], b'K' | b'M') {
+        let shifts = if bytes[at] == b'M' { 2 } else { 1 };
+        for _ in 0..shifts {
+            value = value
+                .checked_mul(1024)
+                .ok_or_else(|| eyre!("numeric value `{text}` overflows 32-bit unsigned int"))?;
+        }
+        at += 1;
+        // `KiB` and `KB` are the same multiplier spelled longer.
+        at += usize::from(bytes.get(at) == Some(&b'i'));
+        at += usize::from(bytes.get(at) == Some(&b'B'));
+    }
+    Ok((value, &text[at..]))
+}
+
 /// Parse a `-M` / `--memory` value into bytes, or `None` for "the default".
 ///
 /// The default unit is MiB, as upstream documents, so `-M256` is 256 MiB. A
@@ -518,17 +549,22 @@ fn parse_args(
                         // `--fast` is the level -1 alias.
                         opts.level = -1;
                     } else if let Some(v) = long.strip_prefix("fast=") {
-                        // `--fast=N` is level -N for a positive N. Parse as
-                        // unsigned so `--fast=-5` is rejected rather than flipping
-                        // sign into a positive level. Exact-match the prefix so a
-                        // typo like `--faster` falls through to unknown-option.
-                        let n = v.parse::<u32>().wrap_err("invalid --fast level")?;
+                        // `--fast=N` is level -N (upstream zstd,
+                        // zstdcli.c:1133-1153). The factor is the LEADING
+                        // number only, so `--fast=3.5` is level -3 and the tail
+                        // is dropped; a factor past the minimum level clamps
+                        // rather than failing; only a zero factor is an error.
+                        // Exact-match the prefix so a typo like `--faster`
+                        // falls through to unknown-option.
+                        let (n, _tail) = read_leading_u32(v).wrap_err("invalid --fast level")?;
                         // Zero would negate to level 0, which is the ordinary
                         // default rather than a fast one.
                         if n == 0 {
                             bail!("--fast level must be at least 1, got 0");
                         }
-                        opts.level = -i32::try_from(n).wrap_err("--fast level too large")?;
+                        let capped = n.min(CompressionLevel::MIN_LEVEL.unsigned_abs());
+                        opts.level = -i32::try_from(capped)
+                            .expect("capped at |MIN_LEVEL|, which is an i32 magnitude");
                     } else if long.starts_with("use-dict=") {
                         opts.dict = Some(attached_path(arg_os, "--use-dict=".len()));
                     } else if let Some(v) = long.strip_prefix("maxdict=") {

zstd/src/bin/structured-zstd/tests.rs

@@ -1376,6 +1376,44 @@ fn fast_flag_maps_to_negative_level() {
     assert!(parse(&["--fast=0"]).is_err());
 }
 
+#[test]
+fn fast_level_reads_a_digit_run_and_ignores_the_tail() {
+    // upstream zstd (zstdcli.c:1139 -> readU32FromCharChecked, 350-376): the
+    // acceleration factor is the LEADING digit run; the parser stops at the
+    // first byte that is neither a digit nor a K/M multiplier and `--fast`
+    // never looks at what is left. `zstd --fast=3.5` compresses at level -3,
+    // so refusing it turns a working command line into an error.
+    assert_eq!(parse(&["--fast=3.5"]).unwrap().level, -3);
+    assert_eq!(parse(&["--fast=3x"]).unwrap().level, -3);
+    assert_eq!(parse(&["--fast=3G"]).unwrap().level, -3);
+    // A sign is not a digit, so the run is empty and the factor is zero —
+    // which upstream rejects. Rust's own integer parser accepts `+3`, and
+    // accepting it here would take a level upstream refuses.
+    assert!(parse(&["--fast=+3"]).is_err());
+}
+
+#[test]
+fn fast_level_honours_the_k_and_m_multipliers() {
+    // Same reader, suffix half (zstdcli.c:362-373): `K` shifts by 10, `M` by
+    // 20, each with an optional `i` and `B` spelling.
+    assert_eq!(parse(&["--fast=1K"]).unwrap().level, -1024);
+    assert_eq!(parse(&["--fast=2KiB"]).unwrap().level, -2048);
+    // A bare multiplier has no digits before it, so it reads as zero.
+    assert!(parse(&["--fast=K"]).is_err());
+}
+
+#[test]
+fn fast_level_clamps_to_the_minimum_level() {
+    // upstream zstd (zstdcli.c:1136-1140): a factor past `-ZSTD_minCLevel()`
+    // is CLAMPED, not refused — `zstd --fast=200000` compresses at the
+    // lowest level rather than failing.
+    let min = structured_zstd::encoding::CompressionLevel::MIN_LEVEL;
+    assert_eq!(parse(&["--fast=200000"]).unwrap().level, min);
+    assert_eq!(parse(&["--fast=1M"]).unwrap().level, min);
+    // The digit run itself still has to fit 32 bits (zstdcli.c:356-359).
+    assert!(parse(&["--fast=99999999999"]).is_err());
+}
+
 #[test]
 fn clustered_short_flags() {
     // -d (decompress) + -c (stdout) + -k (keep) in one token.

zstd/src/decoding/decode_buffer.rs

@@ -818,6 +818,9 @@ impl<B: BufferBackend> DecodeBuffer<B> {
         }
     }
 
+    // Cold on purpose, and measured: taking the attribute off buys 2.7% on a
+    // dictionary decode (where this IS the common path, 23 matches a frame) and
+    // costs 3.5% on an ordinary one, which is the path that runs far more often.
     #[cold]
     fn repeat_from_dict(
         &mut self,
@@ -876,7 +879,11 @@ impl<B: BufferBackend> DecodeBuffer<B> {
                 self.buffer.extend(dict_slice);
 
                 self.total_output_counter += bytes_from_dict as u64;
-                return self.repeat(dict, self.buffer.len(), match_length - bytes_from_dict);
+                return self.repeat_tail_after_dict(
+                    dict,
+                    self.buffer.len(),
+                    match_length - bytes_from_dict,
+                );
             } else {
                 let low = dict_len - bytes_from_dict;
                 let high = low + match_length;
@@ -894,6 +901,34 @@ impl<B: BufferBackend> DecodeBuffer<B> {
         }
     }
 
+    /// The part of a match that continues out of the dictionary and into the
+    /// output already produced.
+    ///
+    /// Its own function, and deliberately not inlined: `repeat_inner` is
+    /// `inline(always)`, so calling it from the dictionary path pulled the whole
+    /// copy machinery — overlapping copies, the wildcopy variants, their error
+    /// paths — into that path's body. Every dictionary match then paid the
+    /// prologue and epilogue of a frame sized for code most of them never run,
+    /// and on a dictionary-heavy frame that is the common path (23 matches a
+    /// frame on the benchmark's small-10k-random scenario, at 112 instructions
+    /// a call). Behind a call the frame belongs to the tail alone.
+    ///
+    /// Worth 13.2 million retired instructions on 200 000 frames of that
+    /// scenario (3.1032 -> 3.0900 G, i9, `perf stat -r 3`). Cycles and wall
+    /// clock did not move with it (1245-1266 -> 1259-1266 ns a frame, ranges
+    /// overlapping), so this is fewer operations and NOT a speed claim: the
+    /// timer is too coarse to resolve a fraction of a percent, and work that is
+    /// gone is gone.
+    #[inline(never)]
+    fn repeat_tail_after_dict(
+        &mut self,
+        dict: Option<&crate::decoding::dictionary::Dictionary>,
+        offset: usize,
+        match_length: usize,
+    ) -> Result<(), DecodeBufferError> {
+        self.repeat(dict, offset, match_length)
+    }
+
     /// Check if and how many bytes can currently be drawn from the buffer
     pub fn can_drain_to_window_size(&self) -> Option<usize> {
         if self.buffer.len() > self.window_size {

zstd/src/encoding/blocks/compressed.rs

@@ -1013,6 +1013,7 @@ fn estimate_literals_section_bytes(
         last_huff.as_ref(),
         new_desc,
         literals,
+        counts,
         strategy,
     );
     let reuse_payload = if !use_new {
@@ -1276,12 +1277,33 @@ fn decide_huff_reuse_like_encoder(
     last_table: Option<&huff0_encoder::HuffmanTable>,
     new_desc: usize,
     literals: &[u8],
+    counts: &[usize; 256],
     strategy: crate::encoding::strategy::StrategyTag,
 ) -> bool {
     let Some(prev) = last_table else {
         return true;
     };
-    let Some(old_estimate) = prev.estimate_compressed_size(literals) else {
+    // Off the histogram, not the literals: the same sum over at most 256
+    // symbols instead of over every byte of the section, which is where
+    // upstream reads it from too (huf_compress.c:1416-1417). On a 4 KiB
+    // dictionary frame the two per-literal walks this replaces were the
+    // largest single item outside the matcher.
+    //
+    // Three arms in one session on the i9 (before, after, and the C reference
+    // through `ffi_loop_dict`), `perf stat -r 3`, three rounds, 20 000 frames
+    // of the corpus fixture with its dictionary — cycles / instructions / wall:
+    //
+    //             4 KiB frame                    10 KiB frame
+    //   before    2.92-2.94 G / 8.035 G / 0.70 s  4.81-4.83 G / 12.837 G / 1.15 s
+    //   after     2.27-2.31 G / 6.664 G / 0.54 s  3.41-3.46 G /  9.556 G / 0.82 s
+    //   reference 1.35-1.37 G / 4.307 G / 0.32 s  2.53-2.54 G /  7.128 G / 0.61 s
+    //
+    // So 22% of the cycles and 17% of the instructions on the 4 KiB frame, 29%
+    // and 26% on the 10 KiB one, taking this path from 2.16x of the reference
+    // to 1.68x and from 1.90x to 1.36x. On input the literal stage writes off
+    // as incompressible the decision never runs, and the change measures as
+    // nothing there (instructions 4.8813 G against 4.8843 G) — as expected.
+    let Some(old_estimate) = prev.estimate_compressed_size_from_counts_checked(counts) else {
         return true;
     };
     // Late-stage `HUF_flags_preferRepeat` mirror — kept here for
@@ -1296,7 +1318,7 @@ fn decide_huff_reuse_like_encoder(
         return false;
     }
     let new_estimate = new_table
-        .estimate_compressed_size(literals)
+        .estimate_compressed_size_from_counts_checked(counts)
         .unwrap_or(literals.len());
     !(old_estimate <= new_desc + new_estimate || new_desc + 12 >= literals.len())
 }
@@ -2819,6 +2841,7 @@ fn compress_literals(
         last_table,
         new_table_description_size,
         literals,
+        &counts,
         strategy,
     );
     let encoder_table = if new_table {