fix(dictionary): load raw content where upstream does, and scan dictionary blocks the way it scans them

c-api/src/attach.rs

@@ -159,7 +159,13 @@ fn encode_raw_content(dict: &[u8], content_type: c_int) -> Result<bool, ZSTD_Err
         ZSTD_DCT_RAW_CONTENT => Ok(true),
         ZSTD_DCT_FULL_DICT => {
             if !has_magic {
-                return Err(ZSTD_ErrorCode::ZSTD_error_dictionary_corrupted);
+                // `dictionary_wrong`, not `corrupted`: on the compression side
+                // fullDict over bytes that are not a dictionary is the caller
+                // having named the wrong kind (`ZSTD_compress_insertDictionary`,
+                // zstd_compress.c:5223). The decode side answers `corrupted`
+                // for the same bytes (`ZSTD_loadEntropy_intoDDict`,
+                // zstd_ddict.c:105) — see `parse_decode_dict`.
+                return Err(ZSTD_ErrorCode::ZSTD_error_dictionary_wrong);
             }
             Ok(false)
         }

c-api/src/tests.rs

@@ -585,6 +585,40 @@ fn create_cdict_treats_unmagicked_bytes_as_raw_content() {
     assert!(cdict.is_null(), "corrupt full dict must fail");
 }
 
+/// The two sides report a `ZSTD_dct_fullDict` selector over bytes that are not
+/// a dictionary with DIFFERENT codes, and a caller that branches on the code
+/// sees the difference: the compression side answers `dictionary_wrong`
+/// (`ZSTD_compress_insertDictionary`, zstd_compress.c:5207 and 5223), the
+/// decompression side `dictionary_corrupted` (`ZSTD_loadEntropy_intoDDict`,
+/// zstd_ddict.c:99 and 105).
+#[test]
+fn full_dict_over_unmagicked_bytes_reports_the_side_it_came_from() {
+    let raw = [0xABu8; 64];
+    const FULL_DICT: c_int = 2;
+
+    let cctx = ZSTD_createCCtx();
+    let rc =
+        unsafe { ZSTD_CCtx_loadDictionary_advanced(cctx, raw.as_ptr(), raw.len(), 0, FULL_DICT) };
+    assert_ne!(ZSTD_isError(rc), 0);
+    assert_eq!(
+        ZSTD_getErrorCode(rc),
+        ZSTD_ErrorCode::ZSTD_error_dictionary_wrong,
+        "the compression side calls a non-dictionary under fullDict `wrong`",
+    );
+    unsafe { ZSTD_freeCCtx(cctx) };
+
+    let dctx = ZSTD_createDCtx();
+    let rc =
+        unsafe { ZSTD_DCtx_loadDictionary_advanced(dctx, raw.as_ptr(), raw.len(), 0, FULL_DICT) };
+    assert_ne!(ZSTD_isError(rc), 0);
+    assert_eq!(
+        ZSTD_getErrorCode(rc),
+        ZSTD_ErrorCode::ZSTD_error_dictionary_corrupted,
+        "the decompression side calls the same bytes `corrupted`",
+    );
+    unsafe { ZSTD_freeDCtx(dctx) };
+}
+
 // ---- Phase 6.2: advanced parameters + streaming ----
 
 use crate::params::{

zstd/examples/encode_loop_dict.rs

@@ -91,16 +91,12 @@ fn main() {
     let mut cctx: FrameCompressor = FrameCompressor::new(CompressionLevel::from_level(level));
     if let Some(path) = dict_path {
         let dict = std::fs::read(path).expect("read dict file");
-        // A finalized dict carries the zstd magic; a non-magic blob is raw
-        // content (the `ZSTD_createCDict`-on-raw-bytes path the dict_matrix
-        // bench uses), attached with the id flag off so the frame omits a id.
-        if cctx.set_dictionary_from_bytes(&dict).is_err() {
-            let dict_obj = structured_zstd::decoding::Dictionary::from_raw_content(1, dict)
-                .expect("raw-content dictionary should build");
-            cctx.set_dictionary_id_flag(false);
-            cctx.set_dictionary(dict_obj)
-                .expect("raw-content dictionary should attach");
-        }
+        // Either kind, the way `zstd -D` takes it: a finalized dict carries
+        // the zstd magic, a non-magic blob is raw content (the
+        // `ZSTD_createCDict`-on-raw-bytes path the dict_matrix bench uses) and
+        // has no id, so the frame omits the field on its own.
+        cctx.set_dictionary_from_bytes(&dict)
+            .expect("dictionary should attach");
     }
 
     // Output buffer reused across iterations (allocated once, replaced in

zstd/src/bin/structured-zstd/main.rs

@@ -66,6 +66,11 @@ macro_rules! info {
 
 const ZSTD_SUFFIX: &str = ".zst";
 
+/// Highest level the CLI compresses at when `--ultra` was not given (upstream
+/// `ZSTDCLI_CLEVEL_MAX`). Asking for more without naming `--ultra` reduces to
+/// this with a warning rather than failing.
+const CLI_MAX_LEVEL_WITHOUT_ULTRA: i32 = 19;
+
 /// Operation selected by mode flags / `argv[0]`.
 #[derive(Clone, Copy, PartialEq, Eq, Debug)]
 enum Mode {
@@ -781,8 +786,16 @@ fn parse_args(
     // read every input before the first pass refused it.
     validate_level(lowest_level)?;
     validate_level(highest_level)?;
-    if !ultra && highest_level > 19 {
-        bail!("level {highest_level} requires --ultra (levels 20-22)");
+    // Unnamed, an ultra level is not refused but reduced, with a warning, the
+    // way upstream reduces it — a script that runs `zstd -22` compresses at 19
+    // rather than failing, and refusing here is what would break it.
+    if !ultra && highest_level > CLI_MAX_LEVEL_WITHOUT_ULTRA {
+        info!(
+            "Warning : compression level higher than max, reduced to {CLI_MAX_LEVEL_WITHOUT_ULTRA} "
+        );
+        opts.level = opts.level.min(CLI_MAX_LEVEL_WITHOUT_ULTRA);
+        opts.bench_start = opts.bench_start.min(CLI_MAX_LEVEL_WITHOUT_ULTRA);
+        opts.bench_end = opts.bench_end.min(CLI_MAX_LEVEL_WITHOUT_ULTRA);
     }
     // Long-distance matching runs on the optimal parser here, so below it the
     // flag would widen the window and never run the matcher it names. Settled

zstd/src/bin/structured-zstd/tests.rs

@@ -1340,16 +1340,30 @@ fn bare_numeric_flag_is_a_level() {
     assert_eq!(opts.inputs, vec![PathBuf::from("in.txt")]);
 }
 
-#[test]
-fn levels_above_19_require_ultra() {
-    assert!(parse(&["-22", "in.txt"]).is_err());
-    let opts = parse(&["--ultra", "-22", "in.txt"]).unwrap();
-    assert_eq!(opts.level, 22);
-    // Benchmarking compresses the range `-b`/`-e` name, so that is the range
-    // the gate has to read: `-b20` runs an ultra level as surely as `-20` does.
-    assert!(parse(&["-b20", "in.txt"]).is_err());
-    assert!(parse(&["-b3", "-e22", "in.txt"]).is_err());
-    assert!(parse(&["--ultra", "-b20", "in.txt"]).is_ok());
+/// Levels 20-22 are expensive enough that they have to be asked for by name,
+/// but asking without `--ultra` is not an error: upstream warns and compresses
+/// at 19 ("Warning : compression level higher than max, reduced to 19", exit
+/// 0), so a script that runs `zstd -22` keeps working. Refusing instead breaks
+/// it against us.
+#[test]
+fn levels_above_19_without_ultra_fall_back_to_19() {
+    assert_eq!(parse(&["-22", "in.txt"]).unwrap().level, 19);
+    assert_eq!(parse(&["-20", "in.txt"]).unwrap().level, 19);
+    // Named, they run as asked.
+    assert_eq!(parse(&["--ultra", "-22", "in.txt"]).unwrap().level, 22);
+    // Benchmarking compresses the range `-b`/`-e` name rather than the level
+    // `-N` sets, so the range is what gets clamped: `-b20` reaches an ultra
+    // level as surely as `-20` does.
+    let opts = parse(&["-b20", "in.txt"]).unwrap();
+    assert_eq!((opts.bench_start, opts.bench_end), (19, 19));
+    let opts = parse(&["-b3", "-e22", "in.txt"]).unwrap();
+    assert_eq!((opts.bench_start, opts.bench_end), (3, 19));
+    assert_eq!(
+        parse(&["--ultra", "-b20", "in.txt"]).unwrap().bench_start,
+        20
+    );
+    // Below the ultra band nothing moves.
+    assert_eq!(parse(&["-19", "in.txt"]).unwrap().level, 19);
 }
 
 #[test]

zstd/src/decoding/dictionary.rs

@@ -121,6 +121,21 @@ impl Dictionary {
     pub fn from_serialized_or_raw_content(raw: &[u8]) -> Result<Dictionary, DictionaryDecodeError> {
         if raw.starts_with(&MAGIC_NUM) {
             Self::decode_dict(raw)
+        } else if raw.is_empty() {
+            // A zero-sized buffer is a dictionary with nothing in it rather
+            // than a malformed one: `ZSTD_createDDict(NULL, 0)` builds a
+            // usable `DDict` referencing no content, and
+            // `ZSTD_CCtx_loadDictionary` with an empty buffer is how a caller
+            // says "no dictionary". [`Self::from_raw_content`] still refuses
+            // it, because naming raw content and handing over none is the
+            // caller asking for a dictionary that cannot exist.
+            Ok(Dictionary {
+                id: 0,
+                fse: FSEScratch::new(),
+                huf: HuffmanScratch::new(),
+                dict_content: Vec::new(),
+                offset_hist: [1, 4, 8],
+            })
         } else {
             Self::from_raw_content(0, raw.to_vec())
         }
@@ -313,6 +328,14 @@ impl DictionaryHandle {
         Dictionary::decode_dict(raw).map(Self::from_dictionary)
     }
 
+    /// Load whichever kind of dictionary `raw` holds, as `ZSTD_createDDict`
+    /// does: a blob starting with [`MAGIC_NUM`] is a serialized dictionary,
+    /// anything else is raw content. See
+    /// [`Dictionary::from_serialized_or_raw_content`].
+    pub fn from_serialized_or_raw_content(raw: &[u8]) -> Result<Self, DictionaryDecodeError> {
+        Dictionary::from_serialized_or_raw_content(raw).map(Self::from_dictionary)
+    }
+
     pub fn id(&self) -> u32 {
         self.inner.id
     }

zstd/src/decoding/dictionary/tests.rs

@@ -105,6 +105,46 @@ fn dictionary_handle_from_raw_content_supports_as_ref() {
     assert_eq!(dict_ref.dict_content.as_slice(), &[42]);
 }
 
+/// `ZSTD_createDDict` loads in `ZSTD_dct_auto` mode (zstd_ddict.c:102-107): a
+/// buffer without the magic is raw content with no id and no entropy tables.
+/// The handle is the shared form the decoder is handed, so it takes the same
+/// two kinds the `Dictionary` constructor does.
+#[test]
+fn dictionary_handle_takes_serialized_or_raw_content() {
+    let serialized = include_bytes!("../../../dict_tests/dictionary");
+    let parsed = DictionaryHandle::from_serialized_or_raw_content(serialized)
+        .expect("a magic-prefixed blob parses as a full dictionary");
+    assert_ne!(parsed.id(), 0, "a full dictionary carries its id");
+
+    let raw = b"tenant=demo table=orders op=put".repeat(8);
+    let handle = DictionaryHandle::from_serialized_or_raw_content(&raw)
+        .expect("anything else is raw content");
+    assert_eq!(handle.id(), 0, "raw content has no header to carry an id");
+    assert_eq!(handle.as_dict().dict_content.as_slice(), raw.as_slice());
+}
+
+/// A zero-sized buffer is a dictionary with nothing in it, not a malformed
+/// one: `ZSTD_createDDict(NULL, 0)` builds a usable `DDict` that references no
+/// content (zstd_ddict.c:123-140), and a caller handed an empty file gets the
+/// same nothing here rather than an error.
+#[test]
+fn an_empty_buffer_is_a_dictionary_with_no_content() {
+    let dict = Dictionary::from_serialized_or_raw_content(&[])
+        .expect("an empty buffer is a dictionary with no content");
+    assert_eq!(dict.id, 0);
+    assert!(dict.dict_content.is_empty());
+    assert_eq!(dict.offset_hist, [1, 4, 8]);
+
+    let handle = DictionaryHandle::from_serialized_or_raw_content(&[])
+        .expect("the handle takes it the same way");
+    assert_eq!(handle.id(), 0);
+    assert!(handle.as_dict().dict_content.is_empty());
+
+    // The constructor that names raw content still refuses it: there its
+    // emptiness is the caller asking for a dictionary that cannot exist.
+    assert!(Dictionary::from_raw_content(1, Vec::new()).is_err());
+}
+
 #[test]
 fn dictionary_handle_clones_share_inner() {
     let raw = include_bytes!("../../../dict_tests/dictionary");