diff --git a/crates/gpu-gates/src/bin/gate_qwen35moe_mtp.rs b/crates/gpu-gates/src/bin/gate_qwen35moe_mtp.rs index d3f583a10..c5f90c5b7 100644 --- a/crates/gpu-gates/src/bin/gate_qwen35moe_mtp.rs +++ b/crates/gpu-gates/src/bin/gate_qwen35moe_mtp.rs @@ -66,9 +66,10 @@ mod gate { const WHAT: &str = "gate_qwen35moe_mtp"; - /// The file the gate runs on: unsloth's `UD-Q4_K_M` with its next-token - /// layer. Pinned by `general.name` and by its size in bytes. - const DRAFT_FILE: &str = "/models/qwen36-hf/unsloth-mtp/Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"; + // The file the gate runs on: unsloth's `UD-Q4_K_M` with its next-token + // layer (the MTP step family's file). Pinned by `general.name` and by its + // size in bytes. + use refset::arch::qwen35moe::mtp::MODEL as DRAFT_FILE; const DRAFT_NAME: &str = "Qwen3.6-35B-A3B"; const DRAFT_BYTES: u64 = 22_663_387_424; diff --git a/crates/refset/src/arch/nemotron_h/mod.rs b/crates/refset/src/arch/nemotron_h/mod.rs index 572e23309..22df2c499 100644 --- a/crates/refset/src/arch/nemotron_h/mod.rs +++ b/crates/refset/src/arch/nemotron_h/mod.rs @@ -32,6 +32,7 @@ pub static FAMILIES: &[&Family] = &[&LOGITS]; mod tests { use super::LOGITS; use crate::RefError; + use crate::clefvis::testkit::scratch as dir; use crate::family::Build; use crate::nemotron::LogitsSet; use std::path::Path; @@ -67,13 +68,6 @@ mod tests { std::fs::write(dir.join("MANIFEST.tsv"), text).unwrap_or_else(|e| panic!("{e}")); } - fn dir(tag: &str) -> std::path::PathBuf { - let d = - std::env::temp_dir().join(format!("bloomery-nemotron-{tag}-{}", std::process::id())); - let _ = std::fs::remove_dir_all(&d); - d - } - /// The build pin is the tokenizer family's constant. #[test] fn the_build_pin_is_the_tokenizer_familys() { @@ -93,7 +87,7 @@ mod tests { /// A finished set opens, and `logits` is that column's file. #[test] fn a_finished_set_reads_logits_by_column_name() { - let d = dir("ok"); + let d = dir("nemotron-ok"); write(&d, &runs(), pin(), true); let set = LogitsSet::open(&d, &LOGITS).unwrap_or_else(|e| panic!("{e}")); assert_eq!(set.prompts.len(), 1); @@ -107,7 +101,7 @@ mod tests { /// No `# complete`: unfinished, even when the other lines are well formed. #[test] fn a_set_without_its_trailer_is_unfinished() { - let d = dir("trailer"); + let d = dir("nemotron-trailer"); write(&d, &runs(), pin(), false); match LogitsSet::open(&d, &LOGITS) { Err(RefError::Unfinished { set }) => assert_eq!(set, d.display().to_string()), @@ -119,7 +113,7 @@ mod tests { /// Another model path is stale. #[test] fn a_set_of_another_model_is_stale() { - let d = dir("model"); + let d = dir("nemotron-model"); write(&d, "/models/other/file.gguf", pin(), true); match LogitsSet::open(&d, &LOGITS) { Err(RefError::Stale { dumped_from, .. }) => { @@ -133,7 +127,7 @@ mod tests { /// Another mainline build is foreign. #[test] fn a_set_of_another_build_is_foreign() { - let d = dir("build"); + let d = dir("nemotron-build"); write( &d, &runs(), @@ -150,7 +144,7 @@ mod tests { /// A line that is not a token id is malformed, naming the field and the text. #[test] fn a_token_line_that_is_not_an_id_is_malformed() { - let d = dir("bad-id"); + let d = dir("nemotron-bad-id"); write(&d, &runs(), pin(), true); std::fs::write(d.join("p.ids"), "0\n1\n2\ngarbage\n0\n").unwrap_or_else(|e| panic!("{e}")); match LogitsSet::open(&d, &LOGITS) { @@ -168,7 +162,7 @@ mod tests { /// A NaN or infinite logit is malformed, naming its index. #[test] fn a_non_finite_logit_is_malformed() { - let d = dir("nan-logit"); + let d = dir("nemotron-nan-logit"); write(&d, &runs(), pin(), true); let logits: Vec = [0.0f32, f32::NAN, -0.5, 0.25] .into_iter() @@ -177,7 +171,7 @@ mod tests { std::fs::write(d.join("p.logits.f32"), logits).unwrap_or_else(|e| panic!("{e}")); match LogitsSet::open(&d, &LOGITS) { Err(RefError::Malformed { what, .. }) => { - assert!(what.contains("non-finite logit at index 1"), "{what}"); + assert!(what.contains("non-finite value at index 1"), "{what}"); } other => panic!("a non-finite logit: {other:?}"), } @@ -187,7 +181,7 @@ mod tests { /// An empty line among the ids is malformed, as the C++ reader that writes the set refuses it. #[test] fn an_empty_token_line_is_malformed() { - let d = dir("empty-id"); + let d = dir("nemotron-empty-id"); write(&d, &runs(), pin(), true); std::fs::write(d.join("p.ids"), "0\n\n1\n2\n3\n0\n").unwrap_or_else(|e| panic!("{e}")); match LogitsSet::open(&d, &LOGITS) { @@ -202,7 +196,7 @@ mod tests { /// An id at or past `n_vocab` is malformed, naming the id and the vocabulary. #[test] fn a_token_id_outside_the_vocabulary_is_malformed() { - let d = dir("oob-id"); + let d = dir("nemotron-oob-id"); write(&d, &runs(), pin(), true); std::fs::write(d.join("p.ids"), "0\n1\n2\n3\n4\n").unwrap_or_else(|e| panic!("{e}")); match LogitsSet::open(&d, &LOGITS) { @@ -217,7 +211,7 @@ mod tests { /// The trailer's count is the prompt rows. One row and `# complete 2` is malformed. #[test] fn a_completion_count_that_is_not_the_prompt_rows_is_malformed() { - let d = dir("complete-n"); + let d = dir("nemotron-complete-n"); write(&d, &runs(), pin(), true); let text = std::fs::read_to_string(d.join("MANIFEST.tsv")).unwrap_or_else(|e| panic!("{e}")); @@ -238,7 +232,7 @@ mod tests { /// A completion count that does not parse is malformed, naming the field. #[test] fn a_completion_count_that_does_not_parse_is_malformed() { - let d = dir("complete-bad"); + let d = dir("nemotron-complete-bad"); write(&d, &runs(), pin(), true); let text = std::fs::read_to_string(d.join("MANIFEST.tsv")).unwrap_or_else(|e| panic!("{e}")); diff --git a/crates/refset/src/arch/qwen35moe/mod.rs b/crates/refset/src/arch/qwen35moe/mod.rs index 9c6634b6b..ee76005c3 100644 --- a/crates/refset/src/arch/qwen35moe/mod.rs +++ b/crates/refset/src/arch/qwen35moe/mod.rs @@ -7,6 +7,7 @@ use crate::RefError; use crate::family::{Build, Family, Identity}; pub mod fixture; +pub mod mtp; pub mod vis; /// The file every qwen35moe set is dumped from and the tree runs. The @@ -56,6 +57,7 @@ pub static IK: Family = Family { pub static FAMILIES: &[&Family] = &[ &IK, &fixture::IK, + &mtp::MAINLINE, &vis::TAPS, &vis::CHATIDS, &vis::HIDDEN, diff --git a/crates/refset/src/arch/qwen35moe/mtp.rs b/crates/refset/src/arch/qwen35moe/mtp.rs new file mode 100644 index 000000000..6a890c12e --- /dev/null +++ b/crates/refset/src/arch/qwen35moe/mtp.rs @@ -0,0 +1,37 @@ +//! The Qwen3.6 MTP step oracle: llama.cpp mainline's next-token graph +//! (`llama_model_qwen35moe::graph_mtp`) tap by tap, on the first 64 prose ids fed teacher-forced +//! (`tools/ref/q35mtp_ref.cpp`, `just dump-ref-q35mtp`), read by [`crate::mtptaps::TapSet`], at the mainline tree +//! [`QVIS_LCPP_BUILD`] (`MTP_LCPP` of tools/ref/models/qwen35moe.sh). The file is +//! unsloth's UD-Q4_K_M, the one with the next-token layer (`blk.40`); the node dumps' file has none. + +use crate::RefError; +use crate::clefvis::QVIS_LCPP_BUILD; +use crate::family::{Build, Family, Identity}; + +/// The file the set is dumped from and the tree runs: the Q4_K_M mix that carries the next-token layer. +pub const MODEL: &str = "/models/qwen36-hf/unsloth-mtp/Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"; + +/// The set: 63 steps over the first 64 prose ids, taps at four of them. +pub const SET: &str = "ref-mtp/qwen35moe_mainline_prose64"; + +/// [`MODEL`], as the family's `runs`. +fn model() -> Result { + Ok(MODEL.to_string()) +} + +/// Mainline's MTP step taps (`just dump-ref-q35mtp`). +pub static MAINLINE: Family = Family { + name: "mtp-qwen35moe-mainline", + sets: &[SET], + resolve: None, + recipe: "just dump-ref-q35mtp", + identity: Identity::MtpTaps, + arch: Some(super::ARCH), + build: Some(Build::Is(QVIS_LCPP_BUILD)), + runs: Some(model), + draft_runs: None, + consumers: &[], +}; + +#[cfg(test)] +mod tests; diff --git a/crates/refset/src/arch/qwen35moe/mtp/tests.rs b/crates/refset/src/arch/qwen35moe/mtp/tests.rs new file mode 100644 index 000000000..98d3f6f66 --- /dev/null +++ b/crates/refset/src/arch/qwen35moe/mtp/tests.rs @@ -0,0 +1,496 @@ +use super::{MAINLINE, MODEL}; +use crate::RefError; +use crate::clefvis::QVIS_LCPP_BUILD; +use crate::clefvis::testkit::scratch as dir; +use crate::mtptaps::{Elem, IDS_FILE, TapSet}; +use std::path::Path; + +const IDS: [u32; 4] = [10, 11, 12, 13]; + +/// What the set's header and trailer state; `Default` is a finished set of the family's file, build and +/// architecture, with the MTP context's line. +struct Header<'a> { + model: &'a str, + build: &'a str, + arch: &'a str, + mtp: Option<&'a str>, + complete: bool, +} + +impl Default for Header<'_> { + fn default() -> Self { + Header { + model: MODEL, + build: QVIS_LCPP_BUILD, + arch: "qwen35moe", + mtp: Some("1\t3"), + complete: true, + } + } +} + +fn put(dir: &Path, name: &str, bytes: &[u8]) { + std::fs::write(dir.join(name), bytes).unwrap_or_else(|e| panic!("{name}: {e}")); +} + +/// The manifest at `dir` with its one line `from` replaced by `to`. +fn swap(dir: &Path, from: &str, to: &str) { + let m = std::fs::read_to_string(dir.join("MANIFEST.tsv")).unwrap_or_else(|e| panic!("{e}")); + assert_eq!(m.lines().filter(|l| *l == from).count(), 1, "{from}"); + let out: Vec<&str> = m.lines().map(|l| if l == from { to } else { l }).collect(); + put(dir, "MANIFEST.tsv", (out.join("\n") + "\n").as_bytes()); +} + +fn f32s(v: &[f32]) -> Vec { + v.iter().flat_map(|x| x.to_le_bytes()).collect() +} + +/// A set of four prompt ids, three steps, a trunk of four rows of two, and two taps (`mtp_eh_proj`, f32 of +/// two; `ffn_moe_topk`, i32 of two) at rows 0 and 2. `extra` is appended before the trailer. +fn write_set(dir: &Path, h: &Header<'_>, extra: &[&str]) { + put(dir, IDS_FILE, b"10\n11\n12\n13\n"); + put(dir, "trunk_h.f32", &f32s(&[0., 1., 2., 3., 4., 5., 6., 7.])); + for row in [0, 2] { + put( + dir, + &format!("mtp_eh_proj.r{row}.f32"), + &f32s(&[row as f32, 0.5]), + ); + let topk: Vec = [7i32, -1].into_iter().flat_map(i32::to_le_bytes).collect(); + put(dir, &format!("ffn_moe_topk.r{row}.i32"), &topk); + } + let mut lines = vec![ + "# q35mtp_ref — test".to_string(), + format!("# model\t{}", h.model), + format!("# build\t{}", h.build), + format!("# arch\t{}", h.arch), + "# tokens_count\t4".to_string(), + ]; + if let Some(m) = h.mtp { + lines.push(format!("# mtp\t{m}")); + } + lines.extend( + [ + "# rows\t0\t2", + "# trunk columns\tname\ttype\trows\twidth\tfile", + "trunk\th_nextn\tf32\t4\t2\ttrunk_h.f32", + "# step columns\trow\ttoken\tpos\tmtp_argmax\ttarget_next", + "step\t0\t11\t0\t5\t5", + "step\t1\t12\t1\t6\t7", + "step\t2\t13\t2\t8\t8", + "# tap columns\tname\trow\ttype\tn\tfile", + "tap\tmtp_eh_proj\t0\tf32\t2\tmtp_eh_proj.r0.f32", + "tap\tffn_moe_topk\t0\ti32\t2\tffn_moe_topk.r0.i32", + "tap\tmtp_eh_proj\t2\tf32\t2\tmtp_eh_proj.r2.f32", + "tap\tffn_moe_topk\t2\ti32\t2\tffn_moe_topk.r2.i32", + ] + .map(str::to_string), + ); + lines.extend(extra.iter().map(|l| (*l).to_string())); + if h.complete { + let taps = lines.iter().filter(|l| l.starts_with("tap\t")).count(); + lines.push(format!("# complete\t{taps}\t0")); + } + put(dir, "MANIFEST.tsv", (lines.join("\n") + "\n").as_bytes()); +} + +fn open(d: &Path) -> Result { + TapSet::open(d, &MAINLINE) +} + +/// The refusal of the set at `d`, or a panic naming the case that opened. +fn refused(d: &Path, what: &str) -> RefError { + match open(d) { + Err(e) => e, + Ok(_) => panic!("{what}: the set opened"), + } +} + +/// A finished set opens through the family's check, and a tap is read by its graph name and its row, as f32 or +/// i32 as the node is; a name or a row the set does not hold is refused naming both. +#[test] +fn a_finished_set_reads_a_tap_by_name_and_row() { + let d = dir("mtptaps-ok"); + write_set(&d, &Header::default(), &[]); + let p = MAINLINE.check_set(&d).unwrap_or_else(|e| panic!("{e}")); + assert_eq!(p.dumped_from, MODEL); + assert_eq!(p.build.as_deref(), Some(QVIS_LCPP_BUILD)); + let set = open(&d).unwrap_or_else(|e| panic!("{e}")); + assert_eq!(set.ids().unwrap_or_else(|e| panic!("{e}")), IDS); + assert_eq!( + set.tap_f32("mtp_eh_proj", 2) + .unwrap_or_else(|e| panic!("{e}")), + [2.0, 0.5] + ); + assert_eq!( + set.tap_i32("ffn_moe_topk", 0) + .unwrap_or_else(|e| panic!("{e}")), + [7, -1] + ); + assert_eq!( + set.tap("ffn_moe_topk", 0).map(|t| t.elem).ok(), + Some(Elem::I32) + ); + assert_eq!( + set.trunk_rows().unwrap_or_else(|e| panic!("{e}"))[6..], + [6.0, 7.0] + ); + let s = set.step(1).unwrap_or_else(|e| panic!("{e}")); + assert_eq!((s.token, s.pos, s.mtp_argmax, s.target_next), (12, 1, 6, 7)); + for (name, row) in [("mtp_nothing", 0), ("mtp_eh_proj", 1)] { + let e = set.tap(name, row).err().map(|e| e.to_string()); + let e = e.unwrap_or_else(|| panic!("{name} at {row} was found")); + assert!(e.contains(&format!("no tap {name} at row {row}")), "{e}"); + } + let e = set.tap_f32("ffn_moe_topk", 0).err().map(|e| e.to_string()); + assert!( + e.is_some_and(|e| e.contains("tap ffn_moe_topk is i32, read as f32")), + "an i32 tap read as f32" + ); + std::fs::remove_dir_all(&d).unwrap_or_else(|e| panic!("{e}")); +} + +/// The family refuses by name a set of another file, another mainline build, another architecture, one dumped +/// without the MTP context (no `# mtp` line, or a model with no next-token layer) and one without its trailer. +#[test] +fn the_family_refuses_a_set_of_another_file_build_or_arch_or_without_the_mtp_context() { + let d = dir("mtptaps-refuse"); + let other_build = "0000000aa"; + let cases: [(&str, Header<'_>); 6] = [ + ( + "file", + Header { + model: "/models/Qwen3.6-35B-A3B/Qwen3.6-35B-A3B-Q4_K_M.gguf", + ..Header::default() + }, + ), + ( + "build", + Header { + build: other_build, + ..Header::default() + }, + ), + ( + "arch", + Header { + arch: "qwen35", + ..Header::default() + }, + ), + ( + "mtp", + Header { + mtp: None, + ..Header::default() + }, + ), + ( + "layers", + Header { + mtp: Some("0\t3"), + ..Header::default() + }, + ), + ( + "trailer", + Header { + complete: false, + ..Header::default() + }, + ), + ]; + for (what, h) in &cases { + write_set(&d, h, &[]); + let e = refused(&d, what); + match (*what, &e) { + ( + "file", + RefError::Stale { + dumped_from, runs, .. + }, + ) => { + assert_eq!((dumped_from.as_str(), runs.as_str()), (h.model, MODEL)); + } + ( + "build", + RefError::Foreign { + field: "build", + got, + want, + .. + }, + ) => { + assert_eq!( + (got.as_str(), want.as_str()), + (other_build, QVIS_LCPP_BUILD) + ); + } + ( + "arch", + RefError::Foreign { + field: "arch", got, .. + }, + ) => assert_eq!(got, "qwen35"), + ( + "mtp", + RefError::Foreign { + field: "mtp", got, .. + }, + ) => { + assert_eq!(got, "(no # mtp line)"); + assert!(e.to_string().contains("MTP context"), "{e}"); + } + ("layers", RefError::Malformed { what, .. }) => { + assert!(what.contains("# mtp 0 3"), "{what}"); + } + ("trailer", RefError::Unfinished { set }) => assert_eq!(set, &d.display().to_string()), + _ => panic!("{what}: {e:?}"), + } + } + std::fs::remove_dir_all(&d).unwrap_or_else(|e| panic!("{e}")); +} + +/// A set whose files or rows disagree is `Malformed`, naming the row or file: a tap file of another length, a +/// step that is not the prompt's next id, a tap named twice, a row naming a file outside the directory, a +/// tapped row that is no step, and a tap a row lacks. +#[test] +fn a_set_whose_files_or_rows_disagree_is_malformed() { + let d = dir("mtptaps-malformed"); + let at = |what: &str, extra: &[&str], edit: &dyn Fn(&Path), want: &str| { + write_set(&d, &Header::default(), extra); + edit(&d); + let e = refused(&d, what); + assert!( + matches!(e, RefError::Malformed { .. }) && e.to_string().contains(want), + "{what}: {e}" + ); + }; + at( + "a short tap file", + &[], + &|d| put(d, "mtp_eh_proj.r2.f32", &f32s(&[1.0])), + "mtp_eh_proj.r2.f32 is 4 bytes", + ); + at( + "the trunk file", + &[], + &|d| put(d, "trunk_h.f32", &f32s(&[1.0; 7])), + "trunk_h.f32 is 28 bytes", + ); + at( + "a prompt of another length", + &[], + &|d| put(d, IDS_FILE, b"10\n11\n12\n"), + "holds 3 ids", + ); + at( + "a tap named twice", + &["tap\tmtp_eh_proj\t2\tf32\t2\tmtp_eh_proj.r2.f32"], + &|_| {}, + "tap mtp_eh_proj at row 2 is named twice", + ); + at( + "a file outside the directory", + &["tap\tx\t0\tf32\t2\t../trunk_h.f32"], + &|_| {}, + "does not name a file of the set's directory", + ); + at( + "a prompt whose id is not the step's token", + &[], + &|d| put(d, IDS_FILE, b"10\n11\n99\n13\n"), + "step row 1 is not step 1", + ); + at( + "a tap missing at a tapped row", + &["tap\tmtp_hnorm\t0\tf32\t2\tmtp_eh_proj.r0.f32"], + &|_| {}, + "no tap mtp_hnorm at row 2", + ); + at( + "an MTP context that ran fewer steps", + &[], + &|d| swap(d, "# mtp\t1\t3", "# mtp\t1\t2"), + "# mtp 1 2", + ); + at( + "a step count at the integer's end", + &[], + &|d| swap(d, "# mtp\t1\t3", "# mtp\t1\t4294967295"), + "# mtp 1 4294967295", + ); + at( + "a trunk of another length", + &[], + &|d| { + swap( + d, + "trunk\th_nextn\tf32\t4\t2\ttrunk_h.f32", + "trunk\th_nextn\tf32\t3\t2\ttrunk_h.f32", + ) + }, + "the trunk holds 3 rows", + ); + at( + "a tapped row that is no step", + &[], + &|d| swap(d, "# rows\t0\t2", "# rows\t0\t2\t5"), + "# rows names 5, which is no step", + ); + at( + "a tap at a row # rows does not name", + &["tap\tmtp_eh_proj\t1\tf32\t2\tmtp_eh_proj.r0.f32"], + &|_| {}, + "a tap at a row # rows does not name", + ); + at( + "a tap name gone at every row, the trailer unchanged", + &[], + &|d| { + swap(d, "tap\tffn_moe_topk\t0\ti32\t2\tffn_moe_topk.r0.i32", ""); + swap(d, "tap\tffn_moe_topk\t2\ti32\t2\tffn_moe_topk.r2.i32", ""); + }, + "is not `# complete\t2\t0` on the last line", + ); + at( + "a trailer whose second field is not 0", + &[], + &|d| swap(d, "# complete\t4\t0", "# complete\t4\t1"), + "is not `# complete\t4\t0` on the last line", + ); + at( + "a step at another position than its row", + &[], + &|d| swap(d, "step\t1\t12\t1\t6\t7", "step\t1\t12\t2\t6\t7"), + "step row 1", + ); + at( + "two step rows' numbers swapped", + &[], + &|d| { + swap(d, "step\t1\t12\t1\t6\t7", "step\t2\t12\t2\t6\t7"); + swap(d, "step\t2\t13\t2\t8\t8", "step\t1\t13\t1\t8\t8"); + }, + "step row 2 is not step 1", + ); + at( + "an empty line among the prompt ids", + &[], + &|d| put(d, IDS_FILE, b"10\n\n11\n12\n13\n"), + "empty token id", + ); + at( + "a tap file with a backslash", + &["tap\tx\t0\tf32\t2\ta\\b"], + &|_| {}, + "does not name a file of the set's directory", + ); + at( + "a tap file with a NUL", + &["tap\tx\t0\tf32\t2\ta\0b"], + &|_| {}, + "does not name a file of the set's directory", + ); + at( + "repeated rows hide taps at a nonexistent step", + &[ + "tap\tmtp_eh_proj\t5\tf32\t2\tmtp_eh_proj.r0.f32", + "tap\tffn_moe_topk\t5\ti32\t2\tffn_moe_topk.r0.i32", + ], + &|d| swap(d, "# rows\t0\t2", "# rows\t0\t0\t2"), + "duplicate # rows entry", + ); + at( + "no # rows and no tap rows", + &[], + &|d| { + swap(d, "# rows\t0\t2", ""); + for t in [ + "tap\tmtp_eh_proj\t0\tf32\t2\tmtp_eh_proj.r0.f32", + "tap\tffn_moe_topk\t0\ti32\t2\tffn_moe_topk.r0.i32", + "tap\tmtp_eh_proj\t2\tf32\t2\tmtp_eh_proj.r2.f32", + "tap\tffn_moe_topk\t2\ti32\t2\tffn_moe_topk.r2.i32", + ] { + swap(d, t, ""); + } + swap(d, "# complete\t4\t0", "# complete\t0\t0"); + }, + "names nothing to check", + ); + at( + "no tap rows under # rows", + &[], + &|d| { + for t in [ + "tap\tmtp_eh_proj\t0\tf32\t2\tmtp_eh_proj.r0.f32", + "tap\tffn_moe_topk\t0\ti32\t2\tffn_moe_topk.r0.i32", + "tap\tmtp_eh_proj\t2\tf32\t2\tmtp_eh_proj.r2.f32", + "tap\tffn_moe_topk\t2\ti32\t2\tffn_moe_topk.r2.i32", + ] { + swap(d, t, ""); + } + swap(d, "# complete\t4\t0", "# complete\t0\t0"); + }, + "names nothing to check", + ); + at( + "no # tokens_count line", + &[], + &|d| swap(d, "# tokens_count\t4", ""), + "no # tokens_count line", + ); + at( + "no trunk row", + &[], + &|d| swap(d, "trunk\th_nextn\tf32\t4\t2\ttrunk_h.f32", ""), + "no trunk row", + ); + at( + "a NaN in a tap file", + &[], + &|d| put(d, "mtp_eh_proj.r2.f32", &f32s(&[f32::NAN, 0.5])), + "non-finite value at index 0", + ); + at( + "an infinity in the trunk file", + &[], + &|d| { + put( + d, + "trunk_h.f32", + &f32s(&[0., 1., 2., f32::INFINITY, 4., 5., 6., 7.]), + ) + }, + "non-finite value at index 3", + ); + at( + "a step row gone", + &[], + &|d| swap(d, "step\t2\t13\t2\t8\t8", ""), + "2 step rows, # mtp says 3 steps ran", + ); + at( + "a trunk file outside the directory", + &[], + &|d| { + swap( + d, + "trunk\th_nextn\tf32\t4\t2\ttrunk_h.f32", + "trunk\th_nextn\tf32\t4\t2\t../trunk_h.f32", + ) + }, + "does not name a file of the set's directory", + ); + at( + "a line after the trailer", + &[], + &|d| { + let m = + std::fs::read_to_string(d.join("MANIFEST.tsv")).unwrap_or_else(|e| panic!("{e}")); + put(d, "MANIFEST.tsv", (m + "# arch\tqwen35moe\n").as_bytes()); + }, + "on the last line", + ); + std::fs::remove_dir_all(&d).unwrap_or_else(|e| panic!("{e}")); +} diff --git a/crates/refset/src/clefvis.rs b/crates/refset/src/clefvis.rs index faa5fe83b..d8894a117 100644 --- a/crates/refset/src/clefvis.rs +++ b/crates/refset/src/clefvis.rs @@ -207,7 +207,8 @@ impl Projector { } } -/// The mainline commit the Qwen3.6 and Qwen3.8 image-input sets name in their `# build` line. +/// The mainline commit the Qwen3.6 and Qwen3.8 image-input sets and the Qwen3.6 MTP step sets name in their `# build` +/// line. // PIN(2026-10-10): /home/user/llama.cpp-36a73916 at its HEAD 36a73916ee0c (ggml-org master, 2026-10-07), the tree that // carries the qwen4exp and GLM5-Next fixes after Clef's pin (`crate::arch::qwen35::LCPP_BUILD`); both trees stay on the box. pub const QVIS_LCPP_BUILD: &str = "36a73916e"; diff --git a/crates/refset/src/columns.rs b/crates/refset/src/columns.rs index 22f16bcd7..f3aea9126 100644 --- a/crates/refset/src/columns.rs +++ b/crates/refset/src/columns.rs @@ -5,6 +5,7 @@ use crate::RefError; use std::fmt::Display; +use std::path::Path; use std::str::FromStr; /// The names of one row kind's fields, from its column line: field 0 is the @@ -113,7 +114,7 @@ where /// `name` is one file of the set's own directory. Empty, `.`, `..`, or a /// slash is `Malformed`: the manifest must not name a path outside it. pub(crate) fn file_name(name: &str, at: &dyn Display) -> Result<(), RefError> { - if name.is_empty() || name.contains('/') || matches!(name, "." | "..") { + if name.is_empty() || name.contains(['/', '\\', '\0']) || matches!(name, "." | "..") { return Err(RefError::malformed( at, format!("{name:?} does not name a file of the set's directory"), @@ -121,3 +122,65 @@ pub(crate) fn file_name(name: &str, at: &dyn Display) -> Result<(), RefError> { } Ok(()) } + +/// The column line a data row of kind `what` is read by, or a malformed-set error naming the row +/// that came before it. +pub(crate) fn need<'a>( + cols: &'a Option, + what: &str, + at: &dyn Display, +) -> Result<&'a Columns, RefError> { + cols.as_ref() + .ok_or_else(|| RefError::malformed(at, format!("a {what} row before its column line"))) +} + +/// The one value of a `# key\t` header line, or a malformed-set error naming the form `want`. +pub(crate) fn one(f: &[&str], at: &dyn Display, want: &str) -> Result { + match f { + [_, v] if !v.is_empty() => Ok((*v).to_string()), + _ => Err(RefError::malformed(at, format!("want {want}"))), + } +} + +/// `want` four-byte values of `path`, as bytes. +pub(crate) fn read_bytes(path: &Path, want: u64, file: &str) -> Result, RefError> { + let bytes = std::fs::read(path) + .map_err(|e| RefError::missing(path, format!("cannot read {}: {e}", path.display())))?; + if bytes.len() as u64 != want * 4 { + return Err(RefError::malformed( + path.display(), + format!( + "{file} is {} bytes, {want} values want {}", + bytes.len(), + want * 4 + ), + )); + } + Ok(bytes) +} + +/// `want` little-endian f32 values of `path`, each finite (a reference value is never NaN or infinite). +pub(crate) fn read_f32(path: &Path, want: u64, file: &str) -> Result, RefError> { + let values: Vec = read_bytes(path, want, file)? + .as_chunks::<4>() + .0 + .iter() + .map(|c| f32::from_le_bytes(*c)) + .collect(); + if let Some(i) = values.iter().position(|v| !v.is_finite()) { + return Err(RefError::malformed( + path.display(), + format!("non-finite value at index {i}"), + )); + } + Ok(values) +} + +/// Little-endian i32 values, four bytes each (the caller has checked the length). +pub(crate) fn i32s_of(b: &[u8]) -> Vec { + b.as_chunks::<4>() + .0 + .iter() + .map(|c| i32::from_le_bytes(*c)) + .collect() +} diff --git a/crates/refset/src/dsref.rs b/crates/refset/src/dsref.rs index 2f6bead70..8491c16d5 100644 --- a/crates/refset/src/dsref.rs +++ b/crates/refset/src/dsref.rs @@ -13,6 +13,7 @@ use crate::RefError; use crate::columns::Columns; +use crate::columns::i32s_of; use crate::family::Family; use crate::ik::{ FileElem, IntRow, Layout, RefRow, RowKind, dump_file_name, parse_int_row, parse_ref_row, @@ -570,13 +571,5 @@ fn place(c: &Columns, line: &str, at: &dyn fmt::Display) -> Result<(i32, Graph), Ok((r.parse("block")?, Graph::parse(r.text("graph")?, at)?)) } -fn i32s_of(b: &[u8]) -> Vec { - b.as_chunks::<4>() - .0 - .iter() - .map(|c| i32::from_le_bytes(*c)) - .collect() -} - #[cfg(test)] mod tests; diff --git a/crates/refset/src/family.rs b/crates/refset/src/family.rs index 898e6657d..2c38c338b 100644 --- a/crates/refset/src/family.rs +++ b/crates/refset/src/family.rs @@ -31,6 +31,10 @@ pub enum Identity { /// names [`crate::fixture::first_shard`], and `# fixture` equals /// [`crate::fixture::line`] of that file. FixtureManifest, + /// `# model\t` in an MTP step set's `MANIFEST.tsv` (`tools/ref/q35mtp_ref.cpp`): the target file, which + /// carries the next-token layer; `# build` naming the mainline commit and `# mtp` the next-token layers and the + /// steps the MTP context ran ([`crate::mtptaps::TapSet`]). + MtpTaps, /// [`Identity::MtpManifest`] of a set dumped on a fixture file: `# model` /// and `# fixture` as [`Identity::FixtureManifest`]'s, and /// `# draft_model\t` too for a family whose draft is a file of its @@ -329,6 +333,14 @@ impl Family { build: set.build, }) } + Identity::MtpTaps => { + let set = crate::mtptaps::TapSet::open(path, self)?; + Ok(Provenance { + dumped_from: stated(set.model.as_deref()), + draft: None, + build: set.build, + }) + } Identity::ArgmaxHeader => { crate::greedy::check(path, self)?; Ok(Provenance { diff --git a/crates/refset/src/lib.rs b/crates/refset/src/lib.rs index 5dd176596..cbc1101a0 100644 --- a/crates/refset/src/lib.rs +++ b/crates/refset/src/lib.rs @@ -19,6 +19,7 @@ //! `MANIFEST.tsv` and the files its rows name. //! - [`dsref`]: ik's DSpark draft sets (`tools/ref/dump_draft.cpp`). //! - [`mtpref`]: ik's MTP draft sets (`tools/ref/dump_mtp.cpp`). +//! - [`mtptaps`]: mainline's MTP step, tap by tap (`tools/ref/q35mtp_ref.cpp`). //! - [`greedy`]: ik's greedy continuations (`tools/ref/argmax_ref.cpp`). //! - [`kld`]: ik's KL-divergence base runs (`tools/ref/ik-ppl.sh`). //! - [`vision`]: the vision encoder's oracle (`tools/ref/vision/dump_vision.py`). @@ -53,6 +54,7 @@ pub mod ik; pub mod kld; mod md5; pub mod mtpref; +pub mod mtptaps; pub mod nemotron; pub mod tokenizer; pub mod vision; diff --git a/crates/refset/src/mtpref.rs b/crates/refset/src/mtpref.rs index ee52796a4..880f5153e 100644 --- a/crates/refset/src/mtpref.rs +++ b/crates/refset/src/mtpref.rs @@ -16,6 +16,7 @@ use crate::RefError; use crate::columns::Columns; +use crate::columns::i32s_of; use crate::dsref::{Draft, Plain, Verify}; use crate::family::Family; use crate::ik::{ @@ -634,13 +635,5 @@ fn place(c: &Columns, line: &str, at: &dyn fmt::Display) -> Result<(i32, Graph), Ok((r.parse("block")?, Graph::parse(r.text("graph")?, at)?)) } -fn i32s_of(b: &[u8]) -> Vec { - b.as_chunks::<4>() - .0 - .iter() - .map(|c| i32::from_le_bytes(*c)) - .collect() -} - #[cfg(test)] mod tests; diff --git a/crates/refset/src/mtptaps.rs b/crates/refset/src/mtptaps.rs new file mode 100644 index 000000000..de79c2db7 --- /dev/null +++ b/crates/refset/src/mtptaps.rs @@ -0,0 +1,583 @@ +//! The MTP step oracle's sets (`tools/ref/q35mtp_ref.cpp`, `just dump-ref-q35mtp`): llama.cpp mainline's +//! next-token (MTP) graph run on a teacher-forced prompt, with the graph's own nodes dumped at some of its +//! steps. Step `i` takes token `ids[i + 1]`, position `i` and the trunk's `h_nextn` row `i`; every step runs in +//! order and the taps are read at the steps `# rows` names. A tap is read by its graph name and its row. +//! +//! ```text +//! # model +//! # build +//! # arch qwen35moe +//! # mtp +//! # tokens_count 64 +//! # rows 0 5 31 62 +//! # trunk columns name type rows width file +//! trunk h_nextn f32 64 2048 trunk_h.f32 +//! # step columns row token pos mtp_argmax target_next +//! step 0 0 +//! # tap columns name row type n file +//! tap mtp_eh_proj 0 f32 2048 mtp_eh_proj.r0.f32 +//! # complete 0 +//! ``` +//! +//! `prompt.ids` beside the manifest is the prompt, one id a line. `# mtp` is written only when the model has a +//! next-token layer and the MTP context ran every step: a set without it was dumped without the MTP context. +//! +//! [`TapSet::open`] refuses a set that did not finish, one dumped from another file or by another mainline +//! build, for another architecture, without the MTP context, and one whose files disagree with its rows. + +use crate::RefError; +use crate::columns::{Columns, Row, file_name, need, one, read_bytes, read_f32}; +use crate::family::{Family, Identity}; +use std::collections::{BTreeSet, HashSet}; +use std::fmt; +use std::path::{Path, PathBuf}; + +/// The prompt's ids, one a line, beside the manifest. +pub const IDS_FILE: &str = "prompt.ids"; + +/// A tap's element type. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Elem { + F32, + I32, +} + +impl Elem { + fn parse(v: &str, at: &dyn fmt::Display) -> Result { + match v { + "f32" => Ok(Elem::F32), + "i32" => Ok(Elem::I32), + v => Err(RefError::malformed( + at, + format!("type {v:?}, want f32 or i32"), + )), + } + } +} + +/// The trunk's `h_nextn` rows: the states the steps are fed. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Trunk { + /// The rows' name (`h_nextn`). + pub name: String, + /// One row a prompt position. + pub rows: u32, + /// The row's width. + pub width: u32, + /// The f32 file under the set's directory, row `t` at `t * width`. + pub file: String, +} + +/// One MTP step's token and the two argmaxes the oracle kept beside it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Step { + /// The step's row, `i`. + pub row: u32, + /// The token fed, `ids[i + 1]`. + pub token: u32, + /// The position fed, `i`. + pub pos: u32, + /// The MTP head's argmax at this step (the first id on a tie). + pub mtp_argmax: u32, + /// The trunk's own argmax at position `i + 1`: the token the target takes after `token`. + pub target_next: u32, +} + +/// One tapped node at one row. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Tap { + /// The graph's name of the node, without its layer suffix. + pub name: String, + pub row: u32, + pub elem: Elem, + /// Elements. + pub n: u32, + /// The file under the set's directory, `n` little-endian values. + pub file: String, +} + +/// `# mtp`: the model's next-token layers and the steps the MTP context ran. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct MtpRan { + pub layers: u32, + pub steps: u32, +} + +/// A tap set's manifest, and once opened, its files checked. +#[derive(Debug, Clone)] +pub struct TapSet { + /// The set's directory. + pub dir: PathBuf, + /// `# model`. + pub model: Option, + /// `# build`. + pub build: Option, + /// `# arch`. + pub arch: Option, + /// `# mtp`; `None` for a set dumped without the MTP context. + pub mtp: Option, + /// `# tokens_count`: the prompt's length. + pub tokens_count: Option, + /// `# rows`: the steps the taps were read at. + pub rows: Vec, + pub trunk: Option, + /// One row a step, in the manifest's order. + pub steps: Vec, + /// One row a (tap, row), in the manifest's order. + pub taps: Vec, + /// Whether the `# complete` trailer is there. + pub complete: bool, +} + +/// A manifest line's position, formatted only into an error. +struct At<'a>(&'a Path, usize); + +impl fmt::Display for At<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "mtptaps: {}:{}", self.0.display(), self.1) + } +} + +impl TapSet { + /// Parse `dir/MANIFEST.tsv`. A set without one is `Missing`, naming the recipe that writes it. A set without + /// its trailer comes back unread, `complete` false. A line of the wrong width, a row before its column line, + /// a count that does not parse, a file name that leaves the directory or a (tap, row) named twice is + /// `Malformed`. + pub fn read(dir: &Path, family: &Family) -> Result { + let path = dir.join("MANIFEST.tsv"); + let text = std::fs::read_to_string(&path).map_err(|e| { + RefError::missing( + &path, + format!("{}: {e} — run: {}", path.display(), family.recipe), + ) + })?; + let mut set = TapSet { + dir: dir.to_path_buf(), + model: None, + build: None, + arch: None, + mtp: None, + tokens_count: None, + rows: Vec::new(), + trunk: None, + steps: Vec::new(), + taps: Vec::new(), + complete: false, + }; + // A manifest without its trailer is a dump that did not finish: its rows are not read. + if !text + .lines() + .any(|l| l.split('\t').next() == Some("# complete")) + { + return Ok(set); + } + let mut trunk_cols: Option = None; + let mut step_cols: Option = None; + let mut tap_cols: Option = None; + let mut seen = HashSet::new(); + for (i, line) in text.lines().enumerate() { + if line.is_empty() { + continue; + } + let at = At(&path, i + 1); + let f: Vec<&str> = line.split('\t').collect(); + match f[0] { + "# model" => set.model = Some(one(&f, &at, "# model\t")?), + "# build" => set.build = Some(one(&f, &at, "# build\t")?), + "# arch" => set.arch = Some(one(&f, &at, "# arch\t")?), + "# tokens_count" => { + set.tokens_count = Some(number(&one(&f, &at, "# tokens_count\t")?, &at)?); + } + "# mtp" => { + let &[_, layers, steps] = f.as_slice() else { + return Err(RefError::malformed(&at, "want # mtp\t\t")); + }; + set.mtp = Some(MtpRan { + layers: number(layers, &at)?, + steps: number(steps, &at)?, + }); + } + "# rows" => { + set.rows = f[1..] + .iter() + .map(|v| number(v, &at)) + .collect::>()?; + } + "# trunk columns" => { + trunk_cols = Some(Columns::new("trunk", f[1..].iter().copied())) + } + "# step columns" => step_cols = Some(Columns::new("step", f[1..].iter().copied())), + "# tap columns" => tap_cols = Some(Columns::new("tap", f[1..].iter().copied())), + "# complete" => { + check_trailer(&f, set.taps.len(), text.lines().skip(i + 1), &at)?; + set.complete = true; + } + h if h.starts_with('#') => {} + "trunk" => { + let r = need(&trunk_cols, "trunk", &at)?.row(line, &at)?; + if set.trunk.is_some() { + return Err(RefError::malformed(&at, "a second trunk row")); + } + set.trunk = Some(trunk_row(&r, &at)?); + } + "step" => { + let r = need(&step_cols, "step", &at)?.row(line, &at)?; + let step = step_row(&r)?; + if !seen.insert(("step", step.row, String::new())) { + return Err(RefError::malformed( + &at, + format!("step {} is named twice", step.row), + )); + } + set.steps.push(step); + } + "tap" => { + let r = need(&tap_cols, "tap", &at)?.row(line, &at)?; + let tap = tap_row(&r, &at)?; + if !seen.insert(("tap", tap.row, tap.name.clone())) { + return Err(RefError::malformed( + &at, + format!("tap {} at row {} is named twice", tap.name, tap.row), + )); + } + set.taps.push(tap); + } + other => { + return Err(RefError::malformed( + &at, + format!("unknown row kind {other:?}"), + )); + } + } + } + Ok(set) + } + + /// The set at `dir` read and checked against `family`'s row. + pub fn open(dir: &Path, family: &Family) -> Result { + if !matches!(family.identity, Identity::MtpTaps) { + return Err(RefError::missing( + dir, + format!("mtptaps: {} is not an MTP tap family", family.name), + )); + } + let set = TapSet::read(dir, family)?; + set.check_family(family)?; + Ok(set) + } + + /// This set against its family's row: the trailer ([`RefError::Unfinished`]), the model file + /// ([`RefError::Stale`]), the build, the architecture and the MTP context ([`RefError::Foreign`]), then + /// the rows against each other and each file against its row. + pub fn check_family(&self, family: &Family) -> Result<(), RefError> { + if !self.complete { + return Err(RefError::Unfinished { + set: self.dir.display().to_string(), + }); + } + family.check_file(&self.dir, "# model", self.model.as_deref())?; + family.check_build(&self.dir, self.build.as_deref())?; + family.check_arch(&self.dir, self.arch.as_deref())?; + let mtp = self.mtp.ok_or_else(|| RefError::Foreign { + set: self.dir.display().to_string(), + family: family.name, + field: "mtp", + got: "(no # mtp line)".to_string(), + want: "a set dumped with the model's MTP context (llama_model_n_layer_nextn > 0, every step run)" + .to_string(), + })?; + let at = self.dir.join("MANIFEST.tsv"); + let n = self + .tokens_count + .ok_or_else(|| RefError::malformed(at.display(), "no # tokens_count line"))?; + if mtp.layers == 0 || u64::from(mtp.steps) + 1 != u64::from(n) { + return Err(RefError::malformed( + at.display(), + format!( + "# mtp {} {}: the MTP context needs a next-token layer and one step a prompt id but the last ({n} ids)", + mtp.layers, mtp.steps + ), + )); + } + let ids = self.ids()?; + if ids.len() != n as usize { + return Err(RefError::malformed( + self.dir.join(IDS_FILE).display(), + format!( + "{IDS_FILE} holds {} ids, # tokens_count says {n}", + ids.len() + ), + )); + } + self.check_trunk(n)?; + self.check_steps(&ids, mtp.steps)?; + self.check_taps() + } + + fn check_trunk(&self, n: u32) -> Result<(), RefError> { + let at = self.dir.join("MANIFEST.tsv"); + let t = self + .trunk + .as_ref() + .ok_or_else(|| RefError::malformed(at.display(), "no trunk row"))?; + if t.rows != n { + return Err(RefError::malformed( + at.display(), + format!("the trunk holds {} rows, the prompt {n} ids", t.rows), + )); + } + self.trunk_rows().map(|_| ()) + } + + fn check_steps(&self, ids: &[u32], steps: u32) -> Result<(), RefError> { + let at = self.dir.join("MANIFEST.tsv"); + if self.steps.len() != steps as usize { + return Err(RefError::malformed( + at.display(), + format!( + "{} step rows, # mtp says {steps} steps ran", + self.steps.len() + ), + )); + } + for (i, s) in self.steps.iter().enumerate() { + if s.row as usize != i || s.pos != s.row || ids.get(i + 1) != Some(&s.token) { + return Err(RefError::malformed( + at.display(), + format!( + "step row {} is not step {i}: token {} at position {}, the prompt's id {} is {:?}", + s.row, + s.token, + s.pos, + i + 1, + ids.get(i + 1) + ), + )); + } + } + Ok(()) + } + + /// Every tapped row has every tap, every row is a step, and every file is its row's length. + fn check_taps(&self) -> Result<(), RefError> { + let at = self.dir.join("MANIFEST.tsv"); + if self.rows.is_empty() || self.taps.is_empty() { + return Err(RefError::malformed( + at.display(), + "no # rows or no tap rows: the set names nothing to check", + )); + } + if self.rows.iter().collect::>().len() != self.rows.len() { + return Err(RefError::malformed(at.display(), "duplicate # rows entry")); + } + let names: Vec<&str> = { + let mut v: Vec<&str> = Vec::new(); + for t in &self.taps { + if !v.contains(&t.name.as_str()) { + v.push(&t.name); + } + } + v + }; + for &row in &self.rows { + if self.step(row).is_err() { + return Err(RefError::malformed( + at.display(), + format!("# rows names {row}, which is no step"), + )); + } + for name in &names { + if !self.taps.iter().any(|t| t.row == row && t.name == *name) { + return Err(RefError::malformed( + at.display(), + format!("no tap {name} at row {row}, which # rows names"), + )); + } + } + } + if self.taps.len() != self.rows.len() * names.len() { + return Err(RefError::malformed( + at.display(), + format!( + "{} tap rows for {} taps at {} rows: a tap at a row # rows does not name", + self.taps.len(), + names.len(), + self.rows.len() + ), + )); + } + for t in &self.taps { + match t.elem { + Elem::F32 => { + self.tap_f32(&t.name, t.row)?; + } + Elem::I32 => { + read_bytes(&self.dir.join(&t.file), u64::from(t.n), &t.file)?; + } + } + } + Ok(()) + } + + /// The prompt's ids. + pub fn ids(&self) -> Result, RefError> { + let path = self.dir.join(IDS_FILE); + let text = std::fs::read_to_string(&path).map_err(|e| { + RefError::missing( + &path, + format!("mtptaps: cannot read {}: {e}", path.display()), + ) + })?; + text.lines() + .map(|l| { + if l.is_empty() { + return Err(RefError::malformed( + path.display(), + format!("{IDS_FILE}: empty token id"), + )); + } + l.parse().map_err(|e| { + RefError::malformed(path.display(), format!("{IDS_FILE}: id {l:?}: {e}")) + }) + }) + .collect() + } + + /// The trunk's `h_nextn` rows, `rows * width` f32, row `t` at `t * width`. + pub fn trunk_rows(&self) -> Result, RefError> { + let t = self.trunk.as_ref().ok_or_else(|| { + RefError::missing( + &self.dir, + format!("mtptaps: {} has no trunk row", self.dir.display()), + ) + })?; + read_f32( + &self.dir.join(&t.file), + u64::from(t.rows) * u64::from(t.width), + &t.file, + ) + } + + /// Step `row`. + pub fn step(&self, row: u32) -> Result<&Step, RefError> { + self.steps.iter().find(|s| s.row == row).ok_or_else(|| { + RefError::missing( + &self.dir, + format!("mtptaps: {} has no step {row}", self.dir.display()), + ) + }) + } + + /// The tap `name` at `row`; a name or a row the set does not hold is `Missing`, naming both. + pub fn tap(&self, name: &str, row: u32) -> Result<&Tap, RefError> { + self.taps + .iter() + .find(|t| t.row == row && t.name == name) + .ok_or_else(|| { + RefError::missing( + &self.dir, + format!( + "mtptaps: {} has no tap {name} at row {row} (taps {:?}, rows {:?})", + self.dir.display(), + self.taps + .iter() + .map(|t| t.name.as_str()) + .collect::>(), + self.rows + ), + ) + }) + } + + /// The f32 tap `name` at `row`; an i32 tap is `Malformed` here, as a count of another length is. + pub fn tap_f32(&self, name: &str, row: u32) -> Result, RefError> { + let t = self.tap(name, row)?; + if t.elem != Elem::F32 { + return Err(RefError::malformed( + self.dir.join(&t.file).display(), + format!("tap {name} is i32, read as f32"), + )); + } + read_f32(&self.dir.join(&t.file), u64::from(t.n), &t.file) + } + + /// The i32 tap `name` at `row` (the routed expert ids). + pub fn tap_i32(&self, name: &str, row: u32) -> Result, RefError> { + let t = self.tap(name, row)?; + if t.elem != Elem::I32 { + return Err(RefError::malformed( + self.dir.join(&t.file).display(), + format!("tap {name} is f32, read as i32"), + )); + } + let bytes = read_bytes(&self.dir.join(&t.file), u64::from(t.n), &t.file)?; + Ok(crate::columns::i32s_of(&bytes)) + } +} + +/// `# complete 0`, the manifest's last line: a set whose trailer counts other tap rows +/// than it holds lost rows after the writer counted them. `after` is the lines after the trailer. +fn check_trailer<'a>( + f: &[&str], + taps: usize, + mut after: impl Iterator, + at: &dyn fmt::Display, +) -> Result<(), RefError> { + if f.len() != 3 + || f[2] != "0" + || number(f[1], at)? as usize != taps + || after.any(|l| !l.is_empty()) + { + return Err(RefError::malformed( + at, + format!( + "the trailer `{}` is not `# complete\t{taps}\t0` on the last line", + f.join("\t") + ), + )); + } + Ok(()) +} + +fn number(v: &str, at: &dyn fmt::Display) -> Result { + crate::columns::parse_field(v, "count", at) +} + +/// The trunk row: its rows are f32 and its file is one name of the set's directory. +fn trunk_row(r: &Row<'_>, at: &dyn fmt::Display) -> Result { + let file = r.text("file")?; + file_name(file, at)?; + if r.text("type")? != "f32" { + return Err(RefError::malformed(at, "the trunk rows are f32")); + } + Ok(Trunk { + name: r.text("name")?.to_string(), + rows: r.parse("rows")?, + width: r.parse("width")?, + file: file.to_string(), + }) +} + +fn step_row(r: &Row<'_>) -> Result { + Ok(Step { + row: r.parse("row")?, + token: r.parse("token")?, + pos: r.parse("pos")?, + mtp_argmax: r.parse("mtp_argmax")?, + target_next: r.parse("target_next")?, + }) +} + +fn tap_row(r: &Row<'_>, at: &dyn fmt::Display) -> Result { + let name = r.text("name")?; + let file = r.text("file")?; + file_name(name, at)?; + file_name(file, at)?; + Ok(Tap { + name: name.to_string(), + row: r.parse("row")?, + elem: Elem::parse(r.text("type")?, at)?, + n: r.parse("n")?, + file: file.to_string(), + }) +} diff --git a/crates/refset/src/nemotron.rs b/crates/refset/src/nemotron.rs index b1979cc94..8fd726a1c 100644 --- a/crates/refset/src/nemotron.rs +++ b/crates/refset/src/nemotron.rs @@ -16,7 +16,7 @@ //! another file, and one written by another mainline build. use crate::RefError; -use crate::columns::{Columns, Row, file_name}; +use crate::columns::{Columns, Row, file_name, need, one}; use crate::family::{Family, Identity}; use std::collections::HashSet; use std::fmt; @@ -118,9 +118,7 @@ impl LogitsSet { } h if h.starts_with('#') => {} "prompt" => { - let c = cols.as_ref().ok_or_else(|| { - RefError::malformed(&at, "a prompt row before its `# prompt columns` line") - })?; + let c = need(&cols, "prompt", &at)?; let prompt = prompt_row(&c.row(line, &at)?, &at)?; if !seen.insert(prompt.name.clone()) { return Err(RefError::malformed( @@ -189,39 +187,11 @@ impl LogitsSet { /// Prompt `p`'s last-position logits, `n_vocab` f32, little-endian. pub fn logits(&self, p: &Prompt) -> Result, RefError> { - let path = self.dir.join(&p.logits_file); - let bytes = std::fs::read(&path).map_err(|e| { - RefError::missing( - &path, - format!("nemotron: cannot read {}: {e}", path.display()), - ) - })?; - let want = p.n_vocab as usize * 4; - if bytes.len() != want { - return Err(RefError::malformed( - path.display(), - format!( - "{} is {} bytes, {} logits of {} want {want}", - p.logits_file, - bytes.len(), - p.name, - p.n_vocab - ), - )); - } - let mut out = Vec::with_capacity(p.n_vocab as usize); - for chunk in bytes.chunks_exact(4) { - out.push(f32::from_le_bytes( - chunk.try_into().expect("chunks_exact(4)"), - )); - } - if let Some(i) = out.iter().position(|v| !v.is_finite()) { - return Err(RefError::malformed( - path.display(), - format!("non-finite logit at index {i}"), - )); - } - Ok(out) + crate::columns::read_f32( + &self.dir.join(&p.logits_file), + u64::from(p.n_vocab), + &p.logits_file, + ) } fn check_prompt(&self, p: &Prompt) -> Result<(), RefError> { @@ -263,13 +233,6 @@ impl LogitsSet { } } -fn one(f: &[&str], at: &dyn fmt::Display, want: &str) -> Result { - match f { - [_, v] if !v.is_empty() => Ok((*v).to_string()), - _ => Err(RefError::malformed(at, format!("want {want}"))), - } -} - fn prompt_row(r: &Row<'_>, at: &dyn fmt::Display) -> Result { let name = r.text("name")?; let ids_file = r.text("ids")?; diff --git a/crates/refset/src/vision.rs b/crates/refset/src/vision.rs index 330e4917a..9d92cd4f1 100644 --- a/crates/refset/src/vision.rs +++ b/crates/refset/src/vision.rs @@ -10,7 +10,7 @@ //! those names. use crate::RefError; -use crate::columns::{Columns, Row, parse_field}; +use crate::columns::{Columns, Row, need, parse_field}; use crate::family::Family; use std::fmt; use std::path::{Path, PathBuf}; @@ -253,15 +253,6 @@ fn columns(kind: &str, f: &[&str], at: &dyn fmt::Display) -> Result( - cols: &'a Option, - what: &str, - at: &dyn fmt::Display, -) -> Result<&'a Columns, RefError> { - cols.as_ref() - .ok_or_else(|| RefError::malformed(at, format!("a {what} row before its column line"))) -} - fn image_row(r: &Row<'_>) -> Result { Ok(Image { name: r.text("name")?.to_string(), diff --git a/crates/refset/src/visref.rs b/crates/refset/src/visref.rs index 76126acde..229786c25 100644 --- a/crates/refset/src/visref.rs +++ b/crates/refset/src/visref.rs @@ -16,6 +16,7 @@ //! (`[logits, n_vocab]` f32). use crate::RefError; +use crate::columns::need as need_cols; use crate::columns::{Columns, Row, parse_field}; use crate::family::{Family, Identity}; use std::fmt; @@ -355,15 +356,6 @@ fn columns(kind: &str, f: &[&str], at: &dyn fmt::Display) -> Result( - cols: &'a Option, - what: &str, - at: &dyn fmt::Display, -) -> Result<&'a Columns, RefError> { - cols.as_ref() - .ok_or_else(|| RefError::malformed(at, format!("a {what} row before its column line"))) -} - fn case_row(r: &Row<'_>) -> Result { let eog: u8 = r.parse("eog")?; if eog > 1 { diff --git a/justfile b/justfile index 594f19bbf..f54ed2d9a 100644 --- a/justfile +++ b/justfile @@ -683,6 +683,19 @@ build-ref-hidden: dump-hidden-qwen35 *SETS: BLOOMERY_MODEL=qwen35 ./tools/box.sh 'bash tools/ref/hidden.sh {{SETS}}' +# The Qwen3.6 MTP step oracle's binary: q35mtp_ref (tools/ref/q35mtp_ref.cpp) linked against llama.cpp mainline's libllama in the +# qwen35moe profile's MTP_LCPP tree (tools/ref/build-q35mtp-ref.sh); builds nothing in that tree. +build-ref-q35mtp: + BLOOMERY_MODEL=qwen35moe ./tools/box.sh 'bash tools/ref/build-q35mtp-ref.sh' + +# The Qwen3.6 MTP step oracle set (refset family mtp-qwen35moe-mainline): llama.cpp mainline's graph_mtp over the first 64 prose +# ids fed teacher-forced (step i: token ids[i+1], position i, the trunk's h_nextn row i), every step run in order, the profile's +# MTP_TAPS nodes dumped at MTP_ROWS = 0, 5, 31, 62, into $BLOOMERY_DATA/ref-mtp/qwen35moe_mainline_prose64 (tools/ref/dump-q35mtp.sh). +# Trunk and MTP context on the card box.sh puts in view, through its gate lock; no lease (a functional oracle). `--cpu` runs on the +# host (no card, no lock); `--tap NAME` (repeatable) replaces the default tap list. +dump-ref-q35mtp *ARGS: + BLOOMERY_MODEL=qwen35moe ./tools/box.sh 'bash tools/ref/real-only.sh dump-ref-q35mtp && bash tools/ref/dump-q35mtp.sh {{ARGS}}' + # Clef-Flash image input, the oracle's binary: dump_mtmd (tools/ref/clefvis/dump_mtmd.cpp) linked against llama.cpp mainline's libmtmd in the qwen35 profile's tree (tools/ref/clefvis/build-clefvis.sh). build-clefvis-ref: BLOOMERY_MODEL=qwen35 ./tools/box.sh 'bash tools/ref/clefvis/build-clefvis.sh' diff --git a/tools/ref/build-dump-mtp.sh b/tools/ref/build-dump-mtp.sh index ba991aa5e..a9f64fcaa 100644 --- a/tools/ref/build-dump-mtp.sh +++ b/tools/ref/build-dump-mtp.sh @@ -54,7 +54,7 @@ echo "build-dump-mtp: ik libraries current in $(($(date +%s) - t0)) s" mkdir -p "$OUT" TMP=$OUT/dump_mtp.tmp.$$ trap 'rm -f "$TMP" "$TMP.build"' EXIT -SRC_SHA=$(sha256sum "$SRC" | cut -d' ' -f1) +SRC_SHA=$(cat "$SRC" "$HERE/tools/ref/ref_io.h" | sha256sum | cut -d' ' -f1) # -rdynamic exports dump_mtp's ggml_backend_sched_graph_compute_async and llama_set_mtp_op_type so # libllama's calls bind to them (its header says why); -ldl for the dlsym(RTLD_NEXT) that forwards to # the libraries' own. diff --git a/tools/ref/build-dump.sh b/tools/ref/build-dump.sh index 5828672b5..b70f31f57 100755 --- a/tools/ref/build-dump.sh +++ b/tools/ref/build-dump.sh @@ -14,7 +14,7 @@ SRC=$HERE/tools/ref/dump_ref.cpp mkdir -p "$OUT" TMP=$OUT/dump_ref.tmp.$$ trap 'rm -f "$TMP" "$TMP.build"' EXIT -SRC_SHA=$(sha256sum "$SRC" | cut -d' ' -f1) +SRC_SHA=$(cat "$SRC" "$HERE/tools/ref/ref_io.h" | sha256sum | cut -d' ' -f1) ref_cxx -o "$TMP" "$SRC" "${REF_LLAMA_INC[@]}" "${REF_LLAMA_LINK[@]}" printf 'source_sha256 %s\nik %s\n' "$SRC_SHA" "$(readlink -f "$IK")" > "$TMP.build" mv -f "$TMP" "$OUT/dump_ref" diff --git a/tools/ref/build-q35mtp-ref.sh b/tools/ref/build-q35mtp-ref.sh new file mode 100755 index 000000000..36e208f77 --- /dev/null +++ b/tools/ref/build-q35mtp-ref.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Build the MTP step oracle on the box: q35mtp_ref (tools/ref/q35mtp_ref.cpp) linked against llama.cpp mainline's +# libllama and libggml in the qwen35moe profile's MTP_LCPP tree (tools/ref/models/qwen35moe.sh), not the ik tree the +# node dumpers link. Under the qwen35moe profile only. +# +# The tree is checked, not trusted: no tracked change (a hand edit there would reach the oracle under the commit's +# name). Its build is the one that exists; this links against it and builds nothing in it. q35mtp_ref includes the +# tree's staging header src/llama-ext.h (the nextn embeddings), so it is built from that tree's sources at that commit. +# Output: $BLOOMERY_DATA/bin/q35mtp_ref (Q35MTP_OUT moves it) and q35mtp_ref.build beside it, naming the sha256 of the +# q35mtp_ref.cpp and ref_io.h it was built from, the mainline commit and the md5 of the libllama it links; dump-q35mtp.sh refuses a +# binary whose record does not match its own tree. The compile writes a temporary name and only a finished link +# replaces the installed binary. +set -euo pipefail +# shellcheck source=tools/ref/ref-paths.sh +source "$(dirname "${BASH_SOURCE[0]}")/ref-paths.sh" +[ "$MODEL_NAME" = qwen35moe ] || + { echo "build-q35mtp-ref: q35mtp_ref is built under the qwen35moe profile; this command picked $MODEL_NAME" >&2; exit 2; } +HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +OUT=${Q35MTP_OUT:-$BLOOMERY_DATA/bin} +SRC=$HERE/tools/ref/q35mtp_ref.cpp +L=$MTP_LCPP +git_lcpp() { git -c safe.directory="$L" -C "$L" "$@"; } +# A failed query ends the build (set -e on the assignment), so an unreadable index is not read as a clean tree. +dirty=$(git_lcpp status --porcelain --untracked-files=no) +if [ -n "$dirty" ]; then + echo "build-q35mtp-ref: $L has tracked changes; the oracle is the commit's build, not an edited tree" >&2 + exit 2 +fi +COMMIT=$(git_lcpp rev-parse --short=9 HEAD) +LIB=$L/build/bin +[ -f "$LIB/libllama.so.0" ] || { echo "build-q35mtp-ref: no $LIB/libllama.so.0: $L is not built" >&2; exit 66; } +[ -f "$LIB/libggml-cuda.so" ] || { echo "build-q35mtp-ref: no $LIB/libggml-cuda.so: $L has no CUDA backend" >&2; exit 66; } +LIB_MD5=$(md5sum < "$LIB/libllama.so.0" | cut -d' ' -f1) +mkdir -p "$OUT" +TMP=$OUT/q35mtp_ref.tmp.$$ +trap 'rm -f "$TMP" "$TMP.build"' EXIT +SRC_SHA=$(cat "$SRC" "$HERE/tools/ref/ref_io.h" | sha256sum | cut -d' ' -f1) +g++ -std=c++17 -O2 -o "$TMP" "$SRC" -I"$L/include" -I"$L/ggml/include" -I"$L/src" \ + -L"$LIB" -lllama -lggml -lggml-base -Wl,-rpath,"$LIB" +printf 'source_sha256 %s\nlcpp %s\nlcpp_commit %s\nlibllama_md5 %s\n' "$SRC_SHA" "$(readlink -f "$L")" "$COMMIT" "$LIB_MD5" > "$TMP.build" +mv -f "$TMP" "$OUT/q35mtp_ref" +mv -f "$TMP.build" "$OUT/q35mtp_ref.build" +echo "built $OUT/q35mtp_ref from q35mtp_ref.cpp sha256 ${SRC_SHA:0:12} against $L at $COMMIT" diff --git a/tools/ref/dump-mtp.sh b/tools/ref/dump-mtp.sh index 65e760727..4a32c52f3 100644 --- a/tools/ref/dump-mtp.sh +++ b/tools/ref/dump-mtp.sh @@ -102,7 +102,7 @@ for lib in libllama.so libggml.so; do esac done SRC="${BASH_SOURCE[0]%/*}/dump_mtp.cpp" -SRC_SHA=$(sha256sum "$SRC" | cut -d' ' -f1) +SRC_SHA=$(cat "$SRC" "${BASH_SOURCE[0]%/*}/ref_io.h" | sha256sum | cut -d' ' -f1) rec() { awk -v k="$1" '$1 == k { print $2 }' "$BIN.build" 2>/dev/null || true; } stale=() [ "$(rec source_sha256)" = "$SRC_SHA" ] || stale+=("$SRC (sha256 ${SRC_SHA:0:12}; the binary's record names '$(rec source_sha256 | cut -c1-12)')") diff --git a/tools/ref/dump-q35mtp.sh b/tools/ref/dump-q35mtp.sh new file mode 100755 index 000000000..c70b6bd98 --- /dev/null +++ b/tools/ref/dump-q35mtp.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# Write the Qwen3.6 MTP step oracle set (tools/ref/models/qwen35moe.sh MTP_*): llama.cpp mainline's MTP draft step of +# the prompt's first MTP_PROMPT prose ids, fed teacher-forced (step i: token ids[i+1], position i, the trunk's h_nextn +# row i), every step run in order and the graph nodes MTP_TAPS dumped at the steps MTP_ROWS, by q35mtp_ref +# (tools/ref/q35mtp_ref.cpp), into $BLOOMERY_DATA/ref-mtp/qwen35moe_mainline_prose64 — the set the refset family +# mtp-qwen35moe-mainline checks. +# +# The binary must be the one built from this tree's q35mtp_ref.cpp and ref_io.h (its .build record's source sha256) against the profile's +# MTP_LCPP at its current commit and libllama (build-q35mtp-ref.sh), and the ids file must have the profile's sha256: +# either refused names what differs. The set is written to .staging and moved over the old set only when q35mtp_ref +# exits 0 and its manifest has the `# mtp` line and the `# complete` trailer. Every layer and the head are on the card +# (-ngl 999), the context the prompt's length, one ubatch. The run goes through tools/gpu-gate.sh, as lev-dump.sh's does: +# the card's gate lock, the bound (Q35MTP_BOUND, default 1500 s) and the card pick (BLOOMERY_GATE_CARD); no timing lease +# (a functional oracle). The script links itself as target/release/q35mtp_dump and runs under that name inside the runner. +# +# Q35MTP_SET moves the set (a trial run installs nothing under the family's name). `--cpu` runs the same binary on the host instead +# (no card in view, -ngl 0, -t 16, no gate lock), only into a set Q35MTP_SET names; the set's `# backend` line says which one wrote it. +# +# dump-q35mtp.sh [--cpu] [--tap NAME]... the default taps are the profile's MTP_TAPS; --tap replaces them +set -euo pipefail +REAL=$(readlink -f "${BASH_SOURCE[0]}") +cd "$(dirname "$REAL")/../.." +cpu=0 +if [ "${1:-}" = --cpu ]; then cpu=1; shift; fi +# The family's set is the card's (mmvq over q8_1); a --cpu run writes only to a set Q35MTP_SET names. +if [ "$cpu" = 1 ] && [ -z "${Q35MTP_SET:-}" ]; then + echo "dump-q35mtp: --cpu writes a trial set; name it with Q35MTP_SET (the family's set is the card's)" >&2 + exit 64 +fi +if [ "$(basename "${BASH_SOURCE[0]}")" != q35mtp_dump ] && [ "$cpu" = 0 ]; then + mkdir -p target/release + ln -sf ../../tools/ref/dump-q35mtp.sh target/release/q35mtp_dump + BLOOMERY_GATE_BOUND=${Q35MTP_BOUND:-1500} BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh q35mtp_dump "$@" + exit $? +fi +# shellcheck source=tools/ref/ref-paths.sh +source tools/ref/ref-paths.sh +[ "$MODEL_NAME" = qwen35moe ] || + { echo "dump-q35mtp: the MTP set is the qwen35moe profile's; this command picked $MODEL_NAME" >&2; exit 2; } +BIN=$BLOOMERY_DATA/bin/q35mtp_ref +want_src=$(cat tools/ref/q35mtp_ref.cpp tools/ref/ref_io.h | sha256sum | cut -d' ' -f1) +want_commit=$(git -c safe.directory="$MTP_LCPP" -C "$MTP_LCPP" rev-parse --short=9 HEAD) +want_lib=$(md5sum < "$MTP_LCPP/build/bin/libllama.so.0" | cut -d' ' -f1) +if ! grep -qx "source_sha256 $want_src" "$BIN.build" 2>/dev/null || + ! grep -qx "lcpp_commit $want_commit" "$BIN.build" 2>/dev/null || + ! grep -qx "libllama_md5 $want_lib" "$BIN.build" 2>/dev/null; then + echo "dump-q35mtp: $BIN is not this tree's q35mtp_ref at $MTP_LCPP $want_commit (rebuild: just build-ref-q35mtp)" >&2 + exit 2 +fi +got_sha=$(sha256sum "$MTP_IDS" | cut -d' ' -f1) +[ "$got_sha" = "$MTP_IDS_SHA256" ] || + { echo "dump-q35mtp: $MTP_IDS has sha256 $got_sha, the profile pins $MTP_IDS_SHA256" >&2; exit 2; } +[ -f "$MTP_MODEL" ] || { echo "dump-q35mtp: no $MTP_MODEL" >&2; exit 66; } +taps=() +while [ $# -gt 0 ]; do + case $1 in + --tap) [ $# -ge 2 ] || { echo "dump-q35mtp: --tap wants a name" >&2; exit 64; }; taps+=(--tap "$2"); shift 2 ;; + *) echo "dump-q35mtp: unknown argument '$1' (--cpu, --tap NAME)" >&2; exit 64 ;; + esac +done +if [ ${#taps[@]} = 0 ]; then for t in "${MTP_TAPS[@]}"; do taps+=(--tap "$t"); done; fi +SET=${Q35MTP_SET:-$BLOOMERY_DATA/ref-mtp/qwen35moe_mainline_prose64} +mkdir -p "$(dirname "$SET")" +# One writer a destination, card or --cpu, held through the install and the staging cleanup. +exec {set_lock}> "$SET.lock" +flock -n "$set_lock" || { echo "dump-q35mtp: $SET already has a writer" >&2; exit 75; } +STAGE=$SET.staging +rm -rf "$STAGE" +# ref-paths.sh's drop_incomplete_stage; a finished stage the checks below refuse is removed by the next run. +trap drop_incomplete_stage EXIT +where=(-ngl 999) run=(env) +if [ "$cpu" = 1 ]; then where=(-ngl 0 -t 16) run=(env CUDA_VISIBLE_DEVICES=); fi +"${run[@]}" BLOOMERY_REF_BUILD="$want_commit" BLOOMERY_REF_TOKENS_SHA256="$MTP_IDS_SHA256" \ + timeout --kill-after=10 "${Q35MTP_RUN_BOUND:-1200}" "$BIN" -m "$MTP_MODEL" --arch qwen35moe \ + --tokens-file "$MTP_IDS" --tokens-count "$MTP_PROMPT" --rows "$MTP_ROWS" "${taps[@]}" \ + --out "$STAGE" -c "$MTP_PROMPT" "${where[@]}" +grep -q '^# complete' "$STAGE/MANIFEST.tsv" || + { echo "dump-q35mtp: the staged set has no completion trailer — not installing it" >&2; exit 1; } +grep -q $'^# mtp\t' "$STAGE/MANIFEST.tsv" || + { echo "dump-q35mtp: the staged set has no # mtp line (no MTP context ran) — not installing it" >&2; exit 1; } +rm -rf "$SET" +mv "$STAGE" "$SET" +echo "dump-q35mtp: installed $SET" diff --git a/tools/ref/dump.sh b/tools/ref/dump.sh index 5044bcc0f..01a1449cc 100755 --- a/tools/ref/dump.sh +++ b/tools/ref/dump.sh @@ -171,7 +171,7 @@ SRC="${BASH_SOURCE[0]%/*}/dump_ref.cpp" BUILD_REC="$BIN.build" BIN_SHA=$(sha256sum "$BIN" | cut -c1-12) BIN_MTIME=$(date -u -r "$BIN" +%Y-%m-%dT%H:%M:%SZ) -SRC_SHA=$(sha256sum "$SRC" | cut -d' ' -f1) +SRC_SHA=$(cat "$SRC" "$(dirname "$SRC")/ref_io.h" | sha256sum | cut -d' ' -f1) REC_SHA=$(awk '$1 == "source_sha256" { print $2 }' "$BUILD_REC" 2>/dev/null || true) stale=() if [ -z "$REC_SHA" ]; then diff --git a/tools/ref/dump_draft.cpp b/tools/ref/dump_draft.cpp index 832fcfd1a..2e974448a 100644 --- a/tools/ref/dump_draft.cpp +++ b/tools/ref/dump_draft.cpp @@ -52,6 +52,7 @@ #include "ggml.h" #include "sampling.h" #include "speculative.h" +#include "ref_io.h" #include #include @@ -171,24 +172,6 @@ static void gather(const ggml_tensor * t, const uint8_t * src, bool logical, } } -static bool write_raw(const std::string & path, const void * data, size_t size, size_t count) { - FILE * f = fopen(path.c_str(), "wb"); - if (!f) { - fprintf(stderr, "dump_draft: cannot write %s\n", path.c_str()); - return false; - } - if (fwrite(data, size, count, f) != count) { - fprintf(stderr, "dump_draft: short write on %s\n", path.c_str()); - fclose(f); - return false; - } - if (fclose(f) != 0) { - fprintf(stderr, "dump_draft: cannot flush %s\n", path.c_str()); - return false; - } - return true; -} - static bool write_twin(dump_ctx * d, const ggml_tensor * t, const char * name, int occurrence, bool input, const std::string & stem, bool logical, const std::vector & ints) { const bool wide = t->type == GGML_TYPE_I64; @@ -421,33 +404,6 @@ static void report_unhandled(const dump_ctx & d) { fprintf(stderr, "dump_draft: %d tensors skipped as unhandled: %s\n", total, types.empty() ? "none" : types.c_str()); } -static bool read_token_file(const std::string & path, long long count, std::vector & ids) { - std::ifstream in(path); - if (!in) { - fprintf(stderr, "dump_draft: cannot read %s\n", path.c_str()); - return false; - } - std::string line; - long long lines = 0; - while ((long long) ids.size() < count && std::getline(in, line)) { - ++lines; - char * end = nullptr; - errno = 0; - const long v = strtol(line.c_str(), &end, 10); - if (line.empty() || *end != '\0' || errno != 0 || v < 0 || v > INT32_MAX) { - fprintf(stderr, "dump_draft: %s:%lld is not a token id: '%s'\n", path.c_str(), lines, line.c_str()); - return false; - } - ids.push_back((llama_token) v); - } - if ((long long) ids.size() < count) { - fprintf(stderr, "dump_draft: %s holds %zu token ids and --tokens-count asks for %lld\n", path.c_str(), - ids.size(), count); - return false; - } - return true; -} - static std::string file_arch(const std::string & path) { gguf_init_params gp = { /*.no_alloc =*/ true, /*.ctx =*/ nullptr }; gguf_context * g = gguf_init_from_file(path.c_str(), gp); @@ -526,7 +482,7 @@ int main(int argc, char ** argv) { if (strcmp(argv[i], "--tokens-file") == 0 && i + 1 < argc) { tokens_file = argv[++i]; } else if (strcmp(argv[i], "--tokens-count") == 0 && i + 1 < argc) { - tokens_count = atoll(argv[++i]); + if (!parse_i64("dump_draft", "--tokens-count", argv[++i], 1, LLONG_MAX, tokens_count)) return 2; } else if (strcmp(argv[i], "--expect-arch") == 0 && i + 1 < argc) { expect_arch = argv[++i]; } else if (strcmp(argv[i], "--expect-draft-arch") == 0 && i + 1 < argc) { diff --git a/tools/ref/dump_mtp.cpp b/tools/ref/dump_mtp.cpp index e0e3ea710..dfac4e222 100644 --- a/tools/ref/dump_mtp.cpp +++ b/tools/ref/dump_mtp.cpp @@ -64,6 +64,7 @@ #include "ggml.h" #include "sampling.h" #include "speculative.h" +#include "ref_io.h" #include #include @@ -225,24 +226,6 @@ static void gather(const ggml_tensor * t, const uint8_t * src, bool logical, } } -static bool write_raw(const std::string & path, const void * data, size_t size, size_t count) { - FILE * f = fopen(path.c_str(), "wb"); - if (!f) { - fprintf(stderr, "dump_mtp: cannot write %s\n", path.c_str()); - return false; - } - if (fwrite(data, size, count, f) != count) { - fprintf(stderr, "dump_mtp: short write on %s\n", path.c_str()); - fclose(f); - return false; - } - if (fclose(f) != 0) { - fprintf(stderr, "dump_mtp: cannot flush %s\n", path.c_str()); - return false; - } - return true; -} - static bool write_twin(dump_ctx * d, const ggml_tensor * t, const char * name, int occurrence, bool input, const std::string & stem, bool logical, const std::vector & ints) { const bool wide = t->type == GGML_TYPE_I64; @@ -498,33 +481,6 @@ static void report_unhandled(const dump_ctx & d) { fprintf(stderr, "dump_mtp: %d tensors skipped as unhandled: %s\n", total, types.empty() ? "none" : types.c_str()); } -static bool read_token_file(const std::string & path, long long count, std::vector & ids) { - std::ifstream in(path); - if (!in) { - fprintf(stderr, "dump_mtp: cannot read %s\n", path.c_str()); - return false; - } - std::string line; - long long lines = 0; - while ((long long) ids.size() < count && std::getline(in, line)) { - ++lines; - char * end = nullptr; - errno = 0; - const long v = strtol(line.c_str(), &end, 10); - if (line.empty() || *end != '\0' || errno != 0 || v < 0 || v > INT32_MAX) { - fprintf(stderr, "dump_mtp: %s:%lld is not a token id: '%s'\n", path.c_str(), lines, line.c_str()); - return false; - } - ids.push_back((llama_token) v); - } - if ((long long) ids.size() < count) { - fprintf(stderr, "dump_mtp: %s holds %zu token ids and --tokens-count asks for %lld\n", path.c_str(), - ids.size(), count); - return false; - } - return true; -} - static std::string file_arch(const std::string & path) { gguf_init_params gp = { /*.no_alloc =*/ true, /*.ctx =*/ nullptr }; gguf_context * g = gguf_init_from_file(path.c_str(), gp); @@ -615,7 +571,7 @@ int main(int argc, char ** argv) { if (strcmp(argv[i], "--tokens-file") == 0 && i + 1 < argc) { tokens_file = argv[++i]; } else if (strcmp(argv[i], "--tokens-count") == 0 && i + 1 < argc) { - tokens_count = atoll(argv[++i]); + if (!parse_i64("dump_mtp", "--tokens-count", argv[++i], 1, LLONG_MAX, tokens_count)) return 2; } else if (strcmp(argv[i], "--expect-arch") == 0 && i + 1 < argc) { expect_arch = argv[++i]; } else if (strcmp(argv[i], "--fixture-line") == 0 && i + 1 < argc) { diff --git a/tools/ref/dump_ref.cpp b/tools/ref/dump_ref.cpp index cda69ad70..4b3e1284a 100644 --- a/tools/ref/dump_ref.cpp +++ b/tools/ref/dump_ref.cpp @@ -105,6 +105,7 @@ #include "common.h" #include "llama.h" #include "ggml.h" +#include "ref_io.h" #include #include @@ -199,24 +200,6 @@ static void gather(const ggml_tensor * t, const uint8_t * src, bool logical, } } -static bool write_raw(const std::string & path, const void * data, size_t size, size_t count) { - FILE * f = fopen(path.c_str(), "wb"); - if (!f) { - fprintf(stderr, "dump_ref: cannot write %s\n", path.c_str()); - return false; - } - if (fwrite(data, size, count, f) != count) { - fprintf(stderr, "dump_ref: short write on %s\n", path.c_str()); - fclose(f); - return false; - } - if (fclose(f) != 0) { - fprintf(stderr, "dump_ref: cannot flush %s\n", path.c_str()); - return false; - } - return true; -} - // An integer tensor's lossless twin for one layout, and its `int` row. i64 stays 64-bit; // every narrower integer widens to i32, which is exact. static bool write_twin(dump_ctx * d, const ggml_tensor * t, const char * name, int occurrence, bool input, @@ -447,36 +430,6 @@ static void report_unhandled(const dump_ctx & d) { fprintf(stderr, "dump_ref: %d tensors skipped as unhandled: %s\n", total, types.empty() ? "none" : types.c_str()); } -// One decimal token id per line, the format engram-corpus.sh writes and router_trace reads: the -// first `count` ids, or every line for a negative count. A shorter file is an error, not a shorter -// set — a decode step at another position is another set. -static bool read_token_file(const std::string & path, long long count, std::vector & ids) { - std::ifstream in(path); - if (!in) { - fprintf(stderr, "dump_ref: cannot read %s\n", path.c_str()); - return false; - } - std::string line; - long long lines = 0; - while ((count < 0 || (long long) ids.size() < count) && std::getline(in, line)) { - ++lines; - char * end = nullptr; - errno = 0; - const long v = strtol(line.c_str(), &end, 10); - if (line.empty() || *end != '\0' || errno != 0 || v < 0 || v > INT32_MAX) { - fprintf(stderr, "dump_ref: %s:%lld is not a token id: '%s'\n", path.c_str(), lines, line.c_str()); - return false; - } - ids.push_back((llama_token) v); - } - if (count >= 0 && (long long) ids.size() < count) { - fprintf(stderr, "dump_ref: %s holds %zu token ids and --tokens-count asks for %lld\n", path.c_str(), - ids.size(), count); - return false; - } - return true; -} - static double seconds_since(std::chrono::steady_clock::time_point t0) { return std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); } @@ -557,14 +510,20 @@ int main(int argc, char ** argv) { passthrough.push_back(argv[0]); for (int i = 1; i < argc; ++i) { if (strcmp(argv[i], "--tokens") == 0 && i + 1 < argc) { - char * list = argv[++i]; - for (char * p = strtok(list, ","); p; p = strtok(nullptr, ",")) { - tokens.push_back((llama_token) atoi(p)); + const std::string list(argv[++i]); + for (size_t at = 0;;) { + const size_t end = list.find(',', at); + const std::string field = list.substr(at, end == std::string::npos ? end : end - at); + int id = 0; + if (!parse_i32("dump_ref", "--tokens", field.c_str(), 0, id)) return 2; + tokens.push_back((llama_token) id); + if (end == std::string::npos) break; + at = end + 1; } } else if (strcmp(argv[i], "--tokens-file") == 0 && i + 1 < argc) { tokens_file = argv[++i]; } else if (strcmp(argv[i], "--tokens-count") == 0 && i + 1 < argc) { - tokens_count = std::max(0LL, atoll(argv[++i])); // 0 is refused below + if (!parse_i64("dump_ref", "--tokens-count", argv[++i], 1, LLONG_MAX, tokens_count)) return 2; } else if (strcmp(argv[i], "--decode-step") == 0) { decode_step = true; } else if (strcmp(argv[i], "--prefill-every-node") == 0) { @@ -593,8 +552,8 @@ int main(int argc, char ** argv) { fprintf(stderr, "dump_ref: --tokens and --tokens-file are exclusive\n"); return 2; } - if (tokens_count == 0) { - fprintf(stderr, "dump_ref: --tokens-count must be positive\n"); + if (tokens_count < 0) { + fprintf(stderr, "dump_ref: --tokens-file needs --tokens-count\n"); return 2; } if (!read_token_file(tokens_file, tokens_count, tokens)) return 2; diff --git a/tools/ref/models/qwen35moe.sh b/tools/ref/models/qwen35moe.sh index 59bd16b68..cf89ab47f 100644 --- a/tools/ref/models/qwen35moe.sh +++ b/tools/ref/models/qwen35moe.sh @@ -41,6 +41,15 @@ # budget)), not the up to 4096 a step qwen3moe's prompt gets. An mrspp row of this # profile is a prefill in 512-token chunks # +# MTP_LCPP the mainline tree the MTP step oracle (tools/ref/q35mtp_ref.cpp, `just dump-ref-q35mtp`) links +# against: it carries llama_model_qwen35moe::graph_mtp, the MTP context type and the nextn +# embeddings (llama-ext.h), which LCPP's older tree gives through another batch call +# MTP_MODEL the file that carries the next-token layer (blk.40 and nextn.*): unsloth's UD-Q4_K_M. The +# refset family mtp-qwen35moe-mainline states this path as its identity +# MTP_IDS the prose ids the MTP set feeds (the first MTP_PROMPT of them; the first 64 equal the d1k set's +# prefix), MTP_IDS_SHA256 the check; MTP_ROWS the steps whose taps are dumped, MTP_TAPS the graph's +# own node names read at them (the recipe's default list; q35mtp_ref takes any) +# # Deliberately unset: IK_GPU_FLAGS, IK_GPU_DEFAULT_FLAGS, IK_BEST_FLAGS and REF_PROMPTS. Nobody has # read whether ik's fused flags (-fmoe, -mqkv, -muge) apply to build_qwen35moe, and no CPU flag sweep # and no prompt set exist for this model. Every script that reads them runs under `set -u` @@ -65,6 +74,15 @@ REF_DUMP_LEASE=1 : "${LCPP:=/home/user/llama.cpp-mainline}" : "${LCPPBIN:=$LCPP/build/bin/llama-bench}" : "${LCPP_GPU_FLAGS:=-ngl 99 -fa on}" +: "${MTP_LCPP:=/home/user/llama.cpp-36a73916}" +MTP_MODEL=/models/qwen36-hf/unsloth-mtp/Qwen3.6-35B-A3B-UD-Q4_K_M.gguf +MTP_IDS=${BLOOMERY_DATA:-/root/bloomery-data}/qwen35moe/corpus-prose.ids +MTP_IDS_SHA256=dca5f89b2903f9ffd2f4e20eec18a9fdf3561bcb531a721a114e4fab68e925be +MTP_PROMPT=64 +MTP_ROWS=0,5,31,62 +MTP_TAPS=(mtp_hnorm mtp_enorm mtp_concat mtp_eh_proj mtp_attn_out mtp_attn_residual + ffn_moe_logits ffn_moe_topk ffn_moe_weights_norm mtp_ffn_moe_out mtp_shared_expert_gate_sigmoid + mtp_ffn_shexp_gated mtp_ffn_out mtp_post_ffn h_nextn mtp_shared_head_norm result_output) : "${MRS:=/home/user/mistral.rs}" : "${MRSBIN:=$MRS/target/release/mistralrs}" : "${MRS_FLAGS:=--format gguf}" @@ -105,8 +123,8 @@ ref_step_variant() { *) return 1 ;; esac case $name in - d1k|u1s4|u1s5) STEP_TOKENS_FILE=$BLOOMERY_DATA/qwen35moe/corpus-prose.ids - STEP_TOKENS_SHA256=dca5f89b2903f9ffd2f4e20eec18a9fdf3561bcb531a721a114e4fab68e925be ;; + d1k|u1s4|u1s5) STEP_TOKENS_FILE=$MTP_IDS + STEP_TOKENS_SHA256=$MTP_IDS_SHA256 ;; esac # The fixture tier's d1k set also holds ik's routes of the prefill's positions (dump_ref --prefill-routes), as # qwen4exp.sh's long sets do: the picks, the raw gathered scores and the final weights a gate plants into a batch-fed run of diff --git a/tools/ref/nemotron_ref.cpp b/tools/ref/nemotron_ref.cpp index 85661579a..4853e1f48 100644 --- a/tools/ref/nemotron_ref.cpp +++ b/tools/ref/nemotron_ref.cpp @@ -49,26 +49,6 @@ static bool parse_prompt(const char * spec, Prompt & p) { return true; } -static bool write_ids(const std::string & path, const std::vector & ids) { - FILE * f = fopen(path.c_str(), "w"); - if (!f) { - fprintf(stderr, "nemotron_ref: cannot write %s\n", path.c_str()); - return false; - } - for (llama_token id : ids) { - if (fprintf(f, "%d\n", (int) id) < 0) { - fclose(f); - fprintf(stderr, "nemotron_ref: short write on %s\n", path.c_str()); - return false; - } - } - if (fclose(f) != 0) { - fprintf(stderr, "nemotron_ref: short write on %s\n", path.c_str()); - return false; - } - return true; -} - static void print_top5(const std::string & name, const float * row, int n_vocab) { const int k = std::min(n_vocab, 5); std::vector order(n_vocab); @@ -113,7 +93,7 @@ int main(int argc, char ** argv) { } } const char * build = getenv("BLOOMERY_REF_BUILD"); - if (model_path.empty() || out.empty() || prompts.empty() || n_ubatch <= 0 || !build || !*build) { + if (model_path.empty() || out.empty() || prompts.empty() || !build || !*build) { fprintf(stderr, "usage: nemotron_ref -m --out --prompt :: ... " "[-c ] [-ub ] [-t ]\n" "BLOOMERY_REF_BUILD must name the mainline build\n"); diff --git a/tools/ref/q35mtp_ref.cpp b/tools/ref/q35mtp_ref.cpp new file mode 100644 index 000000000..c207435e7 --- /dev/null +++ b/tools/ref/q35mtp_ref.cpp @@ -0,0 +1,477 @@ +// q35mtp_ref — llama.cpp mainline's MTP draft step, tap by tap, on a teacher-forced prompt. +// +// What it runs. The target's trunk takes the prompt's first ids in one llama_decode with the nextn +// embeddings on and every position an output: row t of the trunk's `h_nextn` (the post-output_norm hidden state +// the MTP head is seeded with) and the trunk's logits of every position are kept. A second context of type +// LLAMA_CONTEXT_TYPE_MTP, over the same model loaded with load_mtp, then takes one token a step, in order, +// i = 0 .. n-2: token ids[i+1], position i, state h[i]. The MTP context's K/V therefore covers the prompt as a +// draft that walked it does. Each step's own argmax is kept beside the trunk's argmax at position i+1 (the token +// the target itself takes after ids[i+1]). +// +// What it dumps. At the steps named by --rows only, the graph nodes named by --tap (repeatable; the name is the +// graph's own, `cb(tensor, name, il)`, without the `-` suffix the context adds), f32 or i32 as the node is, +// one file a (tap, row): `.r.`; the prompt ids (`prompt.ids`); the trunk's h rows +// (`trunk_h.f32`, row t at t*n_embd); and MANIFEST.tsv. The taps are read by the scheduler's eval callback, which +// ends a scheduler split at each tapped node: an op the backend would have fused across a tapped node runs +// unfused. A second MTP context with no callback runs every step first, and the tapped run's logits are held to +// it step by step (`# untapped`): the distance is what tapping costs the oracle. +// +// The manifest's header lines the reader checks: `# model` (the path as given), `# build` +// (BLOOMERY_REF_BUILD, the mainline commit the runner names), `# arch` (the file's general.architecture, which +// --arch must equal) and `# mtp `, written only when the model has a next-token layer +// and the MTP context ran every step. `# complete` is written last, by renaming the manifest. +// +// The hidden state rides on the token's entry as llama_batch_ext_set_embd_token does for +// common/speculative.cpp's draft (llama_batch_ext_set_embd_state is a stub in this tree). +// +// q35mtp_ref -m --arch --tokens-file --tokens-count --rows --tap ... +// --out [-ngl ] [-t ] [-c ] +// +// Build: tools/ref/build-q35mtp-ref.sh Run: tools/ref/dump-q35mtp.sh (just dump-ref-q35mtp) + +#include "llama.h" +#include "llama-ext.h" +#include "ggml.h" +#include "ggml-backend.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ref_io.h" + +static int fail(const std::string & what, const std::string & detail = "") { + fprintf(stderr, "q35mtp_ref: %s%s%s\n", what.c_str(), detail.empty() ? "" : ": ", detail.c_str()); + return 1; +} + +// Comma-separated rows, each one whole integer in [0, 1000000] (parse_i64's rule); an empty piece is refused. +static bool parse_rows(const char * spec, std::vector & rows) { + const std::string s(spec); + size_t at = 0; + while (true) { + const size_t comma = s.find(',', at); + const std::string piece = s.substr(at, comma == std::string::npos ? std::string::npos : comma - at); + long long v = 0; + if (!parse_i64("q35mtp_ref", "--rows", piece.c_str(), 0, 1000000, v)) return false; + rows.push_back((int) v); + if (comma == std::string::npos) return true; + at = comma + 1; + } +} + +// The first index of the largest value: the argmax a tie gives to the lower id. A non-finite logit ends the run +// with 1 naming its id: no comparison orders a NaN, so no token or distance is written from such a row. +static int first_argmax(const float * x, int n) { + int best = 0; + for (int i = 0; i < n; ++i) { + if (!std::isfinite(x[i])) { + fprintf(stderr, "q35mtp_ref: non-finite logit %g at id %d\n", (double) x[i], i); + std::exit(1); + } + if (x[i] > x[best]) best = i; + } + return best; +} + +// The graph's own name of a node, without the `-` the context appends to a layered one. +static std::string base_name(const char * name) { + std::string s(name); + const size_t dash = s.rfind('-'); + if (dash == std::string::npos || dash + 1 >= s.size()) return s; + for (size_t i = dash + 1; i < s.size(); ++i) { + if (s[i] < '0' || s[i] > '9') return s; + } + return s.substr(0, dash); +} + +struct Capture { + int type = 0; // GGML_TYPE_F32 or GGML_TYPE_I32 + int64_t count = 0; // elements + std::vector bytes; +}; + +struct TapState { + std::set want; + bool armed = false; + std::map got; + std::string failure; // the first refusal, by name +}; + +// Read tensor `t` row by row so a view (a top-k's first columns of an argsort) comes out dense. +static bool fetch(const ggml_tensor * t, Capture & out, std::string & why) { + if (t->type != GGML_TYPE_F32 && t->type != GGML_TYPE_I32) { + why = std::string(t->name) + " is " + ggml_type_name(t->type) + ", want f32 or i32"; + return false; + } + const size_t es = ggml_type_size(t->type); + if (t->nb[0] != es) { + why = std::string(t->name) + " has a strided first dimension"; + return false; + } + out.type = t->type; + out.count = ggml_nelements(t); + out.bytes.resize((size_t) out.count * es); + if (ggml_is_contiguous(t)) { + ggml_backend_tensor_get(t, out.bytes.data(), 0, out.bytes.size()); + return true; + } + char * dst = out.bytes.data(); + const size_t row = (size_t) t->ne[0] * es; + for (int64_t i3 = 0; i3 < t->ne[3]; ++i3) { + for (int64_t i2 = 0; i2 < t->ne[2]; ++i2) { + for (int64_t i1 = 0; i1 < t->ne[1]; ++i1) { + ggml_backend_tensor_get(t, dst, (size_t) i3 * t->nb[3] + (size_t) i2 * t->nb[2] + (size_t) i1 * t->nb[1], row); + dst += row; + } + } + } + return true; +} + +// Ask phase: observe the nodes the caller named while a tapped row runs. Data phase: copy each out. +// Returns true always: false would cancel the graph. +static bool on_node(struct ggml_tensor * t, bool ask, void * user) { + auto * s = static_cast(user); + const std::string base = base_name(t->name); + if (ask) return s->armed && s->failure.empty() && s->want.count(base) != 0; + if (s->got.count(base)) { + s->failure = "tap " + base + " is named by two nodes of one step"; + return true; + } + Capture c; + std::string why; + if (!fetch(t, c, why)) { + s->failure = why; + return true; + } + s->got[base] = std::move(c); + return true; +} + +struct Mtp { + llama_context * ctx = nullptr; + llama_batch_ext * batch = nullptr; +}; + +// One teacher-forced step: token `id` at position `pos`, state `h` (n_embd floats). +static bool mtp_step(Mtp & m, llama_token id, int pos, const float * h, int n_embd) { + llama_batch_ext_clear(m.batch); + const int32_t idx = llama_batch_ext_add_token(m.batch, 0, id); + if (idx < 0) { + fprintf(stderr, "q35mtp_ref: llama_batch_ext_add_token rc %d\n", idx); + return false; + } + const llama_pos p[GGML_MROPE_SECTIONS] = { pos, 0, 0, 0 }; + if (!llama_batch_ext_set_pos(m.batch, idx, p)) { + fprintf(stderr, "q35mtp_ref: llama_batch_ext_set_pos refused position %d\n", pos); + return false; + } + const llama_embd state = { h, 1, (size_t) n_embd }; + if (!llama_batch_ext_set_embd_token(m.batch, idx, state)) { + fprintf(stderr, "q35mtp_ref: llama_batch_ext_set_embd_token refused the state at position %d\n", pos); + return false; + } + if (!llama_batch_ext_set_output_logits(m.batch, idx, true)) { + fprintf(stderr, "q35mtp_ref: llama_batch_ext_set_output_logits refused position %d\n", pos); + return false; + } + const int32_t rc = llama_process(m.ctx, LLAMA_PROCESS_TYPE_DECODE, m.batch); + if (rc != 0) { + fprintf(stderr, "q35mtp_ref: llama_process rc %d at position %d\n", rc, pos); + return false; + } + return true; +} + +int main(int argc, char ** argv) { + std::string model_path, out, arch_want, tokens_file; + int n_ctx = 0, ngl = 999, threads = 0, n = 0; + std::vector rows; + std::vector taps; + for (int i = 1; i < argc; ++i) { + const bool more = i + 1 < argc; + if (strcmp(argv[i], "-m") == 0 && more) { + model_path = argv[++i]; + } else if (strcmp(argv[i], "--out") == 0 && more) { + out = argv[++i]; + } else if (strcmp(argv[i], "--arch") == 0 && more) { + arch_want = argv[++i]; + } else if (strcmp(argv[i], "--tokens-file") == 0 && more) { + tokens_file = argv[++i]; + } else if (strcmp(argv[i], "--tokens-count") == 0 && more) { + if (!parse_i32("q35mtp_ref", "--tokens-count", argv[++i], 2, n)) return 2; + } else if (strcmp(argv[i], "--rows") == 0 && more) { + if (!parse_rows(argv[++i], rows)) return fail("--rows wants i,i,...", argv[i]); + } else if (strcmp(argv[i], "--tap") == 0 && more) { + taps.push_back(argv[++i]); + } else if (strcmp(argv[i], "-c") == 0 && more) { + if (!parse_i32("q35mtp_ref", "-c", argv[++i], 0, n_ctx)) return 2; + } else if (strcmp(argv[i], "-ngl") == 0 && more) { + if (!parse_i32("q35mtp_ref", "-ngl", argv[++i], 0, ngl)) return 2; + } else if (strcmp(argv[i], "-t") == 0 && more) { + if (!parse_i32("q35mtp_ref", "-t", argv[++i], 0, threads)) return 2; + } else { + return fail("unknown or incomplete argument", argv[i]); + } + } + const char * build = getenv("BLOOMERY_REF_BUILD"); + const char * sha = getenv("BLOOMERY_REF_TOKENS_SHA256"); + if (model_path.empty() || out.empty() || arch_want.empty() || tokens_file.empty() || n < 2 || rows.empty() || + taps.empty() || !build || !*build) { + fprintf(stderr, "usage: q35mtp_ref -m --arch --tokens-file --tokens-count " + "--rows --tap ... --out [-ngl ] [-t ] [-c ]\n" + "BLOOMERY_REF_BUILD must name the mainline build; --tokens-count is at least 2\n"); + return 2; + } + std::sort(rows.begin(), rows.end()); + if (std::adjacent_find(rows.begin(), rows.end()) != rows.end()) return fail("--rows names a row twice"); + if (rows.back() > n - 2) { + return fail("a row feeds ids[row+1]: the last row is at most tokens-count - 2", std::to_string(rows.back())); + } + { + std::set uniq(taps.begin(), taps.end()); + if (uniq.size() != taps.size()) return fail("--tap names a tap twice"); + } + std::vector ids; + if (!read_token_file(tokens_file, n, ids)) return 2; + if (n_ctx == 0) n_ctx = n; + if (n_ctx < n) return fail("the prompt does not fit in -c", std::to_string(n_ctx)); + + llama_backend_init(); + llama_model_params mp = llama_model_default_params(); + mp.n_gpu_layers = ngl; + mp.load_mtp = true; + llama_model * model = llama_model_load_from_file(model_path.c_str(), mp); + if (!model) return fail("failed to load", model_path); + char arch[128] = {}; + if (llama_model_meta_val_str(model, "general.architecture", arch, sizeof(arch)) < 0 || arch_want != arch) { + return fail("general.architecture is not the --arch", std::string(arch) + " vs " + arch_want); + } + const int n_nextn = llama_model_n_layer_nextn(model); + if (n_nextn <= 0) return fail("the model has no next-token layer (llama_model_n_layer_nextn)", model_path); + const int n_embd = llama_model_n_embd_out(model); + const int n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(model)); + if (n_embd <= 0 || n_vocab <= 0) return fail("the model names no width or vocabulary"); + for (llama_token id : ids) { + if (id >= n_vocab) return fail("a prompt id is past the vocabulary", std::to_string(id)); + } + std::string devices; + for (size_t d = 0; d < ggml_backend_dev_count(); ++d) { + ggml_backend_dev_t dev = ggml_backend_dev_get(d); + if (!devices.empty()) devices += "; "; + devices += std::string(ggml_backend_dev_name(dev)) + " " + ggml_backend_dev_description(dev); + } + + // The trunk: nextn rows for every position, logits for every position. + llama_context_params tp = llama_context_default_params(); + tp.n_ctx = (uint32_t) n_ctx; + tp.n_batch = (uint32_t) n; + tp.n_ubatch = (uint32_t) n; + tp.n_seq_max = 1; + if (threads > 0) { + tp.n_threads = threads; + tp.n_threads_batch = threads; + } + llama_context * trunk = llama_init_from_model(model, tp); + if (!trunk) return fail("failed to make the trunk context"); + llama_set_embeddings_nextn(trunk, true, /*masked*/ false); + llama_batch tb = llama_batch_init(n, 0, 1); + for (int t = 0; t < n; ++t) { + tb.token[t] = ids[t]; + tb.pos[t] = t; + tb.n_seq_id[t] = 1; + tb.seq_id[t][0] = 0; + tb.logits[t] = 1; + } + tb.n_tokens = n; + if (llama_decode(trunk, tb) != 0) return fail("the trunk decode failed"); + llama_batch_free(tb); + std::vector h((size_t) n * n_embd); + std::vector target_next(n); + for (int t = 0; t < n; ++t) { + const float * row = llama_get_embeddings_nextn_ith(trunk, t); + if (!row) return fail("no nextn row for position", std::to_string(t)); + for (int j = 0; j < n_embd; ++j) { + if (!std::isfinite(row[j])) return fail("a trunk h_nextn value is not finite", "position " + std::to_string(t)); + } + memcpy(h.data() + (size_t) t * n_embd, row, (size_t) n_embd * sizeof(float)); + const float * lg = llama_get_logits_ith(trunk, t); + if (!lg) return fail("no trunk logits for position", std::to_string(t)); + target_next[t] = first_argmax(lg, n_vocab); + } + llama_free(trunk); + + // The MTP contexts, as common/speculative.cpp makes its draft's. + auto make_mtp = [&](Mtp & m, TapState * taps_state) -> bool { + llama_context_params cp = llama_context_default_params(); + cp.ctx_type = LLAMA_CONTEXT_TYPE_MTP; + cp.n_ctx = (uint32_t) n_ctx; + cp.n_batch = (uint32_t) n; + cp.n_ubatch = (uint32_t) n; + cp.n_seq_max = 1; + cp.n_rs_seq = 0; + if (threads > 0) { + cp.n_threads = threads; + cp.n_threads_batch = threads; + } + if (taps_state) { + cp.cb_eval = on_node; + cp.cb_eval_user_data = taps_state; + } + m.ctx = llama_init_from_model(model, cp); + if (!m.ctx) return false; + llama_set_embeddings_nextn(m.ctx, true, /*masked*/ true); + m.batch = llama_batch_ext_init(m.ctx); + return m.batch != nullptr; + }; + + const int steps = n - 1; + // The untapped run first: every step's logits. + std::vector plain((size_t) steps * n_vocab); + { + Mtp m; + if (!make_mtp(m, nullptr)) return fail("failed to make the MTP context (llama_model_n_layer_nextn > 0, ctx_type MTP)"); + for (int i = 0; i < steps; ++i) { + if (!mtp_step(m, ids[i + 1], i, h.data() + (size_t) i * n_embd, n_embd)) return fail("the untapped MTP step failed"); + const float * lg = llama_get_logits_ith(m.ctx, 0); + if (!lg) return fail("no MTP logits at step", std::to_string(i)); + memcpy(plain.data() + (size_t) i * n_vocab, lg, (size_t) n_vocab * sizeof(float)); + } + llama_batch_ext_free(m.batch); + llama_free(m.ctx); + } + + // The tapped run. + TapState st; + st.want.insert(taps.begin(), taps.end()); + Mtp m; + if (!make_mtp(m, &st)) return fail("failed to make the tapped MTP context"); + if (mkdir(out.c_str(), 0755) != 0 && errno != EEXIST) return fail("cannot create", out); + const std::set want_rows(rows.begin(), rows.end()); + std::vector mtp_argmax(steps); + std::vector tap_rows; // manifest lines + double worst_all = 0.0, worst_rows = 0.0; + int same_all = 0, same_rows = 0; + for (int i = 0; i < steps; ++i) { + const bool tapped = want_rows.count(i) != 0; + st.armed = tapped; + st.got.clear(); + if (!mtp_step(m, ids[i + 1], i, h.data() + (size_t) i * n_embd, n_embd)) return fail("the MTP step failed"); + if (!st.failure.empty()) return fail("a tap is refused", st.failure); + const float * lg = llama_get_logits_ith(m.ctx, 0); + if (!lg) return fail("no MTP logits at step", std::to_string(i)); + for (int j = 0; j < n_vocab; ++j) { + if (!std::isfinite(lg[j])) return fail("an MTP logit is not finite", "step " + std::to_string(i)); + } + mtp_argmax[i] = first_argmax(lg, n_vocab); + const float * pl = plain.data() + (size_t) i * n_vocab; + double worst = 0.0; + for (int j = 0; j < n_vocab; ++j) worst = std::max(worst, (double) std::fabs(lg[j] - pl[j])); + worst_all = std::max(worst_all, worst); + same_all += first_argmax(pl, n_vocab) == mtp_argmax[i]; + if (tapped) { + worst_rows = std::max(worst_rows, worst); + same_rows += first_argmax(pl, n_vocab) == mtp_argmax[i]; + for (const std::string & name : taps) { + auto it = st.got.find(name); + if (it == st.got.end()) return fail("tap " + name + " was not seen at row", std::to_string(i)); + const Capture & c = it->second; + const char * ext = c.type == GGML_TYPE_F32 ? "f32" : "i32"; + const std::string file = name + ".r" + std::to_string(i) + "." + ext; + if (!write_bytes(out + "/" + file, c.bytes.data(), c.bytes.size())) return 1; + if (c.type == GGML_TYPE_F32) { + const float * v = (const float *) c.bytes.data(); + for (int64_t j = 0; j < c.count; ++j) { + if (!std::isfinite(v[j])) return fail("tap " + name + " holds a value that is not finite", "row " + std::to_string(i)); + } + } + tap_rows.push_back("tap\t" + name + "\t" + std::to_string(i) + "\t" + ext + "\t" + std::to_string(c.count) + "\t" + file); + } + } + } + llama_batch_ext_free(m.batch); + llama_free(m.ctx); + + if (!write_ids(out + "/prompt.ids", ids)) return 1; + if (!write_raw(out + "/trunk_h.f32", h.data(), h.size())) return 1; + + // The sanity line: the draft's token against the target's own next token after the fed id, as the gate's + // acceptance reads it (the last 16 steps, and every step). + const int last = std::min(16, steps); + int hits_last = 0, hits_all = 0; + for (int i = 0; i < steps; ++i) { + const bool hit = mtp_argmax[i] == target_next[i + 1]; + hits_all += hit; + if (i >= steps - last) hits_last += hit; + } + for (int r : rows) { + fprintf(stderr, "q35mtp_ref: row %d token %d pos %d: mtp argmax %d, target next %d%s\n", r, (int) ids[r + 1], r, + mtp_argmax[r], target_next[r + 1], mtp_argmax[r] == target_next[r + 1] ? "" : " (differs)"); + } + fprintf(stderr, "q35mtp_ref: mtp argmax = target next at %d/%d of the last steps, %d/%d of every step\n", hits_last, last, + hits_all, steps); + fprintf(stderr, "q35mtp_ref: tapped vs untapped logits: worst |d| %.6g over every step, %.6g over the tapped rows; " + "argmax equal at %d/%d steps, %d/%zu tapped rows\n", + worst_all, worst_rows, same_all, steps, same_rows, rows.size()); + // Tapping must not move the oracle: the tapped run's logits equal the untapped run's on every step. + if (worst_all != 0.0 || same_all != steps) { + return fail("the tapped run's logits differ from the untapped run's; the set is not written", + std::to_string(worst_all)); + } + + const std::string final_path = out + "/MANIFEST.tsv"; + const std::string partial_path = final_path + ".partial"; + FILE * mf = fopen(partial_path.c_str(), "w"); + if (!mf) return fail("cannot write", partial_path); + const char * slash = strrchr(model_path.c_str(), '/'); + std::string flags; + for (int i = 1; i < argc; ++i) flags += (i > 1 ? " " : "") + std::string(argv[i]); + fprintf(mf, "# q35mtp_ref — llama.cpp mainline's MTP draft step, taps at the named rows, raw little-endian\n"); + fprintf(mf, "# model\t%s\n", model_path.c_str()); + fprintf(mf, "# build\t%s\n", build); + fprintf(mf, "# arch\t%s\n", arch); + fprintf(mf, "# model_file\t%s\n", slash ? slash + 1 : model_path.c_str()); + fprintf(mf, "# flags\t%s\n", flags.c_str()); + fprintf(mf, "# tokens_file\t%s\n", tokens_file.c_str()); + fprintf(mf, "# tokens_file_sha256\t%s\n", sha && *sha ? sha : "unknown"); + fprintf(mf, "# tokens_count\t%d\n", n); + fprintf(mf, "# mtp\t%d\t%d\n", n_nextn, steps); + fprintf(mf, "# backend\tngl=%d\t%s\n", ngl, devices.c_str()); + fprintf(mf, "# n_embd\t%d\n", n_embd); + fprintf(mf, "# n_vocab\t%d\n", n_vocab); + fprintf(mf, "# rows"); + for (int r : rows) fprintf(mf, "\t%d", r); + fprintf(mf, "\n"); + fprintf(mf, "# feed\tstep i: token ids[i+1], position i, state trunk h_nextn row i; steps 0..%d in order, taps at the rows\n", + steps - 1); + fprintf(mf, "# untapped\t%.9g\t%.9g\t%d\t%d\n", worst_all, worst_rows, same_all, same_rows); + fprintf(mf, "# trunk columns\tname\ttype\trows\twidth\tfile\n"); + fprintf(mf, "trunk\th_nextn\tf32\t%d\t%d\ttrunk_h.f32\n", n, n_embd); + fprintf(mf, "# step columns\trow\ttoken\tpos\tmtp_argmax\ttarget_next\n"); + for (int i = 0; i < steps; ++i) { + fprintf(mf, "step\t%d\t%d\t%d\t%d\t%d\n", i, (int) ids[i + 1], i, mtp_argmax[i], target_next[i + 1]); + } + fprintf(mf, "# tap columns\tname\trow\ttype\tn\tfile\n"); + for (const std::string & line : tap_rows) fprintf(mf, "%s\n", line.c_str()); + fprintf(mf, "# complete\t%zu\t0\n", tap_rows.size()); + if (fclose(mf) != 0 || rename(partial_path.c_str(), final_path.c_str()) != 0) { + return fail("cannot install", final_path); + } + fprintf(stderr, "q35mtp_ref: %zu tap rows (%zu taps at %zu rows), %d steps, into %s\n", tap_rows.size(), taps.size(), + rows.size(), steps, out.c_str()); + llama_model_free(model); + llama_backend_free(); + return 0; +} diff --git a/tools/ref/ref_io.h b/tools/ref/ref_io.h index 38ee88d84..c915f0f5a 100644 --- a/tools/ref/ref_io.h +++ b/tools/ref/ref_io.h @@ -1,5 +1,5 @@ #pragma once -// Token files and raw f32 rows for the mainline oracles (hidden_ref, nemotron_ref). +// Token files and raw f32 rows for the mainline oracles (hidden_ref, nemotron_ref, q35mtp_ref). // A token id is a non-negative 32-bit integer. The header does not include llama.h: // llama_token is that integer, so a vector of it is a vector of int32_t. @@ -67,16 +67,43 @@ bool read_token_file(const std::string & path, long long count, std::vector +bool write_ids(const std::string & path, const std::vector & ids) { + FILE * f = fopen(path.c_str(), "w"); + if (!f) { + perror(path.c_str()); + return false; + } + bool ok = true; + for (Token id : ids) ok = ok && fprintf(f, "%d\n", (int) id) >= 0; + if (fclose(f) != 0 || !ok) { + fprintf(stderr, "%s: short write\n", path.c_str()); + return false; + } + return true; +} + +// `bytes` bytes to `path`, every byte written and the file closed, or named on stderr. +inline bool write_bytes(const std::string & path, const void * data, size_t bytes) { FILE * f = fopen(path.c_str(), "wb"); if (!f) { perror(path.c_str()); return false; } - const bool ok = fwrite(data, sizeof(float), count, f) == count; + const bool ok = fwrite(data, 1, bytes, f) == bytes; if (fclose(f) != 0 || !ok) { fprintf(stderr, "%s: short write\n", path.c_str()); return false; } return true; } + +// `count` elements of `size` bytes each, the ik dumpers' typed rows. +inline bool write_raw(const std::string & path, const void * data, size_t size, size_t count) { + return write_bytes(path, data, size * count); +} + +inline bool write_raw(const std::string & path, const float * data, size_t count) { + return write_bytes(path, data, count * sizeof(float)); +}