diff --git a/crates/refset/src/arch/mod.rs b/crates/refset/src/arch/mod.rs index aaaef8a06..dcd4b9f98 100644 --- a/crates/refset/src/arch/mod.rs +++ b/crates/refset/src/arch/mod.rs @@ -15,6 +15,7 @@ pub mod draft; pub mod glm5next; pub mod mimo2; pub mod minicpmv4_7; +pub mod nemotron_h; pub mod qwen35; pub mod qwen35moe; pub mod qwen4exp; @@ -30,6 +31,7 @@ static BY_ARCH: &[(&str, &[&Family])] = &[ (mimo2::ARCH, mimo2::FAMILIES), (minicpmv4_7::ARCH, minicpmv4_7::FAMILIES), (qwen4exp::ARCH, qwen4exp::FAMILIES), + (nemotron_h::ARCH, nemotron_h::FAMILIES), (tokenizer::ARCH, tokenizer::FAMILIES), (dequant::ARCH, dequant::FAMILIES), ]; diff --git a/crates/refset/src/arch/nemotron_h/mod.rs b/crates/refset/src/arch/nemotron_h/mod.rs new file mode 100644 index 000000000..572e23309 --- /dev/null +++ b/crates/refset/src/arch/nemotron_h/mod.rs @@ -0,0 +1,261 @@ +//! The Nemotron 3 Nano family. The logits set is dumped from the tokenizer +//! family's model ([`super::tokenizer::nemotron_model`]) by mainline llama.cpp +//! ([`super::tokenizer::NEMOTRON_BUILD`]), the build that family already pins. +//! ik has no `nemotron_h`. + +use crate::family::{Build, Family, Identity}; + +/// The architecture every nemotron manifest names in its `# arch` line. +pub const ARCH: &str = "nemotron_h"; + +/// Last-position logits of four prompts, mainline on the CPU. +pub const LOGITS_SET: &str = "ref-nemotron-logits"; + +/// Mainline's last-position logits (`just dump-ref-nemotron`). +pub static LOGITS: Family = Family { + name: "ref-nemotron-logits", + sets: &[LOGITS_SET], + resolve: None, + recipe: "just dump-ref-nemotron", + identity: Identity::NemotronLogits, + arch: Some(ARCH), + build: Some(Build::Is(super::tokenizer::NEMOTRON_BUILD)), + runs: Some(super::tokenizer::nemotron_model), + draft_runs: None, + consumers: &[], +}; + +/// The architecture's families, in the order `refset-check` lists them. +pub static FAMILIES: &[&Family] = &[&LOGITS]; + +#[cfg(test)] +mod tests { + use super::LOGITS; + use crate::RefError; + use crate::family::Build; + use crate::nemotron::LogitsSet; + use std::path::Path; + + fn pin() -> &'static str { + match LOGITS.build { + Some(Build::Is(p)) => p, + _ => panic!("the logits family pins no build"), + } + } + + fn runs() -> String { + LOGITS.runs().unwrap_or_else(|e| panic!("{e}")) + } + + /// Write one prompt's ids and logits and the manifest. `complete` adds the trailer. + fn write(dir: &Path, model: &str, build: &str, complete: bool) { + std::fs::create_dir_all(dir).unwrap_or_else(|e| panic!("{e}")); + std::fs::write(dir.join("p.ids"), "0\n1\n2\n3\n0\n").unwrap_or_else(|e| panic!("{e}")); + let logits: Vec = [0.0f32, 1.5, -0.5, 0.25] + .into_iter() + .flat_map(f32::to_le_bytes) + .collect(); + std::fs::write(dir.join("p.logits.f32"), logits).unwrap_or_else(|e| panic!("{e}")); + let mut text = format!( + "# model\t{model}\n# build\t{build}\n# arch\tnemotron_h\n\ + # prompt columns\tname\tn_vocab\tn_ids\tlogits\tids\n\ + prompt\tp\t4\t5\tp.logits.f32\tp.ids\n" + ); + if complete { + text.push_str("# complete\t1\t0\n"); + } + std::fs::write(dir.join("MANIFEST.tsv"), text).unwrap_or_else(|e| panic!("{e}")); + } + + fn dir(tag: &str) -> std::path::PathBuf { + let d = + std::env::temp_dir().join(format!("bloomery-nemotron-{tag}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + d + } + + /// The build pin is the tokenizer family's constant. + #[test] + fn the_build_pin_is_the_tokenizer_familys() { + let Some(Build::Is(p)) = LOGITS.build else { + panic!("no pin"); + }; + assert_eq!(p, crate::arch::tokenizer::NEMOTRON_BUILD); + assert_eq!(LOGITS.sets, &["ref-nemotron-logits"]); + assert_eq!( + runs(), + crate::arch::tokenizer::NEMOTRON + .runs() + .unwrap_or_else(|e| panic!("{e}")) + ); + } + + /// A finished set opens, and `logits` is that column's file. + #[test] + fn a_finished_set_reads_logits_by_column_name() { + let d = dir("ok"); + write(&d, &runs(), pin(), true); + let set = LogitsSet::open(&d, &LOGITS).unwrap_or_else(|e| panic!("{e}")); + assert_eq!(set.prompts.len(), 1); + let row = set + .logits(&set.prompts[0]) + .unwrap_or_else(|e| panic!("{e}")); + assert_eq!(row[1], 1.5); + std::fs::remove_dir_all(&d).unwrap_or_else(|e| panic!("{e}")); + } + + /// No `# complete`: unfinished, even when the other lines are well formed. + #[test] + fn a_set_without_its_trailer_is_unfinished() { + let d = dir("trailer"); + write(&d, &runs(), pin(), false); + match LogitsSet::open(&d, &LOGITS) { + Err(RefError::Unfinished { set }) => assert_eq!(set, d.display().to_string()), + other => panic!("a set without its trailer: {other:?}"), + } + std::fs::remove_dir_all(&d).unwrap_or_else(|e| panic!("{e}")); + } + + /// Another model path is stale. + #[test] + fn a_set_of_another_model_is_stale() { + let d = dir("model"); + write(&d, "/models/other/file.gguf", pin(), true); + match LogitsSet::open(&d, &LOGITS) { + Err(RefError::Stale { dumped_from, .. }) => { + assert_eq!(dumped_from, "/models/other/file.gguf"); + } + other => panic!("another file: {other:?}"), + } + std::fs::remove_dir_all(&d).unwrap_or_else(|e| panic!("{e}")); + } + + /// Another mainline build is foreign. + #[test] + fn a_set_of_another_build_is_foreign() { + let d = dir("build"); + write( + &d, + &runs(), + "llama-tokenize 00000000000000000000000000000000 libllama.so 00000000000000000000000000000000", + true, + ); + match LogitsSet::open(&d, &LOGITS) { + Err(RefError::Foreign { field, .. }) => assert_eq!(field, "build"), + other => panic!("another build: {other:?}"), + } + std::fs::remove_dir_all(&d).unwrap_or_else(|e| panic!("{e}")); + } + + /// A line that is not a token id is malformed, naming the field and the text. + #[test] + fn a_token_line_that_is_not_an_id_is_malformed() { + let d = dir("bad-id"); + write(&d, &runs(), pin(), true); + std::fs::write(d.join("p.ids"), "0\n1\n2\ngarbage\n0\n").unwrap_or_else(|e| panic!("{e}")); + match LogitsSet::open(&d, &LOGITS) { + Err(RefError::Malformed { what, .. }) => { + assert!( + what.contains("token id") && what.contains("garbage"), + "{what}" + ); + } + other => panic!("a token line that is not an id: {other:?}"), + } + std::fs::remove_dir_all(&d).unwrap_or_else(|e| panic!("{e}")); + } + + /// A NaN or infinite logit is malformed, naming its index. + #[test] + fn a_non_finite_logit_is_malformed() { + let d = dir("nan-logit"); + write(&d, &runs(), pin(), true); + let logits: Vec = [0.0f32, f32::NAN, -0.5, 0.25] + .into_iter() + .flat_map(f32::to_le_bytes) + .collect(); + std::fs::write(d.join("p.logits.f32"), logits).unwrap_or_else(|e| panic!("{e}")); + match LogitsSet::open(&d, &LOGITS) { + Err(RefError::Malformed { what, .. }) => { + assert!(what.contains("non-finite logit at index 1"), "{what}"); + } + other => panic!("a non-finite logit: {other:?}"), + } + std::fs::remove_dir_all(&d).unwrap_or_else(|e| panic!("{e}")); + } + + /// An empty line among the ids is malformed, as the C++ reader that writes the set refuses it. + #[test] + fn an_empty_token_line_is_malformed() { + let d = dir("empty-id"); + write(&d, &runs(), pin(), true); + std::fs::write(d.join("p.ids"), "0\n\n1\n2\n3\n0\n").unwrap_or_else(|e| panic!("{e}")); + match LogitsSet::open(&d, &LOGITS) { + Err(RefError::Malformed { what, .. }) => { + assert!(what.contains("empty token id"), "{what}") + } + other => panic!("an empty token line: {other:?}"), + } + std::fs::remove_dir_all(&d).unwrap_or_else(|e| panic!("{e}")); + } + + /// An id at or past `n_vocab` is malformed, naming the id and the vocabulary. + #[test] + fn a_token_id_outside_the_vocabulary_is_malformed() { + let d = dir("oob-id"); + write(&d, &runs(), pin(), true); + std::fs::write(d.join("p.ids"), "0\n1\n2\n3\n4\n").unwrap_or_else(|e| panic!("{e}")); + match LogitsSet::open(&d, &LOGITS) { + Err(RefError::Malformed { what, .. }) => { + assert!(what.contains("outside vocabulary"), "{what}"); + } + other => panic!("a token id outside the vocabulary: {other:?}"), + } + std::fs::remove_dir_all(&d).unwrap_or_else(|e| panic!("{e}")); + } + + /// The trailer's count is the prompt rows. One row and `# complete 2` is malformed. + #[test] + fn a_completion_count_that_is_not_the_prompt_rows_is_malformed() { + let d = dir("complete-n"); + write(&d, &runs(), pin(), true); + let text = + std::fs::read_to_string(d.join("MANIFEST.tsv")).unwrap_or_else(|e| panic!("{e}")); + std::fs::write( + d.join("MANIFEST.tsv"), + text.replace("# complete\t1\t0\n", "# complete\t2\t0\n"), + ) + .unwrap_or_else(|e| panic!("{e}")); + match LogitsSet::open(&d, &LOGITS) { + Err(RefError::Malformed { what, .. }) => { + assert!(what.contains("completion count"), "{what}"); + } + other => panic!("a completion count that is not the prompt rows: {other:?}"), + } + std::fs::remove_dir_all(&d).unwrap_or_else(|e| panic!("{e}")); + } + + /// A completion count that does not parse is malformed, naming the field. + #[test] + fn a_completion_count_that_does_not_parse_is_malformed() { + let d = dir("complete-bad"); + write(&d, &runs(), pin(), true); + let text = + std::fs::read_to_string(d.join("MANIFEST.tsv")).unwrap_or_else(|e| panic!("{e}")); + std::fs::write( + d.join("MANIFEST.tsv"), + text.replace("# complete\t1\t0\n", "# complete\tgarbage\t0\n"), + ) + .unwrap_or_else(|e| panic!("{e}")); + match LogitsSet::open(&d, &LOGITS) { + Err(RefError::Malformed { what, .. }) => { + assert!( + what.contains("complete count") && what.contains("garbage"), + "{what}" + ); + } + other => panic!("a completion count that does not parse: {other:?}"), + } + std::fs::remove_dir_all(&d).unwrap_or_else(|e| panic!("{e}")); + } +} diff --git a/crates/refset/src/arch/tokenizer.rs b/crates/refset/src/arch/tokenizer.rs index a99b038f1..0e36e16bd 100644 --- a/crates/refset/src/arch/tokenizer.rs +++ b/crates/refset/src/arch/tokenizer.rs @@ -47,7 +47,7 @@ pub const TOKREF_BUILD: &str = "llama-tokenize 0128e83c5e71e5f2b72f3d5d2c2de6b2 /// `nemotron_h` architecture. The soname is `libllama.so.0`; the pin's word /// is the one [`crate::tokenizer::build_id`] spells. // PIN(2026-10-14): /home/user/llama.cpp-mainline's llama-tokenize and libllama.so.0. -const NEMOTRON_BUILD: &str = "llama-tokenize 5df95d3a942effc971ff170432d840fd \ +pub(crate) const NEMOTRON_BUILD: &str = "llama-tokenize 5df95d3a942effc971ff170432d840fd \ libllama.so 3575b1f39e7773379fec011d39f32a77"; /// V4.1's set (`deepseek-v3` pre-tokenizer), from the file the tree runs. @@ -141,7 +141,7 @@ fn qwen38_model() -> Result { Ok(super::qwen4exp::MODEL.to_string()) } -fn nemotron_model() -> Result { +pub(crate) fn nemotron_model() -> Result { Ok(NEMOTRON_MODEL.to_string()) } diff --git a/crates/refset/src/columns.rs b/crates/refset/src/columns.rs index 62850f220..22f16bcd7 100644 --- a/crates/refset/src/columns.rs +++ b/crates/refset/src/columns.rs @@ -109,3 +109,15 @@ where v.parse() .map_err(|e| RefError::malformed(at, format!("{name} {v:?}: {e}"))) } + +/// `name` is one file of the set's own directory. Empty, `.`, `..`, or a +/// slash is `Malformed`: the manifest must not name a path outside it. +pub(crate) fn file_name(name: &str, at: &dyn Display) -> Result<(), RefError> { + if name.is_empty() || name.contains('/') || matches!(name, "." | "..") { + return Err(RefError::malformed( + at, + format!("{name:?} does not name a file of the set's directory"), + )); + } + Ok(()) +} diff --git a/crates/refset/src/dequant.rs b/crates/refset/src/dequant.rs index 064ec8a26..836ffee14 100644 --- a/crates/refset/src/dequant.rs +++ b/crates/refset/src/dequant.rs @@ -198,12 +198,7 @@ impl DequantSet { fn file_row(r: &Row<'_>, at: &dyn fmt::Display) -> Result { let name = r.text("file")?; - if name.is_empty() || name.contains('/') || matches!(name, "." | "..") { - return Err(RefError::malformed( - at, - format!("{name:?} does not name a file of the set's directory"), - )); - } + crate::columns::file_name(name, at)?; let md5 = r.text("md5")?; check_hex("md5", md5, at)?; Ok(SetFile { diff --git a/crates/refset/src/family.rs b/crates/refset/src/family.rs index 9170d9a96..dd25838e2 100644 --- a/crates/refset/src/family.rs +++ b/crates/refset/src/family.rs @@ -63,6 +63,10 @@ pub enum Identity { /// naming the harness and ggml's library by md5: the family's `build` /// pins both ([`crate::dequant::build_id`]). DequantManifest, + /// `# model`, `# build` and `# arch` in a last-position logits set's + /// `MANIFEST.tsv` (`tools/ref/nemotron_ref.cpp`). A prompt row is read by + /// the `# prompt columns` line; `logits` is that prompt's last position. + NemotronLogits, } /// The ik build a family's sets must name. @@ -390,6 +394,14 @@ impl Family { build: set.build(), }) } + Identity::NemotronLogits => { + let set = crate::nemotron::LogitsSet::open(path, self)?; + Ok(Provenance { + dumped_from: stated(set.model.as_deref()), + draft: None, + build: set.build, + }) + } } } } diff --git a/crates/refset/src/lib.rs b/crates/refset/src/lib.rs index 3ac87f196..368d8d6cc 100644 --- a/crates/refset/src/lib.rs +++ b/crates/refset/src/lib.rs @@ -26,6 +26,8 @@ //! - [`tokenizer`]: `llama-tokenize`'s ids per vocabulary //! (`crates/tokenizer/tools/oracle.sh`). //! - [`dequant`]: ggml's dequantized rows (`tools/ref/dump-dequant.sh`). +//! - [`nemotron`]: mainline's last-position logits of Nemotron 3 Nano +//! (`tools/ref/nemotron_ref.cpp`). //! - [`census`]: the census stubs — header-only copies of many users' model //! files, pinned by `tools/census/stubs.tsv` (the writer //! `tools/type-census.py --stubs`) and checked against it. @@ -48,6 +50,7 @@ pub mod ik; pub mod kld; mod md5; pub mod mtpref; +pub mod nemotron; pub mod tokenizer; pub mod vision; pub mod visref; diff --git a/crates/refset/src/nemotron.rs b/crates/refset/src/nemotron.rs new file mode 100644 index 000000000..b1979cc94 --- /dev/null +++ b/crates/refset/src/nemotron.rs @@ -0,0 +1,287 @@ +//! Last-position logits of Nemotron 3 Nano (`tools/ref/nemotron_ref.cpp`, +//! `just dump-ref-nemotron`). One prompt is one row, read by the names on +//! `# prompt columns`. `logits` is the last position only, `n_vocab` f32, +//! little-endian; `ids` is that prompt, one id a line. +//! +//! ```text +//! # model +//! # build llama-tokenize libllama.so +//! # arch nemotron_h +//! # prompt columns name n_ids n_vocab ids logits +//! prompt code5 5 131072 code5.ids code5.logits.f32 +//! # complete 4 0 +//! ``` +//! +//! [`LogitsSet::open`] refuses a set that did not finish, one dumped from +//! another file, and one written by another mainline build. + +use crate::RefError; +use crate::columns::{Columns, Row, file_name}; +use crate::family::{Family, Identity}; +use std::collections::HashSet; +use std::fmt; +use std::path::{Path, PathBuf}; + +/// One prompt of the set. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Prompt { + /// The row's `name`. + pub name: String, + /// How many ids the prompt holds, BOS included when the file adds one. + pub n_ids: u32, + /// The logits row's width. + pub n_vocab: u32, + /// The ids file under the set's directory. + pub ids_file: String, + /// The logits file under the set's directory (`n_vocab` f32). + pub logits_file: String, +} + +/// A logits set's manifest, and once opened, its files checked. +#[derive(Debug, Clone)] +pub struct LogitsSet { + /// The set's directory. + pub dir: PathBuf, + /// `# model`. + pub model: Option, + /// `# build`, spelled as [`crate::tokenizer::build_id`]. + pub build: Option, + /// `# arch`. + pub arch: Option, + /// One row a prompt, in the manifest's order. + pub prompts: Vec, + /// Whether the `# complete` trailer is there. + pub complete: bool, +} + +/// A manifest line's position, formatted only into an error. +struct At<'a>(&'a Path, usize); + +impl fmt::Display for At<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "nemotron: {}:{}", self.0.display(), self.1) + } +} + +impl LogitsSet { + /// Parse `dir/MANIFEST.tsv`. A set without one is `Missing`, naming the + /// recipe that writes it. A set without its trailer comes back unread, + /// `complete` false. A row before its column line, of another width, with + /// a count that does not parse, a `# complete` count other than the prompt + /// rows, or naming a prompt twice, is `Malformed`. + pub fn read(dir: &Path, family: &Family) -> Result { + let path = dir.join("MANIFEST.tsv"); + let text = std::fs::read_to_string(&path).map_err(|e| { + RefError::missing( + &path, + format!("{}: {e} — run: {}", path.display(), family.recipe), + ) + })?; + let mut set = LogitsSet { + dir: dir.to_path_buf(), + model: None, + build: None, + arch: None, + prompts: Vec::new(), + complete: false, + }; + // A manifest without its trailer is a dump that did not finish: its rows are not read, so + // the refusal is the one that says so. + if !text + .lines() + .any(|l| l.split('\t').next() == Some("# complete")) + { + return Ok(set); + } + let mut cols: Option = None; + let mut seen = HashSet::new(); + let mut complete_rows = None; + for (i, line) in text.lines().enumerate() { + if line.is_empty() { + continue; + } + let at = At(&path, i + 1); + let f: Vec<&str> = line.split('\t').collect(); + match f[0] { + "# model" => set.model = Some(one(&f, &at, "# model\t")?), + "# build" => set.build = Some(one(&f, &at, "# build\t")?), + "# arch" => set.arch = Some(one(&f, &at, "# arch\t")?), + "# prompt columns" => { + cols = Some(Columns::new("prompt", f[1..].iter().copied())); + } + "# complete" => { + if f.len() != 3 || f[2] != "0" { + return Err(RefError::malformed(&at, "want # complete\t\t0")); + } + complete_rows = Some(crate::columns::parse_field(f[1], "complete count", &at)?); + set.complete = true; + } + h if h.starts_with('#') => {} + "prompt" => { + let c = cols.as_ref().ok_or_else(|| { + RefError::malformed(&at, "a prompt row before its `# prompt columns` line") + })?; + let prompt = prompt_row(&c.row(line, &at)?, &at)?; + if !seen.insert(prompt.name.clone()) { + return Err(RefError::malformed( + &at, + format!("{} is named twice", prompt.name), + )); + } + set.prompts.push(prompt); + } + other => { + return Err(RefError::malformed( + &at, + format!("unknown row kind {other:?}"), + )); + } + } + } + if complete_rows != Some(set.prompts.len()) { + return Err(RefError::malformed( + path.display(), + "completion count differs from prompt rows", + )); + } + Ok(set) + } + + /// The set at `dir` read and checked against `family`'s row. + pub fn open(dir: &Path, family: &Family) -> Result { + if !matches!(family.identity, Identity::NemotronLogits) { + return Err(RefError::missing( + dir, + format!( + "nemotron: {} is not a last-position logits family", + family.name + ), + )); + } + let set = LogitsSet::read(dir, family)?; + set.check_family(family)?; + Ok(set) + } + + /// This set against its family's row: the trailer ([`RefError::Unfinished`]), + /// the model file ([`RefError::Stale`]), the build and the architecture + /// ([`RefError::Foreign`]), then each prompt's ids and logits files. + pub fn check_family(&self, family: &Family) -> Result<(), RefError> { + if !self.complete { + return Err(RefError::Unfinished { + set: self.dir.display().to_string(), + }); + } + family.check_file(&self.dir, "# model", self.model.as_deref())?; + family.check_build(&self.dir, self.build.as_deref())?; + family.check_arch(&self.dir, self.arch.as_deref())?; + if self.prompts.is_empty() { + return Err(RefError::malformed( + self.dir.join("MANIFEST.tsv").display(), + "no prompt rows: the set names nothing to check", + )); + } + for p in &self.prompts { + self.check_prompt(p)?; + } + Ok(()) + } + + /// Prompt `p`'s last-position logits, `n_vocab` f32, little-endian. + pub fn logits(&self, p: &Prompt) -> Result, RefError> { + let path = self.dir.join(&p.logits_file); + let bytes = std::fs::read(&path).map_err(|e| { + RefError::missing( + &path, + format!("nemotron: cannot read {}: {e}", path.display()), + ) + })?; + let want = p.n_vocab as usize * 4; + if bytes.len() != want { + return Err(RefError::malformed( + path.display(), + format!( + "{} is {} bytes, {} logits of {} want {want}", + p.logits_file, + bytes.len(), + p.name, + p.n_vocab + ), + )); + } + let mut out = Vec::with_capacity(p.n_vocab as usize); + for chunk in bytes.chunks_exact(4) { + out.push(f32::from_le_bytes( + chunk.try_into().expect("chunks_exact(4)"), + )); + } + if let Some(i) = out.iter().position(|v| !v.is_finite()) { + return Err(RefError::malformed( + path.display(), + format!("non-finite logit at index {i}"), + )); + } + Ok(out) + } + + fn check_prompt(&self, p: &Prompt) -> Result<(), RefError> { + let ids = self.dir.join(&p.ids_file); + let text = std::fs::read_to_string(&ids).map_err(|e| { + RefError::missing( + &ids, + format!("nemotron: cannot read {}: {e}", ids.display()), + ) + })?; + if p.n_ids == 0 || p.n_vocab == 0 { + return Err(RefError::malformed( + ids.display(), + "empty prompt or vocabulary", + )); + } + let mut n = 0usize; + for (i, line) in text.lines().enumerate() { + if line.is_empty() { + return Err(RefError::malformed(At(&ids, i + 1), "empty token id")); + } + n += 1; + let id: u32 = crate::columns::parse_field(line, "token id", &At(&ids, i + 1))?; + if id >= p.n_vocab { + return Err(RefError::malformed( + At(&ids, i + 1), + format!("token id {id} outside vocabulary {}", p.n_vocab), + )); + } + } + if n != p.n_ids as usize { + return Err(RefError::malformed( + ids.display(), + format!("{} holds {n} ids, {} wants {}", p.ids_file, p.name, p.n_ids), + )); + } + self.logits(p)?; + Ok(()) + } +} + +fn one(f: &[&str], at: &dyn fmt::Display, want: &str) -> Result { + match f { + [_, v] if !v.is_empty() => Ok((*v).to_string()), + _ => Err(RefError::malformed(at, format!("want {want}"))), + } +} + +fn prompt_row(r: &Row<'_>, at: &dyn fmt::Display) -> Result { + let name = r.text("name")?; + let ids_file = r.text("ids")?; + let logits_file = r.text("logits")?; + file_name(name, at)?; + file_name(ids_file, at)?; + file_name(logits_file, at)?; + Ok(Prompt { + name: name.to_string(), + n_ids: r.parse("n_ids")?, + n_vocab: r.parse("n_vocab")?, + ids_file: ids_file.to_string(), + logits_file: logits_file.to_string(), + }) +} diff --git a/justfile b/justfile index b3d624c6a..45e5df11a 100644 --- a/justfile +++ b/justfile @@ -817,6 +817,20 @@ dump-ref-mimo2 *VARIANT: dump-ref-mimo2-cuda: BLOOMERY_MODEL=mimo2 ./tools/box.sh 'BLOOMERY_REF_BACKEND=cuda bash tools/ref/dump.sh' +# ref_io.h's option parser, token reader and raw writer (tools/ref/ref_io_test.cpp). No card, no llama.cpp. +test-ref-io: + g++ -std=c++17 -Wall -o "${TMPDIR:-/tmp}/ref_io_test" tools/ref/ref_io_test.cpp && "${TMPDIR:-/tmp}/ref_io_test" + +# Nemotron 3 Nano 4B last-position logits, mainline on the CPU (ik has no nemotron_h). nemotron_ref +# links the existing mainline build (tools/ref/build-nemotron-ref.sh; it does not rebuild it) and +# writes $BLOOMERY_DATA/ref-nemotron-logits (refset family ref-nemotron-logits). CUDA stays hidden. +# No card: the lane is none. +build-ref-nemotron: + BLOOMERY_Q_CARD=none BLOOMERY_Q_PRED_S=90 BLOOMERY_MODEL=nemotron_h ./tools/box.sh 'bash tools/ref/build-nemotron-ref.sh' + +dump-ref-nemotron: + BLOOMERY_Q_CARD=none BLOOMERY_Q_PRED_S=600 BLOOMERY_MODEL=nemotron_h ./tools/box.sh 'bash tools/ref/dump-nemotron.sh' + # DSpark draft 오라클: ik `db517b69`에 `ik-dsv41-draft.py`만 얹은 트리(`/home/user/ik-dspark-draft`)를 짓고 # dump_draft를 그 트리의 libllama·libggml에 링크한다. ik 빌드는 CPU 임대 아래서 돈다. build-ref-dump-draft: diff --git a/tools/ref/build-hidden.sh b/tools/ref/build-hidden.sh index 6b1c6fca5..ae07398b9 100755 --- a/tools/ref/build-hidden.sh +++ b/tools/ref/build-hidden.sh @@ -6,8 +6,8 @@ # The tree is checked, not trusted: no tracked change (a hand edit there would reach the oracle under the # commit's name). Its build is the one the depth runners time; this links against it and builds nothing in it. # Output: $BLOOMERY_DATA/bin/hidden_ref (HIDDEN_OUT moves it) and hidden_ref.build beside it, naming the sha256 of -# the hidden_ref.cpp it was built from and the mainline commit; hidden.sh refuses a binary whose record does not -# match its own tree. The compile writes a temporary name and only a finished link replaces the installed binary. +# hidden_ref.cpp then ref_io.h and the mainline commit; hidden.sh refuses a binary whose record does not match +# its own tree. The compile writes a temporary name and only a finished link replaces the installed binary. set -euo pipefail # shellcheck source=tools/ref/ref-paths.sh source "$(dirname "${BASH_SOURCE[0]}")/ref-paths.sh" @@ -16,6 +16,7 @@ source "$(dirname "${BASH_SOURCE[0]}")/ref-paths.sh" HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) OUT=${HIDDEN_OUT:-$BLOOMERY_DATA/bin} SRC=$HERE/tools/ref/hidden_ref.cpp +IO=$HERE/tools/ref/ref_io.h git_lcpp() { git -c safe.directory="$LCPP" -C "$LCPP" "$@"; } if [ -n "$(git_lcpp status --porcelain --untracked-files=no)" ]; then echo "build-hidden: $LCPP has tracked changes; the oracle is the commit's build, not an edited tree" >&2 @@ -26,10 +27,11 @@ LIB=$LCPP/build/bin mkdir -p "$OUT" TMP=$OUT/hidden_ref.tmp.$$ trap 'rm -f "$TMP" "$TMP.build"' EXIT -SRC_SHA=$(sha256sum "$SRC" | cut -d' ' -f1) +# cpp then the shared header, the same order hidden.sh hashes. +SRC_SHA=$(cat "$SRC" "$IO" | sha256sum | cut -d' ' -f1) g++ -std=c++17 -O2 -o "$TMP" "$SRC" -I"$LCPP/include" -I"$LCPP/ggml/include" \ -L"$LIB" -lllama -lggml -lggml-base -Wl,-rpath,"$LIB" printf 'source_sha256 %s\nlcpp %s\nlcpp_commit %s\n' "$SRC_SHA" "$(readlink -f "$LCPP")" "$COMMIT" > "$TMP.build" mv -f "$TMP" "$OUT/hidden_ref" mv -f "$TMP.build" "$OUT/hidden_ref.build" -echo "built $OUT/hidden_ref from hidden_ref.cpp sha256 ${SRC_SHA:0:12} against $LCPP at $COMMIT" +echo "built $OUT/hidden_ref from hidden_ref.cpp and ref_io.h sha256 ${SRC_SHA:0:12} against $LCPP at $COMMIT" diff --git a/tools/ref/build-nemotron-ref.sh b/tools/ref/build-nemotron-ref.sh new file mode 100755 index 000000000..f9734fc1c --- /dev/null +++ b/tools/ref/build-nemotron-ref.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Link nemotron_ref (tools/ref/nemotron_ref.cpp, with tools/ref/ref_io.h) against the nemotron_h profile's existing +# mainline build. This does not configure or rebuild that tree: a rebuild moves the md5s the +# tokenizer family pins. The .build record spells those md5s as crate::tokenizer::build_id does +# (llama-tokenize libllama.so ; the file on disk is libllama.so.0). +# Output: $BLOOMERY_DATA/bin/nemotron_ref and nemotron_ref.build beside it. +set -euo pipefail +# shellcheck source=tools/ref/ref-paths.sh +source "$(dirname "${BASH_SOURCE[0]}")/ref-paths.sh" +[ "$MODEL_NAME" = nemotron_h ] || { + echo "build-nemotron-ref: this binary is built under the nemotron_h profile; this command picked $MODEL_NAME" >&2 + exit 2 +} +HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +OUT=${NEMOTRON_OUT:-$BLOOMERY_DATA/bin} +SRC=$HERE/tools/ref/nemotron_ref.cpp +IO=$HERE/tools/ref/ref_io.h +git_lcpp() { git -c safe.directory="$LCPP" -C "$LCPP" "$@"; } +if [ -n "$(git_lcpp status --porcelain --untracked-files=no)" ]; then + echo "build-nemotron-ref: $LCPP has tracked changes; the oracle is the commit's build, not an edited tree" >&2 + exit 2 +fi +TOKENIZE=$LCPP/build/bin/llama-tokenize +[ -x "$TOKENIZE" ] || { echo "build-nemotron-ref: no llama-tokenize at $TOKENIZE" >&2; exit 66; } +LIBLLAMA=$(ldd "$TOKENIZE" | awk '$1 == "libllama.so" || $1 == "libllama.so.0" { print $3; exit }') +[ -f "$LIBLLAMA" ] || { echo "build-nemotron-ref: $TOKENIZE loads no libllama.so (ldd names '$LIBLLAMA')" >&2; exit 66; } +TOK_MD5=$(md5sum < "$TOKENIZE" | cut -d' ' -f1) +LIB_MD5=$(md5sum < "$LIBLLAMA" | cut -d' ' -f1) +BUILD_ID="llama-tokenize $TOK_MD5 libllama.so $LIB_MD5" +COMMIT=$(git_lcpp rev-parse --short=9 HEAD) +LIBDIR=$LCPP/build/bin +mkdir -p "$OUT" +TMP=$OUT/nemotron_ref.tmp.$$ +trap 'rm -f "$TMP" "$TMP.build"' EXIT +# cpp then the shared header, the same order dump-nemotron.sh hashes. +SRC_SHA=$(cat "$SRC" "$IO" | sha256sum | cut -d' ' -f1) +g++ -std=c++17 -O2 -o "$TMP" "$SRC" -I"$LCPP/include" -I"$LCPP/ggml/include" \ + -L"$LIBDIR" -lllama -lggml -lggml-base -Wl,-rpath,"$LIBDIR" +printf 'source_sha256 %s\nlcpp_commit %s\nbuild %s\n' "$SRC_SHA" "$COMMIT" "$BUILD_ID" > "$TMP.build" +mv -f "$TMP" "$OUT/nemotron_ref" +mv -f "$TMP.build" "$OUT/nemotron_ref.build" +echo "built $OUT/nemotron_ref from nemotron_ref.cpp and ref_io.h sha256 ${SRC_SHA:0:12} against mainline at $COMMIT" +echo "build $BUILD_ID" diff --git a/tools/ref/dump-mtp.sh b/tools/ref/dump-mtp.sh index 976fd8e46..65e760727 100644 --- a/tools/ref/dump-mtp.sh +++ b/tools/ref/dump-mtp.sh @@ -178,12 +178,6 @@ mkdir -p "$ROOT" rm -rf "$STAGE" mkdir -p "$STAGE" # A staged set without its trailer is removed on every exit, with a line naming it (dump.sh's rule). -drop_incomplete_stage() { - [ -d "$STAGE" ] || return 0 - grep -qs '^# complete' "$STAGE/MANIFEST.tsv" && return 0 - rm -rf "$STAGE" - echo "[staging] removed the incomplete $STAGE" >&2 -} trap drop_incomplete_stage EXIT trap 'exit 130' INT trap 'exit 143' TERM diff --git a/tools/ref/dump-nemotron.sh b/tools/ref/dump-nemotron.sh new file mode 100755 index 000000000..697ee949c --- /dev/null +++ b/tools/ref/dump-nemotron.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# Dump four last-position logit rows into $BLOOMERY_DATA/ref-nemotron-logits. +# CPU only: CUDA_VISIBLE_DEVICES is emptied and nemotron_ref asks for no GPU layers. +# The prompts are prefixes of the tokenizer oracle's copies of the corpora the other +# families use (engram-corpus.sh): code.txt is IK src/*.cpp, prose.txt is IK docs and +# README. llama-tokenize adds BOS because the file sets add_bos_token; the counts +# include that id. Mainline is not rebuilt. +set -euo pipefail +# shellcheck source=tools/ref/ref-paths.sh +source "$(dirname "${BASH_SOURCE[0]}")/ref-paths.sh" +[ "$MODEL_NAME" = nemotron_h ] || { + echo "dump-nemotron: this dump runs under the nemotron_h profile; this command picked $MODEL_NAME" >&2 + exit 2 +} +HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +SRC=$HERE/tools/ref/nemotron_ref.cpp +IO=$HERE/tools/ref/ref_io.h +BIN=${NEMOTRON_BIN:-$BLOOMERY_DATA/bin/nemotron_ref} +REC=$BIN.build +[ -x "$BIN" ] || { echo "dump-nemotron: no $BIN — run just build-ref-nemotron" >&2; exit 2; } +# cpp then the shared header, the same order build-nemotron-ref.sh records. A header-only edit is a mismatch. +want=$(cat "$SRC" "$IO" | sha256sum | cut -d' ' -f1) +got=$(awk '$1 == "source_sha256" { print $2; exit }' "$REC") +[ "$want" = "$got" ] || { + echo "dump-nemotron: $BIN was linked from another nemotron_ref.cpp or ref_io.h — run just build-ref-nemotron" >&2 + exit 2 +} +TOKENIZE=$LCPP/build/bin/llama-tokenize +[ -x "$TOKENIZE" ] || { echo "dump-nemotron: no llama-tokenize at $TOKENIZE" >&2; exit 66; } +LIBLLAMA=$(ldd "$TOKENIZE" | awk '$1 == "libllama.so" || $1 == "libllama.so.0" { print $3; exit }') +[ -f "$LIBLLAMA" ] || { echo "dump-nemotron: $TOKENIZE loads no libllama.so (ldd names '$LIBLLAMA')" >&2; exit 66; } +TOK_MD5=$(md5sum < "$TOKENIZE" | cut -d' ' -f1) +LIB_MD5=$(md5sum < "$LIBLLAMA" | cut -d' ' -f1) +# The pin literals live in the build script, which wrote this record. A live +# binary that does not match it is not the one nemotron_ref was linked against. +want_build=$(awk '$1 == "build" { $1 = ""; sub(/^ /, ""); print; exit }' "$REC") +[ "llama-tokenize $TOK_MD5 libllama.so $LIB_MD5" = "$want_build" ] || { + echo "dump-nemotron: $TOKENIZE or $LIBLLAMA differs from the build $BIN was linked against ($want_build)" >&2 + exit 2 +} +export BLOOMERY_REF_BUILD="$want_build" +export CUDA_VISIBLE_DEVICES="" + +TEXT=$BLOOMERY_DATA/ref-tokenizer-nemotron +[ -f "$TEXT/code.txt" ] || { echo "dump-nemotron: no $TEXT/code.txt (the tokenizer oracle's code corpus)" >&2; exit 66; } +[ -f "$TEXT/prose.txt" ] || { echo "dump-nemotron: no $TEXT/prose.txt (the tokenizer oracle's prose corpus)" >&2; exit 66; } + +# ids_of — tokenize a line prefix. No --no-bos: the file adds BOS. +ids_of() { + local text=$1 lines=$2 count=$3 out=$4 raw got + raw=$(mktemp) + head -n "$lines" "$text" > "$raw.in" + "$TOKENIZE" -m "$MODEL" -f "$raw.in" --ids --log-disable --no-escape > "$raw.bracket" + { tr -d '[] ' < "$raw.bracket" | tr ',' '\n' | grep -v '^$' || true; } > "$raw" + got=$(wc -l < "$raw") + if [ "$got" -lt "$count" ]; then + echo "dump-nemotron: the first $lines lines of $text tokenize to $got ids, need $count" >&2 + rm -f "$raw" "$raw.in" "$raw.bracket" + exit 2 + fi + head -n "$count" "$raw" > "$out" + rm -f "$raw" "$raw.in" "$raw.bracket" +} + +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"' EXIT +ids_of "$TEXT/code.txt" 80 29 "$WORK/code.ids" +ids_of "$TEXT/prose.txt" 400 512 "$WORK/prose.ids" +head -n 5 "$WORK/code.ids" > "$WORK/code5.ids" +head -n 29 "$WORK/code.ids" > "$WORK/code29.ids" +head -n 64 "$WORK/prose.ids" > "$WORK/prose64.ids" +cp "$WORK/prose.ids" "$WORK/prose512.ids" + +SET=$BLOOMERY_DATA/ref-nemotron-logits +STAGE=$BLOOMERY_DATA/ref-nemotron-logits.staging +rm -rf "$STAGE" +mkdir -p "$STAGE" +trap 'drop_incomplete_stage; rm -rf "$WORK"' EXIT +"$BIN" -m "$MODEL" --out "$STAGE" -c 512 -ub 512 -t 8 \ + --prompt "code5:5:$WORK/code5.ids" \ + --prompt "code29:29:$WORK/code29.ids" \ + --prompt "prose64:64:$WORK/prose64.ids" \ + --prompt "prose512:512:$WORK/prose512.ids" +grep -q '^# complete' "$STAGE/MANIFEST.tsv" || { + echo "dump-nemotron: the staged set has no completion trailer — not installing it" >&2 + exit 1 +} +rows=$(grep -c '^prompt ' "$STAGE/MANIFEST.tsv") +[ "$rows" = 4 ] || { echo "dump-nemotron: the staged set has $rows prompt rows, want 4" >&2; exit 1; } +rm -rf "$SET" +mv "$STAGE" "$SET" +echo "dump-nemotron: installed $SET (4 prompts)" diff --git a/tools/ref/dump.sh b/tools/ref/dump.sh index b7db0b830..5044bcc0f 100755 --- a/tools/ref/dump.sh +++ b/tools/ref/dump.sh @@ -254,12 +254,6 @@ mkdir -p "$STAGE" # `set -e`, a refusal below, a signal), with a line naming it: an incomplete set is never left # behind for a reader to mistake for a set. A complete staged set that the swap refuses stays, as # its refusal says. A SIGKILL skips the trap; the next dump of the set clears the directory above. -drop_incomplete_stage() { - [ -d "$STAGE" ] || return 0 - grep -qs '^# complete' "$STAGE/MANIFEST.tsv" && return 0 - rm -rf "$STAGE" - echo "[staging] removed the incomplete $STAGE" >&2 -} trap drop_incomplete_stage EXIT trap 'exit 130' INT trap 'exit 143' TERM diff --git a/tools/ref/hidden.sh b/tools/ref/hidden.sh index 66c5d2a7c..1d4ddc3ac 100755 --- a/tools/ref/hidden.sh +++ b/tools/ref/hidden.sh @@ -4,7 +4,7 @@ # (the profile's MODEL when absent), by hidden_ref (tools/ref/hidden_ref.cpp), into $BLOOMERY_DATA// — the # sets the refset families hidden-qwen35 and hidden-qwen35-flash-q8 check and gate-gpu-clef-hidden reads. # -# The binary must be the one built from this tree's hidden_ref.cpp (its .build record's source sha256) against the +# The binary must be the one built from this tree's hidden_ref.cpp then ref_io.h (its .build record's source sha256) against the # profile's LCPP at its current commit, and the ids file must have the profile's sha256: either refused names what # differs. Every layer on the card box.sh puts in view (-ngl 99), the context the longest set's P, ubatches of 512 # (llama.cpp's default). Each set is written to .staging and moved over the old set only when hidden_ref @@ -26,7 +26,8 @@ HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) twin=0 if [ "${1:-}" = --cpu-twin ]; then twin=1; shift; fi BIN=$BLOOMERY_DATA/bin/hidden_ref -want_src=$(sha256sum "$HERE/tools/ref/hidden_ref.cpp" | cut -d' ' -f1) +# cpp then the shared header, the same order build-hidden.sh records. +want_src=$(cat "$HERE/tools/ref/hidden_ref.cpp" "$HERE/tools/ref/ref_io.h" | sha256sum | cut -d' ' -f1) want_commit=$(git -c safe.directory="$LCPP" -C "$LCPP" rev-parse --short=9 HEAD) if ! grep -qx "source_sha256 $want_src" "$BIN.build" 2>/dev/null || ! grep -qx "lcpp_commit $want_commit" "$BIN.build" 2>/dev/null; then diff --git a/tools/ref/hidden_ref.cpp b/tools/ref/hidden_ref.cpp index d4e3d1964..73c3fd6d1 100644 --- a/tools/ref/hidden_ref.cpp +++ b/tools/ref/hidden_ref.cpp @@ -28,15 +28,12 @@ #include "llama.h" #include "ggml.h" +#include "ref_io.h" #include -#include -#include #include #include -#include #include -#include #include #include #include @@ -67,47 +64,6 @@ static bool on_node(struct ggml_tensor * t, bool ask, void * user) { return false; } -static bool read_token_file(const std::string & path, long long count, std::vector & ids) { - std::ifstream in(path); - if (!in) { - fprintf(stderr, "hidden_ref: cannot read %s\n", path.c_str()); - return false; - } - std::string line; - long long lines = 0; - while ((long long) ids.size() < count && std::getline(in, line)) { - ++lines; - char * end = nullptr; - errno = 0; - const long v = strtol(line.c_str(), &end, 10); - if (line.empty() || *end != '\0' || errno != 0 || v < 0 || v > INT32_MAX) { - fprintf(stderr, "hidden_ref: %s:%lld is not a token id: '%s'\n", path.c_str(), lines, line.c_str()); - return false; - } - ids.push_back((llama_token) v); - } - if ((long long) ids.size() < count) { - fprintf(stderr, "hidden_ref: %s holds %zu token ids and --tokens-count asks for %lld\n", path.c_str(), - ids.size(), count); - return false; - } - return true; -} - -static bool write_raw(const std::string & path, const float * data, size_t count) { - FILE * f = fopen(path.c_str(), "wb"); - if (!f) { - fprintf(stderr, "hidden_ref: cannot write %s\n", path.c_str()); - return false; - } - const bool ok = fwrite(data, sizeof(float), count, f) == count; - if (fclose(f) != 0 || !ok) { - fprintf(stderr, "hidden_ref: short write on %s\n", path.c_str()); - return false; - } - return true; -} - int main(int argc, char ** argv) { std::string model_path, tokens_file, out; long long count = -1; @@ -123,17 +79,17 @@ int main(int argc, char ** argv) { } else if (strcmp(argv[i], "--tokens-file") == 0 && more) { tokens_file = argv[++i]; } else if (strcmp(argv[i], "--tokens-count") == 0 && more) { - count = atoll(argv[++i]); + if (!parse_i64("hidden_ref", "--tokens-count", argv[++i], 1, LLONG_MAX, count)) return 2; } else if (strcmp(argv[i], "--out") == 0 && more) { out = argv[++i]; } else if (strcmp(argv[i], "-c") == 0 && more) { - n_ctx = atoi(argv[++i]); + if (!parse_i32("hidden_ref", "-c", argv[++i], 0, n_ctx)) return 2; } else if (strcmp(argv[i], "-ub") == 0 && more) { - n_ubatch = atoi(argv[++i]); + if (!parse_i32("hidden_ref", "-ub", argv[++i], 1, n_ubatch)) return 2; } else if (strcmp(argv[i], "-ngl") == 0 && more) { - ngl = atoi(argv[++i]); + if (!parse_i32("hidden_ref", "-ngl", argv[++i], 0, ngl)) return 2; } else if (strcmp(argv[i], "-t") == 0 && more) { - threads = atoi(argv[++i]); + if (!parse_i32("hidden_ref", "-t", argv[++i], 0, threads)) return 2; } else { fprintf(stderr, "hidden_ref: unknown or incomplete argument '%s'\n", argv[i]); return 2; diff --git a/tools/ref/models/nemotron_h.sh b/tools/ref/models/nemotron_h.sh new file mode 100644 index 000000000..6d15fd5fd --- /dev/null +++ b/tools/ref/models/nemotron_h.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# shellcheck shell=bash +# The nemotron_h (Nemotron 3 Nano 4B) profile: what the reference harness needs that is a +# property of the model. Sourced by ref-paths.sh when BLOOMERY_MODEL=nemotron_h; never +# executed, and it exports nothing. +# +# The name is the GGUF general.architecture value, so `grep nemotron_h` finds this profile, +# the metadata keys and the logits set (crates/refset/src/arch/nemotron_h). +# +# MODEL the Q4_K_M file the tokenizer family already names. One shard. +# BLOOMERY_REF_MODEL moves it; a caller's own MODEL= is ignored. +# LCPP mainline llama.cpp. ik has no nemotron_h, so this tree is the oracle. +# nemotron_ref links its existing build and does not rebuild it: a rebuild +# moves the md5s the tokenizer family pins. +# IK no ik oracle. The default is only so IKBIN can be formed; nothing here runs it. +# +# No lease and no card: the file is a few gigabytes and the dump hides CUDA. +# +# SC2034: every name here is read by the file that sources this one. +# shellcheck disable=SC2034 +MODEL_NAME=nemotron_h +: "${IK:=/home/user/ik-idxkey}" +: "${LCPP:=/home/user/llama.cpp-mainline}" +MODEL=${BLOOMERY_REF_MODEL:-/models/NVIDIA-Nemotron-3-Nano-4B-GGUF/NVIDIA-Nemotron3-Nano-4B-Q4_K_M.gguf} diff --git a/tools/ref/nemotron_ref.cpp b/tools/ref/nemotron_ref.cpp new file mode 100644 index 000000000..85661579a --- /dev/null +++ b/tools/ref/nemotron_ref.cpp @@ -0,0 +1,232 @@ +// nemotron_ref — last-position logits of each prompt from llama.cpp mainline, on the CPU. +// +// One llama_decode per prompt, then llama_get_logits_ith(ctx, -1): the last logit row. Only that +// token is marked for output. The row is n_vocab f32, little-endian, plus the prompt's ids. +// Headers are # model, # build (BLOOMERY_REF_BUILD, the tokenizer family's spelling of the +// mainline md5s) and # arch. # complete is written only after every file, by renaming the manifest. +// +// nemotron_ref -m --out --prompt :: ... [-c ] [-ub ] [-t ] +// +// --prompt is repeated, once a prompt. is one decimal id a line, at least of them; the +// first are the prompt. n_gpu_layers is 0. The caller also empties CUDA_VISIBLE_DEVICES. + +#include "llama.h" +#include "ref_io.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct Prompt { + std::string name; + int n = 0; + std::string path; + std::vector ids; +}; + +static bool parse_prompt(const char * spec, Prompt & p) { + const char * c1 = strchr(spec, ':'); + if (!c1 || c1 == spec) return false; + const char * c2 = strchr(c1 + 1, ':'); + if (!c2 || c2 == c1 + 1 || c2[1] == '\0') return false; + p.name.assign(spec, c1); + for (char ch : p.name) { + if (!((ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '0' && ch <= '9'))) return false; + } + const std::string count(c1 + 1, c2); + long long n = 0; + if (!parse_i64("nemotron_ref", "--prompt count", count.c_str(), 1, 1000000, n)) return false; + p.n = (int) n; + p.path = c2 + 1; + return true; +} + +static bool write_ids(const std::string & path, const std::vector & ids) { + FILE * f = fopen(path.c_str(), "w"); + if (!f) { + fprintf(stderr, "nemotron_ref: cannot write %s\n", path.c_str()); + return false; + } + for (llama_token id : ids) { + if (fprintf(f, "%d\n", (int) id) < 0) { + fclose(f); + fprintf(stderr, "nemotron_ref: short write on %s\n", path.c_str()); + return false; + } + } + if (fclose(f) != 0) { + fprintf(stderr, "nemotron_ref: short write on %s\n", path.c_str()); + return false; + } + return true; +} + +static void print_top5(const std::string & name, const float * row, int n_vocab) { + const int k = std::min(n_vocab, 5); + std::vector order(n_vocab); + for (int i = 0; i < n_vocab; ++i) order[i] = i; + std::partial_sort(order.begin(), order.begin() + k, order.end(), [&](int a, int b) { + if (row[a] != row[b]) return row[a] > row[b]; + return a < b; + }); + fprintf(stderr, "nemotron_ref: top5 %s ids=", name.c_str()); + for (int i = 0; i < k; ++i) fprintf(stderr, "%s%d", i ? "," : "", order[i]); + fprintf(stderr, " logits="); + for (int i = 0; i < k; ++i) fprintf(stderr, "%s%.6f", i ? "," : "", row[order[i]]); + fprintf(stderr, "\n"); +} + +int main(int argc, char ** argv) { + std::string model_path, out; + int n_ctx = 0, n_ubatch = 512, threads = 0; + std::vector prompts; + for (int i = 1; i < argc; ++i) { + const bool more = i + 1 < argc; + if (strcmp(argv[i], "-m") == 0 && more) { + model_path = argv[++i]; + } else if (strcmp(argv[i], "--out") == 0 && more) { + out = argv[++i]; + } else if (strcmp(argv[i], "--prompt") == 0 && more) { + Prompt p; + if (!parse_prompt(argv[++i], p)) { + fprintf(stderr, "nemotron_ref: --prompt wants name:count:ids-file, got '%s'\n", argv[i]); + return 2; + } + prompts.push_back(std::move(p)); + } else if (strcmp(argv[i], "-c") == 0 && more) { + if (!parse_i32("nemotron_ref", "-c", argv[++i], 0, n_ctx)) return 2; + } else if (strcmp(argv[i], "-ub") == 0 && more) { + if (!parse_i32("nemotron_ref", "-ub", argv[++i], 1, n_ubatch)) return 2; + } else if (strcmp(argv[i], "-t") == 0 && more) { + if (!parse_i32("nemotron_ref", "-t", argv[++i], 0, threads)) return 2; + } else { + fprintf(stderr, "nemotron_ref: unknown or incomplete argument '%s'\n", argv[i]); + return 2; + } + } + const char * build = getenv("BLOOMERY_REF_BUILD"); + if (model_path.empty() || out.empty() || prompts.empty() || n_ubatch <= 0 || !build || !*build) { + fprintf(stderr, "usage: nemotron_ref -m --out --prompt :: ... " + "[-c ] [-ub ] [-t ]\n" + "BLOOMERY_REF_BUILD must name the mainline build\n"); + return 2; + } + int max_n = 0; + for (Prompt & p : prompts) { + if (!read_token_file(p.path, p.n, p.ids)) return 2; + max_n = std::max(max_n, p.n); + } + if (n_ctx == 0) n_ctx = max_n; + if (n_ctx < max_n) { + fprintf(stderr, "nemotron_ref: a prompt of %d ids does not fit in -c %d\n", max_n, n_ctx); + return 2; + } + + llama_backend_init(); + llama_model_params mp = llama_model_default_params(); + mp.n_gpu_layers = 0; + llama_model * model = llama_model_load_from_file(model_path.c_str(), mp); + if (!model) { + fprintf(stderr, "nemotron_ref: failed to load %s\n", model_path.c_str()); + return 1; + } + llama_context_params cp = llama_context_default_params(); + cp.n_ctx = (uint32_t) n_ctx; + cp.n_batch = (uint32_t) max_n; + cp.n_ubatch = (uint32_t) std::min(n_ubatch, max_n); + cp.n_seq_max = 1; + if (threads > 0) { + cp.n_threads = threads; + cp.n_threads_batch = threads; + } + llama_context * ctx = llama_init_from_model(model, cp); + if (!ctx) { + fprintf(stderr, "nemotron_ref: failed to make the context\n"); + llama_model_free(model); + llama_backend_free(); + return 1; + } + const int n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(model)); + if (n_vocab <= 0) { + fprintf(stderr, "nemotron_ref: the model names no vocabulary\n"); + return 1; + } + char arch[128] = {}; + if (llama_model_meta_val_str(model, "general.architecture", arch, sizeof(arch)) < 0 || strcmp(arch, "nemotron_h") != 0) { + fprintf(stderr, "nemotron_ref: general.architecture is '%s', want nemotron_h\n", arch); + return 1; + } + + if (mkdir(out.c_str(), 0755) != 0 && errno != EEXIST) { + fprintf(stderr, "nemotron_ref: cannot create %s\n", out.c_str()); + return 1; + } + llama_batch batch = llama_batch_init(max_n, 0, 1); + for (int i = 0; i < (int) prompts.size(); ++i) { + const Prompt & p = prompts[i]; + if (i > 0) llama_memory_clear(llama_get_memory(ctx), true); + for (int t = 0; t < p.n; ++t) { + batch.token[t] = p.ids[t]; + batch.pos[t] = t; + batch.n_seq_id[t] = 1; + batch.seq_id[t][0] = 0; + batch.logits[t] = t + 1 == p.n; + } + batch.n_tokens = p.n; + if (llama_decode(ctx, batch) != 0) { + fprintf(stderr, "nemotron_ref: decode failed for %s\n", p.name.c_str()); + return 1; + } + const float * row = llama_get_logits_ith(ctx, -1); + if (!row) { + fprintf(stderr, "nemotron_ref: no last-position logits for %s\n", p.name.c_str()); + return 1; + } + for (int j = 0; j < n_vocab; ++j) { + if (!std::isfinite(row[j])) { + fprintf(stderr, "nemotron_ref: %s logit %d is not finite\n", p.name.c_str(), j); + return 1; + } + } + if (!write_ids(out + "/" + p.name + ".ids", p.ids)) return 1; + if (!write_raw(out + "/" + p.name + ".logits.f32", row, (size_t) n_vocab)) return 1; + print_top5(p.name, row, n_vocab); + } + + const std::string final_path = out + "/MANIFEST.tsv"; + const std::string partial_path = final_path + ".partial"; + FILE * m = fopen(partial_path.c_str(), "w"); + if (!m) { + fprintf(stderr, "nemotron_ref: cannot write %s\n", partial_path.c_str()); + return 1; + } + fprintf(m, "# nemotron_ref — last-position logits, raw f32, little-endian\n"); + fprintf(m, "# model\t%s\n", model_path.c_str()); + fprintf(m, "# build\t%s\n", build); + fprintf(m, "# arch\t%s\n", arch); + fprintf(m, "# source\tllama_get_logits_ith(ctx, -1)\n"); + fprintf(m, "# prompt columns\tname\tn_ids\tn_vocab\tids\tlogits\n"); + for (const Prompt & p : prompts) { + fprintf(m, "prompt\t%s\t%d\t%d\t%s.ids\t%s.logits.f32\n", p.name.c_str(), p.n, n_vocab, p.name.c_str(), + p.name.c_str()); + } + fprintf(m, "# complete\t%zu\t0\n", prompts.size()); + if (fclose(m) != 0 || rename(partial_path.c_str(), final_path.c_str()) != 0) { + fprintf(stderr, "nemotron_ref: cannot install %s\n", final_path.c_str()); + return 1; + } + fprintf(stderr, "nemotron_ref: %zu prompts, vocab %d, into %s\n", prompts.size(), n_vocab, out.c_str()); + llama_batch_free(batch); + llama_free(ctx); + llama_model_free(model); + llama_backend_free(); + return 0; +} diff --git a/tools/ref/ref-paths.sh b/tools/ref/ref-paths.sh index 1f190d3e1..0b3a9dd29 100644 --- a/tools/ref/ref-paths.sh +++ b/tools/ref/ref-paths.sh @@ -262,3 +262,12 @@ fixture_line() { esac printf '%s\n' "$line" } + +# A dump runner's EXIT trap: the set staged at $STAGE is removed unless its manifest carries the +# `# complete` trailer, so a run that dies leaves no half-written staging set. +drop_incomplete_stage() { + [ -d "$STAGE" ] || return 0 + grep -qs '^# complete' "$STAGE/MANIFEST.tsv" && return 0 + rm -rf "$STAGE" + echo "[staging] removed the incomplete $STAGE" >&2 +} diff --git a/tools/ref/ref_io.h b/tools/ref/ref_io.h new file mode 100644 index 000000000..38ee88d84 --- /dev/null +++ b/tools/ref/ref_io.h @@ -0,0 +1,82 @@ +#pragma once +// Token files and raw f32 rows for the mainline oracles (hidden_ref, nemotron_ref). +// A token id is a non-negative 32-bit integer. The header does not include llama.h: +// llama_token is that integer, so a vector of it is a vector of int32_t. + +#include +#include +#include +#include +#include +#include +#include +#include + +// `value` is one decimal integer in [lo, hi], the whole argument. Anything else is named on stderr. +inline bool parse_i64(const char * who, const char * flag, const char * value, long long lo, long long hi, + long long & out) { + if (value == nullptr || value[0] < '0' || value[0] > '9') { + fprintf(stderr, "%s: %s wants an integer from %lld to %lld, got %s\n", who, flag, + lo, hi, value == nullptr ? "" : value); + return false; + } + char * end = nullptr; + errno = 0; + const long long n = strtoll(value, &end, 10); + if (end == value || *end != '\0' || errno != 0 || n < lo || n > hi) { + fprintf(stderr, "%s: %s wants an integer from %lld to %lld, got %s\n", who, flag, lo, hi, value); + return false; + } + out = n; + return true; +} + +// An int in [lo, INT_MAX]. `lo` is 0 where 0 means the harness default, and 1 where 0 is not a size. +inline bool parse_i32(const char * who, const char * flag, const char * value, int lo, int & dest) { + long long n = 0; + if (!parse_i64(who, flag, value, lo, INT_MAX, n)) return false; + dest = (int) n; + return true; +} + +template +bool read_token_file(const std::string & path, long long count, std::vector & ids) { + std::ifstream in(path); + if (!in) { + fprintf(stderr, "%s: cannot read\n", path.c_str()); + return false; + } + std::string line; + long long lines = 0; + while ((long long) ids.size() < count && std::getline(in, line)) { + ++lines; + if (!line.empty() && line.back() == '\r') line.pop_back(); + char * end = nullptr; + errno = 0; + const long id = strtol(line.c_str(), &end, 10); + // The whole record is the number: a NUL inside the line ends the C string but not the record. + if (line.empty() || line[0] < '0' || line[0] > '9' || end != line.c_str() + line.size() || + errno != 0 || id < 0 || id > INT32_MAX) { + fprintf(stderr, "%s:%lld: invalid token id '%s'\n", path.c_str(), lines, line.c_str()); + return false; + } + ids.push_back((Token) id); + } + if (!in.bad() && (long long) ids.size() == count) return true; + fprintf(stderr, "%s: holds %zu token ids, asked for %lld\n", path.c_str(), ids.size(), count); + return false; +} + +inline bool write_raw(const std::string & path, const float * data, size_t count) { + FILE * f = fopen(path.c_str(), "wb"); + if (!f) { + perror(path.c_str()); + return false; + } + const bool ok = fwrite(data, sizeof(float), count, f) == count; + if (fclose(f) != 0 || !ok) { + fprintf(stderr, "%s: short write\n", path.c_str()); + return false; + } + return true; +} diff --git a/tools/ref/ref_io_test.cpp b/tools/ref/ref_io_test.cpp new file mode 100644 index 000000000..0f3bae331 --- /dev/null +++ b/tools/ref/ref_io_test.cpp @@ -0,0 +1,92 @@ +#include "ref_io.h" + +#include +#include +#include +#include +#include + +static int fails = 0; + +static void expect(bool ok, const char * what) { + if (ok) return; + std::fprintf(stderr, "FAIL %s\n", what); + ++fails; +} + +static bool rejected(const char * flag, const char * value, long long lo, long long hi) { + long long n = 123; + const bool ok = parse_i64("ref", flag, value, lo, hi, n); + return !ok && n == 123; +} + +static bool accepted(const char * flag, const char * value, long long lo, long long hi, long long want) { + long long n = -1; + return parse_i64("ref", flag, value, lo, hi, n) && n == want; +} + +int main() { + int n32 = 123; + expect(!parse_i32("ref", "-c", "garbage", 0, n32) && n32 == 123, "i32 rejects garbage"); + expect(parse_i32("ref", "-c", "16", 0, n32) && n32 == 16, "i32 reads 16"); + expect(rejected("-c", "garbage", 0, INT_MAX), "nonnumeric -c"); + expect(rejected("-c", "12x", 0, INT_MAX), "trailing junk -c"); + expect(rejected("-c", "12 ", 0, INT_MAX), "trailing space -c"); + expect(rejected("-t", " 8", 0, INT_MAX), "leading space -t"); + expect(rejected("-t", "-1", 0, INT_MAX), "negative -t"); + expect(rejected("-t", "+8", 0, INT_MAX), "plus -t"); + expect(rejected("-ub", "0", 1, INT_MAX), "zero -ub"); + expect(rejected("--tokens-count", "", 1, LLONG_MAX), "empty count"); + expect(accepted("-c", "0", 0, INT_MAX, 0), "zero context"); + expect(accepted("-t", "16", 0, INT_MAX, 16), "threads"); + expect(accepted("-c", "2147483647", 0, INT_MAX, INT_MAX), "int max"); + expect(rejected("-c", "2147483648", 0, INT_MAX), "past int max"); + + char dir[] = "/tmp/ref-io-test-XXXXXX"; + if (mkdtemp(dir) == nullptr) { + std::perror("mkdtemp"); + return 1; + } + const std::string ids = std::string(dir) + "/ids"; + { + std::ofstream o(ids); + o << "0\n1\r\n2\n"; + } + std::vector got; + expect(read_token_file(ids, 3, got) && got.size() == 3 && got[0] == 0 && got[1] == 1 && got[2] == 2, + "three ids, one with cr"); + got.clear(); + { + std::ofstream o(ids); + o << "0\ngarbage\n"; + } + expect(!read_token_file(ids, 2, got), "garbage token id"); + got.clear(); + { + std::ofstream o(ids, std::ios::binary); + o.write("0\n1\0garbage\n", 12); + } + expect(!read_token_file(ids, 2, got), "a NUL inside a token record"); + got.clear(); + { + std::ofstream o(ids); + o << "1\n"; + } + expect(!read_token_file(ids, 2, got), "short token file"); + const float row[2] = {1.5f, -0.25f}; + const std::string raw = std::string(dir) + "/row.f32"; + expect(write_raw(raw, row, 2), "write raw"); + { + std::ifstream in(raw, std::ios::binary); + float back[2] = {}; + in.read(reinterpret_cast(back), sizeof(back)); + expect(in.gcount() == (std::streamsize) sizeof(back) && back[0] == 1.5f && back[1] == -0.25f, "read raw back"); + } + expect(!write_raw(std::string(dir) + "/missing/row.f32", row, 2), "missing directory"); + if (fails) { + std::fprintf(stderr, "ref_io_test: %d failed\n", fails); + return 1; + } + std::printf("ref_io_test: ok\n"); + return 0; +}