From b59058528755e7a240be04259e5286d6b9a412c6 Mon Sep 17 00:00:00 2001 From: midagedev Date: Thu, 15 Oct 2026 08:56:12 +0900 Subject: [PATCH 1/7] refset: Nemotron 3 Nano last-position logits reader The ref-nemotron-logits family reuses the tokenizer build pin. A set is read by column name and refused when its trailer, model path, or build pin does not match. --- crates/refset/src/arch/mod.rs | 2 + crates/refset/src/arch/nemotron_h/mod.rs | 152 +++++++++++++ crates/refset/src/arch/tokenizer.rs | 4 +- crates/refset/src/family.rs | 12 ++ crates/refset/src/lib.rs | 3 + crates/refset/src/nemotron.rs | 264 +++++++++++++++++++++++ 6 files changed, 435 insertions(+), 2 deletions(-) create mode 100644 crates/refset/src/arch/nemotron_h/mod.rs create mode 100644 crates/refset/src/nemotron.rs diff --git a/crates/refset/src/arch/mod.rs b/crates/refset/src/arch/mod.rs index aaaef8a06..dcd4b9f98 100644 --- a/crates/refset/src/arch/mod.rs +++ b/crates/refset/src/arch/mod.rs @@ -15,6 +15,7 @@ pub mod draft; pub mod glm5next; pub mod mimo2; pub mod minicpmv4_7; +pub mod nemotron_h; pub mod qwen35; pub mod qwen35moe; pub mod qwen4exp; @@ -30,6 +31,7 @@ static BY_ARCH: &[(&str, &[&Family])] = &[ (mimo2::ARCH, mimo2::FAMILIES), (minicpmv4_7::ARCH, minicpmv4_7::FAMILIES), (qwen4exp::ARCH, qwen4exp::FAMILIES), + (nemotron_h::ARCH, nemotron_h::FAMILIES), (tokenizer::ARCH, tokenizer::FAMILIES), (dequant::ARCH, dequant::FAMILIES), ]; diff --git a/crates/refset/src/arch/nemotron_h/mod.rs b/crates/refset/src/arch/nemotron_h/mod.rs new file mode 100644 index 000000000..ae14eae9b --- /dev/null +++ b/crates/refset/src/arch/nemotron_h/mod.rs @@ -0,0 +1,152 @@ +//! The Nemotron 3 Nano family. The logits set is dumped from [`MODEL`] by +//! mainline llama.cpp ([`super::tokenizer::NEMOTRON_BUILD`]), which is the +//! build the tokenizer family already pins. ik has no `nemotron_h`. + +use crate::RefError; +use crate::family::{Build, Family, Identity}; + +/// The architecture every nemotron manifest names in its `# arch` line. +pub const ARCH: &str = "nemotron_h"; + +/// The file the tree runs. The tokenizer family owns the path. +pub const MODEL: &str = super::tokenizer::NEMOTRON_MODEL; + +/// Last-position logits of four prompts, mainline on the CPU. +pub const LOGITS_SET: &str = "ref-nemotron-logits"; + +/// [`MODEL`], as a family's `runs`. +fn model() -> Result { + Ok(MODEL.to_string()) +} + +/// Mainline's last-position logits (`just dump-ref-nemotron`). +pub static LOGITS: Family = Family { + name: "ref-nemotron-logits", + sets: &[LOGITS_SET], + resolve: None, + recipe: "just dump-ref-nemotron", + identity: Identity::NemotronLogits, + arch: Some(ARCH), + build: Some(Build::Is(super::tokenizer::NEMOTRON_BUILD)), + runs: Some(model), + draft_runs: None, + consumers: &[], +}; + +/// The architecture's families, in the order `refset-check` lists them. +pub static FAMILIES: &[&Family] = &[&LOGITS]; + +#[cfg(test)] +mod tests { + use super::LOGITS; + use crate::RefError; + use crate::family::Build; + use crate::nemotron::LogitsSet; + use std::path::Path; + + fn pin() -> &'static str { + match LOGITS.build { + Some(Build::Is(p)) => p, + _ => panic!("the logits family pins no build"), + } + } + + fn runs() -> String { + LOGITS.runs().unwrap_or_else(|e| panic!("{e}")) + } + + /// Write one prompt's ids and logits and the manifest. `complete` adds the trailer. + fn write(dir: &Path, model: &str, build: &str, complete: bool) { + std::fs::create_dir_all(dir).unwrap_or_else(|e| panic!("{e}")); + std::fs::write(dir.join("p.ids"), "1\n2\n3\n4\n5\n").unwrap_or_else(|e| panic!("{e}")); + let logits: Vec = [0.0f32, 1.5, -0.5, 0.25] + .into_iter() + .flat_map(f32::to_le_bytes) + .collect(); + std::fs::write(dir.join("p.logits.f32"), logits).unwrap_or_else(|e| panic!("{e}")); + let mut text = format!( + "# model\t{model}\n# build\t{build}\n# arch\tnemotron_h\n\ + # prompt columns\tname\tn_ids\tn_vocab\tids\tlogits\n\ + prompt\tp\t5\t4\tp.ids\tp.logits.f32\n" + ); + if complete { + text.push_str("# complete\t1\t0\n"); + } + std::fs::write(dir.join("MANIFEST.tsv"), text).unwrap_or_else(|e| panic!("{e}")); + } + + fn dir(tag: &str) -> std::path::PathBuf { + let d = + std::env::temp_dir().join(format!("bloomery-nemotron-{tag}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + d + } + + /// The build pin is the tokenizer family's constant. + #[test] + fn the_build_pin_is_the_tokenizer_familys() { + let Some(Build::Is(p)) = LOGITS.build else { + panic!("no pin"); + }; + assert_eq!(p, crate::arch::tokenizer::NEMOTRON_BUILD); + assert_eq!(LOGITS.sets, &["ref-nemotron-logits"]); + assert_eq!(runs(), crate::arch::tokenizer::NEMOTRON_MODEL); + } + + /// A finished set opens, and `logits` is that column's file. + #[test] + fn a_finished_set_reads_logits_by_column_name() { + let d = dir("ok"); + write(&d, &runs(), pin(), true); + let set = LogitsSet::open(&d, &LOGITS).unwrap_or_else(|e| panic!("{e}")); + assert_eq!(set.prompts.len(), 1); + let row = set + .logits(&set.prompts[0]) + .unwrap_or_else(|e| panic!("{e}")); + assert_eq!(row[1], 1.5); + std::fs::remove_dir_all(&d).unwrap_or_else(|e| panic!("{e}")); + } + + /// No `# complete`: unfinished, even when the other lines are well formed. + #[test] + fn a_set_without_its_trailer_is_unfinished() { + let d = dir("trailer"); + write(&d, &runs(), pin(), false); + match LogitsSet::open(&d, &LOGITS) { + Err(RefError::Unfinished { set }) => assert_eq!(set, d.display().to_string()), + other => panic!("a set without its trailer: {other:?}"), + } + std::fs::remove_dir_all(&d).unwrap_or_else(|e| panic!("{e}")); + } + + /// Another model path is stale. + #[test] + fn a_set_of_another_model_is_stale() { + let d = dir("model"); + write(&d, "/models/other/file.gguf", pin(), true); + match LogitsSet::open(&d, &LOGITS) { + Err(RefError::Stale { dumped_from, .. }) => { + assert_eq!(dumped_from, "/models/other/file.gguf"); + } + other => panic!("another file: {other:?}"), + } + std::fs::remove_dir_all(&d).unwrap_or_else(|e| panic!("{e}")); + } + + /// Another mainline build is foreign. + #[test] + fn a_set_of_another_build_is_foreign() { + let d = dir("build"); + write( + &d, + &runs(), + "llama-tokenize 00000000000000000000000000000000 libllama.so 00000000000000000000000000000000", + true, + ); + match LogitsSet::open(&d, &LOGITS) { + Err(RefError::Foreign { field, .. }) => assert_eq!(field, "build"), + other => panic!("another build: {other:?}"), + } + std::fs::remove_dir_all(&d).unwrap_or_else(|e| panic!("{e}")); + } +} diff --git a/crates/refset/src/arch/tokenizer.rs b/crates/refset/src/arch/tokenizer.rs index a99b038f1..8d25f084f 100644 --- a/crates/refset/src/arch/tokenizer.rs +++ b/crates/refset/src/arch/tokenizer.rs @@ -47,7 +47,7 @@ pub const TOKREF_BUILD: &str = "llama-tokenize 0128e83c5e71e5f2b72f3d5d2c2de6b2 /// `nemotron_h` architecture. The soname is `libllama.so.0`; the pin's word /// is the one [`crate::tokenizer::build_id`] spells. // PIN(2026-10-14): /home/user/llama.cpp-mainline's llama-tokenize and libllama.so.0. -const NEMOTRON_BUILD: &str = "llama-tokenize 5df95d3a942effc971ff170432d840fd \ +pub(crate) const NEMOTRON_BUILD: &str = "llama-tokenize 5df95d3a942effc971ff170432d840fd \ libllama.so 3575b1f39e7773379fec011d39f32a77"; /// V4.1's set (`deepseek-v3` pre-tokenizer), from the file the tree runs. @@ -107,7 +107,7 @@ pub static QWEN38: Family = Family { }; /// The Nemotron 3 Nano 4B file the tokenizer gate loads (`pixtral`). -const NEMOTRON_MODEL: &str = +pub(crate) const NEMOTRON_MODEL: &str = "/models/NVIDIA-Nemotron-3-Nano-4B-GGUF/NVIDIA-Nemotron3-Nano-4B-Q4_K_M.gguf"; /// Nemotron 3 Nano 4B's set (`pixtral` pre-tokenizer). diff --git a/crates/refset/src/family.rs b/crates/refset/src/family.rs index 9170d9a96..dd25838e2 100644 --- a/crates/refset/src/family.rs +++ b/crates/refset/src/family.rs @@ -63,6 +63,10 @@ pub enum Identity { /// naming the harness and ggml's library by md5: the family's `build` /// pins both ([`crate::dequant::build_id`]). DequantManifest, + /// `# model`, `# build` and `# arch` in a last-position logits set's + /// `MANIFEST.tsv` (`tools/ref/nemotron_ref.cpp`). A prompt row is read by + /// the `# prompt columns` line; `logits` is that prompt's last position. + NemotronLogits, } /// The ik build a family's sets must name. @@ -390,6 +394,14 @@ impl Family { build: set.build(), }) } + Identity::NemotronLogits => { + let set = crate::nemotron::LogitsSet::open(path, self)?; + Ok(Provenance { + dumped_from: stated(set.model.as_deref()), + draft: None, + build: set.build, + }) + } } } } diff --git a/crates/refset/src/lib.rs b/crates/refset/src/lib.rs index 3ac87f196..368d8d6cc 100644 --- a/crates/refset/src/lib.rs +++ b/crates/refset/src/lib.rs @@ -26,6 +26,8 @@ //! - [`tokenizer`]: `llama-tokenize`'s ids per vocabulary //! (`crates/tokenizer/tools/oracle.sh`). //! - [`dequant`]: ggml's dequantized rows (`tools/ref/dump-dequant.sh`). +//! - [`nemotron`]: mainline's last-position logits of Nemotron 3 Nano +//! (`tools/ref/nemotron_ref.cpp`). //! - [`census`]: the census stubs — header-only copies of many users' model //! files, pinned by `tools/census/stubs.tsv` (the writer //! `tools/type-census.py --stubs`) and checked against it. @@ -48,6 +50,7 @@ pub mod ik; pub mod kld; mod md5; pub mod mtpref; +pub mod nemotron; pub mod tokenizer; pub mod vision; pub mod visref; diff --git a/crates/refset/src/nemotron.rs b/crates/refset/src/nemotron.rs new file mode 100644 index 000000000..f9a5835fa --- /dev/null +++ b/crates/refset/src/nemotron.rs @@ -0,0 +1,264 @@ +//! Last-position logits of Nemotron 3 Nano (`tools/ref/nemotron_ref.cpp`, +//! `just dump-ref-nemotron`). One prompt is one row, read by the names on +//! `# prompt columns`. `logits` is the last position only, `n_vocab` f32, +//! little-endian; `ids` is that prompt, one id a line. +//! +//! ```text +//! # model +//! # build llama-tokenize libllama.so +//! # arch nemotron_h +//! # prompt columns name n_ids n_vocab ids logits +//! prompt code5 5 131072 code5.ids code5.logits.f32 +//! # complete 4 0 +//! ``` +//! +//! [`LogitsSet::open`] refuses a set that did not finish, one dumped from +//! another file, and one written by another mainline build. + +use crate::RefError; +use crate::columns::{Columns, Row}; +use crate::family::{Family, Identity}; +use std::collections::HashSet; +use std::fmt; +use std::path::{Path, PathBuf}; + +/// One prompt of the set. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Prompt { + /// The row's `name`. + pub name: String, + /// How many ids the prompt holds, BOS included when the file adds one. + pub n_ids: u32, + /// The logits row's width. + pub n_vocab: u32, + /// The ids file under the set's directory. + pub ids_file: String, + /// The logits file under the set's directory (`n_vocab` f32). + pub logits_file: String, +} + +/// A logits set's manifest, and once opened, its files checked. +#[derive(Debug, Clone)] +pub struct LogitsSet { + /// The set's directory. + pub dir: PathBuf, + /// `# model`. + pub model: Option, + /// `# build`, spelled as [`crate::tokenizer::build_id`]. + pub build: Option, + /// `# arch`. + pub arch: Option, + /// One row a prompt, in the manifest's order. + pub prompts: Vec, + /// Whether the `# complete` trailer is there. + pub complete: bool, +} + +/// A manifest line's position, formatted only into an error. +struct At<'a>(&'a Path, usize); + +impl fmt::Display for At<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "nemotron: {}:{}", self.0.display(), self.1) + } +} + +impl LogitsSet { + /// Parse `dir/MANIFEST.tsv`. A set without one is `Missing`, naming the + /// recipe that writes it. A set without its trailer comes back unread, + /// `complete` false. A row before its column line, of another width, with + /// a count that does not parse, or naming a prompt twice, is `Malformed`. + pub fn read(dir: &Path, family: &Family) -> Result { + let path = dir.join("MANIFEST.tsv"); + let text = std::fs::read_to_string(&path).map_err(|e| { + RefError::missing( + &path, + format!("{}: {e} — run: {}", path.display(), family.recipe), + ) + })?; + let mut set = LogitsSet { + dir: dir.to_path_buf(), + model: None, + build: None, + arch: None, + prompts: Vec::new(), + complete: false, + }; + // A manifest without its trailer is a dump that did not finish: its rows are not read, so + // the refusal is the one that says so. + if !text + .lines() + .any(|l| l.split('\t').next() == Some("# complete")) + { + return Ok(set); + } + let mut cols: Option = None; + let mut seen = HashSet::new(); + for (i, line) in text.lines().enumerate() { + if line.is_empty() { + continue; + } + let at = At(&path, i + 1); + let f: Vec<&str> = line.split('\t').collect(); + match f[0] { + "# model" => set.model = Some(one(&f, &at, "# model\t")?), + "# build" => set.build = Some(one(&f, &at, "# build\t")?), + "# arch" => set.arch = Some(one(&f, &at, "# arch\t")?), + "# prompt columns" => { + cols = Some(Columns::new("prompt", f[1..].iter().copied())); + } + "# complete" => set.complete = true, + h if h.starts_with('#') => {} + "prompt" => { + let c = cols.as_ref().ok_or_else(|| { + RefError::malformed(&at, "a prompt row before its `# prompt columns` line") + })?; + let prompt = prompt_row(&c.row(line, &at)?, &at)?; + if !seen.insert(prompt.name.clone()) { + return Err(RefError::malformed( + &at, + format!("{} is named twice", prompt.name), + )); + } + set.prompts.push(prompt); + } + other => { + return Err(RefError::malformed( + &at, + format!("unknown row kind {other:?}"), + )); + } + } + } + Ok(set) + } + + /// The set at `dir` read and checked against `family`'s row. + pub fn open(dir: &Path, family: &Family) -> Result { + if !matches!(family.identity, Identity::NemotronLogits) { + return Err(RefError::missing( + dir, + format!( + "nemotron: {} is not a last-position logits family", + family.name + ), + )); + } + let set = LogitsSet::read(dir, family)?; + set.check_family(family)?; + Ok(set) + } + + /// This set against its family's row: the trailer ([`RefError::Unfinished`]), + /// the model file ([`RefError::Stale`]), the build and the architecture + /// ([`RefError::Foreign`]), then each prompt's ids and logits files. + pub fn check_family(&self, family: &Family) -> Result<(), RefError> { + if !self.complete { + return Err(RefError::Unfinished { + set: self.dir.display().to_string(), + }); + } + family.check_file(&self.dir, "# model", self.model.as_deref())?; + family.check_build(&self.dir, self.build.as_deref())?; + family.check_arch(&self.dir, self.arch.as_deref())?; + if self.prompts.is_empty() { + return Err(RefError::malformed( + self.dir.join("MANIFEST.tsv").display(), + "no prompt rows: the set names nothing to check", + )); + } + for p in &self.prompts { + self.check_prompt(p)?; + } + Ok(()) + } + + /// Prompt `p`'s last-position logits, `n_vocab` f32, little-endian. + pub fn logits(&self, p: &Prompt) -> Result, RefError> { + let path = self.dir.join(&p.logits_file); + let bytes = std::fs::read(&path).map_err(|e| { + RefError::missing( + &path, + format!("nemotron: cannot read {}: {e}", path.display()), + ) + })?; + let want = p.n_vocab as usize * 4; + if bytes.len() != want { + return Err(RefError::malformed( + path.display(), + format!( + "{} is {} bytes, {} logits of {} want {want}", + p.logits_file, + bytes.len(), + p.name, + p.n_vocab + ), + )); + } + let mut out = Vec::with_capacity(p.n_vocab as usize); + for chunk in bytes.chunks_exact(4) { + out.push(f32::from_le_bytes( + chunk.try_into().expect("chunks_exact(4)"), + )); + } + Ok(out) + } + + fn check_prompt(&self, p: &Prompt) -> Result<(), RefError> { + let ids = self.dir.join(&p.ids_file); + let text = std::fs::read_to_string(&ids).map_err(|e| { + RefError::missing( + &ids, + format!("nemotron: cannot read {}: {e}", ids.display()), + ) + })?; + let n = text.lines().filter(|l| !l.is_empty()).count(); + if n != p.n_ids as usize { + return Err(RefError::malformed( + ids.display(), + format!("{} holds {n} ids, {} wants {}", p.ids_file, p.name, p.n_ids), + )); + } + let row = self.logits(p)?; + if row.len() != p.n_vocab as usize { + return Err(RefError::malformed( + self.dir.join(&p.logits_file).display(), + format!("{} logits, {} wants {}", row.len(), p.name, p.n_vocab), + )); + } + Ok(()) + } +} + +fn one(f: &[&str], at: &dyn fmt::Display, want: &str) -> Result { + match f { + [_, v] if !v.is_empty() => Ok((*v).to_string()), + _ => Err(RefError::malformed(at, format!("want {want}"))), + } +} + +fn file_name(name: &str, at: &dyn fmt::Display) -> Result<(), RefError> { + if name.is_empty() || name.contains('/') || matches!(name, "." | "..") { + return Err(RefError::malformed( + at, + format!("{name:?} does not name a file of the set's directory"), + )); + } + Ok(()) +} + +fn prompt_row(r: &Row<'_>, at: &dyn fmt::Display) -> Result { + let name = r.text("name")?; + let ids_file = r.text("ids")?; + let logits_file = r.text("logits")?; + file_name(name, at)?; + file_name(ids_file, at)?; + file_name(logits_file, at)?; + Ok(Prompt { + name: name.to_string(), + n_ids: r.parse("n_ids")?, + n_vocab: r.parse("n_vocab")?, + ids_file: ids_file.to_string(), + logits_file: logits_file.to_string(), + }) +} From 2fd7b881ab90827ab98be590287143b2414106df Mon Sep 17 00:00:00 2001 From: midagedev Date: Thu, 15 Oct 2026 08:56:38 +0900 Subject: [PATCH 2/7] refset: dump Nemotron 3 Nano last-position logits from mainline ik has no nemotron_h. nemotron_ref links the pinned mainline build and writes four CPU prompts into ref-nemotron-logits. CUDA stays hidden. --- justfile | 10 ++ tools/ref/build-nemotron-ref.sh | 51 ++++++ tools/ref/dump-nemotron.sh | 97 +++++++++++ tools/ref/models/nemotron_h.sh | 34 ++++ tools/ref/nemotron_ref.cpp | 274 ++++++++++++++++++++++++++++++++ 5 files changed, 466 insertions(+) create mode 100755 tools/ref/build-nemotron-ref.sh create mode 100755 tools/ref/dump-nemotron.sh create mode 100644 tools/ref/models/nemotron_h.sh create mode 100644 tools/ref/nemotron_ref.cpp diff --git a/justfile b/justfile index b3d624c6a..3112a3c77 100644 --- a/justfile +++ b/justfile @@ -817,6 +817,16 @@ dump-ref-mimo2 *VARIANT: dump-ref-mimo2-cuda: BLOOMERY_MODEL=mimo2 ./tools/box.sh 'BLOOMERY_REF_BACKEND=cuda bash tools/ref/dump.sh' +# Nemotron 3 Nano 4B last-position logits, mainline on the CPU (ik has no nemotron_h). nemotron_ref +# links the existing mainline build (tools/ref/build-nemotron-ref.sh; it does not rebuild it) and +# writes $BLOOMERY_DATA/ref-nemotron-logits (refset family ref-nemotron-logits). CUDA stays hidden. +# No card: the lane is none. +build-ref-nemotron: + BLOOMERY_Q_CARD=none BLOOMERY_Q_PRED_S=90 BLOOMERY_MODEL=nemotron_h ./tools/box.sh 'bash tools/ref/build-nemotron-ref.sh' + +dump-ref-nemotron: + BLOOMERY_Q_CARD=none BLOOMERY_Q_PRED_S=600 BLOOMERY_MODEL=nemotron_h ./tools/box.sh 'bash tools/ref/dump-nemotron.sh' + # DSpark draft 오라클: ik `db517b69`에 `ik-dsv41-draft.py`만 얹은 트리(`/home/user/ik-dspark-draft`)를 짓고 # dump_draft를 그 트리의 libllama·libggml에 링크한다. ik 빌드는 CPU 임대 아래서 돈다. build-ref-dump-draft: diff --git a/tools/ref/build-nemotron-ref.sh b/tools/ref/build-nemotron-ref.sh new file mode 100755 index 000000000..69523fbc7 --- /dev/null +++ b/tools/ref/build-nemotron-ref.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# Link nemotron_ref (tools/ref/nemotron_ref.cpp) against the nemotron_h profile's existing +# mainline build. This does not configure or rebuild that tree: a rebuild moves the md5s the +# tokenizer family pins. The .build record spells those md5s as crate::tokenizer::build_id does +# (llama-tokenize libllama.so ; the file on disk is libllama.so.0). +# Output: $BLOOMERY_DATA/bin/nemotron_ref and nemotron_ref.build beside it. +set -euo pipefail +# shellcheck source=tools/ref/ref-paths.sh +source "$(dirname "${BASH_SOURCE[0]}")/ref-paths.sh" +[ "$MODEL_NAME" = nemotron_h ] || { + echo "build-nemotron-ref: this binary is built under the nemotron_h profile; this command picked $MODEL_NAME" >&2 + exit 2 +} +HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +OUT=${NEMOTRON_OUT:-$BLOOMERY_DATA/bin} +SRC=$HERE/tools/ref/nemotron_ref.cpp +git_lcpp() { git -c safe.directory="$LCPP" -C "$LCPP" "$@"; } +if [ -n "$(git_lcpp status --porcelain --untracked-files=no)" ]; then + echo "build-nemotron-ref: $LCPP has tracked changes; the oracle is the commit's build, not an edited tree" >&2 + exit 2 +fi +TOKENIZE=$LCPP/build/bin/llama-tokenize +[ -x "$TOKENIZE" ] || { echo "build-nemotron-ref: no llama-tokenize at $TOKENIZE" >&2; exit 66; } +LIBLLAMA=$(ldd "$TOKENIZE" | awk '$1 == "libllama.so" || $1 == "libllama.so.0" { print $3; exit }') +[ -f "$LIBLLAMA" ] || { echo "build-nemotron-ref: $TOKENIZE loads no libllama.so (ldd names '$LIBLLAMA')" >&2; exit 66; } +TOK_MD5=$(md5sum < "$TOKENIZE" | cut -d' ' -f1) +LIB_MD5=$(md5sum < "$LIBLLAMA" | cut -d' ' -f1) +# The tokenizer family's pin. Recording a different pair would make every nemotron set foreign +# and would mean this tree was rebuilt. +[ "$TOK_MD5" = 5df95d3a942effc971ff170432d840fd ] || { + echo "build-nemotron-ref: llama-tokenize md5 is $TOK_MD5, the pinned mainline is 5df95d3a942effc971ff170432d840fd" >&2 + exit 2 +} +[ "$LIB_MD5" = 3575b1f39e7773379fec011d39f32a77 ] || { + echo "build-nemotron-ref: libllama md5 is $LIB_MD5, the pinned mainline is 3575b1f39e7773379fec011d39f32a77" >&2 + exit 2 +} +BUILD_ID="llama-tokenize $TOK_MD5 libllama.so $LIB_MD5" +COMMIT=$(git_lcpp rev-parse --short=9 HEAD) +LIBDIR=$LCPP/build/bin +mkdir -p "$OUT" +TMP=$OUT/nemotron_ref.tmp.$$ +trap 'rm -f "$TMP" "$TMP.build"' EXIT +SRC_SHA=$(sha256sum "$SRC" | cut -d' ' -f1) +g++ -std=c++17 -O2 -o "$TMP" "$SRC" -I"$LCPP/include" -I"$LCPP/ggml/include" \ + -L"$LIBDIR" -lllama -lggml -lggml-base -Wl,-rpath,"$LIBDIR" +printf 'source_sha256 %s\nlcpp_commit %s\nbuild %s\n' "$SRC_SHA" "$COMMIT" "$BUILD_ID" > "$TMP.build" +mv -f "$TMP" "$OUT/nemotron_ref" +mv -f "$TMP.build" "$OUT/nemotron_ref.build" +echo "built $OUT/nemotron_ref from nemotron_ref.cpp sha256 ${SRC_SHA:0:12} against mainline at $COMMIT" +echo "build $BUILD_ID" diff --git a/tools/ref/dump-nemotron.sh b/tools/ref/dump-nemotron.sh new file mode 100755 index 000000000..c80c3d855 --- /dev/null +++ b/tools/ref/dump-nemotron.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +# Dump four last-position logit rows into $BLOOMERY_DATA/ref-nemotron-logits. +# CPU only: CUDA_VISIBLE_DEVICES is emptied and nemotron_ref asks for no GPU layers. +# The prompts are prefixes of the tokenizer oracle's copies of the corpora the other +# families use (engram-corpus.sh): code.txt is IK src/*.cpp, prose.txt is IK docs and +# README. llama-tokenize adds BOS because the file sets add_bos_token; the counts +# include that id. Mainline is not rebuilt. +set -euo pipefail +# shellcheck source=tools/ref/ref-paths.sh +source "$(dirname "${BASH_SOURCE[0]}")/ref-paths.sh" +[ "$MODEL_NAME" = nemotron_h ] || { + echo "dump-nemotron: this dump runs under the nemotron_h profile; this command picked $MODEL_NAME" >&2 + exit 2 +} +HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +SRC=$HERE/tools/ref/nemotron_ref.cpp +BIN=${NEMOTRON_BIN:-$BLOOMERY_DATA/bin/nemotron_ref} +REC=$BIN.build +[ -x "$BIN" ] || { echo "dump-nemotron: no $BIN — run just build-ref-nemotron" >&2; exit 2; } +want=$(sha256sum "$SRC" | cut -d' ' -f1) +got=$(awk '$1 == "source_sha256" { print $2; exit }' "$REC") +[ "$want" = "$got" ] || { + echo "dump-nemotron: $BIN was linked from another nemotron_ref.cpp — run just build-ref-nemotron" >&2 + exit 2 +} +TOKENIZE=$LCPP/build/bin/llama-tokenize +[ -x "$TOKENIZE" ] || { echo "dump-nemotron: no llama-tokenize at $TOKENIZE" >&2; exit 66; } +LIBLLAMA=$(ldd "$TOKENIZE" | awk '$1 == "libllama.so" || $1 == "libllama.so.0" { print $3; exit }') +[ -f "$LIBLLAMA" ] || { echo "dump-nemotron: $TOKENIZE loads no libllama.so (ldd names '$LIBLLAMA')" >&2; exit 66; } +TOK_MD5=$(md5sum < "$TOKENIZE" | cut -d' ' -f1) +LIB_MD5=$(md5sum < "$LIBLLAMA" | cut -d' ' -f1) +[ "$TOK_MD5" = 5df95d3a942effc971ff170432d840fd ] || { + echo "dump-nemotron: llama-tokenize md5 is $TOK_MD5, the pinned mainline is 5df95d3a942effc971ff170432d840fd" >&2 + exit 2 +} +[ "$LIB_MD5" = 3575b1f39e7773379fec011d39f32a77 ] || { + echo "dump-nemotron: libllama md5 is $LIB_MD5, the pinned mainline is 3575b1f39e7773379fec011d39f32a77" >&2 + exit 2 +} +export BLOOMERY_REF_BUILD="llama-tokenize $TOK_MD5 libllama.so $LIB_MD5" +export CUDA_VISIBLE_DEVICES="" + +TEXT=$BLOOMERY_DATA/ref-tokenizer-nemotron +[ -f "$TEXT/code.txt" ] || { echo "dump-nemotron: no $TEXT/code.txt (the tokenizer oracle's code corpus)" >&2; exit 66; } +[ -f "$TEXT/prose.txt" ] || { echo "dump-nemotron: no $TEXT/prose.txt (the tokenizer oracle's prose corpus)" >&2; exit 66; } + +# ids_of — tokenize a line prefix. No --no-bos: the file adds BOS. +ids_of() { + local text=$1 lines=$2 count=$3 out=$4 raw got + raw=$(mktemp) + head -n "$lines" "$text" > "$raw.in" + "$TOKENIZE" -m "$MODEL" -f "$raw.in" --ids --log-disable --no-escape > "$raw.bracket" + { tr -d '[] ' < "$raw.bracket" | tr ',' '\n' | grep -v '^$' || true; } > "$raw" + got=$(wc -l < "$raw") + if [ "$got" -lt "$count" ]; then + echo "dump-nemotron: the first $lines lines of $text tokenize to $got ids, need $count" >&2 + rm -f "$raw" "$raw.in" "$raw.bracket" + exit 2 + fi + head -n "$count" "$raw" > "$out" + rm -f "$raw" "$raw.in" "$raw.bracket" +} + +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"' EXIT +ids_of "$TEXT/code.txt" 80 29 "$WORK/code.ids" +ids_of "$TEXT/prose.txt" 400 512 "$WORK/prose.ids" +head -n 5 "$WORK/code.ids" > "$WORK/code5.ids" +head -n 29 "$WORK/code.ids" > "$WORK/code29.ids" +head -n 64 "$WORK/prose.ids" > "$WORK/prose64.ids" +cp "$WORK/prose.ids" "$WORK/prose512.ids" + +SET=$BLOOMERY_DATA/ref-nemotron-logits +STAGE=$BLOOMERY_DATA/ref-nemotron-logits.staging +rm -rf "$STAGE" +mkdir -p "$STAGE" +drop_incomplete_stage() { + [ -d "$STAGE" ] || return 0 + grep -qs '^# complete' "$STAGE/MANIFEST.tsv" && return 0 + rm -rf "$STAGE" + echo "[staging] removed the incomplete $STAGE" >&2 +} +trap 'drop_incomplete_stage; rm -rf "$WORK"' EXIT +"$BIN" -m "$MODEL" --out "$STAGE" -c 512 -ub 512 -t 8 \ + --prompt "code5:5:$WORK/code5.ids" \ + --prompt "code29:29:$WORK/code29.ids" \ + --prompt "prose64:64:$WORK/prose64.ids" \ + --prompt "prose512:512:$WORK/prose512.ids" +grep -q '^# complete' "$STAGE/MANIFEST.tsv" || { + echo "dump-nemotron: the staged set has no completion trailer — not installing it" >&2 + exit 1 +} +rows=$(grep -c '^prompt ' "$STAGE/MANIFEST.tsv") +[ "$rows" = 4 ] || { echo "dump-nemotron: the staged set has $rows prompt rows, want 4" >&2; exit 1; } +rm -rf "$SET" +mv "$STAGE" "$SET" +echo "dump-nemotron: installed $SET (4 prompts)" diff --git a/tools/ref/models/nemotron_h.sh b/tools/ref/models/nemotron_h.sh new file mode 100644 index 000000000..0bb2097bb --- /dev/null +++ b/tools/ref/models/nemotron_h.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# shellcheck shell=bash +# The nemotron_h (Nemotron 3 Nano 4B) profile: what the reference harness needs that is a +# property of the model. Sourced by ref-paths.sh when BLOOMERY_MODEL=nemotron_h; never +# executed, and it exports nothing. +# +# The name is the GGUF general.architecture value, so `grep nemotron_h` finds this profile, +# the metadata keys and the logits set (crates/refset/src/arch/nemotron_h). +# +# MODEL the Q4_K_M file the tokenizer family already names. One shard. +# BLOOMERY_REF_MODEL moves it; a caller's own MODEL= is ignored. +# LCPP mainline llama.cpp. ik has no nemotron_h, so this tree is the oracle. +# nemotron_ref links its existing build and does not rebuild it: a rebuild +# moves the md5s the tokenizer family pins. +# IK no ik oracle. The default is only so IKBIN can be formed; nothing here runs it. +# LOGITS_SET ref-nemotron-logits under $BLOOMERY_DATA: four last-position rows. +# CODE_TEXT the tokenizer oracle's copy of the code corpus the other families use +# (engram-corpus.sh: the reference tree's src/*.cpp). +# PROSE_TEXT the same for prose (that tree's docs/**/*.md and README). +# PROMPTS name:count:text. The counts include the BOS the file adds. code5 and +# code29 are prefixes of CODE_TEXT; prose64 and prose512 of PROSE_TEXT. +# +# No lease and no card: the file is a few gigabytes and the dump hides CUDA. +# +# SC2034: every name here is read by the file that sources this one. +# shellcheck disable=SC2034 +MODEL_NAME=nemotron_h +: "${IK:=/home/user/ik-idxkey}" +: "${LCPP:=/home/user/llama.cpp-mainline}" +MODEL=${BLOOMERY_REF_MODEL:-/models/NVIDIA-Nemotron-3-Nano-4B-GGUF/NVIDIA-Nemotron3-Nano-4B-Q4_K_M.gguf} +: "${LOGITS_SET:=ref-nemotron-logits}" +CODE_TEXT=${BLOOMERY_DATA:-/root/bloomery-data}/ref-tokenizer-nemotron/code.txt +PROSE_TEXT=${BLOOMERY_DATA:-/root/bloomery-data}/ref-tokenizer-nemotron/prose.txt +PROMPTS=(code5:5:code code29:29:code prose64:64:prose prose512:512:prose) diff --git a/tools/ref/nemotron_ref.cpp b/tools/ref/nemotron_ref.cpp new file mode 100644 index 000000000..e727f3992 --- /dev/null +++ b/tools/ref/nemotron_ref.cpp @@ -0,0 +1,274 @@ +// nemotron_ref — last-position logits of each prompt from llama.cpp mainline, on the CPU. +// +// One llama_decode per prompt, then llama_get_logits_ith(ctx, -1): the last logit row. Only that +// token is marked for output. The row is n_vocab f32, little-endian, plus the prompt's ids. +// Headers are # model, # build (BLOOMERY_REF_BUILD, the tokenizer family's spelling of the +// mainline md5s) and # arch. # complete is written only after every file, by renaming the manifest. +// +// nemotron_ref -m --out --prompt :: ... [-c ] [-ub ] [-t ] +// +// --prompt is repeated, once a prompt. is one decimal id a line, at least of them; the +// first are the prompt. n_gpu_layers is 0. The caller also empties CUDA_VISIBLE_DEVICES. + +#include "llama.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct Prompt { + std::string name; + int n = 0; + std::string path; + std::vector ids; +}; + +static bool read_ids(Prompt & p) { + std::ifstream in(p.path); + if (!in) { + fprintf(stderr, "nemotron_ref: cannot read %s\n", p.path.c_str()); + return false; + } + std::string line; + long long lines = 0; + while ((int) p.ids.size() < p.n && std::getline(in, line)) { + ++lines; + if (!line.empty() && line.back() == '\r') line.pop_back(); + char * end = nullptr; + errno = 0; + const long v = strtol(line.c_str(), &end, 10); + if (line.empty() || *end != '\0' || errno != 0 || v < 0 || v > INT32_MAX) { + fprintf(stderr, "nemotron_ref: %s:%lld is not a token id: '%s'\n", p.path.c_str(), lines, line.c_str()); + return false; + } + p.ids.push_back((llama_token) v); + } + if ((int) p.ids.size() < p.n) { + fprintf(stderr, "nemotron_ref: %s holds %zu ids and the prompt asks for %d\n", p.path.c_str(), p.ids.size(), p.n); + return false; + } + return true; +} + +static bool parse_prompt(const char * spec, Prompt & p) { + const char * c1 = strchr(spec, ':'); + if (!c1 || c1 == spec) return false; + const char * c2 = strchr(c1 + 1, ':'); + if (!c2 || c2 == c1 + 1 || c2[1] == '\0') return false; + p.name.assign(spec, c1); + for (char ch : p.name) { + if (!((ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '0' && ch <= '9'))) return false; + } + char * end = nullptr; + errno = 0; + const long n = strtol(c1 + 1, &end, 10); + if (end != c2 || errno != 0 || n <= 0 || n > 1000000) return false; + p.n = (int) n; + p.path = c2 + 1; + return true; +} + +static bool write_raw(const std::string & path, const float * data, size_t count) { + FILE * f = fopen(path.c_str(), "wb"); + if (!f) { + fprintf(stderr, "nemotron_ref: cannot write %s\n", path.c_str()); + return false; + } + const bool ok = fwrite(data, sizeof(float), count, f) == count; + if (fclose(f) != 0 || !ok) { + fprintf(stderr, "nemotron_ref: short write on %s\n", path.c_str()); + return false; + } + return true; +} + +static bool write_ids(const std::string & path, const std::vector & ids) { + FILE * f = fopen(path.c_str(), "w"); + if (!f) { + fprintf(stderr, "nemotron_ref: cannot write %s\n", path.c_str()); + return false; + } + for (llama_token id : ids) { + if (fprintf(f, "%d\n", (int) id) < 0) { + fclose(f); + fprintf(stderr, "nemotron_ref: short write on %s\n", path.c_str()); + return false; + } + } + if (fclose(f) != 0) { + fprintf(stderr, "nemotron_ref: short write on %s\n", path.c_str()); + return false; + } + return true; +} + +static void print_top5(const std::string & name, const float * row, int n_vocab) { + const int k = std::min(n_vocab, 5); + std::vector order(n_vocab); + for (int i = 0; i < n_vocab; ++i) order[i] = i; + std::partial_sort(order.begin(), order.begin() + k, order.end(), [&](int a, int b) { + if (row[a] != row[b]) return row[a] > row[b]; + return a < b; + }); + fprintf(stderr, "nemotron_ref: top5 %s ids=", name.c_str()); + for (int i = 0; i < k; ++i) fprintf(stderr, "%s%d", i ? "," : "", order[i]); + fprintf(stderr, " logits="); + for (int i = 0; i < k; ++i) fprintf(stderr, "%s%.6f", i ? "," : "", row[order[i]]); + fprintf(stderr, "\n"); +} + +int main(int argc, char ** argv) { + std::string model_path, out; + int n_ctx = 0, n_ubatch = 512, threads = 0; + std::vector prompts; + for (int i = 1; i < argc; ++i) { + const bool more = i + 1 < argc; + if (strcmp(argv[i], "-m") == 0 && more) { + model_path = argv[++i]; + } else if (strcmp(argv[i], "--out") == 0 && more) { + out = argv[++i]; + } else if (strcmp(argv[i], "--prompt") == 0 && more) { + Prompt p; + if (!parse_prompt(argv[++i], p)) { + fprintf(stderr, "nemotron_ref: --prompt wants name:count:ids-file, got '%s'\n", argv[i]); + return 2; + } + prompts.push_back(std::move(p)); + } else if (strcmp(argv[i], "-c") == 0 && more) { + n_ctx = atoi(argv[++i]); + } else if (strcmp(argv[i], "-ub") == 0 && more) { + n_ubatch = atoi(argv[++i]); + } else if (strcmp(argv[i], "-t") == 0 && more) { + threads = atoi(argv[++i]); + } else { + fprintf(stderr, "nemotron_ref: unknown or incomplete argument '%s'\n", argv[i]); + return 2; + } + } + const char * build = getenv("BLOOMERY_REF_BUILD"); + if (model_path.empty() || out.empty() || prompts.empty() || n_ubatch <= 0 || !build || !*build) { + fprintf(stderr, "usage: nemotron_ref -m --out --prompt :: ... " + "[-c ] [-ub ] [-t ]\n" + "BLOOMERY_REF_BUILD must name the mainline build\n"); + return 2; + } + int max_n = 0; + for (Prompt & p : prompts) { + if (!read_ids(p)) return 2; + max_n = std::max(max_n, p.n); + } + if (n_ctx == 0) n_ctx = max_n; + if (n_ctx < max_n) { + fprintf(stderr, "nemotron_ref: a prompt of %d ids does not fit in -c %d\n", max_n, n_ctx); + return 2; + } + + llama_backend_init(); + llama_model_params mp = llama_model_default_params(); + mp.n_gpu_layers = 0; + llama_model * model = llama_model_load_from_file(model_path.c_str(), mp); + if (!model) { + fprintf(stderr, "nemotron_ref: failed to load %s\n", model_path.c_str()); + return 1; + } + llama_context_params cp = llama_context_default_params(); + cp.n_ctx = (uint32_t) n_ctx; + cp.n_batch = (uint32_t) max_n; + cp.n_ubatch = (uint32_t) std::min(n_ubatch, max_n); + cp.n_seq_max = 1; + if (threads > 0) { + cp.n_threads = threads; + cp.n_threads_batch = threads; + } + llama_context * ctx = llama_init_from_model(model, cp); + if (!ctx) { + fprintf(stderr, "nemotron_ref: failed to make the context\n"); + llama_model_free(model); + llama_backend_free(); + return 1; + } + const int n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(model)); + if (n_vocab <= 0) { + fprintf(stderr, "nemotron_ref: the model names no vocabulary\n"); + return 1; + } + char arch[128] = {}; + if (llama_model_meta_val_str(model, "general.architecture", arch, sizeof(arch)) < 0 || strcmp(arch, "nemotron_h") != 0) { + fprintf(stderr, "nemotron_ref: general.architecture is '%s', want nemotron_h\n", arch); + return 1; + } + + if (mkdir(out.c_str(), 0755) != 0 && errno != EEXIST) { + fprintf(stderr, "nemotron_ref: cannot create %s\n", out.c_str()); + return 1; + } + llama_batch batch = llama_batch_init(max_n, 0, 1); + for (int i = 0; i < (int) prompts.size(); ++i) { + const Prompt & p = prompts[i]; + if (i > 0) llama_memory_clear(llama_get_memory(ctx), true); + for (int t = 0; t < p.n; ++t) { + batch.token[t] = p.ids[t]; + batch.pos[t] = t; + batch.n_seq_id[t] = 1; + batch.seq_id[t][0] = 0; + batch.logits[t] = t + 1 == p.n; + } + batch.n_tokens = p.n; + if (llama_decode(ctx, batch) != 0) { + fprintf(stderr, "nemotron_ref: decode failed for %s\n", p.name.c_str()); + return 1; + } + const float * row = llama_get_logits_ith(ctx, -1); + if (!row) { + fprintf(stderr, "nemotron_ref: no last-position logits for %s\n", p.name.c_str()); + return 1; + } + for (int j = 0; j < n_vocab; ++j) { + if (!std::isfinite(row[j])) { + fprintf(stderr, "nemotron_ref: %s logit %d is not finite\n", p.name.c_str(), j); + return 1; + } + } + if (!write_ids(out + "/" + p.name + ".ids", p.ids)) return 1; + if (!write_raw(out + "/" + p.name + ".logits.f32", row, (size_t) n_vocab)) return 1; + print_top5(p.name, row, n_vocab); + } + + const std::string final_path = out + "/MANIFEST.tsv"; + const std::string partial_path = final_path + ".partial"; + FILE * m = fopen(partial_path.c_str(), "w"); + if (!m) { + fprintf(stderr, "nemotron_ref: cannot write %s\n", partial_path.c_str()); + return 1; + } + fprintf(m, "# nemotron_ref — last-position logits, raw f32, little-endian\n"); + fprintf(m, "# model\t%s\n", model_path.c_str()); + fprintf(m, "# build\t%s\n", build); + fprintf(m, "# arch\t%s\n", arch); + fprintf(m, "# source\tllama_get_logits_ith(ctx, -1)\n"); + fprintf(m, "# prompt columns\tname\tn_ids\tn_vocab\tids\tlogits\n"); + for (const Prompt & p : prompts) { + fprintf(m, "prompt\t%s\t%d\t%d\t%s.ids\t%s.logits.f32\n", p.name.c_str(), p.n, n_vocab, p.name.c_str(), + p.name.c_str()); + } + fprintf(m, "# complete\t%zu\t0\n", prompts.size()); + if (fclose(m) != 0 || rename(partial_path.c_str(), final_path.c_str()) != 0) { + fprintf(stderr, "nemotron_ref: cannot install %s\n", final_path.c_str()); + return 1; + } + fprintf(stderr, "nemotron_ref: %zu prompts, vocab %d, into %s\n", prompts.size(), n_vocab, out.c_str()); + llama_batch_free(batch); + llama_free(ctx); + llama_model_free(model); + llama_backend_free(); + return 0; +} From 7df2f37c144235ba9942f9ab5bf84f9b95ab31b0 Mon Sep 17 00:00:00 2001 From: midagedev Date: Thu, 15 Oct 2026 09:16:53 +0900 Subject: [PATCH 3/7] refset: one owner for the nemotron model path, pin, and prompt list The logits family runs the tokenizer family's model function. The dump compares the live mainline md5s with the build record instead of repeating the pin. The profile no longer restates the prompt list the dump script owns, and a prompt check reads its logits through logits. --- crates/refset/src/arch/nemotron_h/mod.rs | 25 +++++++++++------------- crates/refset/src/arch/tokenizer.rs | 4 ++-- crates/refset/src/nemotron.rs | 8 +------- tools/ref/dump-nemotron.sh | 13 ++++++------ tools/ref/models/nemotron_h.sh | 10 ---------- 5 files changed, 20 insertions(+), 40 deletions(-) diff --git a/crates/refset/src/arch/nemotron_h/mod.rs b/crates/refset/src/arch/nemotron_h/mod.rs index ae14eae9b..9a959e6ae 100644 --- a/crates/refset/src/arch/nemotron_h/mod.rs +++ b/crates/refset/src/arch/nemotron_h/mod.rs @@ -1,24 +1,16 @@ -//! The Nemotron 3 Nano family. The logits set is dumped from [`MODEL`] by -//! mainline llama.cpp ([`super::tokenizer::NEMOTRON_BUILD`]), which is the -//! build the tokenizer family already pins. ik has no `nemotron_h`. +//! The Nemotron 3 Nano family. The logits set is dumped from the tokenizer +//! family's model ([`super::tokenizer::nemotron_model`]) by mainline llama.cpp +//! ([`super::tokenizer::NEMOTRON_BUILD`]), the build that family already pins. +//! ik has no `nemotron_h`. -use crate::RefError; use crate::family::{Build, Family, Identity}; /// The architecture every nemotron manifest names in its `# arch` line. pub const ARCH: &str = "nemotron_h"; -/// The file the tree runs. The tokenizer family owns the path. -pub const MODEL: &str = super::tokenizer::NEMOTRON_MODEL; - /// Last-position logits of four prompts, mainline on the CPU. pub const LOGITS_SET: &str = "ref-nemotron-logits"; -/// [`MODEL`], as a family's `runs`. -fn model() -> Result { - Ok(MODEL.to_string()) -} - /// Mainline's last-position logits (`just dump-ref-nemotron`). pub static LOGITS: Family = Family { name: "ref-nemotron-logits", @@ -28,7 +20,7 @@ pub static LOGITS: Family = Family { identity: Identity::NemotronLogits, arch: Some(ARCH), build: Some(Build::Is(super::tokenizer::NEMOTRON_BUILD)), - runs: Some(model), + runs: Some(super::tokenizer::nemotron_model), draft_runs: None, consumers: &[], }; @@ -90,7 +82,12 @@ mod tests { }; assert_eq!(p, crate::arch::tokenizer::NEMOTRON_BUILD); assert_eq!(LOGITS.sets, &["ref-nemotron-logits"]); - assert_eq!(runs(), crate::arch::tokenizer::NEMOTRON_MODEL); + assert_eq!( + runs(), + crate::arch::tokenizer::NEMOTRON + .runs() + .unwrap_or_else(|e| panic!("{e}")) + ); } /// A finished set opens, and `logits` is that column's file. diff --git a/crates/refset/src/arch/tokenizer.rs b/crates/refset/src/arch/tokenizer.rs index 8d25f084f..0e36e16bd 100644 --- a/crates/refset/src/arch/tokenizer.rs +++ b/crates/refset/src/arch/tokenizer.rs @@ -107,7 +107,7 @@ pub static QWEN38: Family = Family { }; /// The Nemotron 3 Nano 4B file the tokenizer gate loads (`pixtral`). -pub(crate) const NEMOTRON_MODEL: &str = +const NEMOTRON_MODEL: &str = "/models/NVIDIA-Nemotron-3-Nano-4B-GGUF/NVIDIA-Nemotron3-Nano-4B-Q4_K_M.gguf"; /// Nemotron 3 Nano 4B's set (`pixtral` pre-tokenizer). @@ -141,7 +141,7 @@ fn qwen38_model() -> Result { Ok(super::qwen4exp::MODEL.to_string()) } -fn nemotron_model() -> Result { +pub(crate) fn nemotron_model() -> Result { Ok(NEMOTRON_MODEL.to_string()) } diff --git a/crates/refset/src/nemotron.rs b/crates/refset/src/nemotron.rs index f9a5835fa..b28e8ff2f 100644 --- a/crates/refset/src/nemotron.rs +++ b/crates/refset/src/nemotron.rs @@ -219,13 +219,7 @@ impl LogitsSet { format!("{} holds {n} ids, {} wants {}", p.ids_file, p.name, p.n_ids), )); } - let row = self.logits(p)?; - if row.len() != p.n_vocab as usize { - return Err(RefError::malformed( - self.dir.join(&p.logits_file).display(), - format!("{} logits, {} wants {}", row.len(), p.name, p.n_vocab), - )); - } + self.logits(p)?; Ok(()) } } diff --git a/tools/ref/dump-nemotron.sh b/tools/ref/dump-nemotron.sh index c80c3d855..89dc53c31 100755 --- a/tools/ref/dump-nemotron.sh +++ b/tools/ref/dump-nemotron.sh @@ -29,15 +29,14 @@ LIBLLAMA=$(ldd "$TOKENIZE" | awk '$1 == "libllama.so" || $1 == "libllama.so.0" { [ -f "$LIBLLAMA" ] || { echo "dump-nemotron: $TOKENIZE loads no libllama.so (ldd names '$LIBLLAMA')" >&2; exit 66; } TOK_MD5=$(md5sum < "$TOKENIZE" | cut -d' ' -f1) LIB_MD5=$(md5sum < "$LIBLLAMA" | cut -d' ' -f1) -[ "$TOK_MD5" = 5df95d3a942effc971ff170432d840fd ] || { - echo "dump-nemotron: llama-tokenize md5 is $TOK_MD5, the pinned mainline is 5df95d3a942effc971ff170432d840fd" >&2 +# The pin literals live in the build script, which wrote this record. A live +# binary that does not match it is not the one nemotron_ref was linked against. +want_build=$(awk '$1 == "build" { $1 = ""; sub(/^ /, ""); print; exit }' "$REC") +[ "llama-tokenize $TOK_MD5 libllama.so $LIB_MD5" = "$want_build" ] || { + echo "dump-nemotron: $TOKENIZE or $LIBLLAMA differs from the build $BIN was linked against ($want_build)" >&2 exit 2 } -[ "$LIB_MD5" = 3575b1f39e7773379fec011d39f32a77 ] || { - echo "dump-nemotron: libllama md5 is $LIB_MD5, the pinned mainline is 3575b1f39e7773379fec011d39f32a77" >&2 - exit 2 -} -export BLOOMERY_REF_BUILD="llama-tokenize $TOK_MD5 libllama.so $LIB_MD5" +export BLOOMERY_REF_BUILD="$want_build" export CUDA_VISIBLE_DEVICES="" TEXT=$BLOOMERY_DATA/ref-tokenizer-nemotron diff --git a/tools/ref/models/nemotron_h.sh b/tools/ref/models/nemotron_h.sh index 0bb2097bb..6d15fd5fd 100644 --- a/tools/ref/models/nemotron_h.sh +++ b/tools/ref/models/nemotron_h.sh @@ -13,12 +13,6 @@ # nemotron_ref links its existing build and does not rebuild it: a rebuild # moves the md5s the tokenizer family pins. # IK no ik oracle. The default is only so IKBIN can be formed; nothing here runs it. -# LOGITS_SET ref-nemotron-logits under $BLOOMERY_DATA: four last-position rows. -# CODE_TEXT the tokenizer oracle's copy of the code corpus the other families use -# (engram-corpus.sh: the reference tree's src/*.cpp). -# PROSE_TEXT the same for prose (that tree's docs/**/*.md and README). -# PROMPTS name:count:text. The counts include the BOS the file adds. code5 and -# code29 are prefixes of CODE_TEXT; prose64 and prose512 of PROSE_TEXT. # # No lease and no card: the file is a few gigabytes and the dump hides CUDA. # @@ -28,7 +22,3 @@ MODEL_NAME=nemotron_h : "${IK:=/home/user/ik-idxkey}" : "${LCPP:=/home/user/llama.cpp-mainline}" MODEL=${BLOOMERY_REF_MODEL:-/models/NVIDIA-Nemotron-3-Nano-4B-GGUF/NVIDIA-Nemotron3-Nano-4B-Q4_K_M.gguf} -: "${LOGITS_SET:=ref-nemotron-logits}" -CODE_TEXT=${BLOOMERY_DATA:-/root/bloomery-data}/ref-tokenizer-nemotron/code.txt -PROSE_TEXT=${BLOOMERY_DATA:-/root/bloomery-data}/ref-tokenizer-nemotron/prose.txt -PROMPTS=(code5:5:code code29:29:code prose64:64:prose prose512:512:prose) From f5bb3253dedd776b81b3b4a7ede98a82056bffb7 Mon Sep 17 00:00:00 2001 From: midagedev Date: Thu, 15 Oct 2026 09:20:11 +0900 Subject: [PATCH 4/7] refset: the nemotron build record is the live mainline, and a set file name has one check The build script records the md5s it linked and does not repeat the tokenizer family's pin. A manifest name that leaves the set directory is columns::file_name, which the dequant reader already applied. --- crates/refset/src/columns.rs | 12 ++++++++++++ crates/refset/src/dequant.rs | 7 +------ crates/refset/src/nemotron.rs | 12 +----------- tools/ref/build-nemotron-ref.sh | 10 ---------- 4 files changed, 14 insertions(+), 27 deletions(-) diff --git a/crates/refset/src/columns.rs b/crates/refset/src/columns.rs index 62850f220..22f16bcd7 100644 --- a/crates/refset/src/columns.rs +++ b/crates/refset/src/columns.rs @@ -109,3 +109,15 @@ where v.parse() .map_err(|e| RefError::malformed(at, format!("{name} {v:?}: {e}"))) } + +/// `name` is one file of the set's own directory. Empty, `.`, `..`, or a +/// slash is `Malformed`: the manifest must not name a path outside it. +pub(crate) fn file_name(name: &str, at: &dyn Display) -> Result<(), RefError> { + if name.is_empty() || name.contains('/') || matches!(name, "." | "..") { + return Err(RefError::malformed( + at, + format!("{name:?} does not name a file of the set's directory"), + )); + } + Ok(()) +} diff --git a/crates/refset/src/dequant.rs b/crates/refset/src/dequant.rs index 064ec8a26..836ffee14 100644 --- a/crates/refset/src/dequant.rs +++ b/crates/refset/src/dequant.rs @@ -198,12 +198,7 @@ impl DequantSet { fn file_row(r: &Row<'_>, at: &dyn fmt::Display) -> Result { let name = r.text("file")?; - if name.is_empty() || name.contains('/') || matches!(name, "." | "..") { - return Err(RefError::malformed( - at, - format!("{name:?} does not name a file of the set's directory"), - )); - } + crate::columns::file_name(name, at)?; let md5 = r.text("md5")?; check_hex("md5", md5, at)?; Ok(SetFile { diff --git a/crates/refset/src/nemotron.rs b/crates/refset/src/nemotron.rs index b28e8ff2f..b3d7ddb34 100644 --- a/crates/refset/src/nemotron.rs +++ b/crates/refset/src/nemotron.rs @@ -16,7 +16,7 @@ //! another file, and one written by another mainline build. use crate::RefError; -use crate::columns::{Columns, Row}; +use crate::columns::{Columns, Row, file_name}; use crate::family::{Family, Identity}; use std::collections::HashSet; use std::fmt; @@ -231,16 +231,6 @@ fn one(f: &[&str], at: &dyn fmt::Display, want: &str) -> Result Result<(), RefError> { - if name.is_empty() || name.contains('/') || matches!(name, "." | "..") { - return Err(RefError::malformed( - at, - format!("{name:?} does not name a file of the set's directory"), - )); - } - Ok(()) -} - fn prompt_row(r: &Row<'_>, at: &dyn fmt::Display) -> Result { let name = r.text("name")?; let ids_file = r.text("ids")?; diff --git a/tools/ref/build-nemotron-ref.sh b/tools/ref/build-nemotron-ref.sh index 69523fbc7..edbf87d48 100755 --- a/tools/ref/build-nemotron-ref.sh +++ b/tools/ref/build-nemotron-ref.sh @@ -25,16 +25,6 @@ LIBLLAMA=$(ldd "$TOKENIZE" | awk '$1 == "libllama.so" || $1 == "libllama.so.0" { [ -f "$LIBLLAMA" ] || { echo "build-nemotron-ref: $TOKENIZE loads no libllama.so (ldd names '$LIBLLAMA')" >&2; exit 66; } TOK_MD5=$(md5sum < "$TOKENIZE" | cut -d' ' -f1) LIB_MD5=$(md5sum < "$LIBLLAMA" | cut -d' ' -f1) -# The tokenizer family's pin. Recording a different pair would make every nemotron set foreign -# and would mean this tree was rebuilt. -[ "$TOK_MD5" = 5df95d3a942effc971ff170432d840fd ] || { - echo "build-nemotron-ref: llama-tokenize md5 is $TOK_MD5, the pinned mainline is 5df95d3a942effc971ff170432d840fd" >&2 - exit 2 -} -[ "$LIB_MD5" = 3575b1f39e7773379fec011d39f32a77 ] || { - echo "build-nemotron-ref: libllama md5 is $LIB_MD5, the pinned mainline is 3575b1f39e7773379fec011d39f32a77" >&2 - exit 2 -} BUILD_ID="llama-tokenize $TOK_MD5 libllama.so $LIB_MD5" COMMIT=$(git_lcpp rev-parse --short=9 HEAD) LIBDIR=$LCPP/build/bin From 4d6c6a756eb9d1ad5313039bf50c18eb4dd7ee57 Mon Sep 17 00:00:00 2001 From: midagedev Date: Thu, 15 Oct 2026 13:09:20 +0900 Subject: [PATCH 5/7] refset: a nemotron id, completion count, or oracle numeric option that does not parse is refused A token line that is not a whole id, or an id outside the vocabulary, is malformed. The # complete count is the prompt rows. hidden_ref and nemotron_ref share one parser: a numeric option is the whole argument, an integer in range, or the harness names the flag and the text and exits 2. The build record hashes the .cpp and then ref_io.h. just test-ref-io compiles and runs that parser's checks. No card and no llama.cpp. --- crates/refset/src/arch/nemotron_h/mod.rs | 80 ++++++++++++++++++++++- crates/refset/src/nemotron.rs | 39 ++++++++++- justfile | 4 ++ tools/ref/build-hidden.sh | 10 +-- tools/ref/build-nemotron-ref.sh | 8 ++- tools/ref/dump-nemotron.sh | 6 +- tools/ref/hidden.sh | 5 +- tools/ref/hidden_ref.cpp | 56 ++-------------- tools/ref/nemotron_ref.cpp | 51 ++------------- tools/ref/ref_io.h | 81 +++++++++++++++++++++++ tools/ref/ref_io_test.cpp | 83 ++++++++++++++++++++++++ 11 files changed, 312 insertions(+), 111 deletions(-) create mode 100644 tools/ref/ref_io.h create mode 100644 tools/ref/ref_io_test.cpp diff --git a/crates/refset/src/arch/nemotron_h/mod.rs b/crates/refset/src/arch/nemotron_h/mod.rs index 9a959e6ae..79fa6bf0c 100644 --- a/crates/refset/src/arch/nemotron_h/mod.rs +++ b/crates/refset/src/arch/nemotron_h/mod.rs @@ -50,7 +50,7 @@ mod tests { /// Write one prompt's ids and logits and the manifest. `complete` adds the trailer. fn write(dir: &Path, model: &str, build: &str, complete: bool) { std::fs::create_dir_all(dir).unwrap_or_else(|e| panic!("{e}")); - std::fs::write(dir.join("p.ids"), "1\n2\n3\n4\n5\n").unwrap_or_else(|e| panic!("{e}")); + std::fs::write(dir.join("p.ids"), "0\n1\n2\n3\n0\n").unwrap_or_else(|e| panic!("{e}")); let logits: Vec = [0.0f32, 1.5, -0.5, 0.25] .into_iter() .flat_map(f32::to_le_bytes) @@ -146,4 +146,82 @@ mod tests { } std::fs::remove_dir_all(&d).unwrap_or_else(|e| panic!("{e}")); } + + /// A line that is not a token id is malformed, naming the field and the text. + #[test] + fn a_token_line_that_is_not_an_id_is_malformed() { + let d = dir("bad-id"); + write(&d, &runs(), pin(), true); + std::fs::write(d.join("p.ids"), "0\n1\n2\ngarbage\n0\n").unwrap_or_else(|e| panic!("{e}")); + match LogitsSet::open(&d, &LOGITS) { + Err(RefError::Malformed { what, .. }) => { + assert!( + what.contains("token id") && what.contains("garbage"), + "{what}" + ); + } + other => panic!("a token line that is not an id: {other:?}"), + } + std::fs::remove_dir_all(&d).unwrap_or_else(|e| panic!("{e}")); + } + + /// An id at or past `n_vocab` is malformed, naming the id and the vocabulary. + #[test] + fn a_token_id_outside_the_vocabulary_is_malformed() { + let d = dir("oob-id"); + write(&d, &runs(), pin(), true); + std::fs::write(d.join("p.ids"), "0\n1\n2\n3\n4\n").unwrap_or_else(|e| panic!("{e}")); + match LogitsSet::open(&d, &LOGITS) { + Err(RefError::Malformed { what, .. }) => { + assert!(what.contains("outside vocabulary"), "{what}"); + } + other => panic!("a token id outside the vocabulary: {other:?}"), + } + std::fs::remove_dir_all(&d).unwrap_or_else(|e| panic!("{e}")); + } + + /// The trailer's count is the prompt rows. One row and `# complete 2` is malformed. + #[test] + fn a_completion_count_that_is_not_the_prompt_rows_is_malformed() { + let d = dir("complete-n"); + write(&d, &runs(), pin(), true); + let text = + std::fs::read_to_string(d.join("MANIFEST.tsv")).unwrap_or_else(|e| panic!("{e}")); + std::fs::write( + d.join("MANIFEST.tsv"), + text.replace("# complete\t1\t0\n", "# complete\t2\t0\n"), + ) + .unwrap_or_else(|e| panic!("{e}")); + match LogitsSet::open(&d, &LOGITS) { + Err(RefError::Malformed { what, .. }) => { + assert!(what.contains("completion count"), "{what}"); + } + other => panic!("a completion count that is not the prompt rows: {other:?}"), + } + std::fs::remove_dir_all(&d).unwrap_or_else(|e| panic!("{e}")); + } + + /// A completion count that does not parse is malformed, naming the field. + #[test] + fn a_completion_count_that_does_not_parse_is_malformed() { + let d = dir("complete-bad"); + write(&d, &runs(), pin(), true); + let text = + std::fs::read_to_string(d.join("MANIFEST.tsv")).unwrap_or_else(|e| panic!("{e}")); + std::fs::write( + d.join("MANIFEST.tsv"), + text.replace("# complete\t1\t0\n", "# complete\tgarbage\t0\n"), + ) + .unwrap_or_else(|e| panic!("{e}")); + match LogitsSet::open(&d, &LOGITS) { + Err(RefError::Malformed { what, .. }) => { + assert!( + what.contains("complete count") && what.contains("garbage"), + "{what}" + ); + } + other => panic!("a completion count that does not parse: {other:?}"), + } + std::fs::remove_dir_all(&d).unwrap_or_else(|e| panic!("{e}")); + } } diff --git a/crates/refset/src/nemotron.rs b/crates/refset/src/nemotron.rs index b3d7ddb34..617dcf934 100644 --- a/crates/refset/src/nemotron.rs +++ b/crates/refset/src/nemotron.rs @@ -67,7 +67,8 @@ impl LogitsSet { /// Parse `dir/MANIFEST.tsv`. A set without one is `Missing`, naming the /// recipe that writes it. A set without its trailer comes back unread, /// `complete` false. A row before its column line, of another width, with - /// a count that does not parse, or naming a prompt twice, is `Malformed`. + /// a count that does not parse, a `# complete` count other than the prompt + /// rows, or naming a prompt twice, is `Malformed`. pub fn read(dir: &Path, family: &Family) -> Result { let path = dir.join("MANIFEST.tsv"); let text = std::fs::read_to_string(&path).map_err(|e| { @@ -94,6 +95,7 @@ impl LogitsSet { } let mut cols: Option = None; let mut seen = HashSet::new(); + let mut complete_rows = None; for (i, line) in text.lines().enumerate() { if line.is_empty() { continue; @@ -107,7 +109,13 @@ impl LogitsSet { "# prompt columns" => { cols = Some(Columns::new("prompt", f[1..].iter().copied())); } - "# complete" => set.complete = true, + "# complete" => { + if f.len() != 3 || f[2] != "0" { + return Err(RefError::malformed(&at, "want # complete\t\t0")); + } + complete_rows = Some(crate::columns::parse_field(f[1], "complete count", &at)?); + set.complete = true; + } h if h.starts_with('#') => {} "prompt" => { let c = cols.as_ref().ok_or_else(|| { @@ -130,6 +138,12 @@ impl LogitsSet { } } } + if complete_rows != Some(set.prompts.len()) { + return Err(RefError::malformed( + path.display(), + "completion count differs from prompt rows", + )); + } Ok(set) } @@ -212,7 +226,26 @@ impl LogitsSet { format!("nemotron: cannot read {}: {e}", ids.display()), ) })?; - let n = text.lines().filter(|l| !l.is_empty()).count(); + if p.n_ids == 0 || p.n_vocab == 0 { + return Err(RefError::malformed( + ids.display(), + "empty prompt or vocabulary", + )); + } + let mut n = 0usize; + for (i, line) in text.lines().enumerate() { + if line.is_empty() { + continue; + } + n += 1; + let id: u32 = crate::columns::parse_field(line, "token id", &At(&ids, i + 1))?; + if id >= p.n_vocab { + return Err(RefError::malformed( + At(&ids, i + 1), + format!("token id {id} outside vocabulary {}", p.n_vocab), + )); + } + } if n != p.n_ids as usize { return Err(RefError::malformed( ids.display(), diff --git a/justfile b/justfile index 3112a3c77..45e5df11a 100644 --- a/justfile +++ b/justfile @@ -817,6 +817,10 @@ dump-ref-mimo2 *VARIANT: dump-ref-mimo2-cuda: BLOOMERY_MODEL=mimo2 ./tools/box.sh 'BLOOMERY_REF_BACKEND=cuda bash tools/ref/dump.sh' +# ref_io.h's option parser, token reader and raw writer (tools/ref/ref_io_test.cpp). No card, no llama.cpp. +test-ref-io: + g++ -std=c++17 -Wall -o "${TMPDIR:-/tmp}/ref_io_test" tools/ref/ref_io_test.cpp && "${TMPDIR:-/tmp}/ref_io_test" + # Nemotron 3 Nano 4B last-position logits, mainline on the CPU (ik has no nemotron_h). nemotron_ref # links the existing mainline build (tools/ref/build-nemotron-ref.sh; it does not rebuild it) and # writes $BLOOMERY_DATA/ref-nemotron-logits (refset family ref-nemotron-logits). CUDA stays hidden. diff --git a/tools/ref/build-hidden.sh b/tools/ref/build-hidden.sh index 6b1c6fca5..ae07398b9 100755 --- a/tools/ref/build-hidden.sh +++ b/tools/ref/build-hidden.sh @@ -6,8 +6,8 @@ # The tree is checked, not trusted: no tracked change (a hand edit there would reach the oracle under the # commit's name). Its build is the one the depth runners time; this links against it and builds nothing in it. # Output: $BLOOMERY_DATA/bin/hidden_ref (HIDDEN_OUT moves it) and hidden_ref.build beside it, naming the sha256 of -# the hidden_ref.cpp it was built from and the mainline commit; hidden.sh refuses a binary whose record does not -# match its own tree. The compile writes a temporary name and only a finished link replaces the installed binary. +# hidden_ref.cpp then ref_io.h and the mainline commit; hidden.sh refuses a binary whose record does not match +# its own tree. The compile writes a temporary name and only a finished link replaces the installed binary. set -euo pipefail # shellcheck source=tools/ref/ref-paths.sh source "$(dirname "${BASH_SOURCE[0]}")/ref-paths.sh" @@ -16,6 +16,7 @@ source "$(dirname "${BASH_SOURCE[0]}")/ref-paths.sh" HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) OUT=${HIDDEN_OUT:-$BLOOMERY_DATA/bin} SRC=$HERE/tools/ref/hidden_ref.cpp +IO=$HERE/tools/ref/ref_io.h git_lcpp() { git -c safe.directory="$LCPP" -C "$LCPP" "$@"; } if [ -n "$(git_lcpp status --porcelain --untracked-files=no)" ]; then echo "build-hidden: $LCPP has tracked changes; the oracle is the commit's build, not an edited tree" >&2 @@ -26,10 +27,11 @@ LIB=$LCPP/build/bin mkdir -p "$OUT" TMP=$OUT/hidden_ref.tmp.$$ trap 'rm -f "$TMP" "$TMP.build"' EXIT -SRC_SHA=$(sha256sum "$SRC" | cut -d' ' -f1) +# cpp then the shared header, the same order hidden.sh hashes. +SRC_SHA=$(cat "$SRC" "$IO" | sha256sum | cut -d' ' -f1) g++ -std=c++17 -O2 -o "$TMP" "$SRC" -I"$LCPP/include" -I"$LCPP/ggml/include" \ -L"$LIB" -lllama -lggml -lggml-base -Wl,-rpath,"$LIB" printf 'source_sha256 %s\nlcpp %s\nlcpp_commit %s\n' "$SRC_SHA" "$(readlink -f "$LCPP")" "$COMMIT" > "$TMP.build" mv -f "$TMP" "$OUT/hidden_ref" mv -f "$TMP.build" "$OUT/hidden_ref.build" -echo "built $OUT/hidden_ref from hidden_ref.cpp sha256 ${SRC_SHA:0:12} against $LCPP at $COMMIT" +echo "built $OUT/hidden_ref from hidden_ref.cpp and ref_io.h sha256 ${SRC_SHA:0:12} against $LCPP at $COMMIT" diff --git a/tools/ref/build-nemotron-ref.sh b/tools/ref/build-nemotron-ref.sh index edbf87d48..f9734fc1c 100755 --- a/tools/ref/build-nemotron-ref.sh +++ b/tools/ref/build-nemotron-ref.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Link nemotron_ref (tools/ref/nemotron_ref.cpp) against the nemotron_h profile's existing +# Link nemotron_ref (tools/ref/nemotron_ref.cpp, with tools/ref/ref_io.h) against the nemotron_h profile's existing # mainline build. This does not configure or rebuild that tree: a rebuild moves the md5s the # tokenizer family pins. The .build record spells those md5s as crate::tokenizer::build_id does # (llama-tokenize libllama.so ; the file on disk is libllama.so.0). @@ -14,6 +14,7 @@ source "$(dirname "${BASH_SOURCE[0]}")/ref-paths.sh" HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) OUT=${NEMOTRON_OUT:-$BLOOMERY_DATA/bin} SRC=$HERE/tools/ref/nemotron_ref.cpp +IO=$HERE/tools/ref/ref_io.h git_lcpp() { git -c safe.directory="$LCPP" -C "$LCPP" "$@"; } if [ -n "$(git_lcpp status --porcelain --untracked-files=no)" ]; then echo "build-nemotron-ref: $LCPP has tracked changes; the oracle is the commit's build, not an edited tree" >&2 @@ -31,11 +32,12 @@ LIBDIR=$LCPP/build/bin mkdir -p "$OUT" TMP=$OUT/nemotron_ref.tmp.$$ trap 'rm -f "$TMP" "$TMP.build"' EXIT -SRC_SHA=$(sha256sum "$SRC" | cut -d' ' -f1) +# cpp then the shared header, the same order dump-nemotron.sh hashes. +SRC_SHA=$(cat "$SRC" "$IO" | sha256sum | cut -d' ' -f1) g++ -std=c++17 -O2 -o "$TMP" "$SRC" -I"$LCPP/include" -I"$LCPP/ggml/include" \ -L"$LIBDIR" -lllama -lggml -lggml-base -Wl,-rpath,"$LIBDIR" printf 'source_sha256 %s\nlcpp_commit %s\nbuild %s\n' "$SRC_SHA" "$COMMIT" "$BUILD_ID" > "$TMP.build" mv -f "$TMP" "$OUT/nemotron_ref" mv -f "$TMP.build" "$OUT/nemotron_ref.build" -echo "built $OUT/nemotron_ref from nemotron_ref.cpp sha256 ${SRC_SHA:0:12} against mainline at $COMMIT" +echo "built $OUT/nemotron_ref from nemotron_ref.cpp and ref_io.h sha256 ${SRC_SHA:0:12} against mainline at $COMMIT" echo "build $BUILD_ID" diff --git a/tools/ref/dump-nemotron.sh b/tools/ref/dump-nemotron.sh index 89dc53c31..bf99bb8f9 100755 --- a/tools/ref/dump-nemotron.sh +++ b/tools/ref/dump-nemotron.sh @@ -14,13 +14,15 @@ source "$(dirname "${BASH_SOURCE[0]}")/ref-paths.sh" } HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) SRC=$HERE/tools/ref/nemotron_ref.cpp +IO=$HERE/tools/ref/ref_io.h BIN=${NEMOTRON_BIN:-$BLOOMERY_DATA/bin/nemotron_ref} REC=$BIN.build [ -x "$BIN" ] || { echo "dump-nemotron: no $BIN — run just build-ref-nemotron" >&2; exit 2; } -want=$(sha256sum "$SRC" | cut -d' ' -f1) +# cpp then the shared header, the same order build-nemotron-ref.sh records. A header-only edit is a mismatch. +want=$(cat "$SRC" "$IO" | sha256sum | cut -d' ' -f1) got=$(awk '$1 == "source_sha256" { print $2; exit }' "$REC") [ "$want" = "$got" ] || { - echo "dump-nemotron: $BIN was linked from another nemotron_ref.cpp — run just build-ref-nemotron" >&2 + echo "dump-nemotron: $BIN was linked from another nemotron_ref.cpp or ref_io.h — run just build-ref-nemotron" >&2 exit 2 } TOKENIZE=$LCPP/build/bin/llama-tokenize diff --git a/tools/ref/hidden.sh b/tools/ref/hidden.sh index 66c5d2a7c..1d4ddc3ac 100755 --- a/tools/ref/hidden.sh +++ b/tools/ref/hidden.sh @@ -4,7 +4,7 @@ # (the profile's MODEL when absent), by hidden_ref (tools/ref/hidden_ref.cpp), into $BLOOMERY_DATA// — the # sets the refset families hidden-qwen35 and hidden-qwen35-flash-q8 check and gate-gpu-clef-hidden reads. # -# The binary must be the one built from this tree's hidden_ref.cpp (its .build record's source sha256) against the +# The binary must be the one built from this tree's hidden_ref.cpp then ref_io.h (its .build record's source sha256) against the # profile's LCPP at its current commit, and the ids file must have the profile's sha256: either refused names what # differs. Every layer on the card box.sh puts in view (-ngl 99), the context the longest set's P, ubatches of 512 # (llama.cpp's default). Each set is written to .staging and moved over the old set only when hidden_ref @@ -26,7 +26,8 @@ HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) twin=0 if [ "${1:-}" = --cpu-twin ]; then twin=1; shift; fi BIN=$BLOOMERY_DATA/bin/hidden_ref -want_src=$(sha256sum "$HERE/tools/ref/hidden_ref.cpp" | cut -d' ' -f1) +# cpp then the shared header, the same order build-hidden.sh records. +want_src=$(cat "$HERE/tools/ref/hidden_ref.cpp" "$HERE/tools/ref/ref_io.h" | sha256sum | cut -d' ' -f1) want_commit=$(git -c safe.directory="$LCPP" -C "$LCPP" rev-parse --short=9 HEAD) if ! grep -qx "source_sha256 $want_src" "$BIN.build" 2>/dev/null || ! grep -qx "lcpp_commit $want_commit" "$BIN.build" 2>/dev/null; then diff --git a/tools/ref/hidden_ref.cpp b/tools/ref/hidden_ref.cpp index d4e3d1964..73c3fd6d1 100644 --- a/tools/ref/hidden_ref.cpp +++ b/tools/ref/hidden_ref.cpp @@ -28,15 +28,12 @@ #include "llama.h" #include "ggml.h" +#include "ref_io.h" #include -#include -#include #include #include -#include #include -#include #include #include #include @@ -67,47 +64,6 @@ static bool on_node(struct ggml_tensor * t, bool ask, void * user) { return false; } -static bool read_token_file(const std::string & path, long long count, std::vector & ids) { - std::ifstream in(path); - if (!in) { - fprintf(stderr, "hidden_ref: cannot read %s\n", path.c_str()); - return false; - } - std::string line; - long long lines = 0; - while ((long long) ids.size() < count && std::getline(in, line)) { - ++lines; - char * end = nullptr; - errno = 0; - const long v = strtol(line.c_str(), &end, 10); - if (line.empty() || *end != '\0' || errno != 0 || v < 0 || v > INT32_MAX) { - fprintf(stderr, "hidden_ref: %s:%lld is not a token id: '%s'\n", path.c_str(), lines, line.c_str()); - return false; - } - ids.push_back((llama_token) v); - } - if ((long long) ids.size() < count) { - fprintf(stderr, "hidden_ref: %s holds %zu token ids and --tokens-count asks for %lld\n", path.c_str(), - ids.size(), count); - return false; - } - return true; -} - -static bool write_raw(const std::string & path, const float * data, size_t count) { - FILE * f = fopen(path.c_str(), "wb"); - if (!f) { - fprintf(stderr, "hidden_ref: cannot write %s\n", path.c_str()); - return false; - } - const bool ok = fwrite(data, sizeof(float), count, f) == count; - if (fclose(f) != 0 || !ok) { - fprintf(stderr, "hidden_ref: short write on %s\n", path.c_str()); - return false; - } - return true; -} - int main(int argc, char ** argv) { std::string model_path, tokens_file, out; long long count = -1; @@ -123,17 +79,17 @@ int main(int argc, char ** argv) { } else if (strcmp(argv[i], "--tokens-file") == 0 && more) { tokens_file = argv[++i]; } else if (strcmp(argv[i], "--tokens-count") == 0 && more) { - count = atoll(argv[++i]); + if (!parse_i64("hidden_ref", "--tokens-count", argv[++i], 1, LLONG_MAX, count)) return 2; } else if (strcmp(argv[i], "--out") == 0 && more) { out = argv[++i]; } else if (strcmp(argv[i], "-c") == 0 && more) { - n_ctx = atoi(argv[++i]); + if (!parse_i32("hidden_ref", "-c", argv[++i], 0, n_ctx)) return 2; } else if (strcmp(argv[i], "-ub") == 0 && more) { - n_ubatch = atoi(argv[++i]); + if (!parse_i32("hidden_ref", "-ub", argv[++i], 1, n_ubatch)) return 2; } else if (strcmp(argv[i], "-ngl") == 0 && more) { - ngl = atoi(argv[++i]); + if (!parse_i32("hidden_ref", "-ngl", argv[++i], 0, ngl)) return 2; } else if (strcmp(argv[i], "-t") == 0 && more) { - threads = atoi(argv[++i]); + if (!parse_i32("hidden_ref", "-t", argv[++i], 0, threads)) return 2; } else { fprintf(stderr, "hidden_ref: unknown or incomplete argument '%s'\n", argv[i]); return 2; diff --git a/tools/ref/nemotron_ref.cpp b/tools/ref/nemotron_ref.cpp index e727f3992..b0e20f75f 100644 --- a/tools/ref/nemotron_ref.cpp +++ b/tools/ref/nemotron_ref.cpp @@ -11,6 +11,7 @@ // first are the prompt. n_gpu_layers is 0. The caller also empties CUDA_VISIBLE_DEVICES. #include "llama.h" +#include "ref_io.h" #include #include @@ -20,7 +21,6 @@ #include #include #include -#include #include #include #include @@ -32,33 +32,6 @@ struct Prompt { std::vector ids; }; -static bool read_ids(Prompt & p) { - std::ifstream in(p.path); - if (!in) { - fprintf(stderr, "nemotron_ref: cannot read %s\n", p.path.c_str()); - return false; - } - std::string line; - long long lines = 0; - while ((int) p.ids.size() < p.n && std::getline(in, line)) { - ++lines; - if (!line.empty() && line.back() == '\r') line.pop_back(); - char * end = nullptr; - errno = 0; - const long v = strtol(line.c_str(), &end, 10); - if (line.empty() || *end != '\0' || errno != 0 || v < 0 || v > INT32_MAX) { - fprintf(stderr, "nemotron_ref: %s:%lld is not a token id: '%s'\n", p.path.c_str(), lines, line.c_str()); - return false; - } - p.ids.push_back((llama_token) v); - } - if ((int) p.ids.size() < p.n) { - fprintf(stderr, "nemotron_ref: %s holds %zu ids and the prompt asks for %d\n", p.path.c_str(), p.ids.size(), p.n); - return false; - } - return true; -} - static bool parse_prompt(const char * spec, Prompt & p) { const char * c1 = strchr(spec, ':'); if (!c1 || c1 == spec) return false; @@ -77,20 +50,6 @@ static bool parse_prompt(const char * spec, Prompt & p) { return true; } -static bool write_raw(const std::string & path, const float * data, size_t count) { - FILE * f = fopen(path.c_str(), "wb"); - if (!f) { - fprintf(stderr, "nemotron_ref: cannot write %s\n", path.c_str()); - return false; - } - const bool ok = fwrite(data, sizeof(float), count, f) == count; - if (fclose(f) != 0 || !ok) { - fprintf(stderr, "nemotron_ref: short write on %s\n", path.c_str()); - return false; - } - return true; -} - static bool write_ids(const std::string & path, const std::vector & ids) { FILE * f = fopen(path.c_str(), "w"); if (!f) { @@ -144,11 +103,11 @@ int main(int argc, char ** argv) { } prompts.push_back(std::move(p)); } else if (strcmp(argv[i], "-c") == 0 && more) { - n_ctx = atoi(argv[++i]); + if (!parse_i32("nemotron_ref", "-c", argv[++i], 0, n_ctx)) return 2; } else if (strcmp(argv[i], "-ub") == 0 && more) { - n_ubatch = atoi(argv[++i]); + if (!parse_i32("nemotron_ref", "-ub", argv[++i], 1, n_ubatch)) return 2; } else if (strcmp(argv[i], "-t") == 0 && more) { - threads = atoi(argv[++i]); + if (!parse_i32("nemotron_ref", "-t", argv[++i], 0, threads)) return 2; } else { fprintf(stderr, "nemotron_ref: unknown or incomplete argument '%s'\n", argv[i]); return 2; @@ -163,7 +122,7 @@ int main(int argc, char ** argv) { } int max_n = 0; for (Prompt & p : prompts) { - if (!read_ids(p)) return 2; + if (!read_token_file(p.path, p.n, p.ids)) return 2; max_n = std::max(max_n, p.n); } if (n_ctx == 0) n_ctx = max_n; diff --git a/tools/ref/ref_io.h b/tools/ref/ref_io.h new file mode 100644 index 000000000..59b841c80 --- /dev/null +++ b/tools/ref/ref_io.h @@ -0,0 +1,81 @@ +#pragma once +// Token files and raw f32 rows for the mainline oracles (hidden_ref, nemotron_ref). +// A token id is a non-negative 32-bit integer. The header does not include llama.h: +// llama_token is that integer, so a vector of it is a vector of int32_t. + +#include +#include +#include +#include +#include +#include +#include +#include + +// `value` is one decimal integer in [lo, hi], the whole argument. Anything else is named on stderr. +inline bool parse_i64(const char * who, const char * flag, const char * value, long long lo, long long hi, + long long & out) { + if (value == nullptr || value[0] < '0' || value[0] > '9') { + fprintf(stderr, "%s: %s wants an integer from %lld to %lld, got %s\n", who, flag, + lo, hi, value == nullptr ? "" : value); + return false; + } + char * end = nullptr; + errno = 0; + const long long n = strtoll(value, &end, 10); + if (end == value || *end != '\0' || errno != 0 || n < lo || n > hi) { + fprintf(stderr, "%s: %s wants an integer from %lld to %lld, got %s\n", who, flag, lo, hi, value); + return false; + } + out = n; + return true; +} + +// An int in [lo, INT_MAX]. `lo` is 0 where 0 means the harness default, and 1 where 0 is not a size. +inline bool parse_i32(const char * who, const char * flag, const char * value, int lo, int & dest) { + long long n = 0; + if (!parse_i64(who, flag, value, lo, INT_MAX, n)) return false; + dest = (int) n; + return true; +} + +template +bool read_token_file(const std::string & path, long long count, std::vector & ids) { + std::ifstream in(path); + if (!in) { + fprintf(stderr, "%s: cannot read\n", path.c_str()); + return false; + } + std::string line; + long long lines = 0; + while ((long long) ids.size() < count && std::getline(in, line)) { + ++lines; + if (!line.empty() && line.back() == '\r') line.pop_back(); + char * end = nullptr; + errno = 0; + const long id = strtol(line.c_str(), &end, 10); + if (line.empty() || line[0] < '0' || line[0] > '9' || *end != '\0' || errno != 0 || id < 0 || + id > INT32_MAX) { + fprintf(stderr, "%s:%lld: invalid token id '%s'\n", path.c_str(), lines, line.c_str()); + return false; + } + ids.push_back((Token) id); + } + if (!in.bad() && (long long) ids.size() == count) return true; + fprintf(stderr, "%s: holds %zu token ids, asked for %lld\n", path.c_str(), ids.size(), count); + return false; +} + +inline bool write_raw(const std::string & path, const float * data, size_t count) { + FILE * f = fopen(path.c_str(), "wb"); + if (!f) { + perror(path.c_str()); + return false; + } + const bool ok = fwrite(data, sizeof(float), count, f) == count; + if (fclose(f) != 0 || !ok) { + fprintf(stderr, "%s: short write\n", path.c_str()); + return false; + } + return true; +} diff --git a/tools/ref/ref_io_test.cpp b/tools/ref/ref_io_test.cpp new file mode 100644 index 000000000..f34ad2479 --- /dev/null +++ b/tools/ref/ref_io_test.cpp @@ -0,0 +1,83 @@ +#include "ref_io.h" + +#include +#include +#include +#include +#include + +static int fails = 0; + +static void expect(bool ok, const char * what) { + if (ok) return; + std::fprintf(stderr, "FAIL %s\n", what); + ++fails; +} + +static bool rejected(const char * flag, const char * value, long long lo, long long hi) { + long long n = 123; + const bool ok = parse_i64("ref", flag, value, lo, hi, n); + return !ok && n == 123; +} + +static bool accepted(const char * flag, const char * value, long long lo, long long hi, long long want) { + long long n = -1; + return parse_i64("ref", flag, value, lo, hi, n) && n == want; +} + +int main() { + expect(rejected("-c", "garbage", 0, INT_MAX), "nonnumeric -c"); + expect(rejected("-c", "12x", 0, INT_MAX), "trailing junk -c"); + expect(rejected("-c", "12 ", 0, INT_MAX), "trailing space -c"); + expect(rejected("-t", " 8", 0, INT_MAX), "leading space -t"); + expect(rejected("-t", "-1", 0, INT_MAX), "negative -t"); + expect(rejected("-t", "+8", 0, INT_MAX), "plus -t"); + expect(rejected("-ub", "0", 1, INT_MAX), "zero -ub"); + expect(rejected("--tokens-count", "", 1, LLONG_MAX), "empty count"); + expect(accepted("-c", "0", 0, INT_MAX, 0), "zero context"); + expect(accepted("-t", "16", 0, INT_MAX, 16), "threads"); + expect(accepted("-c", "2147483647", 0, INT_MAX, INT_MAX), "int max"); + expect(rejected("-c", "2147483648", 0, INT_MAX), "past int max"); + + char dir[] = "/tmp/ref-io-test-XXXXXX"; + if (mkdtemp(dir) == nullptr) { + std::perror("mkdtemp"); + return 1; + } + const std::string ids = std::string(dir) + "/ids"; + { + std::ofstream o(ids); + o << "0\n1\r\n2\n"; + } + std::vector got; + expect(read_token_file(ids, 3, got) && got.size() == 3 && got[0] == 0 && got[1] == 1 && got[2] == 2, + "three ids, one with cr"); + got.clear(); + { + std::ofstream o(ids); + o << "0\ngarbage\n"; + } + expect(!read_token_file(ids, 2, got), "garbage token id"); + got.clear(); + { + std::ofstream o(ids); + o << "1\n"; + } + expect(!read_token_file(ids, 2, got), "short token file"); + const float row[2] = {1.5f, -0.25f}; + const std::string raw = std::string(dir) + "/row.f32"; + expect(write_raw(raw, row, 2), "write raw"); + { + std::ifstream in(raw, std::ios::binary); + float back[2] = {}; + in.read(reinterpret_cast(back), sizeof(back)); + expect(in.gcount() == (std::streamsize) sizeof(back) && back[0] == 1.5f && back[1] == -0.25f, "read raw back"); + } + expect(!write_raw(std::string(dir) + "/missing/row.f32", row, 2), "missing directory"); + if (fails) { + std::fprintf(stderr, "ref_io_test: %d failed\n", fails); + return 1; + } + std::printf("ref_io_test: ok\n"); + return 0; +} From 433f8e3d4e4724bfb823d21308c24ec4886bc8d2 Mon Sep 17 00:00:00 2001 From: midagedev Date: Thu, 15 Oct 2026 16:48:06 +0900 Subject: [PATCH 6/7] refset, tools: an empty or NUL-cut token record is refused by both readers The C++ reader takes a token record only when strtol consumed every byte of it, so a NUL inside the line no longer ends the record early. The nemotron set reader refuses an empty id line, as the C++ reader does. Co-Authored-By: Claude Opus 5.5 --- crates/refset/src/arch/nemotron_h/mod.rs | 15 +++++++++++++++ crates/refset/src/nemotron.rs | 2 +- tools/ref/ref_io.h | 5 +++-- tools/ref/ref_io_test.cpp | 6 ++++++ 4 files changed, 25 insertions(+), 3 deletions(-) diff --git a/crates/refset/src/arch/nemotron_h/mod.rs b/crates/refset/src/arch/nemotron_h/mod.rs index 79fa6bf0c..ea60b15a3 100644 --- a/crates/refset/src/arch/nemotron_h/mod.rs +++ b/crates/refset/src/arch/nemotron_h/mod.rs @@ -165,6 +165,21 @@ mod tests { std::fs::remove_dir_all(&d).unwrap_or_else(|e| panic!("{e}")); } + /// An empty line among the ids is malformed, as the C++ reader that writes the set refuses it. + #[test] + fn an_empty_token_line_is_malformed() { + let d = dir("empty-id"); + write(&d, &runs(), pin(), true); + std::fs::write(d.join("p.ids"), "0\n\n1\n2\n3\n0\n").unwrap_or_else(|e| panic!("{e}")); + match LogitsSet::open(&d, &LOGITS) { + Err(RefError::Malformed { what, .. }) => { + assert!(what.contains("empty token id"), "{what}") + } + other => panic!("an empty token line: {other:?}"), + } + std::fs::remove_dir_all(&d).unwrap_or_else(|e| panic!("{e}")); + } + /// An id at or past `n_vocab` is malformed, naming the id and the vocabulary. #[test] fn a_token_id_outside_the_vocabulary_is_malformed() { diff --git a/crates/refset/src/nemotron.rs b/crates/refset/src/nemotron.rs index 617dcf934..c01e60958 100644 --- a/crates/refset/src/nemotron.rs +++ b/crates/refset/src/nemotron.rs @@ -235,7 +235,7 @@ impl LogitsSet { let mut n = 0usize; for (i, line) in text.lines().enumerate() { if line.is_empty() { - continue; + return Err(RefError::malformed(At(&ids, i + 1), "empty token id")); } n += 1; let id: u32 = crate::columns::parse_field(line, "token id", &At(&ids, i + 1))?; diff --git a/tools/ref/ref_io.h b/tools/ref/ref_io.h index 59b841c80..38ee88d84 100644 --- a/tools/ref/ref_io.h +++ b/tools/ref/ref_io.h @@ -54,8 +54,9 @@ bool read_token_file(const std::string & path, long long count, std::vector '9' || *end != '\0' || errno != 0 || id < 0 || - id > INT32_MAX) { + // The whole record is the number: a NUL inside the line ends the C string but not the record. + if (line.empty() || line[0] < '0' || line[0] > '9' || end != line.c_str() + line.size() || + errno != 0 || id < 0 || id > INT32_MAX) { fprintf(stderr, "%s:%lld: invalid token id '%s'\n", path.c_str(), lines, line.c_str()); return false; } diff --git a/tools/ref/ref_io_test.cpp b/tools/ref/ref_io_test.cpp index f34ad2479..13f80077f 100644 --- a/tools/ref/ref_io_test.cpp +++ b/tools/ref/ref_io_test.cpp @@ -59,6 +59,12 @@ int main() { } expect(!read_token_file(ids, 2, got), "garbage token id"); got.clear(); + { + std::ofstream o(ids, std::ios::binary); + o.write("0\n1\0garbage\n", 12); + } + expect(!read_token_file(ids, 2, got), "a NUL inside a token record"); + got.clear(); { std::ofstream o(ids); o << "1\n"; From 07aa96b1cee48bb2451e4b7598cd2970c8355f60 Mon Sep 17 00:00:00 2001 From: midagedev Date: Thu, 15 Oct 2026 17:37:30 +0900 Subject: [PATCH 7/7] refset, tools: a non-finite nemotron logit is refused, and the staging cleanup and prompt count parse have one owner The nemotron reader refuses a NaN or infinite logit by index. ref-paths.sh owns drop_incomplete_stage for dump.sh, dump-mtp.sh and dump-nemotron.sh. nemotron_ref reads a prompt's count with parse_i64, so +5 is refused. The tests read the prompt columns in another order and call parse_i32. Co-Authored-By: Claude Opus 5.5 --- crates/refset/src/arch/nemotron_h/mod.rs | 23 +++++++++++++++++++++-- crates/refset/src/nemotron.rs | 6 ++++++ tools/ref/dump-mtp.sh | 6 ------ tools/ref/dump-nemotron.sh | 6 ------ tools/ref/dump.sh | 6 ------ tools/ref/nemotron_ref.cpp | 7 +++---- tools/ref/ref-paths.sh | 9 +++++++++ tools/ref/ref_io_test.cpp | 3 +++ 8 files changed, 42 insertions(+), 24 deletions(-) diff --git a/crates/refset/src/arch/nemotron_h/mod.rs b/crates/refset/src/arch/nemotron_h/mod.rs index ea60b15a3..572e23309 100644 --- a/crates/refset/src/arch/nemotron_h/mod.rs +++ b/crates/refset/src/arch/nemotron_h/mod.rs @@ -58,8 +58,8 @@ mod tests { std::fs::write(dir.join("p.logits.f32"), logits).unwrap_or_else(|e| panic!("{e}")); let mut text = format!( "# model\t{model}\n# build\t{build}\n# arch\tnemotron_h\n\ - # prompt columns\tname\tn_ids\tn_vocab\tids\tlogits\n\ - prompt\tp\t5\t4\tp.ids\tp.logits.f32\n" + # prompt columns\tname\tn_vocab\tn_ids\tlogits\tids\n\ + prompt\tp\t4\t5\tp.logits.f32\tp.ids\n" ); if complete { text.push_str("# complete\t1\t0\n"); @@ -165,6 +165,25 @@ mod tests { std::fs::remove_dir_all(&d).unwrap_or_else(|e| panic!("{e}")); } + /// A NaN or infinite logit is malformed, naming its index. + #[test] + fn a_non_finite_logit_is_malformed() { + let d = dir("nan-logit"); + write(&d, &runs(), pin(), true); + let logits: Vec = [0.0f32, f32::NAN, -0.5, 0.25] + .into_iter() + .flat_map(f32::to_le_bytes) + .collect(); + std::fs::write(d.join("p.logits.f32"), logits).unwrap_or_else(|e| panic!("{e}")); + match LogitsSet::open(&d, &LOGITS) { + Err(RefError::Malformed { what, .. }) => { + assert!(what.contains("non-finite logit at index 1"), "{what}"); + } + other => panic!("a non-finite logit: {other:?}"), + } + std::fs::remove_dir_all(&d).unwrap_or_else(|e| panic!("{e}")); + } + /// An empty line among the ids is malformed, as the C++ reader that writes the set refuses it. #[test] fn an_empty_token_line_is_malformed() { diff --git a/crates/refset/src/nemotron.rs b/crates/refset/src/nemotron.rs index c01e60958..b1979cc94 100644 --- a/crates/refset/src/nemotron.rs +++ b/crates/refset/src/nemotron.rs @@ -215,6 +215,12 @@ impl LogitsSet { chunk.try_into().expect("chunks_exact(4)"), )); } + if let Some(i) = out.iter().position(|v| !v.is_finite()) { + return Err(RefError::malformed( + path.display(), + format!("non-finite logit at index {i}"), + )); + } Ok(out) } diff --git a/tools/ref/dump-mtp.sh b/tools/ref/dump-mtp.sh index 976fd8e46..65e760727 100644 --- a/tools/ref/dump-mtp.sh +++ b/tools/ref/dump-mtp.sh @@ -178,12 +178,6 @@ mkdir -p "$ROOT" rm -rf "$STAGE" mkdir -p "$STAGE" # A staged set without its trailer is removed on every exit, with a line naming it (dump.sh's rule). -drop_incomplete_stage() { - [ -d "$STAGE" ] || return 0 - grep -qs '^# complete' "$STAGE/MANIFEST.tsv" && return 0 - rm -rf "$STAGE" - echo "[staging] removed the incomplete $STAGE" >&2 -} trap drop_incomplete_stage EXIT trap 'exit 130' INT trap 'exit 143' TERM diff --git a/tools/ref/dump-nemotron.sh b/tools/ref/dump-nemotron.sh index bf99bb8f9..697ee949c 100755 --- a/tools/ref/dump-nemotron.sh +++ b/tools/ref/dump-nemotron.sh @@ -75,12 +75,6 @@ SET=$BLOOMERY_DATA/ref-nemotron-logits STAGE=$BLOOMERY_DATA/ref-nemotron-logits.staging rm -rf "$STAGE" mkdir -p "$STAGE" -drop_incomplete_stage() { - [ -d "$STAGE" ] || return 0 - grep -qs '^# complete' "$STAGE/MANIFEST.tsv" && return 0 - rm -rf "$STAGE" - echo "[staging] removed the incomplete $STAGE" >&2 -} trap 'drop_incomplete_stage; rm -rf "$WORK"' EXIT "$BIN" -m "$MODEL" --out "$STAGE" -c 512 -ub 512 -t 8 \ --prompt "code5:5:$WORK/code5.ids" \ diff --git a/tools/ref/dump.sh b/tools/ref/dump.sh index b7db0b830..5044bcc0f 100755 --- a/tools/ref/dump.sh +++ b/tools/ref/dump.sh @@ -254,12 +254,6 @@ mkdir -p "$STAGE" # `set -e`, a refusal below, a signal), with a line naming it: an incomplete set is never left # behind for a reader to mistake for a set. A complete staged set that the swap refuses stays, as # its refusal says. A SIGKILL skips the trap; the next dump of the set clears the directory above. -drop_incomplete_stage() { - [ -d "$STAGE" ] || return 0 - grep -qs '^# complete' "$STAGE/MANIFEST.tsv" && return 0 - rm -rf "$STAGE" - echo "[staging] removed the incomplete $STAGE" >&2 -} trap drop_incomplete_stage EXIT trap 'exit 130' INT trap 'exit 143' TERM diff --git a/tools/ref/nemotron_ref.cpp b/tools/ref/nemotron_ref.cpp index b0e20f75f..85661579a 100644 --- a/tools/ref/nemotron_ref.cpp +++ b/tools/ref/nemotron_ref.cpp @@ -41,10 +41,9 @@ static bool parse_prompt(const char * spec, Prompt & p) { for (char ch : p.name) { if (!((ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '0' && ch <= '9'))) return false; } - char * end = nullptr; - errno = 0; - const long n = strtol(c1 + 1, &end, 10); - if (end != c2 || errno != 0 || n <= 0 || n > 1000000) return false; + const std::string count(c1 + 1, c2); + long long n = 0; + if (!parse_i64("nemotron_ref", "--prompt count", count.c_str(), 1, 1000000, n)) return false; p.n = (int) n; p.path = c2 + 1; return true; diff --git a/tools/ref/ref-paths.sh b/tools/ref/ref-paths.sh index 1f190d3e1..0b3a9dd29 100644 --- a/tools/ref/ref-paths.sh +++ b/tools/ref/ref-paths.sh @@ -262,3 +262,12 @@ fixture_line() { esac printf '%s\n' "$line" } + +# A dump runner's EXIT trap: the set staged at $STAGE is removed unless its manifest carries the +# `# complete` trailer, so a run that dies leaves no half-written staging set. +drop_incomplete_stage() { + [ -d "$STAGE" ] || return 0 + grep -qs '^# complete' "$STAGE/MANIFEST.tsv" && return 0 + rm -rf "$STAGE" + echo "[staging] removed the incomplete $STAGE" >&2 +} diff --git a/tools/ref/ref_io_test.cpp b/tools/ref/ref_io_test.cpp index 13f80077f..0f3bae331 100644 --- a/tools/ref/ref_io_test.cpp +++ b/tools/ref/ref_io_test.cpp @@ -26,6 +26,9 @@ static bool accepted(const char * flag, const char * value, long long lo, long l } int main() { + int n32 = 123; + expect(!parse_i32("ref", "-c", "garbage", 0, n32) && n32 == 123, "i32 rejects garbage"); + expect(parse_i32("ref", "-c", "16", 0, n32) && n32 == 16, "i32 reads 16"); expect(rejected("-c", "garbage", 0, INT_MAX), "nonnumeric -c"); expect(rejected("-c", "12x", 0, INT_MAX), "trailing junk -c"); expect(rejected("-c", "12 ", 0, INT_MAX), "trailing space -c");