diff --git a/Cargo.lock b/Cargo.lock index aa93ebc66..7bbb722a6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -172,6 +172,7 @@ dependencies = [ "bloomery-gguf", "bloomery-gpu", "bloomery-gpu-deepseek41", + "bloomery-gpu-gemma-embedding2", "bloomery-gpu-glm5next", "bloomery-gpu-mimo2", "bloomery-gpu-vision", @@ -196,6 +197,16 @@ dependencies = [ "serde_json", ] +[[package]] +name = "bloomery-gpu-gemma-embedding2" +version = "0.1.0" +dependencies = [ + "bloomery-gguf", + "bloomery-gpu", + "bloomery-model", + "cuda-core", +] + [[package]] name = "bloomery-gpu-glm5next" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index a13c6941c..e1a92e8fb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] resolver = "3" -members = ["crates/qdot", "crates/gguf", "crates/models", "crates/model", "crates/placement", "crates/threads", "crates/levers", "crates/verbs", "crates/gpu", "crates/gpu-deepseek41", "crates/gpu-glm5next", "crates/gpu-mimo2", "crates/gpu-gates", "crates/engram", "crates/engram-lab", "crates/sampler", "crates/tokenizer", "crates/serve", "crates/jinja", "crates/vision", "crates/gpu-vision", "crates/refset", "crates/runtime", "crates/app", "crates/decision", "crates/hf"] +members = ["crates/qdot", "crates/gguf", "crates/models", "crates/model", "crates/placement", "crates/threads", "crates/levers", "crates/verbs", "crates/gpu", "crates/gpu-deepseek41", "crates/gpu-glm5next", "crates/gpu-mimo2", "crates/gpu-gemma-embedding2", "crates/gpu-gates", "crates/engram", "crates/engram-lab", "crates/sampler", "crates/tokenizer", "crates/serve", "crates/jinja", "crates/vision", "crates/gpu-vision", "crates/refset", "crates/runtime", "crates/app", "crates/decision", "crates/hf"] # 의도적으로 컴파일되지 않는 재현기다(MUL-7). 워크스페이스에 넣으면 --workspace가 영구히 빨갛다. exclude = ["crates/oxide-ice-unroll", "crates/oxide-ice-unroll/variants/unroll_smoke", "fuzz", "labs/hip-native"] diff --git a/crates/gpu-gates/Cargo.toml b/crates/gpu-gates/Cargo.toml index f6af56a28..deec778e4 100644 --- a/crates/gpu-gates/Cargo.toml +++ b/crates/gpu-gates/Cargo.toml @@ -44,6 +44,9 @@ bloomery-gpu-deepseek41 = { path = "../gpu-deepseek41", optional = true } bloomery-gpu-glm5next = { path = "../gpu-glm5next", optional = true } # The MiMo-V2.6-Flash program, reached only through the `mimo2` feature (its end-to-end gate). bloomery-gpu-mimo2 = { path = "../gpu-mimo2", optional = true } +# The EmbeddingGemma 2 encoder, reached only through the `gemma-embedding2` feature (its end-to-end +# gate). +bloomery-gpu-gemma-embedding2 = { path = "../gpu-gemma-embedding2", optional = true } cuda-core = { workspace = true, optional = true } # A gate binary's own `#[cuda_module]` (`gate_kquant`: entries no engine binary links) needs the # device and host halves of cuda-oxide as direct dependencies: the macros emit `::cuda_host` and @@ -119,6 +122,7 @@ vision = ["gpu", "dep:bloomery-gpu-vision", "dep:bloomery-vision"] clef = ["gpu", "dep:bloomery-decision", "dep:bloomery-serve", "dep:serde_json"] glm5next = ["deepseek41", "dep:bloomery-gpu-glm5next", "bloomery-app/glm5next"] mimo2 = ["gpu", "dep:bloomery-gpu-mimo2", "bloomery-app/mimo2"] +gemma-embedding2 = ["gpu", "dep:bloomery-gpu-gemma-embedding2"] # A bin whose `main` sits behind a feature requires that feature: cargo refuses to build it without the # feature before it links anything, instead of linking the feature-off stub `main` over the working @@ -535,6 +539,10 @@ required-features = ["gpu"] name = "gate_gemma_embedding2_head512" required-features = ["gpu"] +[[bin]] +name = "gate_gemma_embedding2_e2e" +required-features = ["gemma-embedding2"] + [[bin]] name = "gate_qwen3moe_router" required-features = ["gpu"] diff --git a/crates/gpu-gates/src/bin/gate_gemma_embedding2_e2e.rs b/crates/gpu-gates/src/bin/gate_gemma_embedding2_e2e.rs new file mode 100644 index 000000000..a7d305710 --- /dev/null +++ b/crates/gpu-gates/src/bin/gate_gemma_embedding2_e2e.rs @@ -0,0 +1,409 @@ +//! GPU gate for EmbeddingGemma 2's encoder (`bloomery-gpu-gemma-embedding2`) +//! end to end on the Q8_0 file the sets are dumped from ([`MODEL`], held to +//! its sha256), against llama.cpp mainline's text set dumped on the card +//! ([`TEXT`]) and its CPU twin ([`TEXT_CPU`]), both read through +//! [`Emb2Set::open`]: each case the set names, its ids read from the +//! tokenizer oracle's gemma4 set ([`IDS_SET`]) and held to the set's count +//! and digest, through one pass at the load's context with its taps read +//! back. +//! +//! A row's band is [`TWIN`] times the CPU twin's own distance from the card +//! set, and at least [`FLOOR`]: the twin is the floor any two realizations of +//! the graph lie apart (the Qwen seats' rule, `shared/glm5next_media.rs`). A +//! row is within its band only when its distance and the twin's are finite. +//! +//! 1. (t) Per case: `inp_scaled` bit for bit (the host's token rows times +//! `√hidden`, the one f32 multiply `ggml_scale` makes); `inp_per_layer`, +//! every layer's `l_out`, `result_norm`, the token rows against `embd` +//! and their mean ([`Pass::pooled`]) against `pooled`, each within its +//! band. A case longer than a sliding layer's reach plus one catches a +//! dropped window; a set with no such case fails by name. The host test +//! `key_bounds_are_the_symmetric_mask` holds the exact key edges; every +//! case holds the global layers. +//! 2. (d) The longest case run again with no taps is bit for bit the first +//! run's rows. +//! 3. (r) Refused by name, the load untouched after each: a pass of no ids, +//! of one id past the load's context and of an id past the vocabulary, +//! and a load at one position past [`CARD_CTX`]. + +#[cfg(feature = "gemma-embedding2")] +fn main() -> std::process::ExitCode { + bloomery_gpu_gates::exit_with("gate_gemma_embedding2_e2e", gate::run()) +} + +#[cfg(feature = "gemma-embedding2")] +#[allow( + dead_code, + reason = "the shared runner serves several gates; each takes the helpers it reads" +)] +#[path = "shared/e2e.rs"] +mod e2e; + +#[cfg(feature = "gemma-embedding2")] +mod gate { + use std::path::Path; + use std::time::Instant; + + use crate::e2e::{clause, elapsed, refused_by, rel, same_bits, tap}; + use bloomery_gpu::Gpu; + use bloomery_gpu_gates::{GateError, RefManifest, checks_failed, data_dir, sha256, verdict}; + use bloomery_gpu_gemma_embedding2::{Encoder, Pass, card_bytes}; + use gguf::Split; + use model::arch::gemma_embedding2::hparams::{CARD_CTX, Hparams}; + use refset::arch::gemma_embedding2::{IDS_SET, MODEL, MODEL_SHA256, TEXT, TEXT_CPU}; + use refset::emb2::Emb2Set; + use refset::family::Family; + use refset::ik::RefHeader; + use refset::tokenizer::read_ids; + + /// A row's band: this many times the CPU twin's distance from the card + /// set (`shared/glm5next_media.rs`'s `TWIN`, the Qwen seats' rule). + const TWIN: f64 = 2.5; + + /// The least band. + // PIN(2026-10-15): 1e-2 over the distances whose twin band is smaller, + // measured on a DGX Spark: `inp_per_layer` 1.344e-3 to 1.594e-3 against + // twins of 1.066e-7 to 2.410e-7 in every case, and the shortest cases' + // `l_out` up to 9.046e-3 at layer 0. + const FLOOR: f64 = 1.0e-2; + + /// One case of the set: its name and its ids. + struct Case { + id: String, + ids: Vec, + } + + fn check_case_identities(card: &RefHeader, cpu: &RefHeader) -> Result<(), GateError> { + if !card + .other + .iter() + .filter(|line| line.starts_with("# case\t")) + .eq(cpu.other.iter().filter(|line| line.starts_with("# case\t"))) + { + return Err("CPU twin case identities differ from the card set".into()); + } + Ok(()) + } + + /// The cases `man`'s `# case ` lines name, each + /// one's ids read from [`IDS_SET`] and refused by name when their count + /// or their file's digest is not the line's. + fn cases(man: &RefManifest) -> Result, GateError> { + let dir = data_dir().join(IDS_SET); + let mut out = Vec::new(); + for line in &man.header.other { + let Some(rest) = line.strip_prefix("# case\t") else { + continue; + }; + let fields: Vec<&str> = rest.split('\t').collect(); + let [id, n, _, sha] = fields[..] else { + return Err( + format!("{}: a case line of another shape: {line:?}", TEXT.name).into(), + ); + }; + let path = dir.join(format!("{id}.ids")); + let want = n + .parse::() + .map_err(|e| format!("{}: case {id}: token count {n:?}: {e}", TEXT.name))?; + let ids = read_ids(&path, want)?; + let digest = sha256(&path)?; + if digest != sha { + return Err(format!( + "{}: {} ids of digest {digest}; the set's case line says {n} of {sha}", + path.display(), + ids.len() + ) + .into()); + } + out.push(Case { + id: id.to_string(), + ids, + }); + } + if out.is_empty() { + return Err(format!("{}: no case line", TEXT.name).into()); + } + Ok(out) + } + + /// One row held to the card set: its distance, the twin's and the band. + struct Held { + r: f64, + twin: f64, + band: f64, + } + + impl Held { + fn of(ours: &[f32], card: &[f32], twin: &[f32]) -> Held { + let (r, twin) = (rel(ours, card), rel(twin, card)); + Held { + r, + twin, + band: (TWIN * twin).max(FLOOR), + } + } + + /// Within the band, both distances finite: `rel` makes a NaN or a + /// length mismatch infinite, and an infinite twin an infinite band. + fn ok(&self) -> bool { + self.r.is_finite() && self.twin.is_finite() && self.r <= self.band + } + + fn line(&self, what: &str) -> String { + format!( + "{what} {:.3e} (twin {:.3e}, band {:.3e})", + self.r, self.twin, self.band + ) + } + } + + /// Clause (t) for one case: its pass with taps, held tap by tap. The + /// pass's rows, for (d). + fn case( + enc: &mut Encoder, + c: &Case, + man: &RefManifest, + cpu: &RefManifest, + hp: &Hparams, + ) -> Result<(bool, Pass), GateError> { + let t0 = Instant::now(); + let pass = enc.pass(&c.ids, true)?; + let taps = pass.taps.as_ref().ok_or("a pass with taps returned none")?; + assert_eq!(taps.l_out.len(), hp.n_layer, "missing l_out taps"); + let row = |m: &RefManifest, name: &str| tap(m, &format!("{}/{name}", c.id)); + let held = |name: &str, ours: &[f32]| -> Result { + Ok(Held::of(ours, &row(man, name)?, &row(cpu, name)?)) + }; + let mut ok = true; + let mut lines = Vec::new(); + let exact = same_bits(&taps.inp_scaled, &row(man, "inp_scaled")?); + lines.push(format!("inp_scaled bits {}", verdict(exact))); + ok &= exact; + let h = held("inp_per_layer", &taps.inp_per_layer)?; + lines.push(h.line("inp_per_layer")); + ok &= h.ok(); + // The worst layer by its distance over its band. + let mut worst = (0.0f64, String::new()); + for (l, ours) in taps.l_out.iter().enumerate() { + let h = held(&format!("l_out-{l}"), ours)?; + if !h.ok() { + println!( + "(t) {}: {} past its band {}", + c.id, + h.line(&format!("layer {l} l_out")), + verdict(false) + ); + ok = false; + } + if h.r / h.band >= worst.0 { + worst = (h.r / h.band, h.line(&format!("l_out worst layer {l}"))); + } + } + lines.push(worst.1); + for (what, ours) in [ + ("result_norm", &taps.result_norm), + ("embd", &pass.rows), + ("pooled", &pass.pooled), + ] { + let h = held(what, ours)?; + lines.push(h.line(what)); + ok &= h.ok(); + } + println!( + "(t) {}: {} ids, {}; {:.1} ms (runtime value) {}", + c.id, + c.ids.len(), + lines.join("; "), + t0.elapsed().as_secs_f64() * 1e3, + verdict(ok) + ); + Ok((ok, pass)) + } + + /// A case long enough for a dropped window to leave the tap bands: + /// some row's keys stop short of the input's ends. The host test holds + /// the exact edges. No sliding layer passes; a set with no such case fails. + fn window(reach: &[Option], cases: &[Case]) -> bool { + let Some(reach) = reach.iter().flatten().copied().max().map(|r| r as usize) else { + return true; + }; + let longest = cases.iter().map(|c| c.ids.len()).max().unwrap_or(0); + let held = longest > reach + 1; + println!( + "(t) the window: the longest case holds {longest} ids, a sliding layer reaches {reach} \ + on each side {}", + verdict(held) + ); + held + } + + /// Clause (r): each refusal by its words, and a pass after them the + /// first one's bits. + fn refusals( + enc: &mut Encoder, + c: &Case, + rows: &[f32], + vocab: usize, + ) -> Result { + const PASS: &str = "Encoder::pass"; + let past = vec![c.ids[0]; enc.ctx() + 1]; + let mut bad = c.ids.clone(); + bad[0] = u32::try_from(vocab)?; + let checks = [ + ( + "no ids", + refused_by(&enc.pass(&[], false), PASS, "0 positions"), + ), + ( + "one past the context", + refused_by(&enc.pass(&past, false), PASS, "a pass takes 1 to"), + ), + ( + "an id past the vocabulary", + refused_by(&enc.pass(&bad, false), PASS, "past the vocabulary"), + ), + ]; + let mut ok = true; + for (what, refused) in checks { + println!("(r) {what}: refused by name {}", verdict(refused)); + ok &= refused; + } + enc.pass(&vec![c.ids[0]; enc.ctx()], false)?; + let again = enc.pass(&c.ids, false)?; + let same = same_bits(&again.rows, rows); + println!( + "(r) {}: after the refusals the rows bit for bit {}", + c.id, + verdict(same) + ); + Ok(ok && same) + } + + /// `family`'s set, read through its reader. + fn open(family: &Family) -> Result { + Ok(Emb2Set::open(&family.path(family.sets[0]), family)?.man) + } + + pub fn run() -> Result<(), GateError> { + bloomery_levers::at_main(&[])?; + let args: Vec = std::env::args().skip(1).collect(); + if !args.is_empty() { + return Err(format!("no arguments; got {args:?}").into()); + } + let digest = sha256(Path::new(MODEL))?; + if digest != MODEL_SHA256 { + return Err(format!( + "{MODEL}: sha256 {digest}, the sets are dumped from {MODEL_SHA256}" + ) + .into()); + } + let split = Split::open(MODEL).map_err(|e| format!("open {MODEL}: {e}"))?; + let hp = Hparams::read(&split).map_err(|e| format!("{MODEL}: {e}"))?; + let read = model::arch::spec(&split).map_err(|e| format!("{MODEL}: {e}"))?; + let encoder = read + .spec + .encoder + .as_ref() + .ok_or("gemma-embedding2 has no encoder description")?; + let ctx = CARD_CTX; + let bytes = card_bytes(&split, ctx)?; + let man = open(&TEXT)?; + let cpu = open(&TEXT_CPU)?; + check_case_identities(&man.header, &cpu.header)?; + let cases = cases(&man)?; + let gpu = Gpu::new()?; + println!( + "gate_gemma_embedding2_e2e: device {}: {MODEL} at {ctx} positions, card figure {bytes} B; \ + {} cases; band {TWIN} x the CPU twin, at least {FLOOR:e}", + gpu.device_name()?, + cases.len() + ); + let mut ok = window(&encoder.reach, &cases); + let t = Instant::now(); + let mut enc = Encoder::open(split, ctx, gpu)?; + elapsed("(open)", &t); + let mut longest: Option<(usize, Pass)> = None; + for (i, c) in cases.iter().enumerate() { + let r = case(&mut enc, c, &man, &cpu, &hp).map(|(pass_ok, pass)| { + if longest + .as_ref() + .is_none_or(|(j, _)| c.ids.len() > cases[*j].ids.len()) + { + longest = Some((i, pass)); + } + pass_ok + }); + ok &= clause("(t)", &c.id, r); + } + if let Some((i, first)) = longest { + let c = &cases[i]; + ok &= clause( + "(d)", + &c.id, + enc.pass(&c.ids, false) + .map_err(GateError::from) + .map(|again| { + let same = same_bits(&again.rows, &first.rows); + println!( + "(d) {}: {} ids, a rerun's rows bit for bit {}", + c.id, + c.ids.len(), + verdict(same) + ); + same + }), + ); + ok &= clause( + "(r)", + "refusals", + refusals(&mut enc, c, &first.rows, hp.n_vocab), + ); + } + let past = Split::open(MODEL).map_err(|e| format!("open {MODEL}: {e}"))?; + let refused = refused_by( + &Encoder::open(past, ctx + 1, Gpu::new()?), + "Encoder::open", + "a context of", + ); + println!( + "(r) a load at {} positions: refused by name {}", + ctx + 1, + verdict(refused) + ); + ok &= refused; + if ok { Ok(()) } else { Err(checks_failed()) } + } + + #[cfg(test)] + mod tests { + use super::{Case, RefHeader, check_case_identities, window}; + + #[test] + fn a_window_case_reaches_past_the_longest_sliding_layer() { + let mut cases = [Case { + id: "window".to_owned(), + ids: vec![0; 3], + }]; + assert!(window(&[None], &[])); + assert!(!window(&[Some(1), None, Some(2)], &cases)); + cases[0].ids.push(0); + assert!(window(&[Some(1), None, Some(2)], &cases)); + } + + #[test] + fn a_cpu_case_of_another_digest_is_refused_by_name() { + let card = RefHeader { + other: vec![format!( + "# case\tcases/01\t2\tcases/01.ids\t{}", + "a".repeat(64) + )], + ..RefHeader::default() + }; + let mut cpu = card.clone(); + check_case_identities(&card, &cpu).unwrap(); + cpu.other[0] = cpu.other[0].replace(&"a".repeat(64), &"b".repeat(64)); + let err = check_case_identities(&card, &cpu).unwrap_err().to_string(); + assert_eq!(err, "CPU twin case identities differ from the card set"); + } + } +} diff --git a/crates/gpu-gates/src/bin/gate_glm5next_serve.rs b/crates/gpu-gates/src/bin/gate_glm5next_serve.rs index ac3eb3582..5ecee1fe6 100644 --- a/crates/gpu-gates/src/bin/gate_glm5next_serve.rs +++ b/crates/gpu-gates/src/bin/gate_glm5next_serve.rs @@ -306,10 +306,10 @@ mod gate { use bloomery_gpu_gates::residency38::{Seqs, reserve_checkpoints}; use bloomery_gpu_gates::serve_client::{ self, PrefixRestore, agree, base64, beside, check, completion, curl, ids_of, json_of, - metric, parse_ids, plan_run, prefix_restore, rendered, server_log, sha256, tokenized, + metric, parse_ids, plan_run, prefix_restore, rendered, server_log, tokenized, }; use bloomery_gpu_gates::tier::{self, Tag}; - use bloomery_gpu_gates::{GateError, checks_failed, ref_model_path}; + use bloomery_gpu_gates::{GateError, checks_failed, ref_model_path, sha256}; use bloomery_gpu_glm5next::{CHECKPOINT_EVERY, checkpoint_price}; use gguf::Split; use model::arch::glm5next::place::{ diff --git a/crates/gpu-gates/src/bin/shared/routed_legs.rs b/crates/gpu-gates/src/bin/shared/routed_legs.rs index 54d048aad..b3a342e17 100644 --- a/crates/gpu-gates/src/bin/shared/routed_legs.rs +++ b/crates/gpu-gates/src/bin/shared/routed_legs.rs @@ -94,11 +94,12 @@ fn family_of(arch: Arch) -> Option<&'static dyn Adapter> { } /// Why a set the gate does not plan is skipped: the Qwen3.5 dense trunk -/// routes nothing, and an architecture no adapter plans is never planned as -/// a model. +/// and the EmbeddingGemma 2 encoder route nothing, and an architecture no +/// adapter plans is never planned as a model. fn skip_of(arch: Arch) -> &'static str { match arch { Arch::Qwen35 => "dense trunk", + Arch::GemmaEmbedding2 => "no routed expert", Arch::Qwen3Moe | Arch::Qwen35Moe | Arch::Qwen4Exp @@ -108,8 +109,7 @@ fn skip_of(arch: Arch) -> &'static str { | Arch::Deepseek4 | Arch::NemotronH | Arch::NemotronHMoe - | Arch::GemmaEmbedding - | Arch::GemmaEmbedding2 => "no program", + | Arch::GemmaEmbedding => "no program", } } @@ -1068,6 +1068,7 @@ fn family_name(program: Program) -> &'static str { Program::Deepseek41Chain => "deepseek41", Program::NemotronHBody => "nemotron_h", Program::NemotronHMoeBody => "nemotron_h_moe", + Program::GemmaEmbedding2Body => "gemma-embedding2", } } diff --git a/crates/gpu-gates/src/lib.rs b/crates/gpu-gates/src/lib.rs index 6309c1a50..c3cd47651 100644 --- a/crates/gpu-gates/src/lib.rs +++ b/crates/gpu-gates/src/lib.rs @@ -67,7 +67,8 @@ pub mod vision_seat; use gguf::quant::{GgmlType, dequant_row}; use gguf::{Gguf, Split, TensorInfo}; use model::arch::Arch; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; +use std::process::Command; /// Kernel gate band against the quantized-input reference. /// PIN(2026-09-21): tightened 1e-2 -> 1e-5. Measured worst on the landed @@ -136,6 +137,17 @@ pub fn checks_failed() -> GateError { "FAILED: one or more checks above did not pass".into() } +/// The sha256 of the file at `path`, as `sha256sum` prints it; a failed `sha256sum` is an error +/// naming the file. +pub fn sha256(path: &Path) -> Result { + let out = Command::new("sha256sum").arg(path).output()?; + let text = String::from_utf8_lossy(&out.stdout).into_owned(); + match text.split_whitespace().next() { + Some(h) if out.status.success() => Ok(h.to_owned()), + _ => Err(format!("sha256sum {}: {:?}", path.display(), text.trim()).into()), + } +} + /// The model file every gate and server opens: the one a server named on /// its command line (`-m`, `--hf`: [`model_file`]), else /// `$BLOOMERY_REF_MODEL`, with no default here. The variable's file is a diff --git a/crates/gpu-gates/src/oracle/mod.rs b/crates/gpu-gates/src/oracle/mod.rs index 53187cc51..b76bd53d1 100644 --- a/crates/gpu-gates/src/oracle/mod.rs +++ b/crates/gpu-gates/src/oracle/mod.rs @@ -131,9 +131,10 @@ pub fn for_arch(a: Arch) -> Result<&'static Oracle, GateError> { Arch::NemotronH => Err("nemotron_h has no oracle row: no program runs it".into()), Arch::NemotronHMoe => Err("nemotron_h_moe has no oracle row: no program runs it".into()), Arch::GemmaEmbedding => Err("gemma-embedding has no oracle row: no program runs it".into()), - Arch::GemmaEmbedding2 => { - Err("gemma-embedding2 has no oracle row: no program runs it".into()) - } + Arch::GemmaEmbedding2 => Err( + "gemma-embedding2 has no oracle row: its gate reads mainline's sets by its family" + .into(), + ), } } diff --git a/crates/gpu-gates/src/serve_client.rs b/crates/gpu-gates/src/serve_client.rs index ec47399d5..8a8d5396a 100644 --- a/crates/gpu-gates/src/serve_client.rs +++ b/crates/gpu-gates/src/serve_client.rs @@ -8,7 +8,7 @@ //! record kinds ([`server_log`]), and the requests the seat gates share: the //! chat turn's ids ([`rendered`], [`tokenized`]), a greedy completion ([`greedy`]), the //! chat answer held to a completion's ids ([`chat_is_those_ids`]), an image -//! file held to its digest and sent as base64 ([`sha256`], [`base64`]) and the +//! file held to its digest and sent as base64 ([`crate::sha256`], [`base64`]) and the //! prefix clauses of a seat that keeps a shared prefix //! ([`edit_and_extension`]) and of one whose prompt cache holds a saved //! state's checkpoints ([`prefix_restore`]). @@ -268,17 +268,6 @@ pub fn base64(bytes: &[u8]) -> String { out } -/// The sha256 of the file at `path`, as `sha256sum` prints it; a failed `sha256sum` is an error -/// naming the file. -pub fn sha256(path: &Path) -> Result { - let out = Command::new("sha256sum").arg(path).output()?; - let text = String::from_utf8_lossy(&out.stdout).into_owned(); - match text.split_whitespace().next() { - Some(h) if out.status.success() => Ok(h.to_owned()), - _ => Err(format!("sha256sum {}: {:?}", path.display(), text.trim()).into()), - } -} - /// One clause's line, `check : PASS|FAIL`, folded into `ok`. pub fn check(ok: &mut bool, name: &str, pass: bool) { println!("check {name}: {}", verdict(pass)); diff --git a/crates/gpu-gemma-embedding2/Cargo.toml b/crates/gpu-gemma-embedding2/Cargo.toml new file mode 100644 index 000000000..4de0a554c --- /dev/null +++ b/crates/gpu-gemma-embedding2/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "bloomery-gpu-gemma-embedding2" +version = "0.1.0" +edition = "2024" +publish = false +license = "MIT" + +[lints] +workspace = true + +# The EmbeddingGemma 2 encoder: host code only, no `#[cuda_module]` of its own. It composes the +# kernels of `bloomery-gpu` (the q8_0 GEMM, the f32 tile, the norms, the GELU GLU, the whole-turn +# rope-and-append at heads 256 and 512, the bidirectional prefill flashes); it is built with +# `cargo oxide` because that crate is. +[dependencies] +cuda-core.workspace = true +bloomery-gpu = { path = "../gpu" } +# Host only: the header read and the token rows. +bloomery-gguf = { path = "../gguf" } +bloomery-model = { path = "../model" } diff --git a/crates/gpu-gemma-embedding2/src/lib.rs b/crates/gpu-gemma-embedding2/src/lib.rs new file mode 100644 index 000000000..0241dc85f --- /dev/null +++ b/crates/gpu-gemma-embedding2/src/lib.rs @@ -0,0 +1,862 @@ +//! bloomery-gpu-gemma-embedding2: the EmbeddingGemma 2 encoder +//! (`gemma-embedding2`, llama.cpp's `src/models/gemma-embedding2.cpp`): one +//! pass over an input's token ids, every position attending both ways, to +//! the token rows of the output projection and their mean. It has no device +//! code of its own: every launch is `bloomery-gpu`'s. +//! +//! The pass, in the graph's order: +//! - the token rows, decoded on the host from the file's table +//! (`gguf::quant::dequant_row`) and multiplied by `√hidden` (`inp_scaled`); +//! - per layer: the attention (RMS norm, the q, k and v GEMMs, the per-head +//! q and k norms with their gains and the v norm with none, the whole NEOX +//! turn, the flash at scale 1 over the keys within the layer's reach on +//! each side of a row (every key on a full layer), the output GEMM, its +//! post norm and the residual); the GELU GLU, its post norm and the +//! residual; the per-layer input, `gelu(inp_gate·x) · ple_l` through +//! `proj` and `post_norm`, added to `x`, where `ple_l` is the layer's +//! slice of `per_layer_model_proj · inp_scaled`, times `1/√hidden` and +//! RMS-normed by `per_layer_proj_norm`; then the layer's output times its +//! `layer_output_scale` (`l_out`); +//! - the output norm (`result_norm`), the `output` GEMM of every position +//! (`result_embd`), and on the host their mean ([`Pass::pooled`]). +//! +//! - [`Encoder`]: the load and the pass ([`Encoder::pass`]); +//! - [`card_bytes`]: what a load at a context holds on its card. +//! +//! Built with `cargo oxide`, as the device crate it links is. + +mod pass; + +pub use pass::{Pass, Taps}; + +use std::collections::BTreeMap; + +use bloomery_gpu::flash_gqa_prefill::FlashGqaPrefill; +use bloomery_gpu::flash_gqa_prefill_512::FlashGqaPrefill512; +use bloomery_gpu::gemm::{GEMM_MAX_SLOTS, Gemm32Kernels, GemmAct32, GemmKernels, GemmRoute}; +use bloomery_gpu::rope_neox::RopeNeoxKernels; +use bloomery_gpu::rope_table::{RopeRows, RopeSpec, RopeTable}; +use bloomery_gpu::weights::{Weights, resident_size}; +use bloomery_gpu::{DeviceTensor, Gpu, GpuError}; +use cuda_core::DeviceBuffer; +use gguf::{GgmlType, Split}; +use model::arch::gemma_embedding2::hparams::{CARD_CTX, Hparams}; +use model::arch::gemma_embedding2::names; + +/// The refusal `what` of a load. +const OPEN: &str = "Encoder::open"; + +fn size_overflow() -> GpuError { + GpuError::Shape { + what: "card_bytes", + detail: "encoder byte count overflows usize".to_string(), + } +} + +fn size_product(factors: &[usize]) -> Result { + factors + .iter() + .try_fold(1usize, |n, &factor| n.checked_mul(factor)) + .ok_or_else(size_overflow) +} + +fn size_sum(sizes: impl IntoIterator) -> Result { + sizes + .into_iter() + .try_fold(0usize, usize::checked_add) + .ok_or_else(size_overflow) +} + +/// How layer `l` attends: its head, its key heads, and the keys a row +/// reaches on each side of its own position (`None`: every key). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Attn { + pub(crate) head: usize, + pub(crate) kv: usize, + pub(crate) reach: Option, +} + +/// Each layer's layout; the program's coverage check owns which shapes run. +pub(crate) fn layers(hp: &Hparams, reach: &[Option]) -> Vec { + (0..hp.n_layer) + .map(|l| Attn { + head: hp.attn(l).head, + kv: hp.kv[l], + reach: reach[l].map(|r| r as usize), + }) + .collect() +} + +/// The distinct reaches of `layers`, each once: a pass uploads one pair of +/// key bound rows a reach. +pub(crate) fn reaches(layers: &[Attn]) -> Vec> { + let mut r: Vec> = layers.iter().map(|a| a.reach).collect(); + r.sort_unstable(); + r.dedup(); + r +} + +/// Whether `name` is resident on the card: every tensor but the token table +/// (read on the host), `per_layer_model_proj` (uploaded per layer as f32, +/// [`Encoder`]'s `ple`) and the layers' output scalars (read on the host). +fn on_card(name: &str) -> bool { + name != names::token_embd() + && name != names::PER_LAYER_MODEL_PROJ + && !name.ends_with(&format!(".{}", names::LAYER_OUTPUT_SCALE)) +} + +/// The tensor types the pass reads, refused by name otherwise: the token +/// table and `per_layer_model_proj` any type the host decodes, every other +/// matrix q8_0 (the GEMM's 32-value path), every vector f32. +fn check_types(split: &Split) -> Result<(), GpuError> { + for (_, t) in split.iter_tensors() { + let ok = if t.name == names::token_embd() || t.name == names::PER_LAYER_MODEL_PROJ { + t.ty.has_dequant() + } else if t.dims.len() >= 2 { + t.ty == GgmlType::Q8_0 + } else { + t.ty == GgmlType::F32 + }; + if !ok { + return Err(GpuError::Tensor { + what: OPEN, + name: t.name.clone(), + need: "q8_0 for a matrix, f32 for a vector, a host-decoded type for the token \ + table and per_layer_model_proj", + }); + } + } + Ok(()) +} + +/// The widths a pass's buffers are sized by, at `ctx` positions. +pub(crate) struct Sizes { + pub(crate) ctx: usize, + pub(crate) hidden: usize, + /// Query values a position: the widest layer's `n_head · head`. + pub(crate) q: usize, + /// Key (and value) values a position: the widest layer's `kv · head`. + pub(crate) kv: usize, + pub(crate) ff: usize, + pub(crate) per_layer: usize, + pub(crate) out: usize, + /// The widest head: the ones the gainless v norm reads. + pub(crate) head: usize, + /// The widths a GEMM's input is quantized at. + pub(crate) act: Vec, + /// The distinct reaches ([`reaches`]), one pair of key bound rows each. + pub(crate) reaches: usize, +} + +impl Sizes { + pub(crate) fn of(hp: &Hparams, layers: &[Attn], ctx: usize) -> Sizes { + let max = |f: fn(&Attn, usize) -> usize| { + layers.iter().map(|a| f(a, hp.n_head)).max().unwrap_or(0) + }; + let mut act: Vec = layers + .iter() + .map(|a| hp.n_head * a.head) + .chain([hp.n_embd, hp.n_ff, hp.per_layer]) + .collect(); + act.sort_unstable(); + act.dedup(); + Sizes { + ctx, + hidden: hp.n_embd, + q: max(|a, h| h * a.head), + kv: max(|a, _| a.kv * a.head), + ff: hp.n_ff, + per_layer: hp.per_layer, + out: hp.n_out, + head: max(|a, _| a.head), + act, + reaches: reaches(layers).len(), + } + } + + /// The f32 scratch values a position: the pass's buffers, six rows of + /// the hidden width and three of the query, key and feed-forward widths + /// and of the per-layer input. + pub(crate) fn f32_per_pos(&self) -> Result { + size_sum([ + size_product(&[6, self.hidden])?, + size_product(&[3, size_sum([self.q, self.kv, self.ff, self.per_layer])?])?, + ]) + } + + /// The scratch bytes at [`Sizes::ctx`]: the f32 rows, the two f16 cache + /// planes, the positions, the two scalar rows and the per-layer scale + /// rows of `n_layer`, the ones, and the per-pass rows (the token rows, + /// the first keys and key counts of each reach, the output rows); the + /// quantizer scratch and the route table bounded above by two bytes a + /// quantized value and sixteen a position. + fn bytes(&self, n_layer: usize) -> Result { + let ctx = self.ctx; + let quant = self.act.iter().try_fold(0usize, |bytes, &k| { + size_sum([bytes, size_product(&[2, k, ctx])?]) + })?; + size_sum([ + size_product(&[4, self.f32_per_pos()?, ctx])?, + size_product(&[4, self.kv, ctx])?, + size_product(&[4, ctx, size_sum([2, n_layer])?])?, + size_product(&[4, self.head])?, + size_product(&[ + 4, + ctx, + size_sum([self.hidden, size_product(&[2, self.reaches])?, self.out])?, + ])?, + quant, + size_product(&[16, ctx])?, + ]) + } +} + +/// The bytes a load of `split` at `ctx` positions holds on its card: the +/// resident tensors in their device formats ([`resident_size`]), the +/// per-layer projection as f32, the two rope tables and the pass's scratch +/// ([`Sizes`]). The figure a plan reserves for the encoder, and the one its +/// open holds itself to. +pub fn card_bytes(split: &Split, ctx: usize) -> Result { + let (hp, layers) = header(split, ctx)?; + let mut bytes = 0usize; + for (_, t) in split.iter_tensors().filter(|(_, t)| on_card(&t.name)) { + let k = t.dims.first().copied().unwrap_or(1); + let rows = t + .dims + .iter() + .skip(1) + .try_fold(1usize, |n, &d| n.checked_mul(usize::try_from(d).ok()?)) + .ok_or_else(size_overflow)?; + let k = usize::try_from(k).map_err(|_| size_overflow())?; + let resident = resident_size(t.ty, k, rows).ok_or_else(|| GpuError::Tensor { + what: "card_bytes", + name: t.name.clone(), + need: "a type and shape with a device format", + })?; + bytes = size_sum([bytes, resident])?; + } + let s = Sizes::of(&hp, &layers, ctx); + card_figure(bytes, &hp, &s) +} + +fn card_figure(bytes: usize, hp: &Hparams, s: &Sizes) -> Result { + let ple = size_product(&[4, hp.n_layer, hp.per_layer, hp.n_embd])?; + let rope = size_product(&[4, s.ctx, size_sum([hp.swa.head, hp.full.head])?])?; + u64::try_from(size_sum([bytes, ple, rope, s.bytes(hp.n_layer)?])?).map_err(|_| size_overflow()) +} + +// A pass's GEMMs take every position as one column. +const _: () = assert!(CARD_CTX <= GEMM_MAX_SLOTS); + +fn check_program(read: &model::arch::Read) -> Result<(), GpuError> { + let hidden = read.spec.hidden as usize; + let line = bloomery_gpu::gemm::F32_TILE_LINE; + if hidden == 0 || !hidden.is_multiple_of(line) { + return Err(GpuError::Shape { + what: OPEN, + detail: format!( + "embedding_length {hidden}: the per-layer f32 projection requires a positive multiple of {line}" + ), + }); + } + let missing = model::arch::coverage::check(&read.spec, &read.tensors); + if missing.is_empty() { + Ok(()) + } else { + Err(GpuError::plan( + OPEN, + model::placement::PlacementError::Unimplemented(missing).to_string(), + )) + } +} + +/// `split`'s header and layers, and `ctx` checked: at least one position, +/// at most the model card's [`CARD_CTX`] and the file's `context_length`. +fn header(split: &Split, ctx: usize) -> Result<(Hparams, Vec), GpuError> { + let hp = Hparams::read(split).map_err(|e| GpuError::plan(OPEN, e.to_string()))?; + if ctx == 0 || ctx > CARD_CTX || ctx > hp.n_ctx { + return Err(GpuError::Shape { + what: OPEN, + detail: format!( + "a context of {ctx} positions: one pass takes 1 to the model card's {CARD_CTX}, \ + within the file's {}", + hp.n_ctx + ), + }); + } + let read = model::arch::spec(split).map_err(|e| GpuError::plan(OPEN, e.to_string()))?; + check_program(&read)?; + check_types(split)?; + let encoder = read.spec.encoder.as_ref().ok_or(GpuError::State { + what: OPEN, + missing: "the encoder description", + })?; + let layers = layers(&hp, &encoder.reach); + Ok((hp, layers)) +} + +/// The kernels a pass launches besides the [`Gpu`]'s own. +pub(crate) struct Kernels { + pub(crate) gemm: GemmKernels, + pub(crate) g32: Gemm32Kernels, + pub(crate) rope: RopeNeoxKernels, + pub(crate) flash: FlashGqaPrefill, + pub(crate) flash512: FlashGqaPrefill512, +} + +/// A pass's device buffers, sized at the load's context ([`Sizes`]). +pub(crate) struct Scratch { + pub(crate) route: GemmRoute, + /// The quantized GEMM input of each width. + pub(crate) acts: BTreeMap, + pub(crate) x: DeviceBuffer, + pub(crate) xn: DeviceBuffer, + pub(crate) a: DeviceBuffer, + pub(crate) c: DeviceBuffer, + pub(crate) o: DeviceBuffer, + pub(crate) on: DeviceBuffer, + pub(crate) qs: DeviceBuffer, + pub(crate) q: DeviceBuffer, + pub(crate) att: DeviceBuffer, + pub(crate) kk: DeviceBuffer, + pub(crate) vv: DeviceBuffer, + pub(crate) vn: DeviceBuffer, + pub(crate) gate: DeviceBuffer, + pub(crate) up: DeviceBuffer, + pub(crate) gu: DeviceBuffer, + pub(crate) p1: DeviceBuffer, + pub(crate) p2: DeviceBuffer, + pub(crate) p3: DeviceBuffer, + pub(crate) cache_k: DeviceBuffer, + pub(crate) cache_v: DeviceBuffer, +} + +impl Scratch { + /// The f32 bytes held by the buffers, independent of their size formula. + fn f32_bytes(&self) -> Result { + let Scratch { + route: _, + acts: _, + x, + xn, + a, + c, + o, + on, + qs, + q, + att, + kk, + vv, + vn, + gate, + up, + gu, + p1, + p2, + p3, + cache_k: _, + cache_v: _, + } = self; + size_product(&[ + 4, + size_sum( + [ + x, xn, a, c, o, on, qs, q, att, kk, vv, vn, gate, up, gu, p1, p2, p3, + ] + .map(DeviceBuffer::len), + )?, + ]) + } + + fn new(gpu: &Gpu, s: &Sizes) -> Result { + let stream = gpu.stream(); + let f = |w: usize| DeviceBuffer::::zeroed(stream, w * s.ctx); + let mut acts = BTreeMap::new(); + for &k in &s.act { + acts.insert(k, GemmAct32::new(stream, s.ctx, k)?); + } + Ok(Scratch { + route: GemmRoute::new(stream, s.ctx, 1)?, + acts, + x: f(s.hidden)?, + xn: f(s.hidden)?, + a: f(s.hidden)?, + c: f(s.hidden)?, + o: f(s.hidden)?, + on: f(s.hidden)?, + qs: f(s.q)?, + q: f(s.q)?, + att: f(s.q)?, + kk: f(s.kv)?, + vv: f(s.kv)?, + vn: f(s.kv)?, + gate: f(s.ff)?, + up: f(s.ff)?, + gu: f(s.ff)?, + p1: f(s.per_layer)?, + p2: f(s.per_layer)?, + p3: f(s.per_layer)?, + cache_k: DeviceBuffer::::zeroed(stream, s.kv * s.ctx)?, + cache_v: DeviceBuffer::::zeroed(stream, s.kv * s.ctx)?, + }) + } +} + +/// The rows every pass reads unchanged: the ones the v norm's gain is, the +/// scalar `1/√hidden` and each layer's output scale, one a position (the +/// per-token weights `weighted_sum` takes), and the positions `0..ctx`. +pub(crate) struct Consts { + pub(crate) ones: DeviceBuffer, + pub(crate) inv: DeviceBuffer, + pub(crate) scale: Vec>, + pub(crate) pos: DeviceBuffer, +} + +/// The encoder of one `gemma-embedding2` file on one card, sized for passes +/// of up to [`Encoder::ctx`] positions. +pub struct Encoder { + pub(crate) gpu: Gpu, + pub(crate) split: Split, + pub(crate) hp: Hparams, + pub(crate) layers: Vec, + pub(crate) w: Weights, + /// Each layer's rows of `per_layer_model_proj`, `per_layer × hidden` f32. + pub(crate) ple: Vec>, + pub(crate) k: Kernels, + /// The rope rows of the sliding layers and of the full layers. + pub(crate) rope_swa: RopeRows, + pub(crate) rope_full: RopeRows, + pub(crate) consts: Consts, + pub(crate) s: Scratch, + pub(crate) ctx: usize, +} + +impl Encoder { + /// The model of `split` on `gpu`, for passes of up to `ctx` positions. + /// A header the reader refuses, a part the program's coverage does not run, a + /// tensor of a type the pass does not read, a context out of range, or a + /// load that holds more than [`card_bytes`] is refused by name. + pub fn open(split: Split, ctx: usize, gpu: Gpu) -> Result { + let (hp, layers) = header(&split, ctx)?; + let budget = card_bytes(&split, ctx)?; + let stream = gpu.stream(); + let w = Weights::load_where(stream, &split, on_card)?; + let ple = ple_rows(&gpu, &split, &hp)?; + let rope = |a: model::arch::gemma_embedding2::hparams::Attn| { + let table = RopeTable::new(&RopeSpec::window(a.rope_base, a.head))?; + RopeRows::new(stream, &table, a.head, ctx) + }; + let (rope_swa, rope_full) = (rope(hp.swa)?, rope(hp.full)?); + let sizes = Sizes::of(&hp, &layers, ctx); + let s = Scratch::new(&gpu, &sizes)?; + let scale = (0..hp.n_layer) + .map(|l| { + let v = output_scale(&split, l)?; + Ok(DeviceBuffer::from_host(stream, &vec![v; ctx])?) + }) + .collect::, GpuError>>()?; + let pos: Vec = (0..u32::try_from(ctx).map_err(|_| GpuError::Shape { + what: OPEN, + detail: format!("{ctx} positions do not fit u32"), + })?) + .collect(); + let consts = Consts { + ones: DeviceBuffer::from_host(stream, &vec![1.0f32; sizes.head])?, + inv: DeviceBuffer::from_host(stream, &vec![1.0 / (hp.n_embd as f32).sqrt(); ctx])?, + scale, + pos: DeviceBuffer::from_host(stream, &pos)?, + }; + let held = w.resident_bytes() + + ple.iter().map(|t| 4 * t.rows() * t.cols()).sum::() + + 4 * (rope_swa.table.len() + rope_full.table.len()) + + s.acts.values().map(GemmAct32::bytes).sum::() + + s.route.bytes() + + s.f32_bytes()? + + 2 * (s.cache_k.len() + s.cache_v.len()) + + 4 * (consts.ones.len() + + consts.inv.len() + + consts.pos.len() + + consts.scale.iter().map(|s| s.len()).sum::()); + if held as u64 > budget { + return Err(GpuError::Shape { + what: OPEN, + detail: format!("the load holds {held} B, past its card figure of {budget} B"), + }); + } + let k = Kernels { + gemm: GemmKernels::load(gpu.context())?, + g32: Gemm32Kernels::load(gpu.context())?, + rope: RopeNeoxKernels::load(gpu.context())?, + flash: FlashGqaPrefill::load(gpu.context())?, + flash512: FlashGqaPrefill512::load(gpu.context())?, + }; + Ok(Encoder { + gpu, + split, + hp, + layers, + w, + ple, + k, + rope_swa, + rope_full, + consts, + s, + ctx, + }) + } + + /// The most positions one pass takes. + #[must_use] + pub fn ctx(&self) -> usize { + self.ctx + } +} + +/// Rows `rows` of `name` in `split`, `k` values each, decoded to f32 and +/// appended to `out` through the common row layout. +pub(crate) fn decode_rows( + split: &Split, + name: &str, + k: usize, + rows: impl Iterator, + out: &mut Vec, +) -> Result<(), GpuError> { + let layout = model::embed::RowLayout::open(split, name, k)?; + for r in rows { + let id = u32::try_from(r).map_err(|_| GpuError::Shape { + what: "decode_rows", + detail: format!("{name}: row {r} exceeds u32"), + })?; + let src = layout.row(split, id)?; + let at = out.len(); + out.resize(at + k, 0.0); + gguf::quant::dequant_row(layout.ty(), src, &mut out[at..]).map_err(|e| { + GpuError::Shape { + what: "decode_rows", + detail: format!("{name}: {e}"), + } + })?; + } + Ok(()) +} + +/// Each layer's `per_layer` rows of `per_layer_model_proj` (`[hidden, +/// per_layer · n_layer]`, layer `l` the rows `l·per_layer ..`), decoded to +/// f32 on the host and uploaded as one tensor a layer: the f32 tile then +/// writes the layer's slice contiguous, each value the whole projection's. +fn ple_rows(gpu: &Gpu, split: &Split, hp: &Hparams) -> Result>, GpuError> { + let (e, pl) = (hp.n_embd, hp.per_layer); + (0..hp.n_layer) + .map(|l| { + let mut rows = Vec::with_capacity(pl * e); + decode_rows( + split, + names::PER_LAYER_MODEL_PROJ, + e, + l * pl..(l + 1) * pl, + &mut rows, + )?; + DeviceTensor::upload(gpu.stream(), &rows, pl, e) + }) + .collect() +} + +/// Layer `l`'s `layer_output_scale`, one f32. +fn output_scale(split: &Split, l: usize) -> Result { + let name = format!("blk.{l}.{}", names::LAYER_OUTPUT_SCALE); + let missing = || GpuError::Tensor { + what: OPEN, + name: name.clone(), + need: "a tensor of the file", + }; + let (shard, t) = split.find(&name).ok_or_else(missing)?; + let data = split.shard(shard).ok_or_else(missing)?.data(t)?; + let b: [u8; 4] = data + .get(..4) + .and_then(|b| b.try_into().ok()) + .ok_or_else(|| GpuError::Shape { + what: OPEN, + detail: format!("{name} holds {} bytes, not one f32", data.len()), + })?; + Ok(f32::from_le_bytes(b)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn geometry() -> (Hparams, Sizes) { + let a = model::arch::gemma_embedding2::hparams::Attn { + head: 256, + rope_dims: 256, + rope_base: 1.0e6, + }; + let hp = Hparams { + n_layer: 3, + n_embd: 32, + n_ctx: 2, + n_vocab: 4, + rms_eps: 1.0e-6, + n_head: 2, + n_ff: 64, + kv: vec![1; 3], + sliding: vec![true, false, true], + full: a, + swa: a, + window: 4, + per_layer: 32, + n_out: 16, + }; + let s = Sizes::of(&hp, &layers(&hp, &[Some(2), None, Some(2)]), 2); + (hp, s) + } + + pub(super) fn with_model( + hidden: usize, + ff: usize, + check: impl FnOnce(Split, &Hparams) -> T, + ) -> T { + use gguf::Value as V; + use gguf::write::{Layout, TensorDecl, Writer}; + use model::arch::names::blk; + + let (mut hp, _) = geometry(); + hp.n_embd = hidden; + hp.n_ff = ff; + hp.per_layer = 512; + hp.n_out = 768; + let mut kvs = vec![ + ( + gguf::GENERAL_ARCHITECTURE.to_owned(), + V::String("gemma-embedding2".to_owned()), + ), + ( + "tokenizer.ggml.model".to_owned(), + V::String("gemma4".to_owned()), + ), + ( + "tokenizer.ggml.tokens".to_owned(), + V::Array((0..hp.n_vocab).map(|i| V::String(i.to_string())).collect()), + ), + ]; + kvs.extend( + [ + ("block_count", hp.n_layer), + ("embedding_length", hp.n_embd), + ("context_length", hp.n_ctx), + ("feed_forward_length", hp.n_ff), + ("attention.head_count", hp.n_head), + ("attention.sliding_window", hp.window), + ("embedding_length_per_layer_input", hp.per_layer), + ("embedding_length_out", hp.n_out), + ("pooling_type", 1), + ] + .map(|(key, n)| (format!("gemma-embedding2.{key}"), V::U64(n as u64))), + ); + kvs.extend([ + ( + "gemma-embedding2.attention.head_count_kv".to_owned(), + V::Array(hp.kv.iter().map(|&n| V::U64(n as u64)).collect()), + ), + ( + "gemma-embedding2.attention.sliding_window_pattern".to_owned(), + V::Array(hp.sliding.iter().map(|&b| V::Bool(b)).collect()), + ), + ( + "gemma-embedding2.attention.layer_norm_rms_epsilon".to_owned(), + V::F32(hp.rms_eps), + ), + ]); + for (suffix, a) in [("", hp.full), ("_swa", hp.swa)] { + kvs.extend( + [ + ("attention.key_length", a.head), + ("attention.value_length", a.head), + ("rope.dimension_count", a.rope_dims), + ] + .map(|(key, n)| (format!("gemma-embedding2.{key}{suffix}"), V::U64(n as u64))), + ); + kvs.push(( + format!("gemma-embedding2.rope.freq_base{suffix}"), + V::F32(a.rope_base), + )); + } + let e = hp.n_embd as u64; + let ple = hp.per_layer as u64; + let mut tensors = vec![ + (names::token_embd(), vec![e, hp.n_vocab as u64]), + ( + names::PER_LAYER_MODEL_PROJ.to_owned(), + vec![e, ple * hp.n_layer as u64], + ), + (names::PER_LAYER_PROJ_NORM.to_owned(), vec![ple]), + (names::output_norm(), vec![e]), + (names::output(), vec![e, hp.n_out as u64]), + ]; + for l in 0..hp.n_layer { + let d = hp.attn(l).head as u64; + let q = d * hp.n_head as u64; + let k = d * hp.kv[l] as u64; + tensors.extend( + [ + (names::ATTN_NORM, vec![e]), + (names::ATTN_Q, vec![e, q]), + (names::ATTN_K, vec![e, k]), + (names::ATTN_V, vec![e, k]), + (names::ATTN_OUTPUT, vec![q, e]), + (names::ATTN_Q_NORM, vec![d]), + (names::ATTN_K_NORM, vec![d]), + (names::POST_ATTENTION_NORM, vec![e]), + (names::FFN_NORM, vec![e]), + (names::FFN_GATE, vec![e, hp.n_ff as u64]), + (names::FFN_UP, vec![e, hp.n_ff as u64]), + (names::FFN_DOWN, vec![hp.n_ff as u64, e]), + (names::POST_FFW_NORM, vec![e]), + (names::INP_GATE, vec![e, ple]), + (names::PROJ, vec![ple, e]), + (names::POST_NORM, vec![e]), + (names::LAYER_OUTPUT_SCALE, vec![1]), + ] + .map(|(stem, dims)| (blk(l, stem), dims)), + ); + } + let tensors: Vec<_> = tensors + .into_iter() + .map(|(name, dims)| { + let ty = if dims.len() >= 2 + && name != names::token_embd() + && name != names::PER_LAYER_MODEL_PROJ + { + GgmlType::Q8_0 + } else { + GgmlType::F32 + }; + TensorDecl { + name, + nbytes: gguf::tensor_bytes( + ty.blck_size().unwrap(), + ty.type_size().unwrap(), + &dims, + ) + .unwrap(), + dims, + type_id: ty.as_u32(), + } + }) + .collect(); + let layout = Layout::new(&kvs, tensors.clone()).unwrap(); + let path = std::env::temp_dir().join(format!( + "gemma-embedding2-coverage-{}-{:?}-{hidden}-{ff}.gguf", + std::process::id(), + std::thread::current().id(), + )); + let file = std::fs::File::create_new(&path).unwrap(); + let mut writer = Writer::new(file, layout).unwrap(); + for t in tensors { + let nbytes = usize::try_from(t.nbytes).unwrap(); + let bytes = match t.name.as_str() { + name if name == names::token_embd() || name == names::PER_LAYER_PROJ_NORM => { + 1.0f32.to_le_bytes().repeat(nbytes / 4) + } + names::PER_LAYER_MODEL_PROJ => (1.0f32 / 65536.0).to_le_bytes().repeat(nbytes / 4), + _ => vec![0; nbytes], + }; + writer.tensor(&t.name, &bytes).unwrap(); + } + drop(writer.finish().unwrap()); + let result = check(Split::open(&path).unwrap(), &hp); + std::fs::remove_file(&path).unwrap(); + result + } + + #[test] + fn a_shape_outside_the_program_is_refused_by_name() { + for (hidden, ff, refused) in [ + (64, 2048, None), + (32, 2048, Some("embedding_length")), + (64, 4096, Some("dense GELU GLU layer: ff 4096")), + ] { + let result = with_model(hidden, ff, |split, hp| { + if refused.is_none() { + let (_, layers) = header(&split, hp.n_ctx).unwrap(); + assert_eq!( + layers.iter().map(|a| a.reach).collect::>(), + [Some(2), None, Some(2)] + ); + } + card_bytes(&split, hp.n_ctx) + }); + if let Some(what) = refused { + let err = result.unwrap_err().to_string(); + assert!(err.contains(OPEN), "{err}"); + assert!(err.contains(what), "{err}"); + } else { + assert!(result.unwrap() > 0); + } + } + } + + #[test] + fn card_figure_counts_the_buffers_once() { + let (hp, s) = geometry(); + assert_eq!(s.f32_per_pos().unwrap(), 2784); + assert_eq!(s.bytes(hp.n_layer).unwrap(), 28264); + assert_eq!(card_figure(0, &hp, &s).unwrap(), 44648); + } + + #[test] + fn byte_counts_refuse_overflow_by_name() { + let (hp, s) = geometry(); + let mut results = vec![ + size_product(&[4, usize::MAX / 4 + 1]), + size_sum([usize::MAX, 1]), + card_figure(usize::MAX, &hp, &s).map(|n| n as usize), + ]; + for change in [ + |hp: &mut Hparams| hp.n_layer = usize::MAX, + |hp: &mut Hparams| hp.per_layer = usize::MAX, + |hp: &mut Hparams| hp.n_embd = usize::MAX, + |hp: &mut Hparams| hp.swa.head = usize::MAX, + ] { + let mut hp = hp.clone(); + change(&mut hp); + results.push(card_figure(0, &hp, &s).map(|n| n as usize)); + } + for change in [ + |s: &mut Sizes| s.hidden = usize::MAX, + |s: &mut Sizes| s.q = usize::MAX, + |s: &mut Sizes| s.kv = usize::MAX, + |s: &mut Sizes| s.ff = usize::MAX, + |s: &mut Sizes| s.per_layer = usize::MAX, + |s: &mut Sizes| s.out = usize::MAX, + |s: &mut Sizes| s.head = usize::MAX, + |s: &mut Sizes| s.ctx = usize::MAX, + |s: &mut Sizes| s.act = vec![usize::MAX], + ] { + let (_, mut s) = geometry(); + change(&mut s); + results.push(s.bytes(hp.n_layer)); + } + for result in results { + assert!(matches!( + result, + Err(GpuError::Shape { what: "card_bytes", detail }) + if detail.contains("overflows") + )); + } + } + + #[test] + #[ignore = "needs a CUDA card"] + fn hw_scratch_census_reads_buffer_lengths() { + let gpu = Gpu::new().unwrap(); + let (_, sizes) = geometry(); + let mut scratch = Scratch::new(&gpu, &sizes).unwrap(); + let bytes = scratch.f32_bytes().unwrap(); + assert_eq!(bytes, 22272); + scratch.x = DeviceBuffer::zeroed(gpu.stream(), scratch.x.len() + 1).unwrap(); + assert_eq!(scratch.f32_bytes().unwrap(), bytes + 4); + } +} diff --git a/crates/gpu-gemma-embedding2/src/pass.rs b/crates/gpu-gemma-embedding2/src/pass.rs new file mode 100644 index 000000000..9be7b1d05 --- /dev/null +++ b/crates/gpu-gemma-embedding2/src/pass.rs @@ -0,0 +1,580 @@ +//! One pass of the encoder over an input's ids (the crate doc's order), its +//! launches all on the [`Gpu`]'s stream, and the rows it reads back. + +use std::collections::BTreeMap; + +use bloomery_gpu::flash_gqa_prefill::GqaPrefillArgs; +use bloomery_gpu::gemm::{GemmAct32, GemmInput, GemmRoute}; +use bloomery_gpu::rope_neox::{HEAD_256, PartialNeoxArgs}; +use bloomery_gpu::rope_table::RopeRows; +use bloomery_gpu::site::{self, SiteTy, SiteWeight, WideIn, WideKernels}; +use bloomery_gpu::weights::Weights; +use bloomery_gpu::{DeviceTensor, FaultSink, Gpu, GpuError}; +use cuda_core::DeviceBuffer; +use model::arch::gemma_embedding2::hparams::{Hparams, key_bounds}; +use model::arch::gemma_embedding2::names; + +use crate::{Attn, Consts, Encoder, Kernels, Scratch, decode_rows, reaches}; + +/// The refusal `what` of a pass. +const PASS: &str = "Encoder::pass"; + +/// One pass's rows read back: the `output` projection of every position +/// (`result_embd`, `n · n_out` f32, token-major), their mean (`pooled`, +/// `n_out` f32), and the taps when asked. +pub struct Pass { + pub rows: Vec, + pub pooled: Vec, + pub taps: Option, +} + +/// A pass's intermediate rows, each token-major in the reference's order. +pub struct Taps { + /// The token rows times `√hidden`, `n · hidden`. + pub inp_scaled: Vec, + /// Every layer's normed input slice, `[per_layer, n_layer, n]`: position + /// `t`'s layer `l` at `(t·n_layer + l)·per_layer`. + pub inp_per_layer: Vec, + /// Each layer's output, `n · hidden`. + pub l_out: Vec>, + /// The output norm's rows, `n · hidden`. + pub result_norm: Vec, +} + +/// What a projection reads besides its input: the card, the kernels, the +/// weights and the one-expert table filled for the pass's columns. +struct Cx<'a> { + gpu: &'a Gpu, + k: &'a Kernels, + w: &'a Weights, + route: &'a GemmRoute, +} + +impl Cx<'_> { + /// `x`'s `n` columns of `width` values quantized into `acts`' scratch of + /// that width, which the next [`Cx::proj`]s at `width` read. + fn quantize( + &self, + acts: &mut BTreeMap, + x: &DeviceBuffer, + width: usize, + n: usize, + sink: FaultSink, + ) -> Result<(), GpuError> { + let act = acts.get_mut(&width).ok_or(GpuError::State { + what: PASS, + missing: "a quantizer scratch of the projection's width", + })?; + self.k + .g32 + .enqueue_quantize_gemm32(self.gpu.stream(), x, n, act, sink) + } + + /// `y = W·x` for the q8_0 weight `name` of `rows` rows over the `n` + /// columns [`Cx::quantize`] last wrote at `width`, token-major. + fn proj( + &self, + acts: &BTreeMap, + name: &str, + (width, rows): (usize, usize), + n: usize, + y: &mut DeviceBuffer, + ) -> Result<(), GpuError> { + let act = acts.get(&width).ok_or(GpuError::State { + what: PASS, + missing: "a quantizer scratch of the projection's width", + })?; + site::gemm( + self.gpu, + &WideKernels { + gemm: &self.k.gemm, + g32: Some(&self.k.g32), + }, + (SiteTy::Q8_0, self.w, SiteWeight::Resident(name)), + rows, + &WideIn { + q128: None, + q32: Some(act), + f32: None, + }, + (self.route, GemmInput::PerSlot), + n, + y, + ) + } +} + +/// A reach's per-row key bounds on the card: row `t` attends keys +/// `[first[t], count[t])` ([`key_bounds`]). +struct Keys { + reach: Option, + first: DeviceBuffer, + count: DeviceBuffer, +} + +impl Keys { + fn new(gpu: &Gpu, reach: Option, n: usize) -> Result { + let narrow = |v: usize| { + u32::try_from(v).map_err(|_| GpuError::Shape { + what: PASS, + detail: format!("a key bound of {v} over {n} positions does not fit u32"), + }) + }; + let mut first = Vec::with_capacity(n); + let mut count = Vec::with_capacity(n); + for t in 0..n { + let (f, c) = key_bounds(reach, t, n); + first.push(narrow(f)?); + count.push(narrow(c)?); + } + Ok(Keys { + reach, + first: DeviceBuffer::from_host(gpu.stream(), &first)?, + count: DeviceBuffer::from_host(gpu.stream(), &count)?, + }) + } +} + +/// What every step of a pass reads unchanged: the card, the kernels, the +/// weights, the header, the constant rows, and the pass's `n` positions in +/// a load of `cap`. +struct Step<'a> { + gpu: &'a Gpu, + k: &'a Kernels, + w: &'a Weights, + hp: &'a Hparams, + consts: &'a Consts, + n: usize, + cap: usize, +} + +impl Step<'_> { + fn gain(&self, name: &str) -> Result<&DeviceBuffer, GpuError> { + self.w.f32_buf(PASS, name) + } + + fn cx<'b>(&'b self, route: &'b GemmRoute) -> Cx<'b> { + Cx { + gpu: self.gpu, + k: self.k, + w: self.w, + route, + } + } + + /// Layer `l`'s attention over `emb` at layer 0, else over `s.x`: `s.a` + /// is the input plus the post-normed attention of its norm, the turn by + /// `turn`'s rows and the flash over `keys`' bounds. + fn attention( + &self, + s: &mut Scratch, + (l, a): (usize, &Attn), + emb: Option<&DeviceBuffer>, + turn: &RopeRows, + keys: &Keys, + ) -> Result<(), GpuError> { + let (stream, elem, sink) = (self.gpu.stream(), self.gpu.elem(), self.gpu.layer_sink(l)?); + let (e, n, n_head, eps) = (self.hp.n_embd, self.n, self.hp.n_head, self.hp.rms_eps); + let (head, kv, qw) = (a.head, a.kv, n_head * a.head); + let blk = |stem: &str| format!("blk.{l}.{stem}"); + let inp = match emb { + Some(emb) => emb, + None => &s.x, + }; + let cx = self.cx(&s.route); + let acts = &mut s.acts; + elem.enqueue_rms_norm( + stream, + inp, + self.gain(&blk(names::ATTN_NORM))?, + eps, + e, + n, + &mut s.xn, + )?; + cx.quantize(acts, &s.xn, e, n, sink)?; + cx.proj(acts, &blk(names::ATTN_Q), (e, qw), n, &mut s.qs)?; + cx.proj(acts, &blk(names::ATTN_K), (e, kv * head), n, &mut s.kk)?; + cx.proj(acts, &blk(names::ATTN_V), (e, kv * head), n, &mut s.vv)?; + elem.enqueue_rms_norm( + stream, + &s.vv, + &self.consts.ones, + eps, + head, + n * kv, + &mut s.vn, + )?; + let rope = PartialNeoxArgs { + qg: &s.qs, + q: &mut s.q, + k: &mut s.kk, + v: &s.vn, + gq: self.gain(&blk(names::ATTN_Q_NORM))?, + gk: self.gain(&blk(names::ATTN_K_NORM))?, + table: &turn.table, + pos: &self.consts.pos, + eps, + n_head, + n_kv: kv, + ctx: self.cap, + m: n, + fault: sink, + cache_k: &mut s.cache_k, + cache_v: &mut s.cache_v, + }; + if head == HEAD_256 { + // SAFETY: `pos` is `0..ctx` (`Consts::pos`) and a pass's `n` is at + // most `ctx`, so the `n` tokens hold distinct positions. + unsafe { + self.k + .rope + .enqueue_head_norm_neox_append_256_whole(stream, rope)?; + } + } else { + // SAFETY: `pos` is `0..ctx` (`Consts::pos`) and a pass's `n` is at + // most `ctx`, so the `n` tokens hold distinct positions. + unsafe { + self.k + .rope + .enqueue_head_norm_neox_append_512(stream, rope)?; + } + } + let args = GqaPrefillArgs { + q: &s.q, + kc: &s.cache_k, + vc: &s.cache_v, + n_keys: &keys.count, + scale: 1.0, + n_head, + n_kv: kv, + ctx: self.cap, + t: n, + fault: sink, + y: &mut s.att, + }; + if head == HEAD_256 { + self.k + .flash + .enqueue_256_p2_lower(stream, args, &keys.first)?; + } else { + self.k.flash512.enqueue_p4(stream, args)?; + } + cx.quantize(acts, &s.att, qw, n, sink)?; + cx.proj(acts, &blk(names::ATTN_OUTPUT), (qw, e), n, &mut s.o)?; + elem.enqueue_rms_norm( + stream, + &s.o, + self.gain(&blk(names::POST_ATTENTION_NORM))?, + eps, + e, + n, + &mut s.on, + )?; + elem.enqueue_add(stream, &s.on, inp, e * n, &mut s.a)?; + Ok(()) + } + + /// Layer `l`'s GELU GLU: `s.c` is `s.a` plus the post-normed + /// feed-forward of its norm. + fn glu(&self, s: &mut Scratch, l: usize) -> Result<(), GpuError> { + let (stream, elem, sink) = (self.gpu.stream(), self.gpu.elem(), self.gpu.layer_sink(l)?); + let (e, n, ff, eps) = (self.hp.n_embd, self.n, self.hp.n_ff, self.hp.rms_eps); + let blk = |stem: &str| format!("blk.{l}.{stem}"); + let cx = self.cx(&s.route); + let acts = &mut s.acts; + elem.enqueue_rms_norm( + stream, + &s.a, + self.gain(&blk(names::FFN_NORM))?, + eps, + e, + n, + &mut s.xn, + )?; + cx.quantize(acts, &s.xn, e, n, sink)?; + cx.proj(acts, &blk(names::FFN_GATE), (e, ff), n, &mut s.gate)?; + cx.proj(acts, &blk(names::FFN_UP), (e, ff), n, &mut s.up)?; + elem.enqueue_geglu(stream, &s.gate, &s.up, ff * n, &mut s.gu)?; + cx.quantize(acts, &s.gu, ff, n, sink)?; + cx.proj(acts, &blk(names::FFN_DOWN), (ff, e), n, &mut s.o)?; + elem.enqueue_rms_norm( + stream, + &s.o, + self.gain(&blk(names::POST_FFW_NORM))?, + eps, + e, + n, + &mut s.on, + )?; + elem.enqueue_add(stream, &s.on, &s.a, e * n, &mut s.c)?; + Ok(()) + } + + /// Layer `l`'s per-layer input and output scale: `s.x` is the layer's + /// scale times `s.c` plus the post-normed `proj` of `gelu(inp_gate · + /// s.c) · ple_l`, where `ple_l` is `ple · emb` times `1/√hidden`, + /// RMS-normed, and copied into `taps` when asked. + fn per_layer( + &self, + s: &mut Scratch, + l: usize, + ple: &DeviceTensor, + emb: &DeviceBuffer, + taps: Option<&mut Taps>, + ) -> Result<(), GpuError> { + let (stream, elem, sink) = (self.gpu.stream(), self.gpu.elem(), self.gpu.layer_sink(l)?); + let (e, n, pl, eps) = (self.hp.n_embd, self.n, self.hp.per_layer, self.hp.rms_eps); + let blk = |stem: &str| format!("blk.{l}.{stem}"); + let cx = self.cx(&s.route); + let acts = &mut s.acts; + cx.quantize(acts, &s.c, e, n, sink)?; + cx.proj(acts, &blk(names::INP_GATE), (e, pl), n, &mut s.p1)?; + self.k + .g32 + .enqueue_f32_tile(stream, ple, emb, n, &mut s.p2)?; + elem.enqueue_weighted_sum(stream, &s.p2, &self.consts.inv, pl, 1, n, &mut s.p3)?; + elem.enqueue_rms_norm( + stream, + &s.p3, + self.gain(names::PER_LAYER_PROJ_NORM)?, + eps, + pl, + n, + &mut s.p2, + )?; + if let Some(t) = taps { + let rows = s.p2.to_host_vec(stream)?; + for (tok, row) in rows[..n * pl].chunks_exact(pl).enumerate() { + let at = (tok * self.hp.n_layer + l) * pl; + t.inp_per_layer[at..at + pl].copy_from_slice(row); + } + } + elem.enqueue_geglu(stream, &s.p1, &s.p2, pl * n, &mut s.p3)?; + cx.quantize(acts, &s.p3, pl, n, sink)?; + cx.proj(acts, &blk(names::PROJ), (pl, e), n, &mut s.o)?; + elem.enqueue_rms_norm( + stream, + &s.o, + self.gain(&blk(names::POST_NORM))?, + eps, + e, + n, + &mut s.on, + )?; + elem.enqueue_add(stream, &s.c, &s.on, e * n, &mut s.a)?; + elem.enqueue_weighted_sum(stream, &s.a, &self.consts.scale[l], e, 1, n, &mut s.x)?; + Ok(()) + } +} + +impl Encoder { + /// The token rows of `ids` from the file's table, each times `√hidden` + /// in f32 (`inp_scaled`); an id past the vocabulary is refused by name. + fn token_rows(&self, ids: &[u32]) -> Result, GpuError> { + let e = self.hp.n_embd; + if let Some(&bad) = ids.iter().find(|&&id| id as usize >= self.hp.n_vocab) { + return Err(GpuError::Shape { + what: PASS, + detail: format!("id {bad} is past the vocabulary of {}", self.hp.n_vocab), + }); + } + let mut rows = Vec::with_capacity(ids.len() * e); + decode_rows( + &self.split, + &names::token_embd(), + e, + ids.iter().map(|&id| id as usize), + &mut rows, + )?; + let scale = (e as f32).sqrt(); + rows.iter_mut().for_each(|v| *v *= scale); + Ok(rows) + } + + /// One pass over `ids` (1 to [`Encoder::ctx`] of them, else refused by + /// name): every position's `output` row and their mean, and with `taps` + /// the rows the reference names on the way. A fault any launch raised is + /// the error, and so is an output value that is not finite, by its + /// position. + pub fn pass(&mut self, ids: &[u32], taps: bool) -> Result { + let n = ids.len(); + if n == 0 || n > self.ctx { + return Err(GpuError::Shape { + what: PASS, + detail: format!("{n} positions; a pass takes 1 to {}", self.ctx), + }); + } + let inp_scaled = self.token_rows(ids)?; + let Encoder { + gpu, + hp, + layers, + w, + ple, + k, + rope_swa, + rope_full, + consts, + s, + ctx, + .. + } = self; + let stream = gpu.stream(); + let (e, n_out) = (hp.n_embd, hp.n_out); + let emb = DeviceBuffer::from_host(stream, &inp_scaled)?; + let keys = reaches(layers) + .into_iter() + .map(|r| Keys::new(gpu, r, n)) + .collect::, _>>()?; + k.gemm + .enqueue_route_dense(stream, n, &mut s.route, gpu.unlabelled_sink())?; + let st = Step { + gpu, + k, + w, + hp, + consts, + n, + cap: *ctx, + }; + let mut tapped = taps.then(|| Taps { + inp_scaled: inp_scaled.clone(), + inp_per_layer: vec![0.0; n * hp.n_layer * hp.per_layer], + l_out: Vec::new(), + result_norm: Vec::new(), + }); + for (l, a) in layers.iter().enumerate() { + let turn = if a.reach.is_some() { + &*rope_swa + } else { + &*rope_full + }; + let bounds = keys + .iter() + .find(|r| r.reach == a.reach) + .ok_or(GpuError::State { + what: PASS, + missing: "the key bounds of the layer's reach", + })?; + st.attention(s, (l, a), (l == 0).then_some(&emb), turn, bounds)?; + st.glu(s, l)?; + st.per_layer(s, l, &ple[l], &emb, tapped.as_mut())?; + if let Some(t) = &mut tapped { + t.l_out.push(s.x.to_host_vec(stream)?[..n * e].to_vec()); + } + } + let sink = gpu.unlabelled_sink(); + gpu.elem().enqueue_rms_norm( + stream, + &s.x, + st.gain(&names::output_norm())?, + hp.rms_eps, + e, + n, + &mut s.xn, + )?; + let cx = st.cx(&s.route); + cx.quantize(&mut s.acts, &s.xn, e, n, sink)?; + let mut out = DeviceBuffer::::zeroed(stream, n * n_out)?; + cx.proj(&s.acts, &names::output(), (e, n_out), n, &mut out)?; + let rows = out.to_host_vec(stream)?; + if let Some(t) = &mut tapped { + t.result_norm = s.xn.to_host_vec(stream)?[..n * e].to_vec(); + } + if let Some(f) = gpu.take_fault()? { + return Err(GpuError::fault(PASS, f)); + } + if let Some(i) = rows.iter().position(|v| !v.is_finite()) { + return Err(GpuError::Shape { + what: PASS, + detail: format!( + "position {}'s output value {} is {}, not finite", + i / n_out, + i % n_out, + rows[i] + ), + }); + } + let pooled = mean(&rows, n_out); + Ok(Pass { + rows, + pooled, + taps: tapped, + }) + } +} + +/// The mean of `rows`, at least one whole row of `width` f32: each column +/// summed in f64 in position order and divided by the row count. +fn mean(rows: &[f32], width: usize) -> Vec { + let n = rows.len() / width; + let mut sum = vec![0.0f64; width]; + for row in rows.chunks_exact(width) { + for (s, &x) in sum.iter_mut().zip(row) { + *s += f64::from(x); + } + } + sum.iter().map(|&s| (s / n as f64) as f32).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + #[ignore = "needs a CUDA card"] + fn hw_per_layer_projection_scale_near_epsilon() { + let (hp, weights, gain, pass) = + crate::tests::with_model(64, 2048, |split, hp| -> Result<_, GpuError> { + let mut weights = Vec::new(); + decode_rows( + &split, + names::PER_LAYER_MODEL_PROJ, + hp.n_embd, + 0..hp.n_layer * hp.per_layer, + &mut weights, + )?; + let mut enc = Encoder::open(split, hp.n_ctx, Gpu::new()?)?; + let gain = enc + .w + .f32_buf(PASS, names::PER_LAYER_PROJ_NORM)? + .to_host_vec(enc.gpu.stream())?; + Ok((hp.clone(), weights, gain, enc.pass(&[0, 1], true)?)) + }) + .unwrap(); + let taps = pass.taps.unwrap(); + assert_eq!(taps.inp_scaled.len(), 2 * hp.n_embd); + assert_eq!(taps.inp_per_layer.len(), 2 * hp.n_layer * hp.per_layer); + let eps = f64::from(hp.rms_eps); + for (t, input) in taps.inp_scaled.chunks_exact(hp.n_embd).enumerate() { + for (l, layer) in weights.chunks_exact(hp.per_layer * hp.n_embd).enumerate() { + let projected: Vec = layer + .chunks_exact(hp.n_embd) + .map(|row| { + row.iter() + .zip(input) + .map(|(&w, &x)| f64::from(w) * f64::from(x)) + .sum::() + / (hp.n_embd as f64).sqrt() + }) + .collect(); + let mean_square = + projected.iter().map(|x| x * x).sum::() / hp.per_layer as f64; + assert!( + (eps / 2.0..eps * 2.0).contains(&mean_square), + "{mean_square:e}" + ); + let at = (t * hp.n_layer + l) * hp.per_layer; + for (i, (&x, &g)) in projected.iter().zip(&gain).enumerate() { + let expected = x / (mean_square + eps).sqrt() * f64::from(g); + let actual = f64::from(taps.inp_per_layer[at + i]); + assert!( + (actual - expected).abs() < 1.0e-5, + "token {t} layer {l} row {i}: {actual} != {expected}" + ); + } + } + } + } +} diff --git a/crates/gpu/src/fault.rs b/crates/gpu/src/fault.rs index 1609a07d6..adbdcf7da 100644 --- a/crates/gpu/src/fault.rs +++ b/crates/gpu/src/fault.rs @@ -490,8 +490,9 @@ pub mod step_order { /// still decodes; it is not a step this architecture runs. pub const NEMOTRON_H: &[FaultSite] = MIMO2; - /// No program runs either Gemma encoder. The order is [`MIMO2`]'s so a - /// fault word still decodes; it is not a step either architecture runs. + /// The Gemma encoders have no step order of their own. The order is + /// [`MIMO2`]'s so a fault word still decodes; it is not the order the + /// `gemma-embedding2` encoder's pass raises its sites in. pub const GEMMA_ENCODER: &[FaultSite] = MIMO2; /// No program runs `nemotron_h_moe`. The order is [`MIMO2`]'s so a fault diff --git a/crates/gpu/src/gemm/f32tile.rs b/crates/gpu/src/gemm/f32tile.rs index c7edab30a..471f2b860 100644 --- a/crates/gpu/src/gemm/f32tile.rs +++ b/crates/gpu/src/gemm/f32tile.rs @@ -38,7 +38,8 @@ const COL_GROUPS: usize = TILE_COLS / TILE; /// 32-value chunks one stage holds of every row and column, and the values /// of one such line: K must be a multiple of it. const STAGE_CHUNKS: usize = 2; -pub(super) const LINE: usize = 32 * STAGE_CHUNKS; +/// Values in a staged F32 tile line; K must be a positive multiple. +pub const LINE: usize = 32 * STAGE_CHUNKS; /// Stages in shared memory: the one the warps read and two being copied in. pub(super) const STAGES: usize = 3; /// One stage: every row's line, then every column's. diff --git a/crates/gpu/src/gemm/mod.rs b/crates/gpu/src/gemm/mod.rs index 80d68fb65..04c4a04cc 100644 --- a/crates/gpu/src/gemm/mod.rs +++ b/crates/gpu/src/gemm/mod.rs @@ -63,6 +63,7 @@ mod kernels32; pub use act::GemmAct; pub use act32::GemmAct32; +pub use f32tile::LINE as F32_TILE_LINE; pub use gemm32::{Gemm32Args, Gemm32Weight}; pub use grouped::{GemmArgs, GemmInput, GemmWeight}; pub use kernels::GemmKernels; diff --git a/crates/gpu/src/rope_neox.rs b/crates/gpu/src/rope_neox.rs index 47299b144..1e5424bc7 100644 --- a/crates/gpu/src/rope_neox.rs +++ b/crates/gpu/src/rope_neox.rs @@ -42,6 +42,9 @@ //! global layers: NEOX pairs `(i, i + 256)`), 256 threads a head, the eight //! warps' sums in warp order, the query read from plain query rows at head //! stride [`HEAD_512`] and written to its own rows. +//! [`RopeNeoxKernels::enqueue_head_norm_neox_append_256_whole`] is that body +//! at [`HEAD_256`] (EmbeddingGemma 2's sliding layers: NEOX pairs `(i, i + +//! 128)`), 128 threads a head, the four warps' sums in warp order. //! //! The `_q8` entries ([`RopeNeoxKernels::enqueue_head_norm_neox_append_q8`], //! [`RopeNeoxKernels::enqueue_head_norm_neox_append_256_q8`]) are the same two @@ -172,7 +175,8 @@ const _: () = assert!(V_HEAD_K192 / 2 <= THREADS_K192 && ROT_K192 / 2 <= THREADS /// over heads of `HEAD` values, the first `ROT` turned, `WARPS = HEAD/64` /// warps a block, into a cache in the format `S` (that entry's body over /// [`F16Sink`], `head_norm_neox_append_256_q8`'s over [`Q8Sink`], and -/// `head_norm_neox_append_512`'s, every value turned, over [`F16Sink`]). +/// `head_norm_neox_append_512`'s and `head_norm_neox_append_256_whole`'s, +/// every value turned, over [`F16Sink`]). /// Block `b = t·(n_head + n_kv) + h`. The query head `h` of token `t` is /// read from its source rows at `qg[(t·n_head + h)·Q_SPAN ..]` (the q+gate /// rows at `Q_SPAN = 2·HEAD`, each head's query and then its gate, or plain @@ -1163,6 +1167,81 @@ mod rope_neox_kernels { }; } + /// [`head_norm_neox_append_512`] over heads of [`HEAD_256`] values + /// (`norm_partial_append` at `ROT = HEAD = 256`): every value turns, the + /// query heads read from plain query rows `qs` (`256` values a head). The + /// rope table holds [`HEAD_256`] f32 per position. + #[allow( + clippy::too_many_arguments, + reason = "kernel entry: the device ABI takes the arguments flat (rust-quality R8)" + )] + #[kernel] + #[launch_bounds(128)] + #[launch_contract( + domain = 1, + block = (128, 1, 1), + requires = ( + qs.len() >= m * n_head * 256, + q.len() >= m * n_head * 256, + k.len() >= m * n_kv * 256, + v.len() >= m * n_kv * 256, + gq.len() >= 256, + gk.len() >= 256, + table.len() >= ctx * 256, + pos.len() >= m, + cache_k.len() >= n_kv * ctx * 256, + cache_v.len() >= n_kv * ctx * 256 + ) + )] + pub fn head_norm_neox_append_256_whole( + gq: &[f32], + gk: &[f32], + table: &[f32], + pos: &[u32], + qs: &[f32], + v: &[f32], + eps: f32, + n_head: u32, + n_kv: u32, + ctx: u32, + m: u32, + fault: FaultSink, + q: DisjointSlice, + k: DisjointSlice, + cache_k: DisjointSlice, + cache_v: DisjointSlice, + ) { + static mut WSUM4W: SharedArray = SharedArray::UNINIT; + + // SAFETY: WSUM4W is this block's own shared allocation of WARPS_256 + // slots; the raw form reaches it without a reference. The launch + // contract is `norm_partial_append`'s at (HEAD_256, HEAD_256) with + // plain query rows over the f16 planes. + unsafe { + norm_partial_append::( + gq, + gk, + table, + pos, + qs, + v, + eps, + n_head, + n_kv, + ctx, + m, + fault, + q, + k, + F16Sink { + k: cache_k, + v: cache_v, + }, + SharedArray::as_raw_mut_ptr(&raw mut WSUM4W), + ) + }; + } + /// [`head_norm_neox_append`] with the append quantized to Q8_0 (the /// module doc's `_q8` paragraph): the norm, the turn and the in-place /// query and key writes are that entry's bit for bit, and each key and @@ -1667,12 +1746,14 @@ pub struct NeoxArgs<'a> { /// The arguments of [`RopeNeoxKernels::enqueue_head_norm_neox_append_256`] /// and [`RopeNeoxKernels::enqueue_head_norm_neox_append_512`] at heads of -/// `HEAD` values ([`HEAD_256`] or [`HEAD_512`]): `m` tokens' query source -/// rows `qg` (`n_head` heads a token; at [`HEAD_256`] a head is its -/// `2·HEAD` q+gate values, the query first, at [`HEAD_512`] its `HEAD` query -/// values), the query output `q` (`n_head` heads of `HEAD`), the `n_kv` key -/// heads in `k` (turned in place) and value heads in `v`, the gains, the rope -/// table (at least `ctx` rows of [`ROT_256`] or [`HEAD_512`] f32), the +/// `HEAD` values ([`HEAD_256`] or [`HEAD_512`]), and of +/// [`RopeNeoxKernels::enqueue_head_norm_neox_append_256_whole`] at +/// [`HEAD_256`]: `m` tokens' query source rows `qg` (`n_head` heads a token; +/// for the 256 entry a head is its `2·HEAD` q+gate values, the query first, +/// for the 512 and whole-turn entries its `HEAD` query values), the query +/// output `q` (`n_head` heads of `HEAD`), the `n_kv` key heads in `k` (turned +/// in place) and value heads in `v`, the gains, the rope table (at least +/// `ctx` rows of [`ROT_256`], or of `HEAD` f32 for a whole turn), the /// positions, the sink, and the layer's two cache planes of `n_kv · ctx` rows /// of `HEAD` f16. pub struct PartialNeoxArgs<'a> { @@ -2081,6 +2162,79 @@ impl RopeNeoxKernels { Ok(()) } + /// [`RopeNeoxKernels::enqueue_head_norm_neox_append_512`] over heads of + /// [`HEAD_256`] values that turn whole (EmbeddingGemma 2's sliding + /// layers: NEOX pairs `(i, i + 128)`): `m·(n_head + n_kv)` blocks of 128 + /// threads. `args` holds the 512 entry's buffers at [`HEAD_256`] values a + /// head: `qg` (plain query rows) and `q` `m·n_head·256` f32, `k` and `v` + /// `m·n_kv·256`, the gains 256, `table` [`HEAD_256`] f32 for each of at + /// least `ctx` positions (`RopeTable::push` over a [`HEAD_256`]-wide + /// spec), the planes `n_kv·ctx·256` f16. A position at or past `ctx` + /// raises [`FaultSite::CachePos`] as there. Asynchronous, + /// allocation-free, capturable. + /// + /// # Safety + /// + /// The `m` positions of `args.pos` are distinct: two tokens at one + /// position write the same cache rows from two blocks at once. + pub unsafe fn enqueue_head_norm_neox_append_256_whole( + &self, + stream: &CudaStream, + args: PartialNeoxArgs<'_>, + ) -> Result<(), GpuError> { + let what = "enqueue_head_norm_neox_append_256_whole"; + let PartialNeoxArgs { + qg, + q, + k, + v, + gq, + gk, + table, + pos, + eps, + n_head, + n_kv, + ctx, + m, + fault, + cache_k, + cache_v, + } = args; + let plane = n_kv * ctx * HEAD_256; + let [grid, n_head, n_kv, ctx, m] = partial_launch( + what, + (HEAD_256, HEAD_256, HEAD_256), + [n_head, n_kv, ctx, m], + [ + qg.len(), + q.len(), + k.len(), + v.len(), + gq.len(), + gk.len(), + table.len(), + pos.len(), + ], + &[ + ("cache_k", cache_k.len(), plane), + ("cache_v", cache_v.len(), plane), + ], + )?; + let prep = self + .module + .prepare_head_norm_neox_append_256_whole(LaunchConfig1D::new( + grid, + THREADS_256_U32, + 0, + ))?; + self.module.head_norm_neox_append_256_whole( + stream, &prep, gq, gk, table, pos, qg, v, eps, n_head, n_kv, ctx, m, fault, q, k, + cache_k, cache_v, + )?; + Ok(()) + } + /// [`RopeNeoxKernels::enqueue_head_norm_neox_append`] with the append /// quantized to Q8_0 (the module doc's `_q8` paragraph): the same norm, /// turn and in-place writes, and each key and value row stored as four diff --git a/crates/model/src/arch/coverage.rs b/crates/model/src/arch/coverage.rs index d3c4ff027..261c090a1 100644 --- a/crates/model/src/arch/coverage.rs +++ b/crates/model/src/arch/coverage.rs @@ -593,6 +593,77 @@ pub const AVAILABLE: &[Available] = &[ ) }, }, + Available { + programs: &[Program::GemmaEmbedding2Body], + at: "gpu/src/flash_gqa_prefill.rs enqueue_256_p2_lower (each row's first key from its \ + reach; two query heads a block)", + runs: |n| matches!(n, Need::Bidirectional { head: 256, group, .. } if group.is_multiple_of(2)), + }, + Available { + programs: &[Program::GemmaEmbedding2Body], + at: "gpu/src/flash_gqa_prefill_512.rs enqueue_p4 (every key; four query heads a block)", + runs: |n| { + matches!( + n, + Need::Bidirectional { + head: 512, + group, + reach: None + } if group.is_multiple_of(4) + ) + }, + }, + Available { + programs: &[Program::GemmaEmbedding2Body], + at: "gpu/src/rope_neox.rs head_norm_neox_append_256_whole, head_norm_neox_append_512", + runs: |n| { + matches!( + n, + Need::QkRope { + qk_norm: true, + head: head @ (256 | 512), + mode: RopeMode::Neox, + dims, + } if dims == head + ) + }, + }, + Available { + programs: &[Program::GemmaEmbedding2Body], + at: "gpu/src/elem.rs enqueue_geglu between gpu/src/site.rs gemm launches", + runs: |n| matches!(n, Need::GeGlu { ff: 2048 }), + }, + Available { + programs: &[Program::GemmaEmbedding2Body], + at: "gpu/src/elem.rs enqueue_rms_norm (a gain of ones)", + runs: |n| matches!(n, Need::ValueNorm { .. }), + }, + Available { + programs: &[Program::GemmaEmbedding2Body], + at: "gpu/src/elem.rs enqueue_rms_norm, enqueue_add (gemma-embedding2's pass)", + runs: |n| matches!(n, Need::PostNorms), + }, + Available { + programs: &[Program::GemmaEmbedding2Body], + at: "gpu/src/elem.rs enqueue_weighted_sum (the layer's scalar)", + runs: |n| matches!(n, Need::LayerScale), + }, + Available { + programs: &[Program::GemmaEmbedding2Body], + at: "gpu/src/gemm/f32tile.rs enqueue_f32_tile; gpu/src/elem.rs enqueue_rms_norm, \ + enqueue_geglu (gemma-embedding2's pass)", + runs: |n| matches!(n, Need::PerLayerInput { width: 512 }), + }, + Available { + programs: &[Program::GemmaEmbedding2Body], + at: "gpu-gemma-embedding2/src/pass.rs token_rows (the host's rows, scaled)", + runs: |n| matches!(n, Need::EmbedScale), + }, + Available { + programs: &[Program::GemmaEmbedding2Body], + at: "gpu/src/site.rs gemm (output); gpu-gemma-embedding2/src/pass.rs mean", + runs: |n| matches!(n, Need::Pooled { proj } if proj.as_slice() == [768]), + }, ]; /// Whether the K192 flash row ([`gqa_row_v`] at head 192, value 128) takes @@ -670,7 +741,8 @@ const NEMOTRON_SITES: &[GgmlType] = &[ /// chain's attention gemvs, whose head-split sites in `chain/attn.rs` take /// q3_K or q8_0, the same head, and the hyper-connection fn `hc.rs` reads as /// q3_K words, `hc_f32.rs` as f32; the glm5next body's sites, each a q8_0 gemv or `hc_pre_q8_0`, the common -/// head's q8_0, q6_K and q4_K arms). +/// head's q8_0, q6_K and q4_K arms; the gemma-embedding2 encoder's GEMMs, +/// each `site::gemm` on the 32-value path). const TYPE_PINS: &[TypePin] = &[ TypePin { programs: &[Program::Qwen3moeBody], @@ -1013,6 +1085,38 @@ const TYPE_PINS: &[TypePin] = &[ what: "shared expert (the body's iq2_xxs_gemv_sel reads iq2_xxs)", reads: &[BOLT_LEG], }, + TypePin { + programs: &[Program::GemmaEmbedding2Body], + role: Role::Attention, + matrices: true, + names: None, + what: "attention matrices (the encoder reads q8_0)", + reads: &[GgmlType::Q8_0], + }, + TypePin { + programs: &[Program::GemmaEmbedding2Body], + role: Role::DenseFfn, + matrices: true, + names: None, + what: "feed-forward matrices (the encoder reads q8_0)", + reads: &[GgmlType::Q8_0], + }, + TypePin { + programs: &[Program::GemmaEmbedding2Body], + role: Role::EngramDense, + matrices: true, + names: Some(|n| !per_layer_model_proj(n)), + what: "per-layer input gate and projection (the encoder reads q8_0)", + reads: &[GgmlType::Q8_0], + }, + TypePin { + programs: &[Program::GemmaEmbedding2Body], + role: Role::Head, + matrices: true, + names: None, + what: "output projection (the encoder reads q8_0)", + reads: &[GgmlType::Q8_0], + }, ]; /// A tensor family whose type a program pins by the card kernel table's @@ -1037,7 +1141,9 @@ struct RowPin { /// dispatch both bodies share), and `Body35`'s FFN ([`qwen35_gate_up`], /// [`qwen35_down`], [`qwen35_shared`]), /// and the glm5next and mimo2 bodies' token embedding (the host's rows, -/// `GgmlType::has_dequant`), each `what` naming the types its row takes or the table row it follows. +/// `GgmlType::has_dequant`), and the gemma-embedding2 encoder's token table +/// and per-layer input projection (the host's rows), each `what` naming the +/// types its row takes or the table row it follows. const ROW_PINS: &[RowPin] = &[ RowPin { program: Program::Qwen3moeBody, @@ -1079,6 +1185,22 @@ const ROW_PINS: &[RowPin] = &[ what: "token embedding (the host's rows: the types dequant_row decodes)", row: GgmlType::has_dequant, }, + RowPin { + program: Program::GemmaEmbedding2Body, + role: Role::TokenEmbedding, + matrices: true, + names: None, + what: "token embedding (the host's rows: the types dequant_row decodes)", + row: GgmlType::has_dequant, + }, + RowPin { + program: Program::GemmaEmbedding2Body, + role: Role::EngramDense, + matrices: true, + names: Some(per_layer_model_proj), + what: "per-layer input projection (the host's rows: the types dequant_row decodes)", + row: GgmlType::has_dequant, + }, RowPin { program: Program::Qwen3moeBody, role: Role::RoutedExperts, @@ -1175,6 +1297,13 @@ fn glm5next_card_routed(ty: GgmlType) -> bool { crate::arch::glm5next::place::card_routed(ty).is_some() } +/// gemma-embedding2's projection of the token rows to the per-layer inputs, +/// which the encoder decodes on the host where it reads the per-layer +/// input's other matrices as q8_0. +fn per_layer_model_proj(name: &str) -> bool { + name == crate::arch::gemma_embedding2::names::PER_LAYER_MODEL_PROJ +} + /// A qwen4exp selector's key or query projection, which `Body38` reads as /// bf16 widened to f32 (`plan38::plans`), where it reads every other /// attention matrix as q8_0. @@ -1338,8 +1467,10 @@ pub fn check_with( // by name and part (the default expert rule's card formats // would pass q3_K and q6_K gate·up to the load's refusal). Some(Program::Qwen3moeBody | Program::Qwen35Body) => {} - // The nemotron_h body is dense: a routed stack is an item. - Some(Program::NemotronHBody) => at(t.layer, Need::RoutedFormat(t.ty)), + // These bodies are dense: a routed stack is an item. + Some(Program::NemotronHBody | Program::GemmaEmbedding2Body) => { + at(t.layer, Need::RoutedFormat(t.ty)); + } // The latent MoE body's rule (`routed_rule`'s arm): its up // and down launches read [`BOLT_LEG`] stacks, every part of // another type is an item. @@ -1679,11 +1810,12 @@ mod tests { ]; for pin in ROW_PINS { // A name the pin takes: the pins scoped by name are the routed - // stacks' parts. + // stacks' parts and gemma-embedding2's per-layer input projection. let name = [ "blk.0.attn_q.weight", "blk.0.ffn_gate_exps.weight", "blk.0.ffn_down_exps.weight", + "per_layer_model_proj.weight", ] .into_iter() .find(|n| pin.names.is_none_or(|takes| takes(n))) @@ -2324,8 +2456,8 @@ mod tests { } } - /// The glm5next and mimo2 embedding pins read the set the host row - /// reader decodes for the table (`GgmlType::has_dequant`, which + /// The glm5next, mimo2 and gemma-embedding2 embedding pins read the set + /// the host row reader decodes for the table (`GgmlType::has_dequant`, which /// `EmbedTable::new` checks too), pinned here by type name; a type /// outside it is an item naming the type. // PIN(2026-10-11): q4_K, q5_K, q6_K and the other types `dequant_row` @@ -2344,7 +2476,11 @@ mod tests { }, ) }; - for program in [Program::Glm5nextBody, Program::Mimo2Body] { + for program in [ + Program::Glm5nextBody, + Program::Mimo2Body, + Program::GemmaEmbedding2Body, + ] { for ty in [ GgmlType::Q8_0, GgmlType::Q3_K, diff --git a/crates/model/src/arch/gemma_embedding/spec.rs b/crates/model/src/arch/gemma_embedding/spec.rs index 6b9d99728..22c4900d5 100644 --- a/crates/model/src/arch/gemma_embedding/spec.rs +++ b/crates/model/src/arch/gemma_embedding/spec.rs @@ -77,6 +77,10 @@ pub fn spec_of(hp: &Hparams, chat: ChatSpec) -> Result Result { crate::arch::gemma_encoder::attn(split, suffix, dims) } +/// Row `t`'s keys among `n` positions at the encoder layer's `reach`: +/// `first..end`, every key `j` with `|t - j| <= reach`, and `0..n` with no +/// reach. +#[must_use] +pub fn key_bounds(reach: Option, t: usize, n: usize) -> (usize, usize) { + match reach { + Some(r) => ( + t.saturating_sub(r), + t.saturating_add(r).saturating_add(1).min(n), + ), + None => (0, n), + } +} + /// `attention.sliding_window_pattern` as one bool per layer. A scalar, a /// length other than `n_layer` or a value that is no bool is refused by /// name: the file's own form is the per-layer array. diff --git a/crates/model/src/arch/gemma_embedding2/mod.rs b/crates/model/src/arch/gemma_embedding2/mod.rs index a54dbc364..693d68a05 100644 --- a/crates/model/src/arch/gemma_embedding2/mod.rs +++ b/crates/model/src/arch/gemma_embedding2/mod.rs @@ -5,8 +5,8 @@ //! trained scalar on the layer's output; after the output norm one //! projection, then the mean of the rows. The file's keys (`hparams`), every //! tensor's role (`roles`), the tensor names (`names`) and the typed -//! description (`spec`). No program runs it. The dialect's authority is -//! llama.cpp's `src/models/gemma-embedding2.cpp`. +//! description (`spec`). The card program is `crates/gpu-gemma-embedding2`. +//! The dialect's authority is llama.cpp's `src/models/gemma-embedding2.cpp`. pub mod hparams; pub mod names; diff --git a/crates/model/src/arch/gemma_embedding2/spec.rs b/crates/model/src/arch/gemma_embedding2/spec.rs index a76e22015..d5780f1a6 100644 --- a/crates/model/src/arch/gemma_embedding2/spec.rs +++ b/crates/model/src/arch/gemma_embedding2/spec.rs @@ -2,8 +2,10 @@ //! query attention with a per-head q/k norm and NeoX rope of its kind's //! head, and a GELU GLU of `feed_forward_length`. The [`EncoderSpec`] holds //! what the trunk does not: the sliding layers' reach, the per-layer input's -//! width and the output projection's. No tool parser and no reasoning span: -//! the model writes no text. +//! width and the output projection's, and the parts llama.cpp's graph runs +//! past a decoder's layer (the token rows' scale, the values' norm, the +//! norms after both blocks, the layer's scalar). No tool parser and no +//! reasoning span: the model writes no text. use gguf::{Split, Value}; use models::{Arch, ChatSpec, EncoderSpec, LayerSpec, ModelSpec}; @@ -92,6 +94,10 @@ pub fn spec_of(hp: &Hparams, chat: ChatSpec) -> Result Result = (0..n) + .filter(|&j| reach.is_none_or(|r| t.abs_diff(j) <= r)) + .collect(); + let (first, end) = key_bounds(reach, t, n); + assert_eq!( + (first..end).collect::>(), + kept, + "{n} positions, reach {reach:?}, row {t}" + ); + } + } + } + } + + /// The coverage check holds the file to the encoder program's shapes: + /// the bidirectional attention of both heads, the GELU GLU, the + /// per-layer input and the output projection at widths it does not run + /// are items, and no causal flash, SwiGLU layer, tool parser or missing + /// program is. The value norm, the post norms, the layer scalar and the + /// token rows' scale run at any width, so none is an item; a reader that + /// drops one of the encoder's parts, or a program row that covers a + /// causal need, fails this. #[test] fn coverage_names_the_encoder_parts() { let read = load("gemma-emb2-cover", keys(), &tensors()).expect("the header reads"); + let encoder = read.spec.encoder.as_ref().expect("an encoder"); + assert!(encoder.embed_scale); + assert!(encoder.value_norm); + assert!(encoder.post_norms); + assert!(encoder.layer_scale); + let needs = models::needs(&read.spec); + assert!(needs.contains(&(Need::EmbedScale, None))); + for (l, layer) in read.spec.layers.iter().enumerate() { + let Mixer::Gqa(gqa) = &layer.mixer else { + panic!("layer {l}: not GQA"); + }; + for need in [ + Need::ValueNorm { head: gqa.head_dim }, + Need::PostNorms, + Need::LayerScale, + ] { + assert!(needs.contains(&(need.clone(), Some(l as u32))), "{need:?}"); + } + } let items: Vec = coverage::check(&read.spec, &read.tensors) .into_iter() .map(|u| u.feature) @@ -370,11 +422,21 @@ mod tests { "bidirectional GQA flash, head 8, group 2, 5 keys each side", "bidirectional GQA flash, head 16, group 4", "dense GELU GLU layer: ff 64", - "a layer program for gemma-embedding2", + "a per-layer input of 16 values from the token rows", + "the output projections [48], then the mean of the rows", ] { assert!(items.iter().any(|i| i == want), "{want}: {items:?}"); } - for not in ["GQA flash, head", "dense SwiGLU", "tool-call parser"] { + for not in [ + "GQA flash, head", + "dense SwiGLU", + "tool-call parser", + "a layer program", + "a gainless", + "RMS norms on", + "a trained scalar", + "token rows scaled", + ] { assert!( !items.iter().any(|i| i.starts_with(not)), "{not}: {items:?}" diff --git a/crates/model/tests/gemma_embedding2_meta.rs b/crates/model/tests/gemma_embedding2_meta.rs index 8e14ca76a..83aa6f69c 100644 --- a/crates/model/tests/gemma_embedding2_meta.rs +++ b/crates/model/tests/gemma_embedding2_meta.rs @@ -14,8 +14,8 @@ use std::fs; use models::{Act, Ffn, Mixer, RopeMode}; -/// The box's path of the file. -const FILE: &str = "/models/embeddinggemma-2-GGUF/embeddinggemma-2-Q8_0.gguf"; +/// The box's path of the file: the one the reference sets are dumped from. +const FILE: &str = refset::arch::gemma_embedding2::MODEL; /// Its length, so a different file at the same path fails before the pins. const BYTES: u64 = 309_855_456; @@ -125,17 +125,10 @@ fn hw_gemma_embedding2_header() { .into_iter() .map(|u| u.feature) .collect(); - for want in [ - "bidirectional GQA flash, head 256, group 2, 512 keys each side", - "bidirectional GQA flash, head 512, group 4", - "dense GELU GLU layer: ff 2048", - "a layer program for gemma-embedding2", - ] { - if !items.iter().any(|i| i == want) { - bad.push(format!("coverage has no {want:?}")); - } - } println!("gemma-embedding2 coverage: {items:?}"); + if !items.is_empty() { + bad.push(format!("coverage lists {} item(s), want none", items.len())); + } if read.tensors.tensors.len() != 413 { bad.push(format!("tensors {}", read.tensors.tensors.len())); } diff --git a/crates/models/src/encoder.rs b/crates/models/src/encoder.rs index 6ebac260e..8e996648f 100644 --- a/crates/models/src/encoder.rs +++ b/crates/models/src/encoder.rs @@ -20,4 +20,14 @@ pub struct EncoderSpec { /// is the embedding's. With no activation between them they commute with /// the mean. pub proj: Vec, + /// The token rows multiplied by the square root of the hidden width before + /// the first layer. + pub embed_scale: bool, + /// Each attention layer's value rows RMS-normed per head with no gain. + pub value_norm: bool, + /// An RMS norm with a gain on the attention's and the feed-forward + /// block's outputs, before each residual add. + pub post_norms: bool, + /// Each layer's output multiplied by a trained scalar. + pub layer_scale: bool, } diff --git a/crates/models/src/need.rs b/crates/models/src/need.rs index e98f735d2..8c6058a48 100644 --- a/crates/models/src/need.rs +++ b/crates/models/src/need.rs @@ -120,6 +120,22 @@ pub enum Need { ReluSqr { ff: u32 }, /// A dense GELU GLU layer of `ff` values. GeGlu { ff: u32 }, + /// An encoder's value rows RMS-normed per head of `head` values, with no + /// gain. + ValueNorm { head: u32 }, + /// An encoder layer's RMS norms on its attention's and its feed-forward + /// block's outputs, before each residual add. + PostNorms, + /// An encoder layer's output multiplied by its trained scalar. + LayerScale, + /// An encoder layer's own input of `width` values, projected from the + /// scaled token rows, gating a block of its own. + PerLayerInput { width: u32 }, + /// An encoder's token rows scaled by the square root of the hidden width. + EmbedScale, + /// An encoder's output: the linear maps of `proj` widths after the output + /// norm, then the mean of the token rows. + Pooled { proj: Vec }, /// A Mamba-2 mixer. The fields are the widths a scan kernel is built for. Mamba2 { d_inner: u32, @@ -320,6 +336,21 @@ impl fmt::Display for Need { Need::DenseFfn { ff } => write!(f, "dense SwiGLU layer: ff {ff}"), Need::ReluSqr { ff } => write!(f, "squared-ReLU dense layer: ff {ff}"), Need::GeGlu { ff } => write!(f, "dense GELU GLU layer: ff {ff}"), + Need::ValueNorm { head } => { + write!(f, "a gainless per-head RMS norm on the values: head {head}") + } + Need::PostNorms => f.write_str("RMS norms on the attention and feed-forward outputs"), + Need::LayerScale => f.write_str("a trained scalar on the layer's output"), + Need::PerLayerInput { width } => { + write!(f, "a per-layer input of {width} values from the token rows") + } + Need::EmbedScale => f.write_str("token rows scaled by the root of the hidden width"), + Need::Pooled { proj } => { + write!( + f, + "the output projections {proj:?}, then the mean of the rows" + ) + } Need::Mamba2 { d_inner, d_state, @@ -384,7 +415,8 @@ fn mode_name(mode: RopeMode) -> String { /// Every part of `spec`'s trunk a program has to run, layer by layer in /// layer order, then the model-wide ones (`None`). The next-token layers /// (`spec.mtp`) are carried and not run, so they need nothing. An encoder's -/// attention is [`Need::Bidirectional`], never a causal [`Need::Gqa`]. +/// attention is [`Need::Bidirectional`], never a causal [`Need::Gqa`], and +/// what its [`crate::EncoderSpec`] carries past the trunk is a need too. #[must_use] pub fn needs(spec: &ModelSpec) -> Vec<(Need, Option)> { let mut out = Vec::new(); @@ -421,6 +453,11 @@ pub fn needs(spec: &ModelSpec) -> Vec<(Need, Option)> { if g.out_gate { at(Need::OutGate); } + if let Some(e) = &spec.encoder + && e.value_norm + { + at(Need::ValueNorm { head: g.head_dim }); + } if let Some(s) = &g.select { select_needs(s, &mut at); } @@ -515,6 +552,17 @@ pub fn needs(spec: &ModelSpec) -> Vec<(Need, Option)> { }, }); } + if let Some(e) = &spec.encoder { + if e.post_norms { + at(Need::PostNorms); + } + if let Some(width) = e.per_layer { + at(Need::PerLayerInput { width }); + } + if e.layer_scale { + at(Need::LayerScale); + } + } if layer.extras.contains(&Extra::Engram) { at(Need::Engram { row: spec.hidden }); } @@ -579,6 +627,14 @@ pub fn needs(spec: &ModelSpec) -> Vec<(Need, Option)> { { model(Need::ImageToken(token)); } + if let Some(e) = &spec.encoder { + if e.embed_scale { + model(Need::EmbedScale); + } + model(Need::Pooled { + proj: e.proj.clone(), + }); + } out } diff --git a/crates/placement/src/accepts.rs b/crates/placement/src/accepts.rs index b1783162f..b3d8b6349 100644 --- a/crates/placement/src/accepts.rs +++ b/crates/placement/src/accepts.rs @@ -42,6 +42,9 @@ pub enum Program { /// whose routed up and down stacks and shared expert are IQ2_XXS read by /// `iq2_xxs_gemv_sel` ([`BOLT_LEG`]). NemotronHMoeBody, + /// The gemma-embedding2 encoder (`crates/gpu-gemma-embedding2`): one + /// pass of a dense trunk on a card, no routed experts. + GemmaEmbedding2Body, } /// The program that runs `arch`'s layers; `None` when none does. @@ -56,7 +59,8 @@ pub fn program_of(arch: Arch) -> Option { Arch::MiMo2 => Some(Program::Mimo2Body), Arch::NemotronH => Some(Program::NemotronHBody), Arch::NemotronHMoe => Some(Program::NemotronHMoeBody), - Arch::GemmaEmbedding | Arch::GemmaEmbedding2 => None, + Arch::GemmaEmbedding => None, + Arch::GemmaEmbedding2 => Some(Program::GemmaEmbedding2Body), } } @@ -180,8 +184,7 @@ pub struct LegArms { } /// Program `p`'s card leg arms, as its load checks each layer it gives card -/// experts. Every program has a leg; the `Option` is for a program that -/// runs no card expert. +/// experts; `None` for a program that runs no routed expert. #[must_use] pub fn leg_arms(p: Program) -> Option { match p { @@ -205,7 +208,7 @@ pub fn leg_arms(p: Program) -> Option { gate_up: |ty| ty == BOLT_LEG, down: |ty| ty == BOLT_LEG, }), - Program::NemotronHBody => None, + Program::NemotronHBody | Program::GemmaEmbedding2Body => None, } } @@ -223,7 +226,8 @@ pub fn leg_runs(p: Program, [gate, up, down]: [GgmlType; 3]) -> Option { /// plan gives card experts is one the load runs. The common leg's plan /// carries a type on the card only where both its gate·up and its down /// launches read it ([`kernels::card_runs`]), in every part. V4.1's chain -/// runs [`DS41_LEG`] alone. +/// runs [`DS41_LEG`] alone. A program with no routed expert puts none on a +/// card. #[must_use] pub fn routed_rule(p: Program) -> RoutedRule { match p { @@ -248,7 +252,7 @@ pub fn routed_rule(p: Program) -> RoutedRule { down: bolt_leg_card, }, // No routed stack: every one is refused by name. - Program::NemotronHBody => RoutedRule::PerType(|_| None), + Program::NemotronHBody | Program::GemmaEmbedding2Body => RoutedRule::PerType(|_| None), } } diff --git a/crates/refset/src/arch/gemma_embedding2/mod.rs b/crates/refset/src/arch/gemma_embedding2/mod.rs index b485e695d..0f10c19bb 100644 --- a/crates/refset/src/arch/gemma_embedding2/mod.rs +++ b/crates/refset/src/arch/gemma_embedding2/mod.rs @@ -47,6 +47,9 @@ pub const MMPROJ_BF16_SHA256: &str = /// The architecture the text and server sets name in their `# arch` line. pub const ARCH: &str = "gemma-embedding2"; +/// The tokenizer oracle set that supplies the text cases. +pub const IDS_SET: &str = "ref-tokenizer-gemma4"; + /// The text sets. pub const TEXT_SET: &str = "ref_gemma_embedding2_text"; pub const TEXT_CPU_SET: &str = "ref_gemma_embedding2_text.cpu"; @@ -77,26 +80,35 @@ fn mmproj() -> Result { Ok(MMPROJ_BF16.to_string()) } +/// The gate that reads the text sets. +const E2E: &[&str] = &["gate-gpu-gemma-embedding2-e2e"]; + /// The text cases' token rows, pooled vectors and layer taps on the card. -pub static TEXT: Family = family( - "text-gemma-embedding2", - &[TEXT_SET], - Kind::Text, - &PROFILE, - ARCH, - model, - RECIPE, -); +pub static TEXT: Family = Family { + consumers: E2E, + ..family( + "text-gemma-embedding2", + &[TEXT_SET], + Kind::Text, + &PROFILE, + ARCH, + model, + RECIPE, + ) +}; /// The same, with no card in view. -pub static TEXT_CPU: Family = family( - "text-gemma-embedding2-cpu", - &[TEXT_CPU_SET], - Kind::TextCpu, - &PROFILE, - ARCH, - model, - RECIPE, -); +pub static TEXT_CPU: Family = Family { + consumers: E2E, + ..family( + "text-gemma-embedding2-cpu", + &[TEXT_CPU_SET], + Kind::TextCpu, + &PROFILE, + ARCH, + model, + RECIPE, + ) +}; /// The image cases' tower taps and rows on the card. pub static TOWER: Family = family( "tower-gemma4v", diff --git a/crates/refset/src/tokenizer.rs b/crates/refset/src/tokenizer.rs index 15ce5f2c5..8d64b3260 100644 --- a/crates/refset/src/tokenizer.rs +++ b/crates/refset/src/tokenizer.rs @@ -76,22 +76,27 @@ impl Text { ("nps.ids", self.nps_ids) }; let path = self.dir.join(format!("{}.{ext}", self.name)); - let bytes = std::fs::read(&path) - .map_err(|e| RefError::missing(&path, format!("{}: {e}", path.display())))?; - let text = String::from_utf8(bytes) - .map_err(|e| RefError::malformed(path.display(), format!("not UTF-8: {e}")))?; - let mut ids = Vec::with_capacity(want); - for (i, line) in text.lines().enumerate() { - ids.push(parse_field(line, "id", &At(&path, i + 1))?); - } - if ids.len() != want { - return Err(RefError::malformed( - path.display(), - format!("{} ids, the manifest's row names {want}", ids.len()), - )); - } - Ok(ids) + read_ids(&path, want) + } +} + +/// Read one token id per line, refusing a malformed id or a count other than `want`. +pub fn read_ids(path: &Path, want: usize) -> Result, RefError> { + let bytes = std::fs::read(path) + .map_err(|e| RefError::missing(path, format!("{}: {e}", path.display())))?; + let text = String::from_utf8(bytes) + .map_err(|e| RefError::malformed(path.display(), format!("not UTF-8: {e}")))?; + let mut ids = Vec::with_capacity(want); + for (i, line) in text.lines().enumerate() { + ids.push(parse_field(line, "id", &At(path, i + 1))?); + } + if ids.len() != want { + return Err(RefError::malformed( + path.display(), + format!("{} ids, the manifest's row names {want}", ids.len()), + )); } + Ok(ids) } /// A tokenizer set's manifest and, once opened, its texts' digests checked. diff --git a/justfile b/justfile index cb21181b2..e6f059ec8 100644 --- a/justfile +++ b/justfile @@ -33,7 +33,7 @@ default: # 빠른 루프: 타입 검사만, 커널은 안 만든다. 의존을 고친 뒤 cargo가 박스 쪽 Cargo.lock을 고쳐 쓰면 lock-back.sh가 # 그것을 이 트리로 가져온다 — box.sh는 한 방향으로만 싣는다. check: - ./tools/box.sh 'cargo check --workspace --all-targets --features gpu,bloomery-gpu-gates/deepseek41,bloomery-gpu-gates/vision,bloomery-gpu-gates/glm5next,bloomery-gpu-gates/mimo2,bloomery-gpu-gates/clef' + ./tools/box.sh 'cargo check --workspace --all-targets --features gpu,bloomery-gpu-gates/deepseek41,bloomery-gpu-gates/vision,bloomery-gpu-gates/glm5next,bloomery-gpu-gates/mimo2,bloomery-gpu-gates/gemma-embedding2,bloomery-gpu-gates/clef' ./tools/lock-back.sh # check와 같은 이유로 `--features gpu`. 이 피처를 켠 것이 기준 계기다(R26). V4.1 op 게이트의 피처 @@ -41,7 +41,7 @@ check: # V4.1 커널의 컴파일러 결함은 여기가 아니라 op 게이트 빌드에서 드러난다. # lint. 에러 0이 계약이고 경고 수는 RESULTS/AGENTS에 적힌 기준선과 비교한다. lint: - ./tools/box.sh 'cargo clippy --workspace --all-targets --features gpu,bloomery-gpu-gates/deepseek41,bloomery-gpu-gates/vision,bloomery-gpu-gates/glm5next,bloomery-gpu-gates/mimo2,bloomery-gpu-gates/clef' + ./tools/box.sh 'cargo clippy --workspace --all-targets --features gpu,bloomery-gpu-gates/deepseek41,bloomery-gpu-gates/vision,bloomery-gpu-gates/glm5next,bloomery-gpu-gates/mimo2,bloomery-gpu-gates/gemma-embedding2,bloomery-gpu-gates/clef' # The build without `vision` of the bins that refuse an image request by name — serve_seats/ds41.rs:444 and glm.rs:825 (--mmproj), # decide.rs:545 (the Qwen3.5 seat's encoder), gate_glm5next_e2e.rs:5146 (--only media-seat). One bin takes one feature set (tools/recipes.py's @@ -1242,6 +1242,17 @@ gate-gpu-qwen35moe-attn: gate-gpu-gemma-embedding2-head512: ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features gpu --release --bin gate_gemma_embedding2_head512 && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_gemma_embedding2_head512' +# EmbeddingGemma 2's encoder (gpu-gemma-embedding2) end to end on the Q8_0 file the sets are dumped from +# (refset's gemma_embedding2::MODEL, held to its sha256; the sets' `# model` line names the same path): every +# text case of mainline's card set (just dump-ref-emb2) through one pass, its taps within the CPU twin's band, +# a rerun bit for bit, and the refusals by name. The +# key bounds first hold every key of the symmetric mask, independently of the tap bands. +gate-gpu-gemma-embedding2-e2e: + ./tools/box.sh 'bash tools/gate.sh --oxide --release -p bloomery-gpu-gemma-embedding2 --lib -- --include-ignored' + ./tools/box.sh 'bash tools/gate.sh --release -p bloomery-model --lib -- arch::gemma_embedding2::spec::tests::key_bounds_are_the_symmetric_mask --exact' + ./tools/box.sh 'bash tools/gate.sh --oxide --release -p bloomery-gpu-gates --features gemma-embedding2 --bin gate_gemma_embedding2_e2e' + ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features gemma-embedding2 --release --bin gate_gemma_embedding2_e2e && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_gemma_embedding2_e2e' + # GLM-5.3-Flash MLA 층의 카드 부분 — 잠재 RMS + f16 추가, 인덱서 키 LayerNorm + [키; 게이트] 추가(`latent`), # 창 0·싱크 −∞·스케일 1/16으로 다시 쓰는 V4.1 어텐션 — 을 모델 없이 합성 입력으로 호스트 규칙에 맞춘다. # 추가는 캐시 비트, 어텐션은 오차 모형에서 끌어낸 밴드, 싱크 −∞는 분할 부분합의 무싱크 접기와 비트로 본다. diff --git a/tools/check-arch.sh b/tools/check-arch.sh index 4d75b4f3a..f02b9ea54 100644 --- a/tools/check-arch.sh +++ b/tools/check-arch.sh @@ -12,6 +12,7 @@ # ⑤ crates/*/src 의 스레드 spawn 은 threads::helper::spawn_helper 를 거친다 — 고정된 스텝 스레드가 # 띄운 스레드는 그 한 CPU 마스크를 물려받는다. 예외는 tools/spawn-allow.txt 의 파일별 개수와 이유다. # 아키텍처 자신의 크레이트(crates/gpu-/src/)는 ①②④에서 그 아키텍처의 arch/ 로 센다. +# 크레이트 이름은 디렉터리 이름의 밑줄을 하이픈으로 쓴다(arch/gemma_embedding2/ 의 크레이트는 crates/gpu-gemma-embedding2/). # 아키텍처 목록은 손으로 적지 않는다: crates/*/src/arch/ 아래 디렉터리 이름이 목록이다. 새 아키텍처는 # 디렉터리가 생기는 순간 넷 모두의 대상이 된다. # @@ -51,7 +52,7 @@ arches crates ALT=$(printf '%s' "$ARCHES" | sed 's/ *$//; s/ /|/g') # deepseek2|deepseek41|… # 모듈 없이 다른 모듈이 읽는 아키텍처 문자열(원칙 3의 예외, docs/arch-split.md)은 규칙 ②의 키 접두에 들어간다. KEYS=$(printf '"%s\\.|' $ARCHES deepseek4 | sed 's/|$//') # "deepseek2\.|"deepseek41\.|"deepseek4\. -OWN="^crates/([^/]+/src/arch/|gpu-($ALT)/src/)" +OWN="^crates/([^/]+/src/arch/|gpu-(${ALT//_/-})/src/)" fail=0 report() { # report <번호> <설명> <위반 줄들> @@ -72,7 +73,7 @@ report() { # report <번호> <설명> <위반 줄들> nov() { echo "check-arch: rule $*, no verdict" >&2; exit 2; } cross_one() { # cross_one : rule ①'s hits into cross local dirs - dirs=$(find "$1" -type d \( -path "*/src/arch/$2" -o -path "$1/gpu-$2/src" \)) || + dirs=$(find "$1" -type d \( -path "*/src/arch/$2" -o -path "$1/gpu-${2//_/-}/src" \)) || nov "1 cannot list the directories of $2 under $1 (find exit $?, above)" [ -n "$dirs" ] || return 0 # shellcheck disable=SC2086 diff --git a/tools/gate-paths.tsv b/tools/gate-paths.tsv index 95c89874a..3590d418c 100644 --- a/tools/gate-paths.tsv +++ b/tools/gate-paths.tsv @@ -59,17 +59,17 @@ crates/app/src/{lib.rs,arch/mod.rs} gate-gpu-ds41-callstream gate-gpu-ds41-resid crates/app/src/arch/deepseek41/** gate-gpu-clef-serve gate-gpu-ds41-callstream gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-media gate-gpu-ds41-residency-a gate-gpu-ds41-twocard gate-gpu-glm5next-serve gate-gpu-qwen3-serve gate-gpu-lev-serve gate-gpu-ds41-step gate-gpu-ds41-prefill gate-gpu-ds41-nvtier gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve the V4.1 session impls and its draft: generate_ds41, the chat and serve bins, and the gates that drive them crates/app/src/arch/glm5next/** gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen3-serve gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve the GLM session impls: the GLM e2e gate crates/app/src/{mtp.rs,mtp/**,arch/qwen3moe/**} gate-app gate-q38-serve-rules gate-gpu-qwen3moe-e2e gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen4exp-mtp gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-glm5next-twocard gate-gpu-qwen3-serve gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-qwen38-nodes the shared MTP window and the qwen3moe session impls: the app tests, the qwen3moe e2e gate, the Qwen3.8 serve seat and MTP gates, and the Qwen3.6 draft gate (its Session taps) -crates/runtime/src/** gate-ops gate-placement gate-union gate-fixture gate-app gate-deepseek4-meta gate-ds41-bind gate-ds41-host gate-ds41-kld gate-ds41-load gate-ds41-meta gate-ds41-oracle gate-ds41-plan gate-dspark-read gate-glm5next-meta gate-qwen35moe-meta gate-qwen3moe-meta gate-qwen4exp-meta gate-ptx-spill gate-gpu-e2e gate-gpu-gemm gate-gpu-lib gate-gpu-gates-lib gate-gpu-head gate-gpu-hc-gated gate-gpu-kquant gate-gpu-mcol gate-gpu-moe gate-gpu-p10 gate-gpu-swap gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-gemm gate-gpu-glm5next-mtp gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-qwen3-serve gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-head gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-attn gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35moe-moe gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-ds41-callstream gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-comp gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-index gate-gpu-ds41-lib gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-moe gate-gpu-ds41-prefill gate-gpu-ds41-residency-a gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-woa gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-q38-serve-rules gate-gpu-nvtier gate-mimo2-meta gate-gpu-hybrid gate-gpu-p8 gate-gpu-linear gate-ds41-place gate-gpu-p8b gate-gpu-iq-sel gate-gpu-xstream gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-hf gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gpu-p1 gate-gpu-mamba2 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gemma-embedding-meta gate-gpu-qwen38-nodes gate-gpu-nemotron-h-moe-e2e the generation loop and its stores: every gate that names a runtime path in its own code (runtime::combine in gate_kquant and the chain-ffn/prefill bins, hc_gated, qsa, seqstate, swaprule, stores), loads weights through bloomery_gpu::weights (whose every loader entry funnels into upload_rows → runtime::words::stream_words), or opens a model or session (arch place.rs → runtime::stores; clef-hidden opens the Qwen3.5 body, whose body35.rs reads runtime::seqstate). The gates left out load no model and name no runtime path: the synthetic kernel gates upload through DeviceTensor::upload/from_host and read reference values host-side, the CPU oracle gates read the model host-side, and gate-runtime is picked as its own lib's tests +crates/runtime/src/** gate-ops gate-placement gate-union gate-fixture gate-app gate-deepseek4-meta gate-ds41-bind gate-ds41-host gate-ds41-kld gate-ds41-load gate-ds41-meta gate-ds41-oracle gate-ds41-plan gate-dspark-read gate-glm5next-meta gate-qwen35moe-meta gate-qwen3moe-meta gate-qwen4exp-meta gate-ptx-spill gate-gpu-e2e gate-gpu-gemm gate-gpu-lib gate-gpu-gates-lib gate-gpu-head gate-gpu-hc-gated gate-gpu-kquant gate-gpu-mcol gate-gpu-moe gate-gpu-p10 gate-gpu-swap gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-gemm gate-gpu-glm5next-mtp gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-qwen3-serve gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-head gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-attn gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35moe-moe gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-ds41-callstream gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-comp gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-index gate-gpu-ds41-lib gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-moe gate-gpu-ds41-prefill gate-gpu-ds41-residency-a gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-woa gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-q38-serve-rules gate-gpu-nvtier gate-mimo2-meta gate-gpu-hybrid gate-gpu-p8 gate-gpu-linear gate-ds41-place gate-gpu-p8b gate-gpu-iq-sel gate-gpu-xstream gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-hf gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gpu-p1 gate-gpu-mamba2 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gemma-embedding-meta gate-gpu-qwen38-nodes gate-gpu-nemotron-h-moe-e2e gate-gpu-gemma-embedding2-e2e the generation loop and its stores: every gate that names a runtime path in its own code (runtime::combine in gate_kquant and the chain-ffn/prefill bins, hc_gated, qsa, seqstate, swaprule, stores), loads weights through bloomery_gpu::weights (whose every loader entry funnels into upload_rows → runtime::words::stream_words), or opens a model or session (arch place.rs → runtime::stores; clef-hidden opens the Qwen3.5 body, whose body35.rs reads runtime::seqstate). The gates left out load no model and name no runtime path: the synthetic kernel gates upload through DeviceTensor::upload/from_host and read reference values host-side, the CPU oracle gates read the model host-side, and gate-runtime is picked as its own lib's tests crates/hf/src/{safetensors.rs,dspark.rs} gate-gpu-qwen35-dspark gate-gpu-clef-serve gate-gpu-lev-serve gate-decision-clef gate-decision-lev gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve hf::safetensors is the Clef head reader (decision re-exports its errors through decision::Error, so the serve seats that open the decide row reach it) and the DSpark draft file reader; hf::dspark is read only by the Qwen3.6 draft gate crates/serve/{src/**,build.rs} gate-ds41-bind gate-gpu-clef-serve gate-gpu-lev-serve gate-serve gate-q38-serve-rules gate-ds41-place gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve the llama-server-compatible API: gpu-gates' bind tests (bloomery-serve-ds41's gate is weekly-gpu-ds41-serve, a trigger below); gate-serve is its own crates/jinja/src/** gate-jinja gate-serve gate-ds41-bind gate-decision-clef gate-decision-lev gate-gpu-clef-serve gate-gpu-lev-serve gate-q38-serve-rules gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve the template engine: its own tests, serve's chat templates (the serve row's readers) and the decision models' templates (Clef's and lev's request renders) -crates/tokenizer/src/** gate-app gate-decision-clef gate-deepseek4-meta gate-ds41-bind gate-ds41-host gate-ds41-kld gate-ds41-load gate-ds41-meta gate-ds41-oracle gate-ds41-plan gate-dspark-read gate-fixture gate-glm5next-meta gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-comp gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-index gate-gpu-ds41-lib gate-gpu-ds41-long gate-gpu-ds41-prefill gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-e2e gate-gpu-gates-lib gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-lib gate-gpu-mcol gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35moe-moe gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-head gate-gpu-qwen3moe-router gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-ops gate-placement gate-qwen35moe-meta gate-qwen3moe-meta gate-qwen4exp-meta gate-union gate-gpu-clef-serve gate-gpu-lev-serve gate-decision-lev gate-q38-serve-rules gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-iq-sel gate-gpu-clef-hidden gate-gpu-qwen4exp-iqleg gate-gpu-glm5next-gemm gate-gpu-ds41-media gate-gpu-nvtier gate-gpu-qwen38-residency gate-mimo2-meta gate-gpu-glm5next-residency gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-ds41-place gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-qwen35-dspark gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-qwen38-nodes gate-gemma-embedding-meta the encoder and decoder, and runs_pre_tokenizer, which coverage::check asks at every placement plan (model::arch::*::place): every gate that plans a placement or tokenizes text +crates/tokenizer/src/** gate-app gate-decision-clef gate-deepseek4-meta gate-ds41-bind gate-ds41-host gate-ds41-kld gate-ds41-load gate-ds41-meta gate-ds41-oracle gate-ds41-plan gate-dspark-read gate-fixture gate-glm5next-meta gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-comp gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-index gate-gpu-ds41-lib gate-gpu-ds41-long gate-gpu-ds41-prefill gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-e2e gate-gpu-gates-lib gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-lib gate-gpu-mcol gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35moe-moe gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-head gate-gpu-qwen3moe-router gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-ops gate-placement gate-qwen35moe-meta gate-qwen3moe-meta gate-qwen4exp-meta gate-union gate-gpu-clef-serve gate-gpu-lev-serve gate-decision-lev gate-q38-serve-rules gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-iq-sel gate-gpu-clef-hidden gate-gpu-qwen4exp-iqleg gate-gpu-glm5next-gemm gate-gpu-ds41-media gate-gpu-nvtier gate-gpu-qwen38-residency gate-mimo2-meta gate-gpu-glm5next-residency gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-ds41-place gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-qwen35-dspark gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e the encoder and decoder, and runs_pre_tokenizer, which coverage::check asks at every placement plan (model::arch::*::place): every gate that plans a placement or tokenizes text # crates/refset: the reference-set readers. Every oracle read goes through refset::arch's dispatch (node_dumps, # families) and a family's check; which family a gate asks for is its run-time arch, so a family's row keeps every # dispatch user. -crates/refset/src/{lib,family,mtpref,dsref,greedy,kld,vision,ik,columns}.rs gate-attn gate-derived gate-ds41-bind gate-ds41-host gate-ds41-kld gate-ds41-load gate-ds41-meta gate-ds41-oracle gate-ds41-plan gate-ffn gate-forward gate-gpu-block gate-gpu-clef-hidden gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-gates-lib gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-mtp gate-gpu-glm5next-residency gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-head gate-gpu-linear gate-gpu-mcol gate-gpu-moe gate-gpu-p0b gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p8 gate-gpu-p8b gate-gpu-qwen35moe-attn gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-qwen35moe-moe gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-vision gate-head gate-kv gate-moe gate-mt gate-ops gate-profile gate-qwen3moe-meta gate-union gate-vision gate-1-1 gate-tokenizer gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-ds41-place gate-q38-serve-rules gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-iq gate-gpu-iq-sel gate-gpu-e2e gate-gpu-hybrid gate-gpu-gemm gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-dspark-read gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-clef-serve gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-minicpmv-tower gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes the readers and the family check every set goes through: every gate that reads a reference set +crates/refset/src/{lib,family,mtpref,dsref,greedy,kld,vision,ik,columns}.rs gate-attn gate-derived gate-ds41-bind gate-ds41-host gate-ds41-kld gate-ds41-load gate-ds41-meta gate-ds41-oracle gate-ds41-plan gate-ffn gate-forward gate-gpu-block gate-gpu-clef-hidden gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-gates-lib gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-mtp gate-gpu-glm5next-residency gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-head gate-gpu-linear gate-gpu-mcol gate-gpu-moe gate-gpu-p0b gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p8 gate-gpu-p8b gate-gpu-qwen35moe-attn gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-qwen35moe-moe gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-vision gate-head gate-kv gate-moe gate-mt gate-ops gate-profile gate-qwen3moe-meta gate-union gate-vision gate-1-1 gate-tokenizer gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-ds41-place gate-q38-serve-rules gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-iq gate-gpu-iq-sel gate-gpu-e2e gate-gpu-hybrid gate-gpu-gemm gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-dspark-read gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-clef-serve gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-minicpmv-tower gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e the readers and the family check every set goes through: every gate that reads a reference set crates/refset/src/arch/mod.rs gate-ds41-bind gate-ds41-kld gate-ds41-load gate-ds41-oracle gate-ds41-plan gate-gpu-block gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-media gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-gates-lib gate-gpu-head gate-gpu-linear gate-gpu-moe gate-gpu-p0b gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p8 gate-gpu-p8b gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-1-1 gate-tokenizer gate-ds41-place gate-gpu-qwen4exp-mtp gate-q38-serve-rules gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-media-seat gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve the per-arch dispatch (node_dumps, families, named): every oracle user except gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-glm5next-serve: bloomery-serve opens one seat a run (--model, or the file's architecture); these run its decide and glm seats, and only the qwen38 seat's module (its DraftFrom) reaches this file | except gate-gpu-serve-start-glm: bloomery-serve opens one seat a run (--model, or the file's architecture); this recipe runs its glm seat, and only another seat's module reaches this file | except gate-gpu-serve-start-ds41: bloomery-serve opens one seat a run (--model, or the file's architecture); this recipe runs its ds41 seat, and only another seat's module reaches this file crates/refset/src/arch/deepseek41/** gate-ds41-bind gate-ds41-host gate-ds41-kld gate-ds41-load gate-ds41-meta gate-ds41-oracle gate-ds41-plan gate-gpu-block gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-long gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-gates-lib gate-gpu-glm5next-serve gate-gpu-head gate-gpu-linear gate-gpu-mcol gate-gpu-moe gate-gpu-p0b gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p8 gate-gpu-p8b gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-union gate-gpu-qwen3-serve gate-gpu-clef-serve gate-gpu-lev-serve gate-ds41-place gate-tokenizer gate-1-1 gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve the deepseek41 family's sets: the dispatch users and the gates that name the family crates/refset/src/arch/deepseek41v/** gate-ds41-bind gate-ds41-kld gate-ds41-load gate-ds41-oracle gate-ds41-plan gate-gpu-block gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-media gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-gates-lib gate-gpu-head gate-gpu-linear gate-gpu-moe gate-gpu-p0b gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p8 gate-gpu-p8b gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-vision gate-vision gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 the deepseek41v family's sets: the dispatch users and the gates that name the family except gate-ds41-place: they read no deepseek41v set: only refset::arch's table (BY_ARCH), which picks a set's family by the file it names at run time, reaches it @@ -77,12 +77,12 @@ crates/refset/src/arch/glm5next/** gate-ds41-bind gate-ds41-kld gate-ds41-load g crates/refset/src/arch/mimo2/** gate-ds41-bind gate-ds41-kld gate-ds41-load gate-ds41-oracle gate-ds41-plan gate-gpu-block gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-gates-lib gate-gpu-head gate-gpu-linear gate-gpu-mimo2-flash gate-gpu-moe gate-gpu-p0b gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p8 gate-gpu-p8b gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router weekly-gpu-mimo2-e2e the mimo2 family's sets: the dispatch users and the gates that name the family except gate-ds41-place: they read no mimo2 set: only refset::arch's table (BY_ARCH), which picks a set's family by the file it names at run time, reaches it crates/refset/src/arch/qwen35moe/** gate-ds41-bind gate-ds41-kld gate-ds41-load gate-ds41-oracle gate-ds41-plan gate-gpu-block gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-gates-lib gate-gpu-head gate-gpu-linear gate-gpu-moe gate-gpu-p0b gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p8 gate-gpu-p8b gate-gpu-qwen35moe-attn gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-qwen35moe-moe gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-qwen3-serve gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp the qwen35moe family's sets: the dispatch users and the gates that name the family except gate-ds41-place: they read no qwen35moe set: only refset::arch's table (BY_ARCH), which picks a set's family by the file it names at run time, reaches it crates/refset/src/arch/qwen4exp/** gate-ds41-bind gate-ds41-kld gate-ds41-load gate-ds41-oracle gate-ds41-plan gate-gpu-block gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-gates-lib gate-gpu-glm5next-serve gate-gpu-head gate-gpu-linear gate-gpu-moe gate-gpu-p0b gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p8 gate-gpu-p8b gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3-serve gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-nvtier gate-q38-serve-rules gate-tokenizer gate-ds41-place gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve the qwen4exp family's sets: the dispatch users and the gates that name the family -crates/refset/src/{clefvis.rs,clefvis/**} gate-refset gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-ds41-hc gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-derived gate-mt gate-profile gate-ds41-meta gate-qwen3moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-tokenizer gate-1-1 gate-gpu-head gate-gpu-mcol gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-vision gate-gpu-vision gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-minicpmv-tower gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-h-moe-e2e the Clef image-input sets' reader: no gate reads a clefvis set yet, gate-refset runs its tests and walks every set in place +crates/refset/src/{clefvis.rs,clefvis/**} gate-refset gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-ds41-hc gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-derived gate-mt gate-profile gate-ds41-meta gate-qwen3moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-tokenizer gate-1-1 gate-gpu-head gate-gpu-mcol gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-vision gate-gpu-vision gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-minicpmv-tower gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-h-moe-e2e gate-gpu-gemma-embedding2-e2e the Clef image-input sets' reader: no gate reads a clefvis set yet, gate-refset runs its tests and walks every set in place crates/refset/src/arch/minicpmv4_7/** gate-ds41-bind gate-ds41-kld gate-ds41-load gate-ds41-oracle gate-ds41-plan gate-gpu-block gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-gates-lib gate-gpu-linear gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-ds41-place gate-gpu-minicpmv-tower gate-refset the MiniCPM-V 4.7 image sets' reader (the tower per tile, the hidden states and four position lanes of an image prompt, the chat ids): the dispatch users, the tower gate that reads the tower sets, and gate-refset, which runs the reader's tests and walks every set in place crates/refset/src/arch/qwen35/** gate-refset gate-gpu-clef-hidden gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media the qwen35 families' sets: the hidden-state gate that names the families, and the table walk except gate-gpu-ds41-hc gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen3moe-down gate-gpu-linear gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-ds41-bind gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-ds41-load: they read no qwen35 set: only refset::arch's table (BY_ARCH), which picks a set's family by the file it names at run time, reaches it -crates/refset/src/emb2.rs gate-refset gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-h-moe-e2e the EmbeddingGemma 2 oracle sets: gate-refset checks these families except gate-1-1 gate-attn gate-derived gate-ds41-bind gate-ds41-host gate-ds41-kld gate-ds41-load gate-ds41-meta gate-ds41-oracle gate-ds41-place gate-ds41-plan gate-ffn gate-forward gate-gpu-block gate-gpu-clef-hidden gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-gates-lib gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-gemm gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-residency gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-glm5v-tower gate-gpu-head gate-gpu-linear gate-gpu-mcol gate-gpu-mimo2-flash gate-gpu-minicpm-text gate-gpu-minicpmv-tower gate-gpu-moe gate-gpu-p0b gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p8 gate-gpu-p8b gate-gpu-qwen3-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-attn gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-media gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-verify gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen3vl-tower gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-serve-start-ds41 gate-gpu-serve-start-glm gate-gpu-vision gate-head gate-kv gate-moe gate-mt gate-ops gate-profile gate-qwen3moe-meta gate-tokenizer gate-union gate-vision: they read no EmbeddingGemma 2 set: only the runtime family dispatch reaches this reader and these families -crates/refset/src/arch/gemma_embedding2/** gate-refset the EmbeddingGemma 2 oracle sets: gate-refset checks these families except gate-ds41-bind gate-ds41-kld gate-ds41-load gate-ds41-oracle gate-ds41-place gate-ds41-plan gate-gpu-block gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-gates-lib gate-gpu-linear gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router: they read no EmbeddingGemma 2 set: only the runtime family dispatch reaches this reader and these families -crates/refset/src/{md5,tokenizer,dequant}.rs gate-tokenizer gate-1-1 gate-gpu-ds41-hc gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-attn gate-gpu-qwen35moe-e2e gate-gpu-minicpm-text gate-gpu-nemotron-h-moe-e2e the tokenizer and dequant set readers and the md5 they check by: the two gates that open those sets (gate-refset runs the crate's own tests) except gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen3moe-down gate-gpu-linear gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-ds41-bind gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-ds41-load: they read no dequant and tokenizer set: only refset::arch's table (BY_ARCH), which picks a set's family by the file it names at run time, reaches it | except gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency: they open their own families' sets through refset::family::Family::check_set, which reaches DequantSet::open only in its DequantManifest arm, an identity no set of theirs carries: they read no dequant set +crates/refset/src/emb2.rs gate-refset gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-h-moe-e2e gate-gpu-gemma-embedding2-e2e the EmbeddingGemma 2 oracle sets: gate-refset checks these families except gate-1-1 gate-attn gate-derived gate-ds41-bind gate-ds41-host gate-ds41-kld gate-ds41-load gate-ds41-meta gate-ds41-oracle gate-ds41-place gate-ds41-plan gate-ffn gate-forward gate-gpu-block gate-gpu-clef-hidden gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-gates-lib gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-gemm gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-residency gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-glm5v-tower gate-gpu-head gate-gpu-linear gate-gpu-mcol gate-gpu-mimo2-flash gate-gpu-minicpm-text gate-gpu-minicpmv-tower gate-gpu-moe gate-gpu-p0b gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p8 gate-gpu-p8b gate-gpu-qwen3-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-attn gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-media gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-verify gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen3vl-tower gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-serve-start-ds41 gate-gpu-serve-start-glm gate-gpu-vision gate-head gate-kv gate-moe gate-mt gate-ops gate-profile gate-qwen3moe-meta gate-tokenizer gate-union gate-vision: they read no EmbeddingGemma 2 set: only the runtime family dispatch reaches this reader and these families +crates/refset/src/arch/gemma_embedding2/** gate-refset gate-gpu-gemma-embedding2-e2e gate-gemma-embedding2-meta the EmbeddingGemma 2 oracle sets: gate-refset checks these families except gate-ds41-bind gate-ds41-kld gate-ds41-load gate-ds41-oracle gate-ds41-place gate-ds41-plan gate-gpu-block gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-gates-lib gate-gpu-linear gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router: they read no EmbeddingGemma 2 set: only the runtime family dispatch reaches this reader and these families +crates/refset/src/{md5,tokenizer,dequant}.rs gate-tokenizer gate-1-1 gate-gpu-ds41-hc gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-attn gate-gpu-qwen35moe-e2e gate-gpu-minicpm-text gate-gpu-nemotron-h-moe-e2e gate-gpu-gemma-embedding2-e2e the tokenizer and dequant set readers and the md5 they check by: the two gates that open those sets (gate-refset runs the crate's own tests) except gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen3moe-down gate-gpu-linear gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-ds41-bind gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-ds41-load: they read no dequant and tokenizer set: only refset::arch's table (BY_ARCH), which picks a set's family by the file it names at run time, reaches it | except gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency: they open their own families' sets through refset::family::Family::check_set, which reaches DequantSet::open only in its DequantManifest arm, an identity no set of theirs carries: they read no dequant set crates/refset/src/arch/{tokenizer,dequant}.rs gate-tokenizer gate-1-1 gate-gpu-nemotron-e2e the tokenizer and dequant families' rows and pins: the two gates that open those sets (gate-refset runs the crate's own tests) except gate-gpu-ds41-hc gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen3moe-down gate-gpu-linear gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-ds41-bind gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-ds41-load: they read no dequant and tokenizer set: only refset::arch's table (BY_ARCH), which picks a set's family by the file it names at run time, reaches it crates/refset/src/census.rs gate-gpu-routed-legs the census stub reader (the manifest and the digest check): the routed-legs gate plans every set through it # The census manifest lands with tools/census (the writer session's branch); the row holds the gate's @@ -98,75 +98,75 @@ tools/census/stubs.tsv gate-gpu-routed-legs the census manifest (refset::census: # files a gate opens. One row: every gate that links the modules, less the gates whose bins only read tensors # through gguf::Gguf or build synthetic blocks and call no arch module (their own code, and the bloomery-gpu # kernel modules and bloomery-gpu-gates helpers they call, name no model::arch:: path). the fixture generator, arch/coverage.rs and arch/qwen35moe/mtp.rs are carved out to their own rows below: their readers are bounded (a leaf the family's fixture module names; the place/engine/plan modules that call them). -crates/model/src/arch/{mod.rs,dspark.rs,classify.rs,stems.rs,deepseek2/**,qwen3moe/**,glm5next/**} gate-gpu-p4 gate-gpu-p6 gate-gpu-kquant gate-gpu-iq-sel gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-ptx-spill gate-dspark-read gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-q38-serve-rules gate-gpu-block gate-gpu-nvtier gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p5 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-cand gate-gpu-iq gate-gpu-q6k-ids gate-gpu-xstream gate-gpu-mimo2-flash gate-gpu-ple gate-gpu-vision gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes every gate that links an arch module, less the gguf-only kernel gates (above): a family's code runs wherever any family's file is opened -crates/model/src/arch/mimo2/** gate-ops gate-mimo2-meta gate-gpu-mimo2-flash gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-ds41-bind gate-ds41-place gate-gpu-routed-legs gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-minicpm-text weekly-gpu-mimo2-e2e weekly-gpu-mimo2-serve gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes MiMo-V2.6-Flash's reader and plan: its code runs only where a mimo2 file opens (its meta test, its flash gate) in bloomery-model's own unit tests (gate-ops runs --lib) and in bloomery-serve's mimo2 seat, which every serve gate's bin links except gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-clef-hidden gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen3moe-e2e gate-fixture gate-gpu-head gate-gpu-lib gate-gpu-ds41-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency: they open no mimo2 file: only model::arch's dispatch by the file's general.architecture (arch/mod.rs) reaches it -crates/model/src/arch/nemotron_h/** gate-ops gate-nemotron-meta gate-fixture gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-iq-sel gate-placement gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-ds41-bind gate-ds41-place gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes Nemotron 3 Nano 4B's header reader and fixture spec: its code runs where a nemotron_h file is described (its meta test), where its fixture is planned and verified (the fixture generator's gate), in bloomery-model's own unit tests, where the card body reads the layer kinds and widths at load (the end-to-end gate), and behind the arch dispatch and, through its plan (place.rs), in bloomery-serve's nemotron seat, which gate-gpu-nemotron-serve starts and every serve gate's bin links except gate-gpu-e2e gate-gpu-hybrid gate-gpu-p6 gate-gpu-ds41-moe gate-gpu-moe gate-gpu-p8 gate-gpu-block gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen35moe-moe gate-gpu-qwen3moe-router gate-qwen35moe-meta gate-gpu-qwen35-dspark gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-minicpm-text gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-types gate-gpu-head gate-gpu-routed-legs gate-gpu-ds41-lib gate-gpu-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-nvtier gate-gpu-ds41-residency-a gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-long gate-gpu-ds41-chat gate-gpu-ds41-media gate-gpu-ds41-prefill gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-ds41-load gate-gpu-qwen38-serve gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-p8b gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-residency gate-gpu-glm5next-twocard: they link the arch dispatch and never open a nemotron_h file -crates/model/src/arch/gemma_embedding2/** gate-ops gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes EmbeddingGemma 2's header reader: no program runs it, so its code runs where a gemma-embedding2 file is described (its meta test), in bloomery-model's own unit tests, and behind the arch dispatch except gate-gpu-e2e gate-gpu-hybrid gate-gpu-p6 gate-gpu-ds41-moe gate-gpu-moe gate-gpu-p8 gate-gpu-block gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen35moe-moe gate-gpu-qwen3moe-router gate-qwen35moe-meta gate-gpu-qwen35-dspark gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-minicpm-text gate-gpu-qwen35-media gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen35moe-media gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-types gate-fixture gate-gpu-head gate-gpu-routed-legs gate-gpu-ds41-lib gate-gpu-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-nvtier gate-gpu-ds41-residency-a gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-long gate-gpu-ds41-chat gate-gpu-ds41-media gate-gpu-ds41-prefill gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-ds41-load gate-gpu-qwen3-serve gate-gpu-qwen38-serve gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-p8b gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-residency gate-gpu-glm5next-twocard gate-gpu-glm5next-serve gate-gpu-serve-start-ds41 gate-gpu-serve-start-glm: they link the arch dispatch and never open a gemma-embedding2 file -crates/model/src/arch/nemotron_h_moe/** gate-ops gate-nemotron-h-moe-meta gate-fixture gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-iq-sel gate-placement gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-ds41-bind gate-ds41-place gate-gpu-nemotron-serve gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 Nemotron 3.5 Bolt's header reader and fixture spec: its code runs where a nemotron_h_moe file is described (its meta test), where its fixture is planned and verified (the fixture generator's gate), in bloomery-model's own unit tests, and behind the arch dispatch and, through its plan (place::read), in bloomery-serve's nemotron seat, which gate-gpu-nemotron-serve and gate-gpu-nemotron-h-moe-serve start and every serve gate's bin links except gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35moe-mtp gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-gpu-clef-hidden gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-head gate-gpu-lib gate-gpu-ds41-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-nvtier gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-twocard gate-gpu-glm5next-residency: they link the arch dispatch and never open a nemotron_h_moe file -crates/model/src/arch/deepseek41/{hparams.rs,host.rs,kv.rs,mod.rs,names.rs,place.rs,plan.rs,roles.rs,spec.rs} gate-gpu-p4 gate-gpu-p6 gate-gpu-kquant gate-gpu-iq-sel gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-ptx-spill gate-dspark-read gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes every gate that links an arch module, less the gguf-only kernel gates (above): a family's code runs wherever any family's file is opened except gate-gpu-block gate-q38-serve-rules gate-gpu-nvtier: they open no deepseek41 file: only model::arch's dispatch by the file's general.architecture (arch/mod.rs) reaches it -crates/model/src/arch/qwen35moe/{head_list.rs,hparams.rs,host.rs,mod.rs,names.rs,place.rs,roles.rs,spec.rs} gate-gpu-p4 gate-gpu-p6 gate-gpu-kquant gate-gpu-iq-sel gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-ptx-spill gate-dspark-read gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-gpu-nvtier gate-q38-serve-rules gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-qwen38-nodes gate-gpu-nemotron-h-moe-e2e every gate that links an arch module, less the gguf-only kernel gates (above): a family's code runs wherever any family's file is opened except gate-gpu-block: they open no qwen35moe file: only model::arch's dispatch by the file's general.architecture (arch/mod.rs) reaches it -crates/model/src/{fixture/**,arch/deepseek41/fixture.rs} gate-ops gate-ds41-meta gate-fixture gate-deepseek4-meta gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-ds41-load gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-qwen35moe-e2e gate-gpu-minicpm-text gate-gpu-qwen38-nodes the fixture generator and V4.1's FixtureSpec: no file but bin fixture, the fixture test and arch/deepseek41/fixture.rs names crate::fixture (grep), so the gates are the fixture gate and the --lib runs that execute arch::deepseek41::fixture's own unit tests (gate-ops unfiltered, gate-ds41-meta and gate-deepseek4-meta filtered arch::deepseek41) -crates/model/src/arch/{coverage.rs,qwen35moe/mtp.rs} gate-gpu-iq-sel gate-gpu-e2e gate-ops gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-glm5next-gemm gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-app gate-gpu-lib gate-gpu-gates-lib gate-ds41-plan gate-gpu-ds41-lib gate-ds41-bind gate-ds41-place gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-ptx-spill gate-deepseek4-meta gate-glm5next-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-gpu-glm-sel gate-fixture gate-q38-serve-rules gate-gpu-nvtier gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-p6 gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-gpu-head gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-qwen38-nodes gate-gpu-nemotron-h-moe-e2e coverage::check's and qwen35moe's mtp readers: deepseek41/glm5next/qwen35moe place.rs (unimplemented, super::mtp), gpu-gates' engine.rs, gpu's qwen3moe plan/plan38/body38 (turns_as_neox, .unimplemented()) and the meta tests that call coverage::check — every gate that opens or plans those families' files; the two modules' lists coincide -crates/model/src/arch/names.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-q38-serve-rules gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-lib gate-gpu-glm5next-nvtier gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-qwen3moe-e2e gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-iq-sel gate-gpu-ds41-moe gate-gpu-moe gate-gpu-p8 gate-gpu-block gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-router gate-placement gate-qwen35moe-meta gate-gpu-clef-hidden gate-gpu-qwen35-dspark gate-gpu-qwen35moe-moe gate-gpu-gemm gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-experts gate-ds41-host gate-fixture gate-union gate-gpu-ds41-comp gate-gpu-head gate-gpu-mcol gate-ds41-bind gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-place gate-gpu-ds41-chain-glue gate-gpu-ds41-engram gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-twocard gate-gpu-ds41-draft gate-gpu-ds41-long gate-gpu-ds41-nvtier gate-gpu-ds41-dspark-loop gate-gpu-ds41-media gate-gpu-ds41-prefill gate-ds41-load gate-gpu-ds41-chat gate-gpu-p8b gate-dspark-read gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-qwen35moe-mtp gate-app gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes the routed stacks' common names (routed_stacks): the GLM, Qwen3.6 and Qwen3.8 stack tables (swap.rs, qwen3moe/swap.rs for swap35.rs and swap38.rs) and the GLM card check (glm5next_card.rs) call it — every gate that opens those bodies or runs that check -crates/model/src/arch/mod.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-ptx-spill gate-dspark-read gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-vision gate-deepseek4-meta gate-glm5next-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-q38-serve-rules gate-gpu-xstream gate-gpu-mimo2-flash gate-gpu-nvtier gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes the dispatch and Arch: every gate that links it, the gguf-only kernel gates too — bloomery-gpu's fault sites (fault.rs) and bloomery-gpu-gates' family check (lib.rs, the oracle) read Arch in them -crates/gpu/src/weights.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq-sel gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-qwen35moe-attn gate-gpu-glm5next-gemm gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-app gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-ds41-place gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-ptx-spill gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-q38-serve-rules gate-gpu-nvtier gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes the card weight loaders: the gates whose own targets name weights, or reach it through an open path (gpu's arch bodies -> model.rs -> host/swap_source.rs, gpu-deepseek41's body/chain/dense/experts/draft, gpu-glm5next's body/ffn; gate-gpu-xstream names GpuModel), less the synthetic kernel gates whose bins build blocks and name no weights path +crates/model/src/arch/{mod.rs,dspark.rs,classify.rs,stems.rs,deepseek2/**,qwen3moe/**,glm5next/**} gate-gpu-p4 gate-gpu-p6 gate-gpu-kquant gate-gpu-iq-sel gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-ptx-spill gate-dspark-read gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-q38-serve-rules gate-gpu-block gate-gpu-nvtier gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p5 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-cand gate-gpu-iq gate-gpu-q6k-ids gate-gpu-xstream gate-gpu-mimo2-flash gate-gpu-ple gate-gpu-vision gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e every gate that links an arch module, less the gguf-only kernel gates (above): a family's code runs wherever any family's file is opened +crates/model/src/arch/mimo2/** gate-ops gate-mimo2-meta gate-gpu-mimo2-flash gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-ds41-bind gate-ds41-place gate-gpu-routed-legs gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-minicpm-text weekly-gpu-mimo2-e2e weekly-gpu-mimo2-serve gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e MiMo-V2.6-Flash's reader and plan: its code runs only where a mimo2 file opens (its meta test, its flash gate) in bloomery-model's own unit tests (gate-ops runs --lib) and in bloomery-serve's mimo2 seat, which every serve gate's bin links except gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-clef-hidden gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen3moe-e2e gate-fixture gate-gpu-head gate-gpu-lib gate-gpu-ds41-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency: they open no mimo2 file: only model::arch's dispatch by the file's general.architecture (arch/mod.rs) reaches it +crates/model/src/arch/nemotron_h/** gate-ops gate-nemotron-meta gate-fixture gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-iq-sel gate-placement gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-ds41-bind gate-ds41-place gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e Nemotron 3 Nano 4B's header reader and fixture spec: its code runs where a nemotron_h file is described (its meta test), where its fixture is planned and verified (the fixture generator's gate), in bloomery-model's own unit tests, where the card body reads the layer kinds and widths at load (the end-to-end gate), and behind the arch dispatch and, through its plan (place.rs), in bloomery-serve's nemotron seat, which gate-gpu-nemotron-serve starts and every serve gate's bin links except gate-gpu-e2e gate-gpu-hybrid gate-gpu-p6 gate-gpu-ds41-moe gate-gpu-moe gate-gpu-p8 gate-gpu-block gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen35moe-moe gate-gpu-qwen3moe-router gate-qwen35moe-meta gate-gpu-qwen35-dspark gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-minicpm-text gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-types gate-gpu-head gate-gpu-routed-legs gate-gpu-ds41-lib gate-gpu-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-nvtier gate-gpu-ds41-residency-a gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-long gate-gpu-ds41-chat gate-gpu-ds41-media gate-gpu-ds41-prefill gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-ds41-load gate-gpu-qwen38-serve gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-p8b gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-residency gate-gpu-glm5next-twocard: they link the arch dispatch and never open a nemotron_h file +crates/model/src/arch/gemma_embedding2/** gate-ops gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-iq-sel gate-placement gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-ds41-bind gate-ds41-place EmbeddingGemma 2's header reader: no program runs it, so its code runs where a gemma-embedding2 file is described (its meta test), in bloomery-model's own unit tests, and behind the arch dispatch except gate-gpu-e2e gate-gpu-hybrid gate-gpu-p6 gate-gpu-ds41-moe gate-gpu-moe gate-gpu-p8 gate-gpu-block gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen35moe-moe gate-gpu-qwen3moe-router gate-qwen35moe-meta gate-gpu-qwen35-dspark gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-minicpm-text gate-gpu-qwen35-media gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen35moe-media gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-types gate-fixture gate-gpu-head gate-gpu-routed-legs gate-gpu-ds41-lib gate-gpu-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-nvtier gate-gpu-ds41-residency-a gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-long gate-gpu-ds41-chat gate-gpu-ds41-media gate-gpu-ds41-prefill gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-ds41-load gate-gpu-qwen3-serve gate-gpu-qwen38-serve gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-p8b gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-residency gate-gpu-glm5next-twocard gate-gpu-glm5next-serve gate-gpu-serve-start-ds41 gate-gpu-serve-start-glm: they link the arch dispatch and never open a gemma-embedding2 file +crates/model/src/arch/nemotron_h_moe/** gate-ops gate-nemotron-h-moe-meta gate-fixture gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-iq-sel gate-placement gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-ds41-bind gate-ds41-place gate-gpu-nemotron-serve gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-gemma-embedding2-e2e Nemotron 3.5 Bolt's header reader and fixture spec: its code runs where a nemotron_h_moe file is described (its meta test), where its fixture is planned and verified (the fixture generator's gate), in bloomery-model's own unit tests, and behind the arch dispatch and, through its plan (place::read), in bloomery-serve's nemotron seat, which gate-gpu-nemotron-serve and gate-gpu-nemotron-h-moe-serve start and every serve gate's bin links except gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35moe-mtp gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-gpu-clef-hidden gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-head gate-gpu-lib gate-gpu-ds41-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-nvtier gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-twocard gate-gpu-glm5next-residency: they link the arch dispatch and never open a nemotron_h_moe file +crates/model/src/arch/deepseek41/{hparams.rs,host.rs,kv.rs,mod.rs,names.rs,place.rs,plan.rs,roles.rs,spec.rs} gate-gpu-p4 gate-gpu-p6 gate-gpu-kquant gate-gpu-iq-sel gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-ptx-spill gate-dspark-read gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e every gate that links an arch module, less the gguf-only kernel gates (above): a family's code runs wherever any family's file is opened except gate-gpu-block gate-q38-serve-rules gate-gpu-nvtier: they open no deepseek41 file: only model::arch's dispatch by the file's general.architecture (arch/mod.rs) reaches it +crates/model/src/arch/qwen35moe/{head_list.rs,hparams.rs,host.rs,mod.rs,names.rs,place.rs,roles.rs,spec.rs} gate-gpu-p4 gate-gpu-p6 gate-gpu-kquant gate-gpu-iq-sel gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-ptx-spill gate-dspark-read gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-gpu-nvtier gate-q38-serve-rules gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-qwen38-nodes gate-gpu-nemotron-h-moe-e2e gate-gpu-gemma-embedding2-e2e every gate that links an arch module, less the gguf-only kernel gates (above): a family's code runs wherever any family's file is opened except gate-gpu-block: they open no qwen35moe file: only model::arch's dispatch by the file's general.architecture (arch/mod.rs) reaches it +crates/model/src/{fixture/**,arch/deepseek41/fixture.rs} gate-ops gate-ds41-meta gate-fixture gate-deepseek4-meta gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-ds41-load gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-qwen35moe-e2e gate-gpu-minicpm-text gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e the fixture generator and V4.1's FixtureSpec: no file but bin fixture, the fixture test and arch/deepseek41/fixture.rs names crate::fixture (grep), so the gates are the fixture gate and the --lib runs that execute arch::deepseek41::fixture's own unit tests (gate-ops unfiltered, gate-ds41-meta and gate-deepseek4-meta filtered arch::deepseek41) +crates/model/src/arch/{coverage.rs,qwen35moe/mtp.rs} gate-gpu-iq-sel gate-gpu-e2e gate-ops gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-glm5next-gemm gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-app gate-gpu-lib gate-gpu-gates-lib gate-ds41-plan gate-gpu-ds41-lib gate-ds41-bind gate-ds41-place gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-ptx-spill gate-deepseek4-meta gate-glm5next-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-gpu-glm-sel gate-fixture gate-q38-serve-rules gate-gpu-nvtier gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-p6 gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-gpu-head gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-qwen38-nodes gate-gpu-nemotron-h-moe-e2e gate-gpu-gemma-embedding2-e2e coverage::check's and qwen35moe's mtp readers: deepseek41/glm5next/qwen35moe place.rs (unimplemented, super::mtp), gpu-gates' engine.rs, gpu's qwen3moe plan/plan38/body38 (turns_as_neox, .unimplemented()) and the meta tests that call coverage::check — every gate that opens or plans those families' files; the two modules' lists coincide +crates/model/src/arch/names.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-q38-serve-rules gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-lib gate-gpu-glm5next-nvtier gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-qwen3moe-e2e gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-iq-sel gate-gpu-ds41-moe gate-gpu-moe gate-gpu-p8 gate-gpu-block gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-router gate-placement gate-qwen35moe-meta gate-gpu-clef-hidden gate-gpu-qwen35-dspark gate-gpu-qwen35moe-moe gate-gpu-gemm gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-experts gate-ds41-host gate-fixture gate-union gate-gpu-ds41-comp gate-gpu-head gate-gpu-mcol gate-ds41-bind gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-place gate-gpu-ds41-chain-glue gate-gpu-ds41-engram gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-twocard gate-gpu-ds41-draft gate-gpu-ds41-long gate-gpu-ds41-nvtier gate-gpu-ds41-dspark-loop gate-gpu-ds41-media gate-gpu-ds41-prefill gate-ds41-load gate-gpu-ds41-chat gate-gpu-p8b gate-dspark-read gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-qwen35moe-mtp gate-app gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e the routed stacks' common names (routed_stacks): the GLM, Qwen3.6 and Qwen3.8 stack tables (swap.rs, qwen3moe/swap.rs for swap35.rs and swap38.rs) and the GLM card check (glm5next_card.rs) call it — every gate that opens those bodies or runs that check +crates/model/src/arch/mod.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-ptx-spill gate-dspark-read gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-vision gate-deepseek4-meta gate-glm5next-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-q38-serve-rules gate-gpu-xstream gate-gpu-mimo2-flash gate-gpu-nvtier gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e the dispatch and Arch: every gate that links it, the gguf-only kernel gates too — bloomery-gpu's fault sites (fault.rs) and bloomery-gpu-gates' family check (lib.rs, the oracle) read Arch in them +crates/gpu/src/weights.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq-sel gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-qwen35moe-attn gate-gpu-glm5next-gemm gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-app gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-ds41-place gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-ptx-spill gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-q38-serve-rules gate-gpu-nvtier gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e the card weight loaders: the gates whose own targets name weights, or reach it through an open path (gpu's arch bodies -> model.rs -> host/swap_source.rs, gpu-deepseek41's body/chain/dense/experts/draft, gpu-glm5next's body/ffn; gate-gpu-xstream names GpuModel), less the synthetic kernel gates whose bins build blocks and name no weights path # crates/gpu's per-arch bodies: GpuModel is generic; a body's impls run where B is that body. crates/gpu/src/arch/qwen3moe/** gate-app gate-ds41-bind gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-e2e gate-gpu-gemm gate-gpu-glm5next-serve gate-gpu-lev-serve gate-gpu-nvtier gate-gpu-qwen3-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-qwen35moe-media gate-gpu-qwen35moe-moe gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-head gate-gpu-qwen3moe-router gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-q38-serve-rules gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-qwen38-nodes the Qwen3, Qwen3.5 and Qwen4 bodies: the bins that name them (the qwen gates, gate_gemm) and AnyEngine (gate_e2e; bind's tests build it under deepseek41) -crates/gpu/src/site.rs gate-ds41-bind gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-ds41-lib gate-gpu-e2e gate-gpu-gemm gate-gpu-glm5next-gemm gate-gpu-glm5next-e2e gate-gpu-glm5next-mtp gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-glm5next-stagger gate-gpu-glm5next-nvtier gate-gpu-glm5next-twocard gate-gpu-lev-serve gate-gpu-nvtier gate-gpu-qwen3-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-qwen35moe-media gate-gpu-qwen35moe-moe gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-head gate-gpu-qwen3moe-router gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-q38-serve-rules gate-gpu-p1 gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e the weight sites' launches by type: read by the Qwen bodies and the Clef hidden gate, the bodies that read a site in more than one type +crates/gpu/src/site.rs gate-ds41-bind gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-ds41-lib gate-gpu-e2e gate-gpu-gemm gate-gpu-glm5next-gemm gate-gpu-glm5next-e2e gate-gpu-glm5next-mtp gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-glm5next-stagger gate-gpu-glm5next-nvtier gate-gpu-glm5next-twocard gate-gpu-lev-serve gate-gpu-nvtier gate-gpu-qwen3-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-qwen35moe-media gate-gpu-qwen35moe-moe gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-head gate-gpu-qwen3moe-router gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-q38-serve-rules gate-gpu-p1 gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-gemma-embedding2-e2e the weight sites' launches by type: read by the Qwen bodies and the Clef hidden gate, the bodies that read a site in more than one type crates/gpu/src/arch/deepseek2/** gate-ds41-bind gate-gpu-e2e gate-gpu-hybrid gate-gpu-moe gate-gpu-p0b gate-gpu-p10 gate-gpu-p4 gate-gpu-p8 gate-gpu-p8b the V2-Lite body: the V2-Lite GPU gates and AnyEngine (gate_e2e) # Derived rows: each row's recipes are `python3 tools/recipes.py reach --row ` at the time of writing, the # gates whose targets can run code of the files through another lib (tools/recipes.py, Reach); files grouped only # where the derivation gives each the same gates. `just check-recipes` holds each to the derivation. -crates/gpu/src/{host/mod.rs,host/swap.rs,lib.rs} gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-app gate-gpu-head gate-gpu-mcol gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-vision gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes the Gpu and GpuError, whose Poison and Unresident variants hold host::Poison and host::swap::Unresident: every gate whose code names a bloomery_gpu error or the Gpu +crates/gpu/src/{host/mod.rs,host/swap.rs,lib.rs} gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-app gate-gpu-head gate-gpu-mcol gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-vision gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e the Gpu and GpuError, whose Poison and Unresident variants hold host::Poison and host::swap::Unresident: every gate whose code names a bloomery_gpu error or the Gpu crates/gpu/src/lane.rs gate-gpu-clef-serve gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-glm5next-serve gate-gpu-glm5next-twocard gate-gpu-lev-serve gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3-serve gate-q38-serve-rules gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve the lane probe a load prints beside its load line (probe_card, record::lane): the seats and generators that call it -crates/gpu/src/{host/lane.rs,watchdog.rs,flash_gqa.rs,elem.rs} gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-app gate-gpu-head gate-gpu-mcol gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes the Gpu's own parts: watchdog's wait_done and the swap lane's Shared, flash's seg_span and ElemKernels, which lib.rs's Gpu and host/mod.rs hold -crates/gpu/src/model.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-head gate-gpu-mcol gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-p2 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes GpuModel, the ChainBody and Rollback traits and StepMode: the gates that open a model body or name its step mode -crates/gpu/src/host/swap_source.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes ResidencyGlue (host/served.rs) and DEADLINE (the qwen3moe bodies' swap38): the bodies that serve a residency -crates/gpu/src/model/slots.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-ds41-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes GpuModel's slot impls (slots, selected) and SlotRows: the gates that open a body with slots -crates/gpu/src/{host/nvtier.rs,host/nvtier/reader.rs,host/step.rs} gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes the host tier's step (hybrid::Chain in model.rs) and the NvTier arena (host/mod.rs): the gates that open a body with a host tier +crates/gpu/src/{host/lane.rs,watchdog.rs,flash_gqa.rs,elem.rs} gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-app gate-gpu-head gate-gpu-mcol gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e the Gpu's own parts: watchdog's wait_done and the swap lane's Shared, flash's seg_span and ElemKernels, which lib.rs's Gpu and host/mod.rs hold +crates/gpu/src/model.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-head gate-gpu-mcol gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-p2 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e GpuModel, the ChainBody and Rollback traits and StepMode: the gates that open a model body or name its step mode +crates/gpu/src/host/swap_source.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e ResidencyGlue (host/served.rs) and DEADLINE (the qwen3moe bodies' swap38): the bodies that serve a residency +crates/gpu/src/model/slots.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-ds41-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e GpuModel's slot impls (slots, selected) and SlotRows: the gates that open a body with slots +crates/gpu/src/{host/nvtier.rs,host/nvtier/reader.rs,host/step.rs} gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e the host tier's step (hybrid::Chain in model.rs) and the NvTier arena (host/mod.rs): the gates that open a body with a host tier crates/gpu/src/host/run.rs gate-gpu-e2e gate-gpu-ds41-chain-ffn gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-gpu-ds41-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-qwen38-nodes HostRun, the host expert leg: the V4.1, Qwen3.8 and GLM bodies that hold one -crates/gpu/src/host/xstream.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes XStream, the host tier's cross-stream lane (host/mod.rs), and XMode in the qwen38 bins +crates/gpu/src/host/xstream.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e XStream, the host tier's cross-stream lane (host/mod.rs), and XMode in the qwen38 bins crates/gpu/src/host/prompt_stream.rs gate-gpu-clef-serve gate-gpu-glm5next-serve gate-gpu-lev-serve gate-gpu-nvtier gate-q38-serve-rules gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-qwen3moe-e2e gate-gpu-clef-hidden gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-qwen38-nodes the prompt call's stream state and its calls (host/mod.rs), reached through the bodies that hold it -crates/gpu/src/host/batch.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-linear gate-gpu-qwen3moe-e2e gate-app gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes BatchPort, the host tier's prompt batch (host/mod.rs), and the legs' ServeTimes -crates/gpu/src/head.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-head gate-gpu-mcol gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-chain-glue gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes Head and HeadNorm, which GpuModel holds (model.rs): the gates that open a model body -crates/gpu/src/rope_table.rs gate-gpu-e2e gate-gpu-qwen3moe-rope gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen35moe-attn gate-gpu-mimo2-flash gate-gpu-qwen3moe-e2e gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-q38-serve-rules gate-gpu-ds41-chain-glue gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-qwen38-nodes RopeTable (V4.1's params) and RopeRows (the qwen3moe bodies' scratch) +crates/gpu/src/host/batch.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-linear gate-gpu-qwen3moe-e2e gate-app gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e BatchPort, the host tier's prompt batch (host/mod.rs), and the legs' ServeTimes +crates/gpu/src/head.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-head gate-gpu-mcol gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-chain-glue gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e Head and HeadNorm, which GpuModel holds (model.rs): the gates that open a model body +crates/gpu/src/rope_table.rs gate-gpu-e2e gate-gpu-qwen3moe-rope gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen35moe-attn gate-gpu-mimo2-flash gate-gpu-qwen3moe-e2e gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-q38-serve-rules gate-gpu-ds41-chain-glue gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e RopeTable (V4.1's params) and RopeRows (the qwen3moe bodies' scratch) crates/gpu/src/media.rs gate-ds41-load gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-ds41-callstream gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-lib gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-prefill gate-gpu-ds41-residency-a gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-e2e gate-gpu-glm5next-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3moe-e2e gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-e2e gate-gpu-glm5next-residency gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-ds41-bind gate-ds41-place gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-qwen4exp-mtp gate-app gate-q38-serve-rules gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-nvtier gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve MediaSpan/ImageSpan/SeqRope and the MediaBody trait: the V4.1 and Qwen3.6/Clef media calls and the sessions that carry them crates/gpu/src/iq_sel.rs gate-gpu-iq-sel gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-q38-serve-rules gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-mtp gate-gpu-glm5next-residency gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes IqSelKernels, the Qwen3.8 card's IQ selection (card38.rs), the common card leg's i-quant arms (card_leg.rs, which GLM's body and mimo2's reach), and its own gate crates/gpu/src/checkpoint/{saved,staging}.rs gate-ds41-load gate-gpu-clef-serve gate-gpu-ds41-callstream gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-lib gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-prefill gate-gpu-ds41-residency-a gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-glm5next-e2e gate-gpu-glm5next-mtp gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-lev-serve gate-gpu-nvtier gate-gpu-qwen3-serve gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-q38-serve-rules gate-ds41-bind gate-ds41-place gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-qwen38-nodes the host copy of a sequence state (SeqState, the pinned staging pool) under the Qwen3.8 and GLM bodies' seq_save and seq_resume, the V4.1 serve seat and the gates that open those bodies crates/gpu/src/checkpoint.rs gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-mtp gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-lev-serve gate-gpu-nvtier gate-gpu-qwen3-serve gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3moe-e2e gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-q38-serve-rules gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-qwen38-nodes the checkpoint ring (Checkpoints: the points a prompt call keeps, adopt, take, the save lane's point copy and its in-flight refusal) under the Qwen3.8 and GLM bodies and the gates that open them -crates/gpu/src/tensor.rs gate-app gate-ds41-bind gate-ds41-load gate-ds41-place gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-ds41-attn gate-gpu-ds41-callstream gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-comp gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-lib gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-moe gate-gpu-ds41-prefill gate-gpu-ds41-residency-a gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-woa gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-e2e gate-gpu-gemm gate-gpu-glm-mla gate-gpu-glm-router gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-gemm gate-gpu-glm5next-mtp gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-hc-gated gate-gpu-head gate-gpu-hybrid gate-gpu-iq-sel gate-gpu-kquant gate-gpu-lev-serve gate-gpu-mcol gate-gpu-moe gate-gpu-nvtier gate-gpu-p0b gate-gpu-p1 gate-gpu-p10 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p8 gate-gpu-p8b gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-q6k-ids gate-gpu-qwen3-serve gate-gpu-qwen35moe-attn gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35moe-moe gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-head gate-gpu-qwen3moe-router gate-gpu-qwen3vl-tower gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-swap gate-gpu-vision gate-q38-serve-rules gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes the device tensor and the 2-D copies (dtoh_2d, htod_2d) every gate that links the gpu crate through a body reads -crates/gpu-gates/src/lib.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-gpu-head gate-gpu-mcol gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-q38-serve-rules gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-dspark-read gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-vision gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes GateError and the family check (the oracle), which the bins' shared modules name -crates/gpu-gates/src/tier.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-gpu-head gate-gpu-mcol gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-q38-serve-rules gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-dspark-read gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-minicpm-text gate-gpu-qwen35moe-attn gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes Tier::from_env (lib.rs's family check) and tier::card, which shared/gate_card.rs runs in the card bins +crates/gpu/src/tensor.rs gate-app gate-ds41-bind gate-ds41-load gate-ds41-place gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-ds41-attn gate-gpu-ds41-callstream gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-comp gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-lib gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-moe gate-gpu-ds41-prefill gate-gpu-ds41-residency-a gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-woa gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-e2e gate-gpu-gemm gate-gpu-glm-mla gate-gpu-glm-router gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-gemm gate-gpu-glm5next-mtp gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-hc-gated gate-gpu-head gate-gpu-hybrid gate-gpu-iq-sel gate-gpu-kquant gate-gpu-lev-serve gate-gpu-mcol gate-gpu-moe gate-gpu-nvtier gate-gpu-p0b gate-gpu-p1 gate-gpu-p10 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p8 gate-gpu-p8b gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-q6k-ids gate-gpu-qwen3-serve gate-gpu-qwen35moe-attn gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35moe-moe gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-head gate-gpu-qwen3moe-router gate-gpu-qwen3vl-tower gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-swap gate-gpu-vision gate-q38-serve-rules gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e the device tensor and the 2-D copies (dtoh_2d, htod_2d) every gate that links the gpu crate through a body reads +crates/gpu-gates/src/lib.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-gpu-head gate-gpu-mcol gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-q38-serve-rules gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-dspark-read gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-vision gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e GateError and the family check (the oracle), which the bins' shared modules name +crates/gpu-gates/src/tier.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-gpu-head gate-gpu-mcol gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-q38-serve-rules gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-dspark-read gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-minicpm-text gate-gpu-qwen35moe-attn gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e Tier::from_env (lib.rs's family check) and tier::card, which shared/gate_card.rs runs in the card bins crates/gpu-gates/src/residency.rs gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3moe-e2e gate-q38-serve-rules gate-gpu-nvtier gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-chat weekly-gpu-mimo2-e2e weekly-gpu-mimo2-serve gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve the unset residency rules' plan inputs (PlanRoom) and the pair rule's call (at_plan): the qwen38, GLM, V4.1 and MiMo residency, serve and placement bins crates/gpu-gates/src/residency38.rs gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3moe-e2e gate-q38-serve-rules gate-gpu-nvtier gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-chat gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve the Qwen3.8 residency figures (checkpoint_bytes, CARD38, glm_seqs): the qwen38 and GLM residency, serve and placement bins crates/gpu-gates/src/q38_fixture.rs gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen38-twocard the Qwen3.8 fixture's Shape, which shared/q38_arch.rs reads crates/gpu-gates/src/q36_fixture.rs gate-gpu-qwen35moe-e2e gate-gpu-minicpm-text the Qwen3.6 body's per-layer launch counts from the header, which gate_qwen35moe_e2e reads, and which the MiniCPM text recipe runs through that bin -crates/gpu-gates/src/act_rule.rs gate-gpu-iq gate-gpu-ds41-hc gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-minicpm-text gate-gpu-ds41-index gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-chain-attn gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-gpu-ds41-step gate-gpu-dspark-experts gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e the activation rules: the MoE, chain, index, comp, hc, iq, gemm and dspark experts bins that name them, and the fixture bands' readers of site_rel (shared/glm5next_tier.rs, the qwen4exp e2e and MTP bins, ds41-step's head; the qwen3moe experts, down and e2e bins and the qwen35moe e2e bin through ik_q8_2's non-finite check) -crates/gpu-gates/src/flip.rs gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-ds41-step gate-gpu-glm5next-mtp gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-h-moe-e2e gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-qwen38-residency Flip, which shared/e2e.rs, the qwen4exp MTP bin, ds41-step's head cap and the GLM MTP replay's margin cap read -crates/model/src/lib.rs gate-gpu-p4 gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes ModelError, which every model reader returns -crates/model/src/moe.rs gate-gpu-p4 gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes TierError in ModelError (lib.rs), and the host expert legs the bodies and tiers call -crates/model/src/ops.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-app gate-prompts gate-gpu-lib gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes DEFER_MAX_COLS (gpu's host/step.rs and host/tier.rs), Tensor2 and the matmul dispatch -crates/model/src/embed.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-ds41-lib gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen3-serve gate-ds41-load gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve RowLayout and EmbedTable: the V4.1 glue, the draft and the GLM body -crates/model/src/placement/workstation.rs gate-gpu-iq-sel gate-placement gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-fixture gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-app gate-gpu-ds41-chain-ffn gate-gpu-ds41-lib gate-gpu-e2e gate-gpu-hybrid gate-gpu-moe gate-gpu-p10 gate-gpu-p8 gate-gpu-p8b gate-gpu-swap gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes host_available, which GpuModel's placement (gpu's model.rs) and the qwen35moe plan read -crates/model/src/arch/qwen35moe/fixture.rs gate-fixture the Qwen3.8 fixture spec, which the fixture test reads -crates/model/data/mtp-head-rows-qwen38-*.txt gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-mtp gate-q38-serve-rules gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve Qwen3.8's shipped MTP head list, which arch/qwen35moe/head_list.rs includes (include_str!): the gates whose targets run its reader (head_rows_of, SHIPPED) -tools/fixture-variants.tsv gate-fixture the fixture variants' table, which fixture/variant.rs reads by path at run time: the fixture test -crates/models/src/lib.rs gate-gpu-p4 gate-gpu-p6 gate-gpu-iq-sel gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-runtime gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-dspark-read gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gpu-qwen35moe-attn gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta its re-exports (Role, Need: every placement path names models::Role through them) and ModelSpec and Score, which the arch readers and models::shape read -crates/models/src/shape.rs gate-gpu-p6 gate-gpu-iq-sel gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-e2e gate-fixture gate-gpu-head gate-gpu-lib gate-ds41-plan gate-gpu-ds41-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-ds41-bind gate-ds41-place gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-placement gate-nemotron-meta gate-nemotron-h-moe-meta gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta MoeShape and the router rows (RouterInst, router_row) the routers' kernels select by -crates/gguf/src/lib.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-qdot gate-engram gate-tokenizer gate-decision-clef gate-decision-lev gate-serve gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-vision gate-gpu-vision gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-refset gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-models gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gpu-mamba2 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta LoadError (in GpuError and placement's errors) and Gguf: every gate that reads a file or names a load error -crates/gguf/src/quant.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-qdot gate-engram gate-tokenizer gate-decision-clef gate-decision-lev gate-serve gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-refset gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-vision gate-gpu-vision gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-dspark-read gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-models gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-hf gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gpu-mamba2 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta GgmlType (gguf's lib.rs, placement) and the f16 conversion the card's fused kernels name; gate-dspark-read also runs gguf's own quant tests -crates/gguf/src/iq_tables.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-kquant gate-gpu-hc-gated gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-ds41-meta gate-qwen3moe-meta gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-gemm gate-gpu-glm5next-gemm gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-qdot gate-decision-clef gate-decision-lev gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta KVALUES_IQ4NL, which quant.rs's dequant and the card's IQ kernels read -crates/qdot/src/lib.rs gate-gpu-p4 gate-gpu-p6 gate-gpu-kquant gate-gpu-iq-sel gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen35moe-attn gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-qdot gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gpu-mamba2 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta QdotError in ModelError, and the fused dots coverage and quant ask for -crates/threads/src/lib.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-ds41-meta gate-qwen3moe-meta gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-glm5next-gemm gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-threads gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-ds41-plan gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gpu-p1 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta the pool: upload's memcpy_pool, ops' matmul and the host tier's legs -crates/engram/src/lib.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-lib gate-gpu-gates-lib gate-ds41-plan gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes EngramError (direct.rs) and Engram, which the V4.1 chain's glue holds -crates/models/src/headpick.rs gate-deepseek4-meta gate-ds41-meta gate-ds41-plan gate-fixture gate-glm5next-meta gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-twocard gate-gpu-glm5next-serve gate-gpu-iq-sel gate-gpu-lev-serve gate-gpu-nvtier gate-gpu-qwen3-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35moe-media gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3moe-e2e gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-mimo2-meta gate-ops gate-q38-serve-rules gate-qwen3moe-meta gate-qwen4exp-meta gate-runtime gate-ds41-bind gate-ds41-place gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35-dspark gate-gpu-gemm gate-gpu-head gate-gpu-lib gate-gpu-ds41-lib gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-nvtier gate-gpu-ds41-long gate-gpu-ds41-prefill gate-gpu-ds41-media gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-nemotron-meta gate-nemotron-h-moe-meta gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta models::headpick, the row-list head's reader, digest and loaded set: every gate whose load reads a head list gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35-dspark gate-gpu-gemm gate-gpu-head gate-gpu-lib gate-gpu-ds41-lib gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-nvtier gate-gpu-ds41-long gate-gpu-ds41-prefill gate-gpu-ds41-media gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-twocard gate-gpu-glm5next-residency +crates/gpu-gates/src/act_rule.rs gate-gpu-iq gate-gpu-ds41-hc gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-minicpm-text gate-gpu-ds41-index gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-chain-attn gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-gpu-ds41-step gate-gpu-dspark-experts gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-gemma-embedding2-e2e the activation rules: the MoE, chain, index, comp, hc, iq, gemm and dspark experts bins that name them, and the fixture bands' readers of site_rel (shared/glm5next_tier.rs, the qwen4exp e2e and MTP bins, ds41-step's head; the qwen3moe experts, down and e2e bins and the qwen35moe e2e bin through ik_q8_2's non-finite check) +crates/gpu-gates/src/flip.rs gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-ds41-step gate-gpu-glm5next-mtp gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-h-moe-e2e gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-qwen38-residency gate-gpu-gemma-embedding2-e2e Flip, which shared/e2e.rs, the qwen4exp MTP bin, ds41-step's head cap and the GLM MTP replay's margin cap read +crates/model/src/lib.rs gate-gpu-p4 gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e ModelError, which every model reader returns +crates/model/src/moe.rs gate-gpu-p4 gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e TierError in ModelError (lib.rs), and the host expert legs the bodies and tiers call +crates/model/src/ops.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-app gate-prompts gate-gpu-lib gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e DEFER_MAX_COLS (gpu's host/step.rs and host/tier.rs), Tensor2 and the matmul dispatch +crates/model/src/embed.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-ds41-lib gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen3-serve gate-ds41-load gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-e2e RowLayout and EmbedTable: the V4.1 glue, the draft and the GLM body +crates/model/src/placement/workstation.rs gate-gpu-iq-sel gate-placement gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-fixture gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-app gate-gpu-ds41-chain-ffn gate-gpu-ds41-lib gate-gpu-e2e gate-gpu-hybrid gate-gpu-moe gate-gpu-p10 gate-gpu-p8 gate-gpu-p8b gate-gpu-swap gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e host_available, which GpuModel's placement (gpu's model.rs) and the qwen35moe plan read +crates/model/src/arch/qwen35moe/fixture.rs gate-fixture gate-gpu-gemma-embedding2-e2e the Qwen3.8 fixture spec, which the fixture test reads +crates/model/data/mtp-head-rows-qwen38-*.txt gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-mtp gate-q38-serve-rules gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-e2e Qwen3.8's shipped MTP head list, which arch/qwen35moe/head_list.rs includes (include_str!): the gates whose targets run its reader (head_rows_of, SHIPPED) +tools/fixture-variants.tsv gate-fixture gate-gpu-gemma-embedding2-e2e the fixture variants' table, which fixture/variant.rs reads by path at run time: the fixture test +crates/models/src/lib.rs gate-gpu-p4 gate-gpu-p6 gate-gpu-iq-sel gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-runtime gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-dspark-read gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gpu-qwen35moe-attn gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e its re-exports (Role, Need: every placement path names models::Role through them) and ModelSpec and Score, which the arch readers and models::shape read +crates/models/src/shape.rs gate-gpu-p6 gate-gpu-iq-sel gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-e2e gate-fixture gate-gpu-head gate-gpu-lib gate-ds41-plan gate-gpu-ds41-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-ds41-bind gate-ds41-place gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-placement gate-nemotron-meta gate-nemotron-h-moe-meta gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e MoeShape and the router rows (RouterInst, router_row) the routers' kernels select by +crates/gguf/src/lib.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-qdot gate-engram gate-tokenizer gate-decision-clef gate-decision-lev gate-serve gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-vision gate-gpu-vision gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-refset gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-models gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gpu-mamba2 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e LoadError (in GpuError and placement's errors) and Gguf: every gate that reads a file or names a load error +crates/gguf/src/quant.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-qdot gate-engram gate-tokenizer gate-decision-clef gate-decision-lev gate-serve gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-refset gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-vision gate-gpu-vision gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-dspark-read gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-models gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-hf gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gpu-mamba2 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e GgmlType (gguf's lib.rs, placement) and the f16 conversion the card's fused kernels name; gate-dspark-read also runs gguf's own quant tests +crates/gguf/src/iq_tables.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-kquant gate-gpu-hc-gated gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-ds41-meta gate-qwen3moe-meta gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-gemm gate-gpu-glm5next-gemm gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-qdot gate-decision-clef gate-decision-lev gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e KVALUES_IQ4NL, which quant.rs's dequant and the card's IQ kernels read +crates/qdot/src/lib.rs gate-gpu-p4 gate-gpu-p6 gate-gpu-kquant gate-gpu-iq-sel gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen35moe-attn gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-qdot gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gpu-mamba2 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e QdotError in ModelError, and the fused dots coverage and quant ask for +crates/threads/src/lib.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-ds41-meta gate-qwen3moe-meta gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-glm5next-gemm gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-threads gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-ds41-plan gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gpu-p1 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e the pool: upload's memcpy_pool, ops' matmul and the host tier's legs +crates/engram/src/lib.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-lib gate-gpu-gates-lib gate-ds41-plan gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e EngramError (direct.rs) and Engram, which the V4.1 chain's glue holds +crates/models/src/headpick.rs gate-deepseek4-meta gate-ds41-meta gate-ds41-plan gate-fixture gate-glm5next-meta gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-twocard gate-gpu-glm5next-serve gate-gpu-iq-sel gate-gpu-lev-serve gate-gpu-nvtier gate-gpu-qwen3-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35moe-media gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3moe-e2e gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-mimo2-meta gate-ops gate-q38-serve-rules gate-qwen3moe-meta gate-qwen4exp-meta gate-runtime gate-ds41-bind gate-ds41-place gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35-dspark gate-gpu-gemm gate-gpu-head gate-gpu-lib gate-gpu-ds41-lib gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-nvtier gate-gpu-ds41-long gate-gpu-ds41-prefill gate-gpu-ds41-media gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-nemotron-meta gate-nemotron-h-moe-meta gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e models::headpick, the row-list head's reader, digest and loaded set: every gate whose load reads a head list gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35-dspark gate-gpu-gemm gate-gpu-head gate-gpu-lib gate-gpu-ds41-lib gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-nvtier gate-gpu-ds41-long gate-gpu-ds41-prefill gate-gpu-ds41-media gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-twocard gate-gpu-glm5next-residency crates/engram/src/rows.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gates-lib gate-q38-serve-rules gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-qwen38-nodes SplitTable, which the Qwen3.8 body opens crates/hf/src/fetch.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-fixture gate-ds41-bind gate-ds41-place gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve the fetch Event that model_file.rs reports; the fixture fetch's range reads crates/decision/src/lib.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-decision-lev gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve the decision crate's Error: the decision gates crates/vision/src/lib.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-serve gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-vision gate-gpu-glm5next-serve gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-decision-clef gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-ds41-media gate-gpu-routed-legs gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve VisionError, which the media and preprocess paths return -crates/gpu/src/gelu.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-mamba2 gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-gpu-clef-hidden gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-app gate-gpu-head gate-gpu-mcol gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-nvtier gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-vision gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-glm5v-tower gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes the shared tanh GELU rule: the registered elementwise module and image towers that use it +crates/gpu/src/gelu.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-mamba2 gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-gpu-clef-hidden gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-app gate-gpu-head gate-gpu-mcol gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-nvtier gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-vision gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-glm5v-tower gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e the shared tanh GELU rule: the registered elementwise module and image towers that use it crates/gpu-vision/src/{aligner,attn,attn_group,chain,encoder,gelu_tanh,gemm_bf16,glm5v,layer_norm,mean_groups,mlp,norm,pos_bilinear,qwen3vl,rope2d}.rs gate-gpu-clef-serve gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-glm5v-tower gate-gpu-qwen3-serve gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-gpu-vision gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-lev-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve the image towers' card kernels: the tower, serve and media-seat gates that run them crates/gpu-vision/src/lib.rs gate-gpu-clef-serve gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-glm5v-tower gate-gpu-qwen3-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-gpu-vision gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-lev-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve the tower crate's root, which the Qwen3.5 media gates also link crates/vision/src/arch/glm5v/{card,hparams,mod,names,tensors}.rs gate-ds41-bind gate-ds41-place gate-gpu-clef-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-glm5next-stagger gate-gpu-glm5v-tower gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-gpu-vision gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve the GLM-5V family's hparams, tensor names and card homes @@ -214,4 +214,6 @@ crates/gpu/src/cand.rs weekly-gpu-ds41-cand the candidate mask's kernels, which crates/gpu-deepseek41/src/{indexer.rs,chain/attn.rs,chain/attn/batch.rs} weekly-gpu-ds41-cand the indexer's split launcher and the select path's candidate launches, per row of words in the step and per chunk in the prompt batch crates/gpu-deepseek41/src/{body.rs,body/ced.rs,body/prefill.rs} weekly-gpu-ds41-cand the decode and pair wiring, the triangle's candidate source, and the prompt batch's kept blocks per batch set crates/gpu-gates/src/bin/{gate_deepseek41_long.rs,gate_deepseek41_prefill.rs,shared/ds41_cand.rs} weekly-gpu-ds41-cand the weekly's two clauses and their candidate rule -crates/placement/src/** gate-app gate-deepseek4-meta gate-ds41-bind gate-ds41-host gate-ds41-kld gate-ds41-load gate-ds41-meta gate-ds41-oracle gate-ds41-plan gate-dspark-read gate-fixture gate-glm5next-meta gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-comp gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-index gate-gpu-ds41-lib gate-gpu-ds41-long gate-gpu-ds41-prefill gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-e2e gate-gpu-gates-lib gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-lib gate-gpu-mcol gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-qwen35moe-moe gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-head gate-gpu-qwen3moe-router gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-ops gate-placement gate-qwen35moe-meta gate-qwen3moe-meta gate-qwen4exp-meta gate-union gate-gpu-p4 gate-gpu-p6 gate-gpu-iq-sel gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-block gate-gpu-p10 gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-gpu-qwen3moe-rope gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-r8 gate-prompts gate-gpu-head gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-woa gate-gpu-ds41-media gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-p8b gate-mimo2-meta gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-nemotron-meta gate-gpu-qwen35moe-attn gate-gpu-p1 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-p2 gate-nemotron-h-moe-meta gate-gpu-nvfp4 gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-nemotron-h-moe-e2e the placement planner, which bloomery-model re-exports as model::placement: every gate that plans a placement, reads a plan's figures or checks the r8 sidecar's refusal type +crates/placement/src/** gate-app gate-deepseek4-meta gate-ds41-bind gate-ds41-host gate-ds41-kld gate-ds41-load gate-ds41-meta gate-ds41-oracle gate-ds41-plan gate-dspark-read gate-fixture gate-glm5next-meta gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-comp gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-index gate-gpu-ds41-lib gate-gpu-ds41-long gate-gpu-ds41-prefill gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-e2e gate-gpu-gates-lib gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-lib gate-gpu-mcol gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-qwen35moe-moe gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-head gate-gpu-qwen3moe-router gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-ops gate-placement gate-qwen35moe-meta gate-qwen3moe-meta gate-qwen4exp-meta gate-union gate-gpu-p4 gate-gpu-p6 gate-gpu-iq-sel gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-block gate-gpu-p10 gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-gpu-qwen3moe-rope gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-r8 gate-prompts gate-gpu-head gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-woa gate-gpu-ds41-media gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-p8b gate-mimo2-meta gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-nemotron-meta gate-gpu-qwen35moe-attn gate-gpu-p1 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-p2 gate-nemotron-h-moe-meta gate-gpu-nvfp4 gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-nemotron-h-moe-e2e gate-gpu-gemma-embedding2-e2e the placement planner, which bloomery-model re-exports as model::placement: every gate that plans a placement, reads a plan's figures or checks the r8 sidecar's refusal type +crates/model/src/arch/gemma_encoder.rs gate-ops gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gemma-embedding-meta gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-nemotron-h-moe-e2e gate-gpu-nemotron-h-moe-serve the shared Gemma encoder header checks, tensor shapes, roles and layer description except gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-gpu-clef-hidden gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-fixture gate-gpu-head gate-gpu-lib gate-gpu-ds41-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-nvtier gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41: they link the arch dispatch and never open a Gemma encoder file +crates/model/src/arch/gemma_embedding/** gate-ops gate-gemma-embedding-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-nemotron-h-moe-e2e gate-gpu-nemotron-h-moe-serve EmbeddingGemma header reader and its synthetic and real-header contracts except gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-gpu-clef-hidden gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-fixture gate-gpu-head gate-gpu-lib gate-gpu-ds41-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-nvtier gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41: they link the arch dispatch and never open a Gemma encoder file diff --git a/tools/recipes.py b/tools/recipes.py index 13e6201ba..ae58b32fa 100755 --- a/tools/recipes.py +++ b/tools/recipes.py @@ -10966,7 +10966,7 @@ def _st_host_crates(fails: list[str], side: Side) -> None: f"host nightly: the real tree's set is {host_now} — a new member of the set is proven by a nightly run before this list takes it", ) left = [n for n, _ in host_left_out(tree, lock_now)] - expect(left == ["bloomery-gpu", "bloomery-gpu-deepseek41", "bloomery-gpu-gates", "bloomery-gpu-glm5next", "bloomery-gpu-mimo2", "bloomery-gpu-vision"], f"host nightly: left out {left}") + expect(left == ["bloomery-gpu", "bloomery-gpu-deepseek41", "bloomery-gpu-gates", "bloomery-gpu-gemma-embedding2", "bloomery-gpu-glm5next", "bloomery-gpu-mimo2", "bloomery-gpu-vision"], f"host nightly: left out {left}") expect(not nightly_problems(tree, lock_now), f"host nightly: {nightly_problems(tree, lock_now)}") ran = {u for u, _ in host_units(tree, lock_now)} expect( diff --git a/tools/ref/ptx-shapes.tsv b/tools/ref/ptx-shapes.tsv index 44391ad54..4247024b5 100644 --- a/tools/ref/ptx-shapes.tsv +++ b/tools/ref/ptx-shapes.tsv @@ -187,6 +187,8 @@ generate_ds41 hc_pre_q8_0 0 0 generate_ds41 head_norm_neox_append 0 0 generate_ds41 head_norm_neox_append_256 0 0 generate_ds41 head_norm_neox_append_256_q8 0 0 +# jit_local is ptxas 13.3.73's sm_86 stack frame, not read on a cc 8.6 card; for this entry the two agree here and in both cc 12.1 tables. +generate_ds41 head_norm_neox_append_256_whole 0 0 generate_ds41 head_norm_neox_append_512 0 0 generate_ds41 head_norm_neox_append_q8 0 0 generate_ds41 head_norm_neox_append_rows 0 0 @@ -459,6 +461,8 @@ gate_e2e hc_gated_up_mix_4x320 0 0 gate_e2e head_norm_neox_append 0 0 gate_e2e head_norm_neox_append_256 0 0 gate_e2e head_norm_neox_append_256_q8 0 0 +# jit_local is ptxas 13.3.73's sm_86 stack frame, not read on a cc 8.6 card; for this entry the two agree here and in both cc 12.1 tables. +gate_e2e head_norm_neox_append_256_whole 0 0 gate_e2e head_norm_neox_append_512 0 0 gate_e2e head_norm_neox_append_q8 0 0 gate_e2e head_norm_neox_append_rows 0 0 diff --git a/tools/ref/ptx-shapes/sm_121-cc12.1.tsv b/tools/ref/ptx-shapes/sm_121-cc12.1.tsv index f4bebd9bb..a21a22263 100644 --- a/tools/ref/ptx-shapes/sm_121-cc12.1.tsv +++ b/tools/ref/ptx-shapes/sm_121-cc12.1.tsv @@ -188,6 +188,7 @@ generate_ds41 hc_pre_q8_0 0 0 generate_ds41 head_norm_neox_append 0 0 generate_ds41 head_norm_neox_append_256 0 0 generate_ds41 head_norm_neox_append_256_q8 0 0 +generate_ds41 head_norm_neox_append_256_whole 0 0 generate_ds41 head_norm_neox_append_512 0 0 generate_ds41 head_norm_neox_append_q8 0 0 generate_ds41 head_norm_neox_append_rows 0 0 @@ -440,6 +441,7 @@ gate_e2e hc_gated_up_mix_4x320 0 0 gate_e2e head_norm_neox_append 0 0 gate_e2e head_norm_neox_append_256 0 0 gate_e2e head_norm_neox_append_256_q8 0 0 +gate_e2e head_norm_neox_append_256_whole 0 0 gate_e2e head_norm_neox_append_512 0 0 gate_e2e head_norm_neox_append_q8 0 0 gate_e2e head_norm_neox_append_rows 0 0 diff --git a/tools/ref/ptx-shapes/sm_86-cc12.1.tsv b/tools/ref/ptx-shapes/sm_86-cc12.1.tsv index b43e9b7b9..2a60a7148 100644 --- a/tools/ref/ptx-shapes/sm_86-cc12.1.tsv +++ b/tools/ref/ptx-shapes/sm_86-cc12.1.tsv @@ -188,6 +188,7 @@ generate_ds41 hc_pre_q8_0 0 0 generate_ds41 head_norm_neox_append 0 0 generate_ds41 head_norm_neox_append_256 0 0 generate_ds41 head_norm_neox_append_256_q8 0 0 +generate_ds41 head_norm_neox_append_256_whole 0 0 generate_ds41 head_norm_neox_append_512 0 0 generate_ds41 head_norm_neox_append_q8 0 0 generate_ds41 head_norm_neox_append_rows 0 0 @@ -440,6 +441,7 @@ gate_e2e hc_gated_up_mix_4x320 0 0 gate_e2e head_norm_neox_append 0 0 gate_e2e head_norm_neox_append_256 0 0 gate_e2e head_norm_neox_append_256_q8 0 0 +gate_e2e head_norm_neox_append_256_whole 0 0 gate_e2e head_norm_neox_append_512 0 0 gate_e2e head_norm_neox_append_q8 0 0 gate_e2e head_norm_neox_append_rows 0 0 diff --git a/tools/unsafe-ratchet.txt b/tools/unsafe-ratchet.txt index cf49d50bb..96040999c 100644 --- a/tools/unsafe-ratchet.txt +++ b/tools/unsafe-ratchet.txt @@ -261,7 +261,12 @@ gguf 9 2 0 # PIN(2026-10-16): gpu 1858 -> 1862 blocks, moved code, not new unsafe: the sigmoid_router_instance! macro (its # entries' body calls and the module load, each with its SAFETY line) moved from gpu-deepseek41/src/router.rs to # gpu/src/route_body.rs beside the bodies it instantiates; gpu-deepseek41 comes down by the same 4. -gpu 1862 201 10 +# PIN(2026-10-16): gpu 1862 -> 1863 blocks, a new kernel entry, not old code relaxed: head_norm_neox_append_256_whole's +# call into norm_partial_append, with its SAFETY line. +# PIN(2026-10-16): gpu 201 -> 202 fns, new code, not old code relaxed: enqueue_head_norm_neox_append_256_whole is an +# unsafe fn whose Safety section names the distinct positions its caller keeps (two tokens at one position write one +# cache row from two blocks). +gpu 1863 202 10 # PIN(2026-10-01): gpu-deepseek41 459 -> 464 blocks, 51 -> 52 fns, a new kernel entry, not old code relaxed: # ds41_ffn_card_acc_8_tier (the tier form of the card sum at 8 slots, chain/ffn/batch.rs) adds its two blocks and # card_acc_tier_at (the one unsafe fn) with its three, each with its SAFETY line. @@ -318,6 +323,11 @@ gpu-deepseek41 388 44 0 # two copies with three blocks (prctl, getppid and pre_exec), and its before-registration test adds raise and # pre_exec, each with its SAFETY line. gpu-gates 78 5 1 +# PIN(2026-10-15): gpu-gemma-embedding2 0 -> 1 blocks, new code, not old code relaxed: the head-512 +# launcher call requires distinct Consts::pos positions, with its SAFETY line. +# PIN(2026-10-15): gpu-gemma-embedding2 1 -> 2 blocks, new code, not old code relaxed: Encoder::pass's call of +# enqueue_head_norm_neox_append_256_whole, its positions 0..ctx distinct, with its SAFETY line. +gpu-gemma-embedding2 2 0 0 gpu-glm5next 4 0 0 gpu-mimo2 0 0 0 # PIN(2026-10-10): gpu-vision 48 -> 86 blocks, new code, not old code relaxed: the Qwen3-VL tower's five kernel entries read and