diff --git a/crates/gpu-gates/Cargo.toml b/crates/gpu-gates/Cargo.toml index bf0bbdc68..aac70fe96 100644 --- a/crates/gpu-gates/Cargo.toml +++ b/crates/gpu-gates/Cargo.toml @@ -404,6 +404,14 @@ required-features = ["gpu"] name = "gate_mamba2" required-features = ["gpu"] +[[bin]] +name = "gate_nemotron_e2e" +required-features = ["gpu"] + +[[bin]] +name = "gate_nemotron_serve" +required-features = ["deepseek41"] + [[bin]] name = "gate_load_v41" required-features = ["gpu"] diff --git a/crates/gpu-gates/src/bin/bloomery_serve.rs b/crates/gpu-gates/src/bin/bloomery_serve.rs index 152ff809a..785c10333 100644 --- a/crates/gpu-gates/src/bin/bloomery_serve.rs +++ b/crates/gpu-gates/src/bin/bloomery_serve.rs @@ -1,12 +1,12 @@ //! `bloomery-serve` — one server binary, one seat a model: `--model -//! ds41|qwen38|glm|qwen3|mimo2|decide` picks the seat (`serve_seats`, the shared +//! ds41|qwen38|glm|qwen3|mimo2|nemotron|decide` picks the seat (`serve_seats`, the shared //! seat modules), every other flag the seat's own, parsed by the seat's own //! parser, so each seat's flag set is exactly its per-model binary's. The //! word is optional with a model file: with none, the file's architecture //! picks the seat, and a bare start or `--help` prints [`drive::MODELS`], //! one line a seat with a working `--hf` example. //! -//! bloomery-serve [--model ds41|qwen38|glm|qwen3|mimo2|decide] +//! bloomery-serve [--model ds41|qwen38|glm|qwen3|mimo2|nemotron|decide] //! [-m PATH | --model-file PATH | --hf [:]] //! [--head [--head-config ]] //! @@ -37,7 +37,7 @@ //! Otherwise, with no `--model`, the architecture of the file's first shard //! picks the seat: deepseek41 and deepseek4 the ds41 seat, glm5next and //! glm5-next the glm seat, qwen4exp the qwen38 seat, qwen3moe and qwen35moe -//! the qwen3 seat, mimo2 the mimo2 seat; +//! the qwen3 seat, mimo2 the mimo2 seat, nemotron_h the nemotron seat; //! a file of a decision row's backbone (qwen35) with no head is refused by //! name, saying what would seat it (`--head`, a row's `--hf` repo, or a file //! that carries its head); a @@ -122,15 +122,15 @@ mod drive { use serve::decide::Ask; use serve::preflight::Driver; - use crate::serve_seats::{decide, ds41, glm, mimo2, qwen3, qwen38}; + use crate::serve_seats::{decide, ds41, glm, mimo2, nemotron, qwen3, qwen38}; const NAME: &str = "bloomery-serve"; - const USAGE: &str = "usage: bloomery-serve [--model ds41|qwen38|glm|qwen3|mimo2|decide] [-m PATH | \ - --model-file PATH | --hf [:]] [--head \ + const USAGE: &str = "usage: bloomery-serve [--model ds41|qwen38|glm|qwen3|mimo2|nemotron|decide] \ + [-m PATH | --model-file PATH | --hf [:]] [--head \ [--head-config ]] | --version"; + mimo2, the nemotron or the decide seat takes them> | --version"; /// One line a seat, printed beside [`USAGE`] on `--help` and a bare /// start: the seat word, what it serves, and a working `--hf` example of @@ -142,6 +142,7 @@ models (the --model word is optional with a model file: the file's architecture glm GLM-5.3-Flash --hf unsloth/GLM-5.3-Flash-GGUF:UD-Q4_K_XL qwen3 Qwen3-30B-A3B and Qwen3.6 --hf unsloth/Qwen3-30B-A3B-Instruct-2507-GGUF:Q4_K_M mimo2 MiMo-V2.6-Flash --hf ggml-org/MiMo-V2.6-Flash-MOPD-GGUF:MXFP4 + nemotron Nemotron 3 Nano --hf nvidia/NVIDIA-Nemotron-3-Nano-4B-GGUF:Q4_K_M decide a decision model by its head --hf bartowski/Cloudflare_clef-flash-GGUF:Q5_K_M decide a decision model in its file --hf ggml-org/lev-GGUF:Q4_K_M"; @@ -162,6 +163,7 @@ models (the --model word is optional with a model file: the file's architecture Glm, Qwen3, Mimo2, + Nemotron, } /// A `--model` word: a generative seat's, or the decide seat's. @@ -179,9 +181,10 @@ models (the --model word is optional with a model file: the file's architecture "glm" => Ok(Word::Seat(Model::Glm)), "qwen3" => Ok(Word::Seat(Model::Qwen3)), "mimo2" => Ok(Word::Seat(Model::Mimo2)), + "nemotron" => Ok(Word::Seat(Model::Nemotron)), serve::decide::WORD => Ok(Word::Decide), other => Err(format!( - "--model is ds41, qwen38, glm, qwen3, mimo2 or {}, not {other}", + "--model is ds41, qwen38, glm, qwen3, mimo2, nemotron or {}, not {other}", serve::decide::WORD ) .into()), @@ -198,12 +201,13 @@ models (the --model word is optional with a model file: the file's architecture impl Model { /// Every generative seat. - const ALL: [Model; 5] = [ + const ALL: [Model; 6] = [ Model::Ds41, Model::Qwen38, Model::Glm, Model::Qwen3, Model::Mimo2, + Model::Nemotron, ]; /// The seat's word, as errors and the usage name it. @@ -214,6 +218,7 @@ models (the --model word is optional with a model file: the file's architecture Model::Glm => "glm", Model::Qwen3 => "qwen3", Model::Mimo2 => "mimo2", + Model::Nemotron => "nemotron", } } @@ -228,6 +233,7 @@ models (the --model word is optional with a model file: the file's architecture Model::Glm => matches!(Arch::from_name(arch), Ok(Arch::Glm5next)), Model::Qwen3 => matches!(arch, "qwen3moe" | "qwen35moe"), Model::Mimo2 => arch == "mimo2", + Model::Nemotron => matches!(Arch::from_name(arch), Ok(Arch::NemotronH)), } } } @@ -245,7 +251,7 @@ models (the --model word is optional with a model file: the file's architecture } let v = it.next().ok_or_else(|| { format!( - "--model needs a value (ds41, qwen38, glm, qwen3, mimo2 or {}): {USAGE}", + "--model needs a value (ds41, qwen38, glm, qwen3, mimo2, nemotron or {}): {USAGE}", serve::decide::WORD ) })?; @@ -436,6 +442,7 @@ models (the --model word is optional with a model file: the file's architecture Model::Glm => glm::run(&rest), Model::Qwen3 => qwen3::run(&rest), Model::Mimo2 => mimo2::run(&rest), + Model::Nemotron => nemotron::run(&rest), }; ended(r) } diff --git a/crates/gpu-gates/src/bin/gate_clef_serve.rs b/crates/gpu-gates/src/bin/gate_clef_serve.rs index db3797b79..a7913f320 100644 --- a/crates/gpu-gates/src/bin/gate_clef_serve.rs +++ b/crates/gpu-gates/src/bin/gate_clef_serve.rs @@ -234,7 +234,8 @@ mod gate { /// header line above them. /// /// PIN(2026-10-10): a seventh row, the mimo2 seat's, joins the table. - const MODELS_LINES: [&str; 7] = [ + /// PIN(2026-10-15): an eighth row, the nemotron seat's, joins the table. + const MODELS_LINES: [&str; 8] = [ "models (the --model word is optional with a model file: the file's architecture picks \ the seat)", "ds41 DeepSeek-V4.1-Flash --hf vcruz305/DeepSeek-V4.1-Flash-GGUF:Q3_K_M", @@ -242,6 +243,7 @@ mod gate { "glm GLM-5.3-Flash --hf unsloth/GLM-5.3-Flash-GGUF:UD-Q4_K_XL", "qwen3 Qwen3-30B-A3B and Qwen3.6 --hf unsloth/Qwen3-30B-A3B-Instruct-2507-GGUF:Q4_K_M", "mimo2 MiMo-V2.6-Flash --hf ggml-org/MiMo-V2.6-Flash-MOPD-GGUF:MXFP4", + "nemotron Nemotron 3 Nano --hf nvidia/NVIDIA-Nemotron-3-Nano-4B-GGUF:Q4_K_M", "decide a decision model by its head --hf bartowski/Cloudflare_clef-flash-GGUF:Q5_K_M", ]; diff --git a/crates/gpu-gates/src/bin/gate_nemotron_e2e.rs b/crates/gpu-gates/src/bin/gate_nemotron_e2e.rs new file mode 100644 index 000000000..fd12f3bcf --- /dev/null +++ b/crates/gpu-gates/src/bin/gate_nemotron_e2e.rs @@ -0,0 +1,525 @@ +//! The nemotron_h end-to-end gate: the whole decode chain — Mamba-2 layers, +//! grouped-query attention with no rope, squared-ReLU MLP layers, the head +//! and the argmax — on one card, over the Nemotron 3 Nano 4B file. +//! +//! The clauses are a list ([`CLAUSES`]); each prints one line ending in PASS +//! or FAIL, and a step that raises the card's fault word is a FAIL naming the +//! fault, never a pass: +//! +//! - `structure`: the captured step holds the entries the layer kinds derive +//! — one Mamba-2 scan a Mamba layer, one squared ReLU an MLP layer, the +//! group-5 flash pass and its merge an attention layer, two appends a key +//! head an attention layer, one Q5_0 embedding row — and no entry of the +//! group-8 flash pass; no host node. +//! - `eager_equals_replay`: eager and graph replay give bit-identical logits +//! over the 16-token prompt, all finite. +//! - `reset_rewinds`: after `reset` the same prompt gives the same ids and the +//! same logits bits. +//! - `state_carries`: after the prompt the last token stepped from the same +//! reset twice gives the same ids; layer 0's scan state is finite and not +//! all zero; and the last token's logits after the prompt differ from the +//! same token's alone from a reset (a body that never carried state would +//! give equal ones). +//! - `fault_word_clean`: the fault word is clean after every run. +//! +//! - `top1_is_mainlines`: for each prompt of the `ref-nemotron-logits` set, +//! the body's argmax at the last position is the set's. +//! - `logits_near_mainlines`: per prompt the max |Δ| and the cosine of the +//! last-position logits against the set's are printed. The floor is cosine +//! at least [`COSINE_FLOOR`] and the same set of top-5 ids; a prompt below +//! it is a FAIL, never a pin. The order inside the top-5 is printed with +//! both sides' logits and not gated: two logits closer than the quantised +//! file's own noise swap places, and the order of rank 1 is the other +//! clause. +//! +//! The own-contract clauses run over a fixed walk of token ids; the mainline +//! clauses run over the set's prompts, read through `crates/refset`. A set +//! absent from the box, dumped from another file or build, or unfinished ends +//! the gate by name before the card loads. + +#[cfg(feature = "gpu")] +fn main() -> std::process::ExitCode { + bloomery_gpu_gates::exit_with("gate_nemotron_e2e", gate::run()) +} + +#[cfg(feature = "gpu")] +#[allow( + dead_code, + reason = "the shared runner serves several gates; each takes the helpers it reads" +)] +#[path = "shared/e2e.rs"] +mod e2e; + +#[cfg(feature = "gpu")] +mod gate { + use bloomery_gpu::model::StepMode; + use bloomery_gpu::{Gpu, GpuError, NemotronHModel, NodeInfo}; + use bloomery_gpu_gates::nodes::count_kinds; + use bloomery_gpu_gates::{GateError, bits_equal, checks_failed, open_split, verdict}; + use cuda_core::sys; + use model::arch::Arch; + use model::arch::nemotron_h::hparams::{Hparams, Kind}; + use refset::arch::nemotron_h::LOGITS; + use refset::nemotron::LogitsSet; + use std::collections::BTreeMap; + + type Model = NemotronHModel; + + /// Rows of the K/V planes a key head holds: the longest prompt of the + /// reference set with room for the clauses' extra steps. + const CTX: usize = 640; + + /// The cosine the last-position logits keep against mainline's. + const COSINE_FLOOR: f64 = 0.999; + + /// How many of the highest logits must be the same ids. + const TOP_K: usize = 5; + + /// Tokens of the prompt every clause runs. + const PROMPT_LEN: usize = 16; + + /// What a clause is run over: the layer kinds the file's header names and + /// the prompts, in order. + struct Case { + hp: Hparams, + prompts: Vec, + set: LogitsSet, + } + + struct Prompt { + name: &'static str, + ids: Vec, + } + + /// One clause: its name and its check. + type Clause = fn(&mut Model, &Case) -> Result; + + /// The clauses, in the order they run. + const CLAUSES: &[(&str, Clause)] = &[ + ("structure", structure), + ("eager_equals_replay", eager_equals_replay), + ("reset_rewinds", reset_rewinds), + ("state_carries", state_carries), + ("top1_is_mainlines", top1_is_mainlines), + ("logits_near_mainlines", logits_near_mainlines), + ("fault_word_clean", fault_word_clean), + ]; + + /// A step run: the argmax id and the logits after each token. + struct Run { + ids: Vec, + logits: Vec>, + } + + /// Feed `ids` one token at a time from wherever `m` stands. + fn feed(m: &mut Model, ids: &[u32]) -> Result { + let mut out = Run { + ids: Vec::new(), + logits: Vec::new(), + }; + for &t in ids { + out.ids.push(m.step(&[t])?); + out.logits.push(m.logits()?); + } + Ok(out) + } + + /// `r`'s run, or — when a step raised the fault word — a line naming the + /// fault and `None`: the clause fails there and the gate goes on to the + /// next one. Any other error ends the gate. + fn or_fault( + clause: &str, + what: &str, + r: Result, + ) -> Result, GateError> { + match r { + Ok(run) => Ok(Some(run)), + Err(GpuError::Fault { fault, .. }) => { + println!("{clause}: {what} raised the fault word: {fault} FAIL"); + Ok(None) + } + Err(e) => Err(e.into()), + } + } + + /// Reset, then [`feed`] `ids`, as [`or_fault`] reads it. + fn fresh( + clause: &str, + what: &str, + m: &mut Model, + ids: &[u32], + ) -> Result, GateError> { + m.reset()?; + or_fault(clause, what, feed(m, ids)) + } + + fn finite(r: &Run) -> bool { + r.logits.iter().flatten().all(|v| v.is_finite()) + } + + fn same(a: &Run, b: &Run) -> bool { + a.ids == b.ids + && a.logits.len() == b.logits.len() + && a.logits + .iter() + .zip(&b.logits) + .all(|(x, y)| bits_equal(x, y)) + } + + // ---------------------------------------------------------- structure + + /// The kernel entries of `nodes` with their counts. + fn histogram(nodes: &[NodeInfo]) -> BTreeMap { + let mut h = BTreeMap::new(); + for k in nodes.iter().filter_map(|n| n.kernel.as_ref()) { + *h.entry(k.name.clone()).or_insert(0) += 1; + } + h + } + + fn structure(m: &mut Model, case: &Case) -> Result { + m.set_mode(StepMode::Graph); + let nodes = m.capture_step()?; + let list = m.step_graph_nodes()?; + let hist = histogram(&list); + let ([host], _) = count_kinds(&list, [sys::CUgraphNodeType_enum_CU_GRAPH_NODE_TYPE_HOST]); + let kinds = |k: Kind| case.hp.kinds.iter().filter(|&&x| x == k).count(); + let (mamba, attn, mlp) = (kinds(Kind::Mamba), kinds(Kind::Attention), kinds(Kind::Mlp)); + let n_kv = case.hp.kv.iter().copied().max().unwrap_or(0); + let got = |name: &str| hist.get(name).copied().unwrap_or(0); + let want = [ + ("mamba2_step", mamba), + ("relu_sqr", mlp), + ("gqa_flash_seg_g5", attn), + ("gqa_flash_merge", attn), + ("kv_append_pos_buf", 2 * n_kv * attn), + ("embed_rows_q5_0", 1), + ]; + let mut pass = host == 0; + for (name, n) in want { + pass &= got(name) == n; + } + // The group-8 pass runs the other segment entries. + let group8: usize = hist + .iter() + .filter(|(k, _)| k.starts_with("gqa_flash_seg") && !k.ends_with("_g5")) + .map(|(_, n)| n) + .sum(); + pass &= group8 == 0; + let listed: Vec = want + .iter() + .map(|(name, n)| format!("{name}={}/{n}", got(name))) + .collect(); + println!( + "structure: layers mamba={mamba} attn={attn} mlp={mlp}, graph_nodes={nodes} host={host}, \ + entries (got/want) {} group8_entries={group8} (want 0) {}", + listed.join(" "), + verdict(pass) + ); + println!( + "structure: kernel entries {}", + hist.iter() + .map(|(k, n)| format!("{k}={n}")) + .collect::>() + .join(" ") + ); + Ok(pass) + } + + // ------------------------------------------------ eager_equals_replay + + fn eager_equals_replay(m: &mut Model, case: &Case) -> Result { + const CLAUSE: &str = "eager_equals_replay"; + let mut ok = true; + for p in &case.prompts { + m.set_mode(StepMode::Eager); + let eager = fresh(CLAUSE, "eager", m, &p.ids)?; + m.set_mode(StepMode::Graph); + let replay = fresh(CLAUSE, "replay", m, &p.ids)?; + let (Some(eager), Some(replay)) = (eager, replay) else { + ok = false; + continue; + }; + let identical = same(&eager, &replay); + let pass = identical && finite(&eager); + println!( + "{CLAUSE}: {} {} tokens, ids {:?}, logits bit-identical {identical}, finite {} {}", + p.name, + p.ids.len(), + &eager.ids[eager.ids.len().saturating_sub(4)..], + finite(&eager), + verdict(pass) + ); + ok &= pass; + } + Ok(ok) + } + + // ------------------------------------------------------ reset_rewinds + + fn reset_rewinds(m: &mut Model, case: &Case) -> Result { + const CLAUSE: &str = "reset_rewinds"; + m.set_mode(StepMode::Graph); + let mut ok = true; + for p in &case.prompts { + let first = fresh(CLAUSE, "the first run", m, &p.ids)?; + let second = fresh(CLAUSE, "the second run", m, &p.ids)?; + let (Some(first), Some(second)) = (first, second) else { + ok = false; + continue; + }; + let ids = first.ids == second.ids; + let bits = same(&first, &second); + let pass = ids && bits; + println!( + "{CLAUSE}: {} ids equal {ids}, logits bit-identical {bits} {}", + p.name, + verdict(pass) + ); + ok &= pass; + } + Ok(ok) + } + + // ------------------------------------------------------ state_carries + + /// Layer 0's scan state: finite and not all zero. + fn layer0_state(m: &Model) -> Result<(bool, bool), GateError> { + let state = m + .body("gate_nemotron_e2e")? + .scan_state(m.gpu().stream(), 0)?; + Ok(( + state.iter().all(|v| v.is_finite()), + state.iter().any(|&v| v != 0.0), + )) + } + + fn state_carries(m: &mut Model, case: &Case) -> Result { + const CLAUSE: &str = "state_carries"; + m.set_mode(StepMode::Graph); + let mut ok = true; + for p in &case.prompts { + let Some(&last) = p.ids.last() else { + continue; + }; + let Some(_) = fresh(CLAUSE, "the first run", m, &p.ids)? else { + ok = false; + continue; + }; + let (finite_state, nonzero) = layer0_state(m)?; + let Some(after_a) = or_fault(CLAUSE, "the last token", feed(m, &[last]))? else { + ok = false; + continue; + }; + let Some(_) = fresh(CLAUSE, "the second run", m, &p.ids)? else { + ok = false; + continue; + }; + let Some(after_b) = or_fault(CLAUSE, "the last token", feed(m, &[last]))? else { + ok = false; + continue; + }; + let Some(alone) = fresh(CLAUSE, "the last token alone", m, &[last])? else { + ok = false; + continue; + }; + let twice = after_a.ids == after_b.ids && same(&after_a, &after_b); + let carried = !same(&after_a, &alone); + let pass = twice && finite_state && nonzero && carried; + println!( + "{CLAUSE}: {} last token {last} stepped twice from the same reset: ids {:?} vs {:?} \ + equal {twice}; layer 0 scan state finite {finite_state} nonzero {nonzero}; \ + logits after the prompt differ from the token alone {carried} {}", + p.name, + after_a.ids, + after_b.ids, + verdict(pass) + ); + ok &= pass; + } + Ok(ok) + } + + // ------------------------------------------------- the mainline set + + /// The ids of the `k` largest values, largest first, lower id first on a tie. + fn top_ids(v: &[f32], k: usize) -> Vec { + let mut ids: Vec = (0..v.len()).collect(); + ids.sort_by(|&a, &b| v[b].total_cmp(&v[a]).then(a.cmp(&b))); + ids.truncate(k); + ids + } + + /// The body's argmax and last-position logits after the set's prompt `p`, + /// from a reset, or `None` after a line naming the fault. + fn body_on( + clause: &str, + m: &mut Model, + set: &LogitsSet, + p: &refset::nemotron::Prompt, + ) -> Result)>, GateError> { + let ids = set.ids(p)?; + m.set_mode(StepMode::Graph); + m.reset()?; + let mut last = 0; + for &t in &ids { + match m.step(&[t]) { + Ok(id) => last = id, + Err(GpuError::Fault { fault, .. }) => { + println!("{clause}: {} raised the fault word: {fault} FAIL", p.name); + return Ok(None); + } + Err(e) => return Err(e.into()), + } + } + Ok(Some((last, m.logits()?))) + } + + fn top1_is_mainlines(m: &mut Model, case: &Case) -> Result { + const CLAUSE: &str = "top1_is_mainlines"; + let mut ok = true; + for p in &case.set.prompts { + let want = crate::e2e::argmax(&case.set.logits(p)?); + let Some((got, logits)) = body_on(CLAUSE, m, &case.set, p)? else { + ok = false; + continue; + }; + let pass = got == want && crate::e2e::argmax(&logits) == want; + println!( + "{CLAUSE}: {} {} tokens, body argmax {got} (logits' {}), mainline {want} {}", + p.name, + p.n_ids, + crate::e2e::argmax(&logits), + verdict(pass) + ); + ok &= pass; + } + Ok(ok) + } + + fn logits_near_mainlines(m: &mut Model, case: &Case) -> Result { + const CLAUSE: &str = "logits_near_mainlines"; + let mut ok = true; + for p in &case.set.prompts { + let want = case.set.logits(p)?; + let Some((_, got)) = body_on(CLAUSE, m, &case.set, p)? else { + ok = false; + continue; + }; + if got.len() != want.len() || !got.iter().all(|v| v.is_finite()) { + println!( + "{CLAUSE}: {} {} logits for the set's {}, finite {} FAIL", + p.name, + got.len(), + want.len(), + got.iter().all(|v| v.is_finite()) + ); + ok = false; + continue; + } + let max_delta = got + .iter() + .zip(&want) + .map(|(a, b)| (f64::from(*a) - f64::from(*b)).abs()) + .fold(0.0, f64::max); + let dot: f64 = got + .iter() + .zip(&want) + .map(|(a, b)| f64::from(*a) * f64::from(*b)) + .sum(); + let norm = |v: &[f32]| v.iter().map(|x| f64::from(*x).powi(2)).sum::().sqrt(); + let cosine = dot / (norm(&got) * norm(&want)); + let (top_got, top_want) = (top_ids(&got, TOP_K), top_ids(&want, TOP_K)); + let (mut set_got, mut set_want) = (top_got.clone(), top_want.clone()); + set_got.sort_unstable(); + set_want.sort_unstable(); + let pass = cosine >= COSINE_FLOOR && set_got == set_want; + let values = |ids: &[usize], v: &[f32]| -> Vec { + ids.iter().map(|&i| format!("{i}:{:.3}", v[i])).collect() + }; + println!( + "{CLAUSE}: {} {} tokens, max |delta| {max_delta:.4}, cosine {cosine:.6} \ + (floor {COSINE_FLOOR}), top-{TOP_K} same ids {} (body {:?}, mainline {:?}) {}", + p.name, + p.n_ids, + set_got == set_want, + values(&top_got, &got), + values(&top_want, &want), + verdict(pass) + ); + ok &= pass; + } + Ok(ok) + } + + // --------------------------------------------------- fault_word_clean + + fn fault_word_clean(m: &mut Model, _case: &Case) -> Result { + let fault = m.gpu().fault()?; + let pass = fault.is_none() && m.poisoned().is_none(); + println!( + "fault_word_clean: word {fault:?}, poisoned {:?} {}", + m.poisoned(), + verdict(pass) + ); + Ok(pass) + } + + // ---------------------------------------------------------------- run + + /// The fixed prompt: a multiplicative walk through ids the vocabulary + /// holds. + fn walk(n: usize, vocab: usize) -> Vec { + (0..n) + .map(|i| u32::try_from(1_000 + (i * 7_919 + 13) % (vocab - 1_000)).unwrap_or(1)) + .collect() + } + + pub fn run() -> Result<(), GateError> { + let file = open_split(Arch::NemotronH, "gate-gpu-nemotron-e2e")?; + let hp = Hparams::read(&file).map_err(|e| format!("nemotron_h header: {e}"))?; + let set = LogitsSet::open(&LOGITS.path(refset::arch::nemotron_h::LOGITS_SET), &LOGITS)?; + let longest = set.prompts.iter().map(|p| p.n_ids as usize).max(); + if longest.is_none_or(|n| n + 1 > CTX) { + return Err(format!( + "the set's longest prompt is {longest:?} tokens, the gate's context holds {CTX}" + ) + .into()); + } + let case = Case { + set, + prompts: vec![Prompt { + name: "walk16", + ids: walk(PROMPT_LEN, hp.n_vocab), + }], + hp, + }; + let t = std::time::Instant::now(); + let mut m = Model::open(Gpu::new()?, file, CTX)?; + println!( + "load resident_bytes={} ctx={CTX} layers={} in {:.1} s (runtime value)", + m.resident_bytes(), + m.layers().len(), + t.elapsed().as_secs_f64() + ); + let mut ok = true; + for (name, clause) in CLAUSES { + let pass = clause(&mut m, &case)?; + if !pass { + println!("clause {name} FAIL"); + } + ok &= pass; + } + if !ok { + return Err(checks_failed()); + } + println!( + "PASSED: gate_nemotron_e2e the captured step holds the entries its layer kinds derive \ + and no group-8 flash entry; eager and replay give bit-identical logits; a reset \ + rewinds the ids and the logits bits; the scan state carries; the body's argmax is \ + mainline's and its logits keep the cosine floor and the top-{TOP_K} ids on every \ + prompt of the set; the fault word stays clean" + ); + Ok(()) + } +} diff --git a/crates/gpu-gates/src/bin/gate_nemotron_serve.rs b/crates/gpu-gates/src/bin/gate_nemotron_serve.rs new file mode 100644 index 000000000..821699c52 --- /dev/null +++ b/crates/gpu-gates/src/bin/gate_nemotron_serve.rs @@ -0,0 +1,473 @@ +//! `gate_nemotron_serve` — the nemotron seat of `bloomery-serve` against the model it serves, +//! on the nemotron_h file of the profile (`BLOOMERY_REF_MODEL`, Nemotron 3 Nano 4B Q4_K_M). +//! +//! gate_nemotron_serve --dir +//! +//! The seat serves one sequence on one card (`--parallel` above 1 is refused by name), feeds a +//! prompt one decode step an id, and keeps a prefix a request shares with the slot only where it is +//! the whole held sequence (the Mamba-2 state cannot step back). Every server is `bloomery-serve +//! --model nemotron -m …` beside this binary, `BLOOMERY_REF_MODEL` removed from its +//! environment. Logs per server in `//` (`server.err`, the responses). +//! +//! Without a load: +//! +//! - `plan_only_names_its_default`: `--plan` with no `--place`, `--ctx` or `--parallel` exits 0 and +//! prints `place unset` (the word `a`), the plan record and the `parallel` line, in that order, +//! and no `load` or `listening` record. The `parallel` line is the default's one slot: +//! `slots=1 slot_ctx= total= from=default pass=one`. The one `ctx` line holds +//! its word (`ctx_ladder::default_line_held`: `target` or `halved`, a rung of the ladder from +//! 262,144 or the trained context down to 4096, the limit the trained context) and names the +//! plan's `ctx_max`; the plan at it stands (`--ctx --plan` exits 0 with the same +//! `ctx_max`); and the rung above it, when there is one, does not (`--ctx --plan` is +//! refused). FAIL-first: a seat that defaults to 4096 regardless names a rung whose rung above +//! stands, and the hold is red. +//! - `parallel_above_one_is_refused`: `--parallel 2 --plan` exits non-zero, its stderr names the +//! body's one sequence, and no `plan` record is printed (the refusal comes before any plan). +//! +//! One server as a user starts it, `--port 0` and no other flag: +//! +//! - `load_and_listen`: it listens; the `load` record comes before the `listening` record; the load +//! names `arch=nemotron_h`, `slots=1` and the file's layer count, and its `ctx` and the +//! `listening` record's are the `ctx` line's: the context the plan chose is the context the load +//! made. FAIL-first: a seat whose load takes a context other than the plan's (a fixed one, or the +//! slot context halved) is red. +//! - `completion_ids`: `/completion` at temperature 0, [`N`] tokens, of [`PROSE`] (`/tokenize`) +//! and of the chat turn the server's own template renders (`/apply-template`, `/tokenize`), +//! each from an empty cache (`cache_prompt: false`), answers the ids `NemotronHModel` gives +//! stepped directly on the same ids (the `gate_nemotron_e2e` way: the prompt one step an id, +//! then each answer fed back), or a prefix of them ending at an end-of-generation id. FAIL-first: +//! a prefill that skips the first id moves the ids and the clause is red. +//! - `chat_is_200`: `/v1/chat/completions` of the same turn at temperature 0 and `max_tokens` +//! [`N`] is a 200 whose `usage` counts the prompt's ids and the completion's tokens, and whose +//! message text is inside the text of the chat run's ids. +//! - `edit_resend_is_a_fresh_run`: the prose prompt answered with `cache_prompt: true`, then +//! resent with its second half changed, answers the ids of the same request at `cache_prompt: +//! false` and keeps nothing (`cache_n` 0): the shared prefix is shorter than the held sequence +//! and the state cannot step back, so the slot is reset. FAIL-first: a `keep` that grants the +//! shared prefix reaches the refused rollback, the request fails and the clause is red. +//! - `extension_keeps_the_held_sequence`: the prose prompt resent with its answer appended answers +//! the ids of the same request at `cache_prompt: false` and keeps the held positions (`cache_n` +//! above 0): the shared prefix is the whole held sequence. + +#[cfg(feature = "deepseek41")] +fn main() -> std::process::ExitCode { + bloomery_gpu_gates::exit_with("gate_nemotron_serve", gate::run()) +} + +#[cfg(feature = "deepseek41")] +mod gate { + use std::path::{Path, PathBuf}; + use std::time::Duration; + + use bloomery_gpu::{Gpu, NemotronHModel}; + use bloomery_gpu_gates::ctx_ladder::{ctx_line, default_line_held, field, rung_above}; + use bloomery_gpu_gates::record::{self, Log}; + use bloomery_gpu_gates::serve_client::{ + Answer, Served, agree, chat_is_those_ids, check, greedy, plan_run, rendered, serve_cmd, + server_log, tokenized, + }; + use bloomery_gpu_gates::{GateError, checks_failed, generate, ref_model_path}; + use gguf::Split; + use model::arch::nemotron_h::hparams::Hparams; + use serde_json::json; + + const USAGE: &str = "usage: gate_nemotron_serve --dir "; + + /// The tokens a completion asks for. + const N: usize = 16; + + /// Polls of a second that bound the wait for a server to listen. + const LISTEN_POLLS: usize = 900; + + /// The cache rows of the session the ids are read from: past every prompt here and its + /// answer. + const SESSION_CTX: usize = 1024; + + /// The default is one slot. + const DEFAULT_SLOTS: u64 = 1; + + /// The prompt the completion and prefix clauses run. + const PROSE: &str = "The lighthouse keeper counted the steps as he climbed: one hundred and twelve, the same as every night, until the hundred and thirteenth"; + + /// The chat turn the file answers. + const TURN: &str = "Name three primary colours."; + + /// The words of the refusal of a second slot. + const ONE_SEQUENCE: &str = "holds one sequence"; + + fn utf8(p: &Path) -> Result<&str, GateError> { + Ok(p.to_str() + .ok_or_else(|| format!("{} is not UTF-8", p.display()))?) + } + + fn parse_args() -> Result { + let mut dir = None; + let mut it = std::env::args().skip(1); + while let Some(flag) = it.next() { + let v = it + .next() + .ok_or_else(|| format!("{flag} needs a value: {USAGE}"))?; + match flag.as_str() { + "--dir" => dir = Some(PathBuf::from(v)), + other => return Err(format!("unknown argument {other:?}: {USAGE}").into()), + } + } + Ok(dir.ok_or(USAGE)?) + } + + /// `bloomery-serve --model nemotron -m ` beside this binary. + fn command(file: &Path) -> Result<(std::process::Command, Vec), GateError> { + Ok(( + serve_cmd()?, + vec![ + "--model".to_owned(), + "nemotron".to_owned(), + "-m".to_owned(), + utf8(file)?.to_owned(), + ], + )) + } + + /// `--plan` of `args`: its exit status and stderr's lines, the log of its records and its + /// text. + fn plan( + dir: &Path, + name: &str, + file: &Path, + args: &[&str], + ) -> Result<(std::process::ExitStatus, Log, String), GateError> { + let d = dir.join(name); + std::fs::create_dir_all(&d)?; + let (cmd, mut all) = command(file)?; + all.extend(args.iter().map(|a| (*a).to_owned())); + let all: Vec<&str> = all.iter().map(String::as_str).collect(); + let err = d.join("server.err"); + let (status, lines) = plan_run(cmd, &all, &err)?; + println!("{name}: {} --plan: {status}", args.join(" ")); + Ok(( + status, + server_log(&err, record::BLOOMERY_SERVE_NEMOTRON)?, + lines.join("\n"), + )) + } + + /// The file's trained context. + fn trained_ctx(file: &Path) -> Result { + let split = Split::open(file).map_err(|e| format!("open {}: {e}", file.display()))?; + let n = generate::trained_ctx(&split) + .ok_or_else(|| format!("{}: no context_length", file.display()))?; + Ok(u64::try_from(n)?) + } + + /// `plan_only_names_its_default` (the module header); the context the default named. + fn plan_only_names_its_default( + dir: &Path, + file: &Path, + ok: &mut bool, + ) -> Result, GateError> { + let limit = trained_ctx(file)?; + let (status, log, text) = plan(dir, "plan", file, &[])?; + let place = log.first(&record::PLACE_UNSET)?; + let planned = log.first(&record::PLAN)?; + let parallel_at = text + .lines() + .position(|l| l.starts_with("parallel ")) + .map(|i| i + 1); + let parallel_line = text.lines().find(|l| l.starts_with("parallel ")); + let in_order = matches!( + (&place, &planned, parallel_at), + (Some(a), Some(b), Some(c)) if a.at() < b.at() && b.at() < c + ); + let word_a = match &place { + Some(p) => p.word("place")? == "a", + None => false, + }; + let no_load = log.first(&record::LOAD_NEMOTRON)?.is_none() + && log.first(&record::LISTENING_NEMOTRON)?.is_none(); + let exits = status.success(); + println!( + "plan only: exit 0 {exits}; place unset, plan, parallel in order {in_order}; place \ + word a {word_a}; no load or listening record {no_load}" + ); + let ctx_max = match &planned { + Some(p) => Some(p.u64("ctx_max")?), + None => None, + }; + let mut default_ok = false; + if let Some(c) = ctx_max { + let want = format!( + "parallel rule=slots slots={DEFAULT_SLOTS} slot_ctx={c} total={c} from=default \ + pass=one" + ); + let line_ok = parallel_line == Some(want.as_str()); + println!("plan only: the parallel line {parallel_line:?} is {want:?} {line_ok}"); + let line = ctx_line(&text); + let held = line.is_some_and(|l| { + default_line_held(l, DEFAULT_SLOTS, (limit, "trained")) == Some(true) + && field(l, "ctx") == Some(c) + }); + println!("plan only: the ctx line {line:?} holds its word at {c} {held}"); + let c_arg = c.to_string(); + let (st, at_log, _) = plan(dir, "plan-at-default", file, &["--ctx", &c_arg])?; + let stands = st.success() + && match at_log.first(&record::PLAN)? { + Some(p) => p.u64("ctx_max")? == c, + None => false, + }; + println!("plan only: --ctx {c} stands {stands}"); + let above_short = match rung_above(limit, c) { + None => { + println!("plan only: the default {c} is the first rung, none above it"); + true + } + Some(above) => { + let above_arg = above.to_string(); + let (st, _, _) = plan(dir, "plan-past-default", file, &["--ctx", &above_arg])?; + println!( + "plan only: --ctx {above}, the rung above, is refused {}", + !st.success() + ); + !st.success() + } + }; + default_ok = line_ok && held && stands && above_short; + } + check( + ok, + "plan_only_names_its_default", + exits && in_order && word_a && no_load && default_ok, + ); + Ok(ctx_line(&text).and_then(|l| field(l, "ctx"))) + } + + /// `parallel_above_one_is_refused` (the module header). + fn parallel_above_one_is_refused( + dir: &Path, + file: &Path, + ok: &mut bool, + ) -> Result<(), GateError> { + let (status, log, text) = plan(dir, "refuse-parallel", file, &["--parallel", "2"])?; + let named = text.contains(ONE_SEQUENCE); + let no_plan = log.first(&record::PLAN)?.is_none(); + println!( + "refuse-parallel: exit non-zero {}; names {ONE_SEQUENCE:?} {named}; no plan record \ + {no_plan}", + !status.success() + ); + if !named { + println!("refuse-parallel stderr:\n{text}"); + } + check( + ok, + "parallel_above_one_is_refused", + !status.success() && named && no_plan, + ); + Ok(()) + } + + /// The ids `NemotronHModel` gives stepped directly: for each prompt, from an empty cache, the + /// prompt one step an id and then each answer fed back, [`N`] tokens. + fn session_ids(file: &Path, prompts: &[&[u32]]) -> Result>, GateError> { + let split = Split::open(file).map_err(|e| format!("open {}: {e}", file.display()))?; + let mut m = NemotronHModel::open(Gpu::new()?, split, SESSION_CTX)?; + let mut out = Vec::new(); + for ids in prompts { + m.reset()?; + let mut answer = vec![m.step(ids)?]; + while answer.len() < N { + let last = *answer.last().ok_or("an empty answer")?; + answer.push(m.step(&[last])?); + } + out.push(answer); + } + Ok(out) + } + + /// What the server answered, held until it is stopped and the session has the card. + struct Answers { + prose: Vec, + turn: Vec, + prose_answer: Answer, + turn_answer: Answer, + } + + /// The prose prompt with its second half replaced by ids the file's vocabulary holds. + fn edited(prose: &[u32]) -> Vec { + let half = prose.len() / 2; + let mut ids = prose[..half].to_vec(); + ids.extend(prose[half..].iter().map(|&id| id + 1)); + ids + } + + /// The clauses on one server: `load_and_listen` through the prefix clauses; `None` when the + /// server never listened. + fn served( + dir: &Path, + file: &Path, + default_ctx: Option, + layers: u64, + ok: &mut bool, + ) -> Result, GateError> { + let d = dir.join("serve"); + std::fs::create_dir_all(&d)?; + let (cmd, mut all) = command(file)?; + all.extend(["--port".to_owned(), "0".to_owned()]); + let all: Vec<&str> = all.iter().map(String::as_str).collect(); + let mut s = Served::spawn_cmd(cmd, &all, &d)?; + let err_log = d.join("server.err"); + let addr = match s.address(&err_log, LISTEN_POLLS, Duration::from_secs(1)) { + Ok(addr) => addr, + Err(e) => { + println!("the server never listened: {e}"); + check(ok, "load_and_listen", false); + return Ok(None); + } + }; + let text = std::fs::read_to_string(&err_log).unwrap_or_default(); + let records = server_log(&err_log, record::BLOOMERY_SERVE_NEMOTRON)?; + let load = records.one(&record::LOAD_NEMOTRON)?; + let listen = records.one(&record::LISTENING_NEMOTRON)?; + let named = ctx_line(&text).and_then(|l| field(l, "ctx")); + let same_ctx = default_ctx.is_none_or(|c| named == Some(c)); + let loaded = load.word("arch")? == "nemotron_h" + && load.u64("slots")? == 1 + && load.u64("layers")? == layers + && listen.u64("slots")? == 1; + let ctx_is_plans = + named.is_some() && named == Some(load.u64("ctx")?) && named == Some(listen.u64("ctx")?); + let in_order = load.at() < listen.at(); + println!( + "serve at {addr}: ctx line {:?} (the plan run's {same_ctx}); load {:?}; listening \ + {:?}; load names arch, one slot and {layers} layers {loaded}; ctx is the line's \ + {ctx_is_plans}; load before listening {in_order}", + ctx_line(&text), + load.line(), + listen.line() + ); + check( + ok, + "load_and_listen", + same_ctx && loaded && ctx_is_plans && in_order, + ); + let url = |p: &str| format!("http://{addr}{p}"); + let prose = tokenized(&url, PROSE)?; + let messages = json!([{ "role": "user", "content": TURN }]); + let turn = rendered(&url, messages.clone())?; + // The clauses that compare with the session (`completion_ids`) read these two, each from + // an empty cache. + let (prose_answer, _) = greedy(&url, prose.clone(), N, &d, "prose", false)?; + let (turn_answer, turn_body) = greedy(&url, turn.clone(), N, &d, "turn", false)?; + // A clause whose requests fail is that clause's FAIL, printed; the gate goes on. + for (name, run) in [ + ( + "chat_is_200", + chat_is_those_ids(&url, &d, messages, N, (&turn_answer, &turn_body)), + ), + ("edit_resend_is_a_fresh_run", edit_resend(&url, &d, &prose)), + ( + "extension_keeps_the_held_sequence", + extension(&url, &d, &prose, &prose_answer), + ), + ] { + let pass = run.unwrap_or_else(|e| { + println!("{name}: {e}"); + false + }); + check(ok, name, pass); + } + println!("server stopped: {}", s.stop()?); + Ok(Some(Answers { + prose, + turn, + prose_answer, + turn_answer, + })) + } + + /// `edit_resend_is_a_fresh_run` (the module header): the slot holds the prose prompt and its + /// answer, and the same prompt with its second half changed is sent. + fn edit_resend( + url: &dyn Fn(&str) -> String, + dir: &Path, + prose: &[u32], + ) -> Result { + let (_, _) = greedy(url, prose.to_vec(), N, dir, "prose-held", true)?; + let changed = edited(prose); + let (fresh, _) = greedy(url, changed.clone(), N, dir, "edit-fresh", false)?; + let (_, _) = greedy(url, prose.to_vec(), N, dir, "prose-held-again", true)?; + let (resent, _) = greedy(url, changed, N, dir, "edit-resent", true)?; + println!( + "edit resend: ids equal the fresh run's {}; cache_n {} is 0", + resent.tokens == fresh.tokens, + resent.cache_n + ); + Ok(resent.tokens == fresh.tokens && resent.cache_n == 0) + } + + /// `extension_keeps_the_held_sequence` (the module header): the slot holds the prose prompt + /// and its answer (`held`'s tokens, from an empty cache), and the prompt with that answer + /// appended is sent. + fn extension( + url: &dyn Fn(&str) -> String, + dir: &Path, + prose: &[u32], + held: &Answer, + ) -> Result { + let (_, _) = greedy(url, prose.to_vec(), N, dir, "extension-held", true)?; + let mut longer = prose.to_vec(); + longer.extend(&held.tokens); + let (fresh, _) = greedy(url, longer.clone(), N, dir, "extension-fresh", false)?; + let (_, _) = greedy(url, prose.to_vec(), N, dir, "extension-held-again", true)?; + let (resent, _) = greedy(url, longer, N, dir, "extension-resent", true)?; + println!( + "extension: ids equal the fresh run's {}; cache_n {} is above 0", + resent.tokens == fresh.tokens, + resent.cache_n + ); + Ok(resent.tokens == fresh.tokens && resent.cache_n > 0) + } + + /// `completion_ids` (the module header), the server stopped and the session on the card. + fn completion_ids(file: &Path, a: &Answers, ok: &mut bool) -> Result<(), GateError> { + let ids = session_ids(file, &[a.prose.as_slice(), a.turn.as_slice()])?; + let prose = agree(&a.prose_answer.tokens, &a.prose_answer.stop, &ids[0]); + let turn = agree(&a.turn_answer.tokens, &a.turn_answer.stop, &ids[1]); + println!( + "completion ids: the prose prompt's {} tokens are the session's {prose}; the chat \ + turn's {} tokens are the session's {turn}", + a.prose_answer.tokens.len(), + a.turn_answer.tokens.len() + ); + check(ok, "completion_ids", prose && turn); + Ok(()) + } + + pub fn run() -> Result<(), GateError> { + let dir = parse_args()?; + std::fs::create_dir_all(&dir)?; + let file = ref_model_path()?; + let split = Split::open(&file).map_err(|e| format!("open {}: {e}", file.display()))?; + let layers = u64::try_from( + Hparams::read(&split) + .map_err(|e| format!("{}: {e}", file.display()))? + .n_layer, + )?; + drop(split); + let mut ok = true; + let default_ctx = plan_only_names_its_default(&dir, &file, &mut ok)?; + parallel_above_one_is_refused(&dir, &file, &mut ok)?; + let answers = served(&dir, &file, default_ctx, layers, &mut ok)?; + if let Some(a) = answers { + completion_ids(&file, &a, &mut ok)?; + } + if !ok { + return Err(checks_failed()); + } + println!( + "PASSED: gate_nemotron_serve the seat plans its default and says so, refuses a second \ + slot, loads at the context it named, answers the session's ids, serves the chat turn, \ + and keeps a prefix only where the sequence stands" + ); + Ok(()) + } +} diff --git a/crates/gpu-gates/src/bin/gate_qwen3moe_experts.rs b/crates/gpu-gates/src/bin/gate_qwen3moe_experts.rs index 689ce2ff5..a2e2c9bbb 100644 --- a/crates/gpu-gates/src/bin/gate_qwen3moe_experts.rs +++ b/crates/gpu-gates/src/bin/gate_qwen3moe_experts.rs @@ -55,7 +55,8 @@ //! eager launch. //! //! Q5_0 embedding rows (`embed_rows_q5_0`) are bit-identical to `dequant_row` -//! on the same bytes. +//! on the same bytes. `relu_sqr` is bit-identical to `max(x, 0)^2`, and a +//! non-finite input is copied through. #[cfg(feature = "gpu")] fn main() -> std::process::ExitCode { @@ -140,6 +141,7 @@ mod gate { ok &= embed(&gpu, &gguf, &sets)?; ok &= embed_kquant(&gpu)?; ok &= embed_q5_0(&gpu)?; + ok &= relu_sqr(&gpu)?; ok &= gate_up_body(&gpu, &gguf, &ex)?; ok &= gate_up_tap(&gpu, &gguf, &ex, &sets)?; ok &= combine(&gpu, &ex, &sets, stream)?; @@ -207,6 +209,53 @@ mod gate { Ok(host_ok && pos_ok && graph_ok && oor_ok) } + /// Host transcription of `relu_sqr`: a non-finite input stays that value, + /// and a finite one is `max(x, 0)^2`. + fn host_relu_sqr(v: f32) -> f32 { + if !v.is_finite() { + v + } else { + let r = if v > 0.0 { v } else { 0.0 }; + r * r + } + } + + /// `relu_sqr` against [`host_relu_sqr`], bit for bit, including a + /// non-finite input, and one captured node. + fn relu_sqr(gpu: &Gpu) -> Result { + let x = [ + 0.0f32, + 1.5, + -2.0, + f32::NAN, + f32::INFINITY, + f32::NEG_INFINITY, + 3.0, + ]; + let want: Vec = x.iter().copied().map(host_relu_sqr).collect(); + let stream = gpu.stream(); + let xb = DeviceBuffer::from_host(stream, &x)?; + let mut y = DeviceBuffer::::zeroed(stream, x.len())?; + gpu.elem().enqueue_relu_sqr(stream, &xb, x.len(), &mut y)?; + stream.synchronize()?; + let got = y.to_host_vec(stream)?; + let host_ok = bits_equal(&got, &want); + // The eager read ends before this closure borrows `y` again. + let mut enqueue = |s: &CudaStream| gpu.elem().enqueue_relu_sqr(s, &xb, x.len(), &mut y); + let g = gpu.capture(&mut enqueue)?; + g.launch(stream)?; + stream.synchronize()?; + let replay = y.to_host_vec(stream)?; + let nodes = g.node_count(); + let graph_ok = bits_equal(&replay, &want) && nodes == 1; + println!( + "relu_sqr n={}: vs host bit_identical={host_ok} graph_nodes={nodes} graph_same={graph_ok} {}", + x.len(), + verdict(host_ok && graph_ok) + ); + Ok(host_ok && graph_ok) + } + // ------------------------------------------------------------ 1. embed /// One gather's outputs, read back: the rows, each row's position and diff --git a/crates/gpu-gates/src/bin/gate_qwen3moe_flash.rs b/crates/gpu-gates/src/bin/gate_qwen3moe_flash.rs index 7d8e1b05e..d164cdf5f 100644 --- a/crates/gpu-gates/src/bin/gate_qwen3moe_flash.rs +++ b/crates/gpu-gates/src/bin/gate_qwen3moe_flash.rs @@ -1704,10 +1704,12 @@ mod gate { /// Head 128, [`GROUP_5`] query heads a key head, the scalar pass, /// against [`exact`]'s head-128 band. Two key heads: a KV index of - /// `h / 8` still reads the first plane for the second group. + /// `h / 8` still reads the first plane for the second group. 100 keys + /// leaves four live keys in the second segment's last tile, so the + /// live-key mask is read. fn group5(k: &FlashGqaKernels, gpu: &Gpu, scale: f32) -> Result { const N_KV: usize = 2; - const KEYS: usize = 96; + const KEYS: usize = 100; const CTX: usize = 128; let n_head = N_KV * GROUP_5; let stream = gpu.stream(); diff --git a/crates/gpu-gates/src/bin/shared/routed_legs.rs b/crates/gpu-gates/src/bin/shared/routed_legs.rs index 9b08a4ddd..330e35979 100644 --- a/crates/gpu-gates/src/bin/shared/routed_legs.rs +++ b/crates/gpu-gates/src/bin/shared/routed_legs.rs @@ -1060,6 +1060,7 @@ fn family_name(program: Program) -> &'static str { Program::Glm5nextBody => "glm5next", Program::Mimo2Body => "mimo2", Program::Deepseek41Chain => "deepseek41", + Program::NemotronHBody => "nemotron_h", } } diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/mimo2.rs b/crates/gpu-gates/src/bin/shared/serve_seats/mimo2.rs index f82597a8e..39ee80086 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/mimo2.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/mimo2.rs @@ -122,8 +122,8 @@ use app::{Loaded, OpenArgs, OpenLog, Session, SessionError}; use bloomery_gpu::host::swap::Residency; use bloomery_gpu::model::StepMode; use bloomery_gpu_gates::bind::{ - ChatSurface, Listen, Pass, Seat, SeatEngine, SlotStep, Vocab, arm_screen, bind_server, - model_props, note_line, parallel_line, placement_of, slots_given, + ChatSurface, Pass, Seat, SeatEngine, SeatFlags, SlotStep, Vocab, arm_screen, model_props, + note_line, parallel_line, placement_of, slots_given, }; use bloomery_gpu_gates::generate::{Lane, Place, choose_ctx, lane_at}; use bloomery_gpu_gates::record::{self, Record}; @@ -135,10 +135,8 @@ use gguf::Split; use model::arch::Arch; use model::arch::mimo2::place::PlanInputs; use model::placement::ctx_ask::{DEFAULT_SLOTS, probe_of}; -use model::placement::workstation::TierBatchBytes; use model::placement::{Machine, Plan, PlanLevers}; use runtime::Target as _; -use serve::flag::number; use serve::{CacheNote, EngineProps, Saved, ServeError}; use tokenizer::Tokenizer; @@ -177,86 +175,24 @@ const ONE_PASS: bool = true; /// Why the seat takes no sequence state: it runs no prompt cache. const NO_CACHE: &str = "the mimo2 seat runs no prompt cache"; -/// MiMo fact: the load refuses any expert tier card (`Body::load`, -/// `PlanInputs::plan_with_slots`), so a tier card serves no prompt batch here. -/// Planning a placement that names one with zero batch bytes reaches the -/// plan's own refusal. -const NO_BATCH: TierBatchBytes = TierBatchBytes { - staging: 0, - scratch: 0, - host: 0, -}; - +/// The flags this seat takes beside the common ones ([`SeatFlags`]). struct Args { - host: String, - port: u16, - ctx: Option, - place: Option, - alias: Option, - /// `--chat-template-file`, replacing the file's own template. - template_file: Option, - /// `--plan`: the records before the load, then exit. - plan_only: bool, + common: SeatFlags, /// `--prefill`: how a prompt call is fed. prefill: PrefillMode, - /// `--parallel`: the resident slots. - parallel: Option, - queue_depth: Option, - /// `--api-key`/`--api-key-file`: the keys every request is checked - /// against ([`serve::flag::ApiKeys`]). - api_keys: serve::flag::ApiKeys, } fn parse_args(args: &[String]) -> Result { - let mut a = Args { - host: "127.0.0.1".to_owned(), - port: 8080, - ctx: None, - place: None, - alias: None, - template_file: None, - plan_only: false, - prefill: PrefillMode::Batch, - parallel: None, - queue_depth: None, - api_keys: serve::flag::ApiKeys::default(), - }; - let mut it = args.iter().map(String::as_str); - while let Some(flag) = it.next() { - if flag == "--help" || flag == "-h" { - return Err(USAGE.into()); - } - if flag == "--plan" { - a.plan_only = true; - continue; - } - let v = it - .next() - .ok_or_else(|| format!("{flag} needs a value, or is unknown: {USAGE}"))?; - match flag { - "--host" => a.host = v.to_owned(), - "--port" => a.port = number(flag, v)?, - f if serve::flag::CTX.contains(&f) => a.ctx = Some(number(flag, v)?), - "--place" => a.place = Some(Place::parse(v)?), - "--alias" => a.alias = Some(v.to_owned()), - "--chat-template-file" => a.template_file = Some(PathBuf::from(v)), - "--prefill" => { - a.prefill = PrefillMode::from_name(v) - .ok_or_else(|| format!("--prefill is batch or steps, not {v}"))?; - } - "--parallel" | "-np" => a.parallel = Some(number(flag, v)?), - "--queue-depth" => a.queue_depth = Some(number(flag, v)?), - f if serve::flag::KEYS.contains(&f) => a.api_keys.add(f, v)?, - other => return Err(format!("unknown argument {other:?}: {USAGE}").into()), + let mut prefill = PrefillMode::Batch; + let common = SeatFlags::parse(args, USAGE, |flag, v| match flag { + "--prefill" => { + prefill = PrefillMode::from_name(v) + .ok_or_else(|| format!("--prefill is batch or steps, not {v}"))?; + Ok(true) } - } - if a.ctx == Some(0) { - return Err("--ctx 0: the stores hold no position".into()); - } - if a.parallel == Some(0) { - return Err("--parallel 0: the server serves no slot".into()); - } - Ok(a) + _ => Ok(false), + })?; + Ok(Args { common, prefill }) } /// The MiMo session on the engine thread and the positions each slot's @@ -373,7 +309,6 @@ impl Mimo { fn open(a: OpenSeat) -> Result { let pinned = a.pin_main && threads::pool().pin_caller(); let file = Split::open(&a.path).map_err(|e| format!("open {}: {e}", a.path.display()))?; - let tier = (!a.place.tier_cards().is_empty()).then_some(NO_BATCH); let mut log = Log { t: Instant::now(), opened: Duration::ZERO, @@ -388,7 +323,7 @@ impl Mimo { }; let args = OpenArgs { place: a.place.name(), - machine: a.place.machine(None, tier)?, + machine: a.place.machine_untiered()?, ctx: a.ctx, mode: StepMode::Graph, cfg: Mimo2Cfg { @@ -503,7 +438,7 @@ pub fn run(args: &[String]) -> Result { model::ops::set_host_lanes(levers.host_lanes()); record::at_main(NAME, record::BLOOMERY_SERVE_MIMO2); arm_screen(levers.screen()); - let a = parse_args(args)?; + let Args { common: a, prefill } = parse_args(args)?; let path = ref_model_path()?; let split = Split::open(&path).map_err(|e| format!("open {}: {e}", path.display()))?; let first = split @@ -518,7 +453,7 @@ pub fn run(args: &[String]) -> Result { ) .into()); } - let ChatSurface { template, name } = ChatSurface::read( + let chat = ChatSurface::read( &path, split.value("tokenizer.chat_template"), split.value("general.name"), @@ -549,9 +484,9 @@ pub fn run(args: &[String]) -> Result { serving_place: place != Place::Gate, check_finite: false, route_trace: false, - prefill_steps: a.prefill == PrefillMode::Steps, + prefill_steps: prefill == PrefillMode::Steps, }); - if pick.why == ResidencyWhy::Set && pick.word != "off" && a.prefill == PrefillMode::Steps { + if pick.why == ResidencyWhy::Set && pick.word != "off" && prefill == PrefillMode::Steps { return Err(format!( "BLOOMERY_RESIDENCY={} is refused beside --prefill steps (each prompt id would end \ a pass the rule counts)", @@ -559,8 +494,7 @@ pub fn run(args: &[String]) -> Result { ) .into()); } - let tier = (!place.tier_cards().is_empty()).then_some(NO_BATCH); - let machine = place.machine(None, tier)?(inputs.model.layers); + let machine = place.machine_untiered()?(inputs.model.layers); // The context, then the plan of every slot around it, each layer's // stores `slots` times at a slot's positions (`choose_ctx`, every seat's). // A tier card, a total the machine cannot hold and a budget that binds @@ -621,25 +555,12 @@ pub fn run(args: &[String]) -> Result { plan: plan_levers, host: levers.host(), pin_main: levers.pin_main(), - prefill: a.prefill, + prefill, residency, group: levers.prefill_group(), }; let engine = SeatEngine::spawn(move || Mimo::open(open), slot_ctx, vocab, card, props, 0)?; - let server = bind_server( - engine, - Listen { - host: &a.host, - port: a.port, - alias: a.alias.unwrap_or(name), - path: &path, - template, - slot_save_path: None, - api_keys: a.api_keys, - parallel: slots, - queue_depth: a.queue_depth, - }, - )?; + let server = a.bind(engine, chat, &path, slots)?; Record::new(&record::LISTENING_MIMO2) .w("place", place.name()) .u("ctx", slot_ctx) diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/mod.rs b/crates/gpu-gates/src/bin/shared/serve_seats/mod.rs index a90583023..d881736c7 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/mod.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/mod.rs @@ -3,7 +3,7 @@ //! behind [`bloomery_gpu_gates::bind::Seat`], with a `run(&args)` a binary //! calls once. The per-model binaries (`bloomery-serve-ds41`, //! `bloomery-serve-qwen38`) include their one seat directly; the one-binary -//! server (`bloomery-serve --model ds41|qwen38|glm|qwen3|mimo2`) includes this +//! server (`bloomery-serve --model ds41|qwen38|glm|qwen3|mimo2|nemotron`) includes this //! registry and is one call into the seat the model names. //! //! A seat's module is its contract's owner: read it for the flags, the @@ -11,7 +11,9 @@ //! seat, opened as `generate_glm5next` opens the model; [`qwen3`] the //! single-card Qwen seat (qwen3moe, qwen35moe), opened as //! `generate_qwen3moe` opens them; [`mimo2`] the MiMo-V2.6-Flash seat, whose -//! every routed expert sits on the host, opened as `gate_mimo2_e2e` opens it. +//! every routed expert sits on the host, opened as `gate_mimo2_e2e` opens it; +//! [`nemotron`] the Nemotron 3 Nano seat, one sequence on one card, opened as +//! `gate_nemotron_e2e` opens it. //! `drafted` is the //! seats' shared MTP draft driver and `served` the owner of the DSpark draft //! the V4.1 and qwen3 seats serve, each one file a seat's per-model binary @@ -43,6 +45,11 @@ pub mod qwen3; // through is scoped to it, and it runs no V4.1 code. #[cfg(all(feature = "deepseek41", feature = "mimo2"))] pub mod mimo2; +// The nemotron seat sits behind `deepseek41` for the same reason: the server +// surface it binds through is scoped to it, and it runs no V4.1 code. The +// body it serves is in the gpu crate, so it needs no feature of its own. +#[cfg(feature = "deepseek41")] +pub mod nemotron; // The seats' one owner of a round run as passes (`step_rows_one_pass`). #[cfg(feature = "deepseek41")] pub mod rounds; diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/nemotron.rs b/crates/gpu-gates/src/bin/shared/serve_seats/nemotron.rs new file mode 100644 index 000000000..cc71fc1c0 --- /dev/null +++ b/crates/gpu-gates/src/bin/shared/serve_seats/nemotron.rs @@ -0,0 +1,312 @@ +//! `bloomery-serve --model nemotron` — the llama-server-compatible HTTP API on +//! the nemotron_h engine: a Nemotron 3 Nano file (Mamba-2, attention and +//! squared-ReLU MLP layers, dense), the whole model on one card, opened as +//! `gate_nemotron_e2e` opens it (`NemotronHModel::open`, the step captured in +//! graph mode). +//! +//! bloomery-serve --model nemotron [-m PATH | --hf [:]] +//! [--host 127.0.0.1] [--port 8080] [--ctx C] [--alias NAME] +//! [--chat-template-file PATH] [--place W] [--plan] +//! [--queue-depth Q] [--api-key KEY] [--api-key-file FNAME] +//! +//! The server takes `-m`/`--hf` out before this seat parses (its module +//! doc); `--ctx-size` and `-c` are `--ctx` under llama-server's spellings. The +//! model's first shard gives the vocabulary, `tokenizer.chat_template` the chat +//! template (`--chat-template-file` replaces it) and `general.name` the alias +//! (`--alias` replaces it; `nemotron` for a file that names none). Sampling is +//! the sampler crate's chain with no repetition penalty, `temperature <= 0` +//! the engine's argmax. Any other flag is refused as an unknown argument. +//! +//! Three facts of the architecture set this seat's shape, and nothing else +//! does: +//! +//! - The body holds one sequence: its attention layers' K and V planes of one +//! context and its Mamba-2 layers' scan state and conv ring. It has no slot +//! rows, so `--parallel` above 1 is refused by name before any plan. +//! - The body has no prompt batch: a prompt call is one decode step an id +//! (`GpuModel::step` over the ids), so the prompt rate is the decode rate. +//! - A Mamba-2 state cannot step back. A request keeps the prefix it shares +//! with what the slot holds only when that prefix is the whole held +//! sequence (the position the model stands at); any other shared prefix is +//! kept 0, the slot is reset and the prompt fed again. A rollback to a +//! position the model does not stand at, a snapshot and a resume are +//! refused by name, and the seat runs no prompt cache. +//! +//! `--place W` is the shared placement word (`generate::Place`): `a` (the +//! largest visible card), `gate`, or one card of the census by name or +//! ordinal. Unset, the common rule every serving seat takes decides +//! (`Place::choose_untiered`: the body serves no tier card), on one census +//! reading. A placement of more than one card or with a tier card is refused +//! before any load by the plan's own refusal (`PlaceError`). +//! +//! `--ctx` is the one slot's context (`--ctx 0` is refused). The context +//! comes first and the plan fills the machine around it +//! (`generate::choose_ctx`, every seat's): a set value past the file's trained +//! context stops by name, and one the card cannot hold stops with the largest +//! it would hold. Unset, the slot asks 262,144 positions or the trained +//! context where that is less, halved while the card cannot hold the stores +//! and the resident bytes. +//! +//! The records on stderr, in order: `place unset` (word and why), one `ctx` +//! line (`placement::ctx_ask::Chosen::line`), the `plan` record (its +//! `ctx_max` the slot's), the `parallel` line (`rule=slots slots=1 +//! slot_ctx=C total=C from= pass=one`); then the `load` record +//! (architecture, resident bytes, the slot's context, slots, layers, the step +//! graph's nodes and the load's wall) and, once the port is bound, the +//! `listening` record (`record::BLOOMERY_SERVE_NEMOTRON`). `--plan` ends the +//! process after the `parallel` line, nothing loaded. An engine error ends the +//! process with the crash block and exit code 70, as every seat's. +//! +//! The levers this seat acts on ([`ACTS_ON`]) are parsed once, at `main` +//! (`bloomery_levers::at_main`), which refuses by name a lever set outside +//! them: the plan's card budget and the start screen. The seat sits behind +//! the `deepseek41` feature: the server surface it binds through (`bind`, the +//! `serve` and `sampler` crates) is scoped to it; it runs no V4.1 code. + +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Instant; + +use bloomery_gpu::model::StepMode; +use bloomery_gpu::{Gpu, NemotronHModel}; +use bloomery_gpu_gates::bind::{ + ChatSurface, Pass, Seat, SeatEngine, SeatFlags, Vocab, arm_screen, model_props, note_line, + parallel_line, placement_of, slots_of, +}; +use bloomery_gpu_gates::generate::{Place, choose_ctx}; +use bloomery_gpu_gates::record::{self, Record}; +use bloomery_gpu_gates::{GateError, gpu_census, ref_model_path}; +use gguf::Split; +use model::arch::Arch; +use model::arch::nemotron_h::place::PlanInputs; +use model::placement::PlanLevers; +use model::placement::ctx_ask::{DEFAULT_SLOTS, probe_of}; +use model::placement::workstation::DeviceId; +use serve::{CacheNote, EngineProps, Saved, ServeError}; +use tokenizer::Tokenizer; + +const NAME: &str = "bloomery-serve-nemotron"; + +const USAGE: &str = "usage: bloomery-serve --model nemotron [-m PATH | --hf [:]] \ + [--host H] [--port P] [--ctx C] [--alias NAME] [--chat-template-file PATH] \ + [--place W] [--plan] [--queue-depth Q] [--api-key KEY] \ + [--api-key-file FNAME]"; + +/// The levers this seat acts on: the plan's card budget and the start screen. +pub const ACTS_ON: &[&str] = &[bloomery_levers::CARD_BUDGET, bloomery_levers::SCREEN]; + +/// Why the seat takes no sequence state: it runs no prompt cache. +const NO_CACHE: &str = "the nemotron seat runs no prompt cache"; + +/// Why a prefix shorter than the held sequence is not kept. +const NO_STEP_BACK: &str = + "the Mamba-2 state cannot step back: a prefix is kept only where the sequence stands"; + +/// What the engine thread opens the seat with. +struct OpenSeat { + /// The planned card, by name and device: the open takes the device the + /// plan resolved. + card: String, + device: Option, + path: PathBuf, + /// The slot's positions the K/V planes are sized for. + ctx: usize, +} + +/// The model on the engine thread. +struct Nemotron { + m: NemotronHModel, +} + +impl Nemotron { + /// The model of the file at `a.path` on the planned card, in graph mode + /// with its step captured, the `load` record on stderr. + fn open(a: OpenSeat) -> Result { + let t = Instant::now(); + let file = Split::open(&a.path).map_err(|e| format!("open {}: {e}", a.path.display()))?; + let opened = t.elapsed(); + let gpu = Gpu::open_card(&a.card, a.device)?; + let mut m = NemotronHModel::open(gpu, file, a.ctx)?; + m.set_mode(StepMode::Graph); + let nodes = m.capture_step()?; + let ctx = m.ctx_max(); + let total = t.elapsed(); + Record::new(&record::LOAD_NEMOTRON) + .w("arch", "nemotron_h") + .u("resident_bytes", m.resident_bytes()) + .u("ctx", ctx) + .u("slots", 1) + .u("layers", m.layers().len()) + .u("graph_nodes", nodes) + .f("load_s", total.as_secs_f64()) + .eprint(); + record::eprint_load_phases(&m, total, opened); + Ok(Nemotron { m }) + } +} + +impl Seat for Nemotron { + fn pos(&self) -> usize { + self.m.pos() as usize + } + + fn ctx_max(&self) -> usize { + self.m.ctx_max() + } + + /// One decode step an id, the argmax after the last: the body has no + /// prompt batch. + fn prefill(&mut self, ids: &[u32]) -> Result { + Ok(self.m.step(ids)?) + } + + fn step(&mut self, last: u32) -> Result { + Ok(self.m.step(&[last])?) + } + + fn logits_into(&self, row: &mut [f32]) -> Result<(), GateError> { + Ok(self.m.logits_into(row)?) + } + + /// The model's reset ([`bloomery_gpu::GpuModel::reset`]): the scan state + /// and the conv rings back to zero, the position to 0. + fn reset(&mut self) -> Result<(), GateError> { + Ok(self.m.reset()?) + } + + /// Never asked for a position other than the one the model stands at + /// ([`Seat::keep`] grants no other, and the engine answers a cut to the + /// standing position itself); any other is refused by name. + fn rollback(&mut self, pos: u32) -> Result<(), GateError> { + Err(format!( + "a rollback to position {pos} from {}: {NO_STEP_BACK}", + self.m.pos() + ) + .into()) + } + + /// `n` when it is the whole held sequence, else nothing: the caller then + /// resets the slot and feeds the prompt again. + fn keep(&self, n: usize) -> (usize, Option) { + if n == self.pos() { + (n, None) + } else { + (0, Some(NO_STEP_BACK.to_owned())) + } + } + + /// Nowhere: no prompt call is cut for a prefix a later request keeps. + fn splits(&self, _first: usize, _end: usize, _marks: &[usize]) -> Vec { + Vec::new() + } + + /// Never asked: the seat's prompt cache budget is 0. + fn snapshot(&mut self) -> Result, GateError> { + Err(format!("a snapshot: {NO_CACHE}").into()) + } + + fn resume(&mut self, _state: &dyn Saved) -> Result<(), GateError> { + Err(format!("a resume: {NO_CACHE}").into()) + } + + fn note(note: &CacheNote) { + note_line(NAME, note); + } +} + +/// Loads the model and serves until the listener or the engine fails; +/// `Ok` carries why the server ended. +pub fn run(args: &[String]) -> Result { + let levers = bloomery_levers::at_main(ACTS_ON)?; + record::at_main(NAME, record::BLOOMERY_SERVE_NEMOTRON); + arm_screen(levers.screen()); + let a = SeatFlags::parse(args, USAGE, |_, _| Ok(false))?; + if let Some(n) = a.parallel.filter(|&n| n > 1) { + return Err(format!( + "--parallel {n}: the nemotron_h body holds one sequence and no resident slots; the \ + seat serves --parallel 1" + ) + .into()); + } + let path = ref_model_path()?; + let split = Split::open(&path).map_err(|e| format!("open {}: {e}", path.display()))?; + let first = split + .shard(0) + .ok_or_else(|| format!("{} opened with no shard", path.display()))?; + let arch = Arch::detect(first).map_err(|e| format!("{}: {e}", path.display()))?; + if !matches!(arch, Arch::NemotronH) { + return Err(format!( + "{} is a {} file; the nemotron seat serves nemotron_h files", + path.display(), + arch.name() + ) + .into()); + } + let chat = ChatSurface::read( + &path, + split.value("tokenizer.chat_template"), + split.value("general.name"), + a.template_file.as_deref(), + "nemotron", + )?; + let inputs = PlanInputs::read(&split)?; + let plan_levers = PlanLevers::from_levers(&levers)?; + // The one slot, and what named it (`from` on the `parallel` line); the + // start's note that `--ctx` is one request's context is left out, since + // `--parallel` has no larger count to offer. + let (slots, from) = slots_of(a.parallel, a.ctx.is_some(), DEFAULT_SLOTS)?; + // The common unset rule, once, on one census reading. + let census = gpu_census::census()?; + let chosen = Place::choose_untiered(a.place, &census, "nemotron")?; + chosen.record().eprint(); + let place = chosen.place; + let machine = place.machine_untiered()?(inputs.model.layers); + // The context, then the plan around it (`choose_ctx`, every seat's): a + // tier card, a second card, a context the card cannot hold and a budget + // that binds are refused here by name, before any load. + let asked = choose_ctx(&split, (a.ctx, slots), (None, None), |c| { + probe_of(inputs.plan(&machine, c, &plan_levers)) + .map_err(|e| format!("--place {}: {e}", place.name())) + })?; + eprintln!("{}", asked.line()); + let slot_ctx = usize::try_from(asked.slot_ctx)?; + let plan = asked.plan; + record::plan(place.name(), &machine, &plan).eprint(); + parallel_line(slots, slot_ctx, from, Pass::One); + if a.plan_only { + // The records before the load are out; nothing was opened on a card. + std::process::exit(0); + } + let props = EngineProps { + model: Some(model_props(&split, &inputs.model)), + placement: placement_of(NAME, &machine, &plan), + ..EngineProps::default() + }; + let card = &machine.cards[0]; + let open = OpenSeat { + card: card.name.clone(), + device: card.device, + path: path.clone(), + ctx: slot_ctx, + }; + let device = card.name.clone(); + drop(plan); + drop(split); + let vocab = Arc::new(Vocab::new(Tokenizer::from_gguf(&path)?)?); + let engine = SeatEngine::spawn( + move || Nemotron::open(open), + slot_ctx, + vocab, + device, + props, + 0, + )?; + let server = a.bind(engine, chat, &path, slots)?; + Record::new(&record::LISTENING_NEMOTRON) + .w("place", place.name()) + .u("ctx", slot_ctx) + .u("slots", slots) + .w("addr", server.local_addr()?) + .eprint(); + Ok(server.run()) +} diff --git a/crates/gpu-gates/src/bind.rs b/crates/gpu-gates/src/bind.rs index d2f07289b..6b0e73934 100644 --- a/crates/gpu-gates/src/bind.rs +++ b/crates/gpu-gates/src/bind.rs @@ -413,6 +413,123 @@ pub fn slots_given( Ok((slots, from)) } +/// The flags every serving seat takes beside its own: the listener (`--host`, +/// `--port`, `--queue-depth`), the keys (`--api-key`, `--api-key-file`), the +/// context (`--ctx`, under every spelling `serve::flag::CTX` lists), the +/// placement word (`--place`), the alias, the chat template file, `--parallel` +/// (and `-np`) and `--plan`. +pub struct SeatFlags { + pub host: String, + pub port: u16, + pub ctx: Option, + pub place: Option, + pub alias: Option, + /// `--chat-template-file`, replacing the file's own template. + pub template_file: Option, + /// `--plan`: the records before the load, then exit. + pub plan_only: bool, + pub parallel: Option, + pub queue_depth: Option, + /// `--api-key`/`--api-key-file`: the keys every request is checked + /// against ([`serve::flag::ApiKeys`]). + pub api_keys: serve::flag::ApiKeys, +} + +impl SeatFlags { + /// The flags of `args`. A flag that is not one of the common set goes to + /// `own` with its value: `Ok(true)` took it, `Ok(false)` is an unknown + /// argument, refused by name with `usage`. `--help` returns `usage` as + /// the error; `--ctx 0` and `--parallel 0` are refused by name. + pub fn parse( + args: &[String], + usage: &str, + mut own: impl FnMut(&str, &str) -> Result, + ) -> Result { + let mut a = SeatFlags { + host: "127.0.0.1".to_owned(), + port: 8080, + ctx: None, + place: None, + alias: None, + template_file: None, + plan_only: false, + parallel: None, + queue_depth: None, + api_keys: serve::flag::ApiKeys::default(), + }; + let mut it = args.iter().map(String::as_str); + while let Some(flag) = it.next() { + if flag == "--help" || flag == "-h" { + return Err(usage.into()); + } + if flag == "--plan" { + a.plan_only = true; + continue; + } + let v = it + .next() + .ok_or_else(|| format!("{flag} needs a value, or is unknown: {usage}"))?; + match flag { + "--host" => a.host = v.to_owned(), + "--port" => a.port = serve::flag::number(flag, v)?, + f if serve::flag::CTX.contains(&f) => a.ctx = Some(serve::flag::number(flag, v)?), + "--place" => a.place = Some(crate::generate::Place::parse(v)?), + "--alias" => a.alias = Some(v.to_owned()), + "--chat-template-file" => a.template_file = Some(PathBuf::from(v)), + "--parallel" | "-np" => a.parallel = Some(serve::flag::number(flag, v)?), + "--queue-depth" => a.queue_depth = Some(serve::flag::number(flag, v)?), + f if serve::flag::KEYS.contains(&f) => a.api_keys.add(f, v)?, + other => { + if !own(other, v)? { + return Err(format!("unknown argument {other:?}: {usage}").into()); + } + } + } + } + if a.ctx == Some(0) { + return Err("--ctx 0: the stores hold no position".into()); + } + if a.parallel == Some(0) { + return Err("--parallel 0: the server serves no slot".into()); + } + Ok(a) + } + + /// [`bind_server`] over `engine` with these flags: the alias the flag + /// gave or `chat`'s, `chat`'s template, no slot file path, `slots` + /// slots the seat made at its open. + pub fn bind( + self, + engine: SeatEngine, + chat: ChatSurface, + path: &Path, + slots: usize, + ) -> Result { + let SeatFlags { + host, + port, + alias, + api_keys, + queue_depth, + .. + } = self; + bind_server( + engine, + Listen { + host: &host, + port, + alias: alias.unwrap_or(chat.name), + path, + template: chat.template, + slot_save_path: None, + api_keys, + parallel: slots, + queue_depth, + }, + ) + } +} + /// The shape a seat's round of several slots runs in, decided once at the /// open: the busy slots' rows as one pass of the body, or a select and a /// step (a pass, drafted) a slot in turn ([`step_rows_in_turn`], diff --git a/crates/gpu-gates/src/generate.rs b/crates/gpu-gates/src/generate.rs index 4885488ca..7b799409d 100644 --- a/crates/gpu-gates/src/generate.rs +++ b/crates/gpu-gates/src/generate.rs @@ -625,6 +625,18 @@ impl Place { .into()) } + /// [`Place::machine`] for a body that serves no tier card, with no draft: + /// a placement that names a tier card is planned with no batch bytes, so + /// the plan's own refusal of the tier reaches the seat. + pub fn machine_untiered(self) -> Result Machine + Copy, GateError> { + let none = TierBatchBytes { + staging: 0, + scratch: 0, + host: 0, + }; + self.machine(None, (!self.tier_cards().is_empty()).then_some(none)) + } + /// The machine the placement plans over, by layer count. `draft_bytes` /// is the DSpark draft's resident bytes when the draft is served on the /// placement's first tier card ([`Place::draft_card`]); `batch` is the diff --git a/crates/gpu-gates/src/record.rs b/crates/gpu-gates/src/record.rs index 9eb3f80a9..5eaf51682 100644 --- a/crates/gpu-gates/src/record.rs +++ b/crates/gpu-gates/src/record.rs @@ -2771,6 +2771,38 @@ pub static LISTENING_MIMO2: Kind = Kind { ], }; +/// The nemotron_h seat's load. +pub static LOAD_NEMOTRON: Kind = Kind { + name: "load_nemotron", + head: "load", + doc: "The nemotron_h seat's model on its card: its architecture, resident bytes (the weights, the K/V planes of `ctx` rows, the scan stores and the step's scratch), the cache rows the model was loaded with (`ctx`), the resident sequences it holds (`slots`: one, the body has no more), layers, the step graph's nodes, and the load's wall.", + parts: &[ + key("arch", Word, ""), + key("resident_bytes", U64, "B"), + key("ctx", U64, "positions"), + key("slots", U64, ""), + key("layers", U64, ""), + key("graph_nodes", U64, ""), + lit(" in "), + pos("load_s", F64(1), "s"), + lit(" s"), + ], +}; + +/// The nemotron_h seat's address. +pub static LISTENING_NEMOTRON: Kind = Kind { + name: "listening_nemotron", + head: "bloomery-serve-nemotron:", + doc: "The nemotron_h seat's placement, the context its one slot serves, the slots it serves and the address it listens on.", + parts: &[ + key("place", Word, ""), + key("ctx", U64, "positions"), + key("slots", U64, ""), + lit(" listening on http://"), + pos("addr", Word, ""), + ], +}; + /// The GGUF set a `--hf` quant picked. pub static HF_SET: Kind = Kind { name: "hf_set", @@ -3521,6 +3553,19 @@ pub static BLOOMERY_SERVE_MIMO2: &[&Kind] = kinds![ groups::bind::CACHE, ]; +/// What the nemotron_h seat of `bloomery-serve` prints, all on stderr: the +/// placement and why (`place unset`), a `--hf` repo's resolve lines (printed by +/// the server before the seat registers), the plan, the load and its phases, +/// the address and each request's note of the prefix it kept less of than it +/// shared (the Mamba-2 state keeps a prefix only at the position it stands). +pub static BLOOMERY_SERVE_NEMOTRON: &[&Kind] = kinds![ + &[&PLACE_UNSET, &PLAN], + groups::model_file::FETCH, + groups::model_file::SET, + &[&LOAD_NEMOTRON, &LOAD_PHASES, &LISTENING_NEMOTRON], + groups::bind::CACHE, +]; + /// What the GLM seat of `bloomery-serve` prints, all on stderr: the /// residency lever's word, the placement and why, the `plan`, `load`, `load /// phases` and `capture` lines `generate_glm5next` prints, the host set's records of a @@ -4452,7 +4497,7 @@ mod tests { residency38, vision_seat, }; type Group = &'static [&'static Kind]; - let table: [(&str, Group, &[Group]); 11] = [ + let table: [(&str, Group, &[Group]); 12] = [ ( "ds41 seat: Flags::encoder, Draft::from_levers, print_widths, bind::spawn", BLOOMERY_SERVE_DS41, @@ -4502,6 +4547,11 @@ mod tests { bind::CACHE, ], ), + ( + "nemotron seat: bind::spawn", + BLOOMERY_SERVE_NEMOTRON, + &[model_file::FETCH, model_file::SET, bind::CACHE], + ), ( "glm seat: Flags::encoder, residency_set, SlotDrafts::open, bind::spawn", BLOOMERY_SERVE_GLM, @@ -4561,6 +4611,11 @@ mod tests { ); } } + // The nemotron_h seat counts no round (`Seat::step_stats` is its default) and has one slot. + assert_eq!( + missing(BLOOMERY_SERVE_NEMOTRON, bind::SLOTS), + ["slots round", "slot call"] + ); // The MiMo seat counts no round (`Seat::step_stats` is its default): no slot record in its list. assert_eq!( missing(BLOOMERY_SERVE_MIMO2, bind::SLOTS), diff --git a/crates/gpu/src/arch/mod.rs b/crates/gpu/src/arch/mod.rs index 14e05a70f..736ae2785 100644 --- a/crates/gpu/src/arch/mod.rs +++ b/crates/gpu/src/arch/mod.rs @@ -12,4 +12,5 @@ //! the gates at once. pub mod deepseek2; +pub mod nemotron_h; pub mod qwen3moe; diff --git a/crates/gpu/src/arch/nemotron_h/body.rs b/crates/gpu/src/arch/nemotron_h/body.rs new file mode 100644 index 000000000..a31ffe2f4 --- /dev/null +++ b/crates/gpu/src/arch/nemotron_h/body.rs @@ -0,0 +1,372 @@ +//! The nemotron_h `ChainBody`: the file's shape read and checked once at +//! load, each layer's weight names and site types resolved once, the caches +//! and the scan stores allocated once, and the per-step record refreshed +//! before every enqueue or replay. + +use super::step::{ + Acts, AttnLayer, Kernels, Layer, MambaLayer, MlpLayer, ROW_WORDS, SITE_TYPES, Scratch, Shape, + Site, WORD_POS, WORD_TOKEN, +}; +use crate::flash_gqa::{FlashGqaKernels, GROUP_5, HEAD}; +use crate::gemm::Gemm32Kernels; +use crate::head::Head; +use crate::linear::CONV_TAPS; +use crate::mamba2::{self, Mamba2Kernels}; +use crate::model::{ChainBody, GpuModel, NoHost, block_count}; +use crate::recur::RecStore; +use crate::site::{self, KGemvKernels, SiteTy}; +use crate::weights::{DevWeight, Weights}; +use crate::{DeviceTensor, Gpu, GpuError}; +use cuda_core::{CudaStream, DeviceBuffer}; +use gguf::Split; +use model::arch::Arch; +use model::arch::names::blk as layer_name; +use model::arch::nemotron_h::hparams::{Hparams, Kind}; +use model::arch::nemotron_h::names; +use model::mamba2::Geom; +use runtime::stores; +use std::ops::Range; + +const WHAT: &str = "nemotron_h::Body::load"; + +// The scan's ring is indexed by the linear module's rows; the plan counts the ring by +// `stores::conv_ring_rows`. +const _: () = assert!(crate::linear::RING_ROWS == stores::conv_ring_rows(CONV_TAPS)); + +/// The input of one decode step. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct DecodeInput { + pub token: u32, + pub pos: u32, +} + +/// The nemotron_h chain over one sequence on one card. +pub struct Body { + /// The Q5_0 token table's name. + pub(super) embed: String, + pub(super) sh: Shape, + pub(super) layers: Vec, + pub(super) k: Kernels, + pub(super) acts: Acts, + pub(super) s: Scratch, + /// The residual stream's second buffer ([`Body::enqueue`]). + pub(super) r: DeviceBuffer, + /// Host zeros as long as the longest recurrent buffer, what a reset + /// copies over every store. + zeros: Vec, + /// The step record's host words, rewritten by every refresh. + host_words: Vec, +} + +impl GpuModel { + /// The whole model of `file` resident on `gpu`, plus the output head: + /// every tensor in its kernels' device format, K/V planes of `ctx` rows + /// a key head, the scan stores and the step scratch. The model takes the + /// file and closes it once the weights are resident. + pub fn open(gpu: Gpu, file: Split, ctx: usize) -> Result, GpuError> { + let n_layers = block_count(&file, "nemotron_h GpuModel::open")?; + GpuModel::load_blocks(gpu, &file, ctx, 0..n_layers, true, |gpu, w| { + Body::load(gpu, &file, w, ctx) + }) + } +} + +impl Body { + fn load(gpu: &Gpu, file: &Split, w: &mut Weights, ctx: usize) -> Result { + let hp = Hparams::read(file).map_err(|e| GpuError::plan(WHAT, e))?; + if ctx == 0 { + return Err(GpuError::shape(WHAT, "a cache of 0 rows")); + } + let ssm = hp.ssm; + let mamba = mamba2::check( + WHAT, + Geom { + d_inner: ssm.d_inner, + n_heads: ssm.n_heads, + n_groups: ssm.n_groups, + d_state: ssm.d_state, + d_conv: ssm.d_conv, + }, + )?; + let n_kv = attention_heads(&hp)?; + let hidden = hp.n_embd; + let mut kinds = Names { file, w }; + let embed = names::token_embd(); + kinds.site(&embed, (hp.n_vocab, hidden), &[SiteTy::Q5_0])?; + let stream = gpu.stream(); + let layers = (0..hp.n_layer) + .map(|l| kinds.layer(&hp, l, ctx, stream)) + .collect::, _>>()?; + let sh = Shape { + hidden, + eps: hp.rms_eps, + mamba, + n_kv, + ctx, + }; + let ff = hp.ff.iter().copied().max().unwrap_or(0); + let longest = layers + .iter() + .filter_map(|l| match l { + Layer::Mamba(m) => Some(m.store.state.len().max(m.store.ring.len())), + _ => None, + }) + .max() + .unwrap_or(0); + let ctx_ = gpu.context(); + Ok(Body { + k: Kernels { + mamba: Mamba2Kernels::load(ctx_)?, + flash: FlashGqaKernels::load(ctx_)?, + g32: Gemm32Kernels::load(ctx_)?, + kgemv: KGemvKernels::load(gpu)?, + }, + acts: Acts::new(stream, layers.iter().flat_map(Layer::sites))?, + s: Scratch::new(stream, &sh, hp.n_head, ff)?, + r: DeviceBuffer::zeroed(stream, hidden)?, + zeros: vec![0.0; longest], + host_words: vec![0; ROW_WORDS + n_kv], + sh, + layers, + embed, + }) + } + + /// Layer `layer`'s scan state read back (a gate's instrument; it + /// synchronizes `stream`). A layer that is not a Mamba-2 layer is refused + /// by name. + pub fn scan_state(&self, stream: &CudaStream, layer: usize) -> Result, GpuError> { + match self.layers.get(layer) { + Some(Layer::Mamba(m)) => Ok(m.store.state.to_host_vec(stream)?), + _ => Err(GpuError::shape( + "nemotron_h::Body::scan_state", + format!("layer {layer} holds no scan state"), + )), + } + } +} + +/// The key heads of the attention layers, which the body takes equal across +/// layers: the planes, the row words and the flash group are cut for one. +/// Heads of 128 values and five query heads a key head are what the flash +/// entry runs; any other is refused by name. +fn attention_heads(hp: &Hparams) -> Result { + let mut found = None; + for (l, &kind) in hp.kinds.iter().enumerate() { + if kind != Kind::Attention { + continue; + } + let kv = hp.kv[l]; + if *found.get_or_insert(kv) != kv { + return Err(GpuError::shape( + WHAT, + format!("attention layer {l} has {kv} key heads, an earlier one {found:?}"), + )); + } + if hp.key_len != HEAD || hp.value_len != HEAD || hp.n_head != kv * GROUP_5 { + return Err(GpuError::shape( + WHAT, + format!( + "attention of {} heads over {kv} key heads of {}/{} values: the flash runs \ + {GROUP_5} query heads a key head of {HEAD} values", + hp.n_head, hp.key_len, hp.value_len + ), + )); + } + } + Ok(found.unwrap_or(0)) +} + +/// The derived conv bias of a file whose Mamba-2 layers hold none. +const ZERO_CONV_BIAS: &str = "derived.nemotron_h.zero_conv_bias"; + +/// The file and its resident weights, what a load resolves names against. +struct Names<'a> { + file: &'a Split, + w: &'a mut Weights, +} + +impl Names<'_> { + /// Site `name` of `rows` rows of `k` values: its type from the file's + /// header (one of `allowed`), checked against the resident weight. + fn site( + &self, + name: &str, + (rows, k): (usize, usize), + allowed: &[SiteTy], + ) -> Result { + let ty = site::file_site(self.file, WHAT, name, (rows, k), allowed)?; + site::site(self.w, WHAT, name, (rows, k), ty)?; + Ok(Site { + name: name.to_string(), + ty, + k, + }) + } + + /// The projection `stem` of layer `l`. + fn proj(&self, l: usize, stem: &str, shape: (usize, usize)) -> Result { + self.site(&layer_name(l, stem), shape, SITE_TYPES) + } + + /// Layer `l`'s conv bias of `len` values: the file's, or where the file + /// holds none (the bias is optional) one zero vector every such layer + /// reads, resident from load. + fn conv_bias(&mut self, l: usize, len: usize, stream: &CudaStream) -> Result { + if self + .file + .find(&layer_name(l, names::SSM_CONV_BIAS)) + .is_some() + { + return self.vector(l, names::SSM_CONV_BIAS, len); + } + let name = ZERO_CONV_BIAS.to_owned(); + if self.w.get(&name).is_none() { + let w = DeviceTensor::zeroed(stream, 1, len)?; + self.w + .insert_derived(name.clone(), DevWeight::F32 { w, k: len })?; + } + Ok(name) + } + + /// The f32 vector `stem` of layer `l`, `len` values: its name. + fn vector(&self, l: usize, stem: &str, len: usize) -> Result { + let name = layer_name(l, stem); + let got = self.w.f32_buf(WHAT, &name)?.len(); + if got != len { + return Err(GpuError::shape( + WHAT, + format!("{name} holds {got} values, the layer reads {len}"), + )); + } + Ok(name) + } + + fn layer( + &mut self, + hp: &Hparams, + l: usize, + ctx: usize, + stream: &CudaStream, + ) -> Result { + let hidden = hp.n_embd; + let norm = self.vector(l, names::ATTN_NORM, hidden)?; + Ok(match hp.kinds[l] { + Kind::Mamba => { + let s = hp.ssm; + let lens = stores::mamba2_lens(s.d_inner, s.d_state, CONV_TAPS, s.conv_dim); + Layer::Mamba(MambaLayer { + ssm_in: self.proj(l, names::SSM_IN, (s.d_in_proj, hidden))?, + ssm_out: self.proj(l, names::SSM_OUT, (hidden, s.d_inner))?, + a: self.vector(l, names::SSM_A, s.n_heads)?, + d: self.vector(l, names::SSM_D, s.n_heads)?, + dt_bias: self.vector(l, names::SSM_DT_BIAS, s.n_heads)?, + conv_w: self.vector(l, names::SSM_CONV, s.d_conv * s.conv_dim)?, + conv_b: self.conv_bias(l, s.conv_dim, stream)?, + ssm_norm: self.vector(l, names::SSM_NORM, s.d_inner)?, + store: RecStore::new(stream, lens, 1)?, + norm, + }) + } + Kind::Attention => { + let kv = hp.kv[l] * HEAD; + Layer::Attn(AttnLayer { + q: self.proj(l, names::ATTN_Q, (hp.n_head * HEAD, hidden))?, + k: self.proj(l, names::ATTN_K, (kv, hidden))?, + v: self.proj(l, names::ATTN_V, (kv, hidden))?, + o: self.proj(l, names::ATTN_OUTPUT, (hidden, hp.n_head * HEAD))?, + kc: DeviceTensor::zeroed(stream, hp.kv[l] * ctx, HEAD)?, + vc: DeviceTensor::zeroed(stream, hp.kv[l] * ctx, HEAD)?, + norm, + }) + } + Kind::Mlp => { + let ff = hp.ff[l]; + Layer::Mlp(MlpLayer { + up: self.proj(l, names::FFN_UP, (ff, hidden))?, + down: self.proj(l, names::FFN_DOWN, (hidden, ff))?, + ff, + norm, + }) + } + }) + } +} + +impl ChainBody for Body { + type Input = DecodeInput; + type Host = NoHost; + const UNHOSTED_POOL: bloomery_levers::UnhostedPool = bloomery_levers::UnhostedPool::NotCarried; + + fn arch() -> Arch { + Arch::NemotronH + } + + fn decode_input(&mut self, token: u32, pos: u32) -> Result { + Ok(DecodeInput { token, pos }) + } + + /// The step record in one copy: the position, the token and, for each key + /// head, the row `h · ctx + pos` its plane's append lands at. + fn refresh(&mut self, stream: &CudaStream, input: &DecodeInput) -> Result<(), GpuError> { + let DecodeInput { token, pos } = *input; + let ctx = self.sh.ctx; + self.host_words[WORD_POS] = pos; + self.host_words[WORD_TOKEN] = token; + for h in 0..self.sh.n_kv { + let row = h + .checked_mul(ctx) + .and_then(|r| r.checked_add(pos as usize)) + .and_then(|r| u32::try_from(r).ok()) + .ok_or_else(|| { + GpuError::shape( + "nemotron_h::Body::refresh", + format!("row {h} · {ctx} + {pos} does not fit a u32 row word"), + ) + })?; + self.host_words[ROW_WORDS + h] = row; + } + Ok(self.s.words.copy_from_host(stream, &self.host_words)?) + } + + fn enqueue_chain(&mut self, gpu: &Gpu, w: &Weights, head: &mut Head) -> Result<(), GpuError> { + self.enqueue(gpu, w, head) + } + + /// Every scan state and conv ring back to zero. The K/V planes need + /// nothing: the flash never reads a row at or past the live count, and + /// every row below it is written by its own step first. + fn reset(&mut self, gpu: &Gpu) -> Result<(), GpuError> { + for layer in &mut self.layers { + if let Layer::Mamba(m) = layer { + m.store.clear(gpu.stream(), &self.zeros)?; + } + } + Ok(()) + } + + fn head_eps(&self) -> f32 { + self.sh.eps + } + + fn resident_bytes(&self) -> usize { + let layers: usize = self + .layers + .iter() + .map(|l| match l { + Layer::Mamba(m) => m.store.bytes(), + Layer::Attn(a) => a.kc.buf().num_bytes() + a.vc.buf().num_bytes(), + Layer::Mlp(_) => 0, + }) + .sum(); + layers + self.s.bytes() + self.acts.bytes() + self.r.num_bytes() + } + + fn layers(&self) -> Range { + 0..self.layers.len() + } + + fn host(&mut self) -> Option<&mut NoHost> { + None + } +} diff --git a/crates/gpu/src/arch/nemotron_h/mod.rs b/crates/gpu/src/arch/nemotron_h/mod.rs new file mode 100644 index 000000000..baf7bd12e --- /dev/null +++ b/crates/gpu/src/arch/nemotron_h/mod.rs @@ -0,0 +1,31 @@ +//! The nemotron_h family's half of the GPU engine: [`Body`], the decode chain +//! of a dense hybrid of Mamba-2 layers, grouped-query attention layers with +//! no rope and squared-ReLU MLP layers (the description is +//! `model::arch::nemotron_h`; the dialect's authority is llama.cpp's +//! `src/models/nemotron-h.cpp`). +//! +//! One layer is a pre-norm mixer and a residual add, by the layer's kind +//! (read from the file's per-layer arrays at load, not from its index): +//! +//! - Mamba-2: norm, the `ssm_in` projection split into `z`, `xBC` and `dt` +//! (three views of one output), the scan step ([`crate::mamba2`]), the +//! `ssm_out` projection; +//! - attention: norm, `q`, `k` and `v` projections, `k` and `v` appended to +//! the layer's head-major planes, the group-5 flash pass, the output +//! projection; +//! - MLP: norm, `ffn_up`, the squared ReLU, `ffn_down`. +//! +//! The sites run the launch their file type selects ([`crate::site`]): the +//! tensors this body reads are Q5_0, Q8_0, Q4_K and Q6_K, and any other type is +//! refused by name at load. The token embedding is a Q5_0 table and the +//! output head a Q8_0 one ([`crate::head::Head`]). +//! +//! Every per-step value (the token, the position, the key heads' row words) +//! lives in device words refreshed before the enqueue or replay, so one +//! captured graph serves every position. The body holds one sequence in one +//! slot; a prompt is one step per token. + +mod body; +mod step; + +pub use body::{Body, DecodeInput}; diff --git a/crates/gpu/src/arch/nemotron_h/step.rs b/crates/gpu/src/arch/nemotron_h/step.rs new file mode 100644 index 000000000..b95fe12a0 --- /dev/null +++ b/crates/gpu/src/arch/nemotron_h/step.rs @@ -0,0 +1,485 @@ +//! The launches of one decode step: the layer kinds, the sites' activation +//! forms and the scratch the chain runs over, and the enqueue of the whole +//! chain (async throughout: nothing here allocates, frees or synchronizes, so +//! it is both the eager body and what a capture records). + +use super::body::Body; +use crate::elem::EmbedRowsArgs; +use crate::flash::PlaneAppend; +use crate::flash_gqa::{FlashGqaKernels, GROUP_5, GqaArgs, HEAD, partials_ms_len, partials_v_len}; +use crate::gemm::Gemm32Kernels; +use crate::head::Head; +use crate::mamba2::{Mamba2Kernels, Mamba2Scratch, StepArgs}; +use crate::q5::Q8Blocks32; +use crate::recur::RecStore; +use crate::site::{self, KGemvKernels, SiteTy}; +use crate::weights::{DevWeight, Weights}; +use crate::{DeviceTensor, Gpu, GpuError, Q8Act, Window}; +use cuda_core::DeviceBuffer; +use model::mamba2::Checked; + +const WHAT: &str = "nemotron_h::Body::enqueue_chain"; + +/// The file types a site launches: Q5_0 through its packed 32-value gemv, +/// Q8_0 through the f32 one, Q4_K and Q6_K through theirs on the q8_1 rows. +pub(super) const SITE_TYPES: &[SiteTy] = &[SiteTy::Q5_0, SiteTy::Q8_0, SiteTy::Q4K, SiteTy::Q6K]; + +/// The step's word record: the position, the token, then one row word a key +/// head ([`ROW_WORDS`]). +pub(super) const WORD_POS: usize = 0; +pub(super) const WORD_TOKEN: usize = 1; +pub(super) const ROW_WORDS: usize = 2; + +/// One projection of a layer: the file tensor's name, the launch its type +/// selects and the width of the rows it reads. +pub(super) struct Site { + pub(super) name: String, + pub(super) ty: SiteTy, + pub(super) k: usize, +} + +pub(super) struct MambaLayer { + pub(super) norm: String, + pub(super) ssm_in: Site, + pub(super) ssm_out: Site, + pub(super) a: String, + pub(super) d: String, + pub(super) dt_bias: String, + pub(super) conv_w: String, + pub(super) conv_b: String, + pub(super) ssm_norm: String, + pub(super) store: RecStore, +} + +pub(super) struct AttnLayer { + pub(super) norm: String, + pub(super) q: Site, + pub(super) k: Site, + pub(super) v: Site, + pub(super) o: Site, + /// The head-major planes: head `h`, position `p` at row `h · ctx + p`. + pub(super) kc: DeviceTensor, + pub(super) vc: DeviceTensor, +} + +pub(super) struct MlpLayer { + pub(super) norm: String, + pub(super) up: Site, + pub(super) down: Site, + pub(super) ff: usize, +} + +/// A layer, by the kind the file's per-layer arrays give it. +pub(super) enum Layer { + Mamba(MambaLayer), + Attn(AttnLayer), + Mlp(MlpLayer), +} + +impl Layer { + pub(super) fn sites(&self) -> Vec<&Site> { + match self { + Layer::Mamba(m) => vec![&m.ssm_in, &m.ssm_out], + Layer::Attn(a) => vec![&a.q, &a.k, &a.v, &a.o], + Layer::Mlp(m) => vec![&m.up, &m.down], + } + } +} + +/// The widths the scratch is cut for and the chain runs at. +#[derive(Clone, Copy)] +pub(super) struct Shape { + pub(super) hidden: usize, + pub(super) eps: f32, + pub(super) mamba: Checked, + pub(super) n_kv: usize, + pub(super) ctx: usize, +} + +/// The kernels the chain launches beyond the `Gpu`'s own. +pub(super) struct Kernels { + pub(super) mamba: Mamba2Kernels, + pub(super) flash: FlashGqaKernels, + pub(super) g32: Gemm32Kernels, + pub(super) kgemv: KGemvKernels, +} + +/// The quantized activation forms the sites read: one of each form a site's +/// type reads, at each width a site reads it at. +pub(super) struct Acts { + blocks32: Vec, + q8_1: Vec, +} + +impl Acts { + /// The forms `sites` read. Load-time only. + pub(super) fn new<'s>( + stream: &cuda_core::CudaStream, + sites: impl Iterator, + ) -> Result { + let mut acts = Acts { + blocks32: Vec::new(), + q8_1: Vec::new(), + }; + for s in sites { + match s.ty { + SiteTy::Q5_0 if !acts.blocks32.iter().any(|a| a.k() == s.k) => { + acts.blocks32.push(Q8Blocks32::new(stream, s.k, 1)?); + } + SiteTy::Q3K | SiteTy::Q4K | SiteTy::Q5K | SiteTy::Q6K + if !acts.q8_1.iter().any(|a| a.k() == s.k) => + { + acts.q8_1.push(Q8Act::with_k(stream, 1, s.k)?); + } + _ => {} + } + } + Ok(acts) + } + + pub(super) fn bytes(&self) -> usize { + self.blocks32 + .iter() + .map(|a| a.q.num_bytes() + a.s8.num_bytes() + a.d8.num_bytes()) + .sum::() + + self.q8_1.iter().map(Q8Act::device_bytes).sum::() + } +} + +/// The buffers one step runs over, m = 1. +pub(super) struct Scratch { + pub(super) xn: DeviceBuffer, + pub(super) delta: DeviceBuffer, + pub(super) zxbcdt: DeviceBuffer, + pub(super) y: DeviceBuffer, + pub(super) q: DeviceBuffer, + pub(super) k: DeviceBuffer, + pub(super) v: DeviceBuffer, + pub(super) attn: DeviceBuffer, + pub(super) up: DeviceBuffer, + pub(super) act: DeviceBuffer, + pub(super) part_v: DeviceBuffer, + pub(super) part_ms: DeviceBuffer, + pub(super) mamba: Mamba2Scratch, + /// The record [`WORD_POS`], [`WORD_TOKEN`] and [`ROW_WORDS`] name. + pub(super) words: DeviceBuffer, + /// The embedding's outputs: the position word the scan reads and the live + /// key count the flash reads. + pub(super) pos: DeviceBuffer, + pub(super) n_keys: DeviceBuffer, +} + +impl Scratch { + pub(super) fn bytes(&self) -> usize { + [ + &self.xn, + &self.delta, + &self.zxbcdt, + &self.y, + &self.q, + &self.k, + &self.v, + &self.attn, + &self.up, + &self.act, + &self.part_v, + &self.part_ms, + ] + .iter() + .map(|b| b.num_bytes()) + .sum::() + + self.words.num_bytes() + + self.pos.num_bytes() + + self.n_keys.num_bytes() + } + + /// The scratch for `sh`, the widest `ff` an MLP layer has and the + /// projection widths `sh`'s layers read. Load-time only. + pub(super) fn new( + stream: &cuda_core::CudaStream, + sh: &Shape, + n_head: usize, + ff: usize, + ) -> Result { + let g = sh.mamba.geom(); + let z = |n: usize| -> Result, GpuError> { + Ok(DeviceBuffer::zeroed(stream, n.max(1))?) + }; + let kv_width = sh.n_kv * HEAD; + Ok(Scratch { + xn: z(sh.hidden)?, + delta: z(sh.hidden)?, + zxbcdt: z(2 * g.d_inner + 2 * g.n_groups * g.d_state + g.n_heads)?, + y: z(g.d_inner)?, + q: z(n_head * HEAD)?, + k: z(kv_width)?, + v: z(kv_width)?, + attn: z(n_head * HEAD)?, + up: z(ff)?, + act: z(ff)?, + part_v: z(partials_v_len(1, n_head))?, + part_ms: z(partials_ms_len(1, n_head))?, + mamba: Mamba2Scratch::new(stream, sh.mamba)?, + words: DeviceBuffer::zeroed(stream, ROW_WORDS + sh.n_kv)?, + pos: DeviceBuffer::zeroed(stream, 1)?, + n_keys: DeviceBuffer::zeroed(stream, 1)?, + }) + } +} + +/// What every launch of a step reads: the card, the resident weights, the +/// kernels and the widths. +struct Cx<'a> { + gpu: &'a Gpu, + w: &'a Weights, + k: &'a Kernels, + sh: Shape, +} + +impl<'a> Cx<'a> { + /// The resident f32 vector `name`. + fn f32(&self, name: &str) -> Result<&'a DeviceBuffer, GpuError> { + self.w.f32_buf(WHAT, name) + } +} + +impl Site { + /// `y = W · x` for this site over the first `k` values of `x`, raising + /// into layer `layer`'s fault sink. + fn project( + &self, + cx: &Cx<'_>, + acts: &mut Acts, + layer: usize, + x: &DeviceBuffer, + y: &mut DeviceBuffer, + ) -> Result<(), GpuError> { + let gpu = cx.gpu; + let key = (self.ty, cx.w, self.name.as_str()); + match self.ty { + SiteTy::Q8_0 | SiteTy::F32 => site::gemv(gpu, &cx.k.g32, key, x, 1, y), + SiteTy::Q5_0 => { + let act = acts + .blocks32 + .iter_mut() + .find(|a| a.k() == self.k) + .ok_or_else(|| { + GpuError::state(WHAT, "a Q8Blocks32 form at the site's width") + })?; + gpu.q5() + .enqueue_quantize_q8(gpu.stream(), x, act, gpu.layer_sink(layer)?)?; + site::blocks32_gemv(gpu, key, act, y) + } + SiteTy::Q3K | SiteTy::Q4K | SiteTy::Q5K | SiteTy::Q6K => { + let act = acts + .q8_1 + .iter_mut() + .find(|a| a.k() == self.k) + .ok_or_else(|| GpuError::state(WHAT, "a q8_1 form at the site's width"))?; + gpu.enqueue_quantize_q8_1_layer(x, act, layer)?; + site::kgemv(gpu, &cx.k.kgemv, key, act, gpu.layer_sink(layer)?, y) + } + } + } +} + +/// `y = rms_norm(x, name)` at the model's epsilon. +fn norm( + cx: &Cx<'_>, + name: &str, + x: &DeviceBuffer, + y: &mut DeviceBuffer, +) -> Result<(), GpuError> { + let gain = cx.f32(name)?; + cx.gpu + .elem() + .enqueue_rms_norm(cx.gpu.stream(), x, gain, cx.sh.eps, cx.sh.hidden, 1, y) +} + +/// `dst = src + delta`, the layer's residual. +fn residual( + cx: &Cx<'_>, + src: &DeviceBuffer, + delta: &DeviceBuffer, + dst: &mut DeviceBuffer, +) -> Result<(), GpuError> { + cx.gpu + .elem() + .enqueue_add(cx.gpu.stream(), src, delta, cx.sh.hidden, dst) +} + +/// One Mamba-2 layer: the `ssm_in` output is `[z | xBC | dt]`, three views of +/// the one buffer at `0`, `d_inner` and `d_inner + conv_dim` values (llama.cpp +/// `build_mamba2_layer`'s views at `0`, `d_inner`, `2·d_inner + 2·n_group·d_state`). +fn mamba_layer( + cx: &Cx<'_>, + (l, m): (usize, &mut MambaLayer), + (s, acts): (&mut Scratch, &mut Acts), + (src, dst): (&DeviceBuffer, &mut DeviceBuffer), +) -> Result<(), GpuError> { + let sh = cx.sh; + let (inner, conv_dim, heads) = ( + sh.mamba.geom().d_inner, + sh.mamba.conv_dim(), + sh.mamba.geom().n_heads, + ); + norm(cx, &m.norm, src, &mut s.xn)?; + m.ssm_in.project(cx, acts, l, &s.xn, &mut s.zxbcdt)?; + let f32s = size_of::(); + let z = Window::::of(&s.zxbcdt, 0, inner)?; + let xbc = Window::::of(&s.zxbcdt, inner * f32s, conv_dim)?; + let dt = Window::::of(&s.zxbcdt, (inner + conv_dim) * f32s, heads)?; + cx.k.mamba.enqueue_step( + cx.gpu.stream(), + StepArgs { + shape: sh.mamba, + xbc: &xbc, + dt: &dt, + dt_bias: cx.f32(&m.dt_bias)?, + z: &z, + a: cx.f32(&m.a)?, + d: cx.f32(&m.d)?, + conv_w: cx.f32(&m.conv_w)?, + conv_b: cx.f32(&m.conv_b)?, + norm: Some(cx.f32(&m.ssm_norm)?), + eps: sh.eps, + pos: &s.pos, + pos_at: 0, + fault: cx.gpu.layer_sink(l)?, + y: &mut s.y, + ring: &mut m.store.ring, + state: &mut m.store.state, + scratch: &mut s.mamba, + }, + )?; + m.ssm_out.project(cx, acts, l, &s.y, &mut s.delta)?; + residual(cx, src, &s.delta, dst) +} + +/// Key head `h`'s row of `buf`, the view an append reads. +fn head_rows<'b>( + buf: &'b DeviceBuffer, +) -> impl FnMut(usize) -> Result, GpuError> { + move |h| Window::::of(buf, h * HEAD * size_of::(), HEAD) +} + +/// One attention layer: no rope and no qk norm, so the projections are the +/// flash's inputs and the planes' rows as they stand. +fn attn_layer( + cx: &Cx<'_>, + (l, a): (usize, &mut AttnLayer), + (s, acts): (&mut Scratch, &mut Acts), + (src, dst): (&DeviceBuffer, &mut DeviceBuffer), +) -> Result<(), GpuError> { + let sh = cx.sh; + let fault = cx.gpu.layer_sink(l)?; + norm(cx, &a.norm, src, &mut s.xn)?; + a.q.project(cx, acts, l, &s.xn, &mut s.q)?; + a.k.project(cx, acts, l, &s.xn, &mut s.k)?; + a.v.project(cx, acts, l, &s.xn, &mut s.v)?; + let at = PlaneAppend { + words: &s.words, + first_word: ROW_WORDS, + heads: sh.n_kv, + m: 1, + fault, + }; + let flash = cx.gpu.flash(); + flash.enqueue_kv_append_planes(cx.gpu.stream(), &at, head_rows(&s.k), &mut a.kc)?; + flash.enqueue_kv_append_planes(cx.gpu.stream(), &at, head_rows(&s.v), &mut a.vc)?; + cx.k.flash.enqueue_pass::( + cx.gpu.stream(), + GqaArgs { + q: &s.q, + kc: a.kc.buf(), + vc: a.vc.buf(), + n_keys: &s.n_keys, + scale: 1.0 / (HEAD as f32).sqrt(), + n_kv: sh.n_kv, + ctx: sh.ctx, + m: 1, + part_v: &mut s.part_v, + part_ms: &mut s.part_ms, + fault, + y: &mut s.attn, + }, + false, + )?; + a.o.project(cx, acts, l, &s.attn, &mut s.delta)?; + residual(cx, src, &s.delta, dst) +} + +/// One MLP layer: `down(relu²(up(x)))`, no gate. +fn mlp_layer( + cx: &Cx<'_>, + (l, m): (usize, &MlpLayer), + (s, acts): (&mut Scratch, &mut Acts), + (src, dst): (&DeviceBuffer, &mut DeviceBuffer), +) -> Result<(), GpuError> { + norm(cx, &m.norm, src, &mut s.xn)?; + m.up.project(cx, acts, l, &s.xn, &mut s.up)?; + cx.gpu + .elem() + .enqueue_relu_sqr(cx.gpu.stream(), &s.up, m.ff, &mut s.act)?; + m.down.project(cx, acts, l, &s.act, &mut s.delta)?; + residual(cx, src, &s.delta, dst) +} + +impl Body { + /// The whole chain: the embedding, every layer, then `head`. The residual + /// stream alternates between the head's input and the body's own buffer, + /// each layer writing the one it did not read, so the last layer's sum is + /// the head's input whatever the layer count's parity. + pub(super) fn enqueue( + &mut self, + gpu: &Gpu, + w: &Weights, + head: &mut Head, + ) -> Result<(), GpuError> { + let cx = Cx { + gpu, + w, + k: &self.k, + sh: self.sh, + }; + let table = match w.get(&self.embed) { + Some(DevWeight::Q5_0 { w, .. }) => w, + _ => return Err(GpuError::tensor(WHAT, &self.embed, "a Q5_0 table")), + }; + let hin = head.input_mut(); + let mut in_head = self.layers.len().is_multiple_of(2); + { + let ids = Window::::of(&self.s.words, WORD_TOKEN * size_of::(), 1)?; + let pos0 = Window::::of(&self.s.words, WORD_POS * size_of::(), 1)?; + let first = if in_head { &mut *hin } else { &mut self.r }; + gpu.elem().enqueue_embed_rows_q5_0( + gpu.stream(), + EmbedRowsArgs { + w: table, + ids: &ids, + pos0: &pos0, + first: 0, + y: first, + pos: &mut self.s.pos, + n_keys: &mut self.s.n_keys, + }, + self.sh.hidden, + )?; + } + for (l, layer) in self.layers.iter_mut().enumerate() { + let (src, dst) = if in_head { + (&*hin, &mut self.r) + } else { + (&self.r as &DeviceBuffer, &mut *hin) + }; + let scratch = (&mut self.s, &mut self.acts); + match layer { + Layer::Mamba(m) => mamba_layer(&cx, (l, m), scratch, (src, dst))?, + Layer::Attn(a) => attn_layer(&cx, (l, a), scratch, (src, dst))?, + Layer::Mlp(m) => mlp_layer(&cx, (l, m), scratch, (src, dst))?, + } + in_head = !in_head; + } + head.enqueue(gpu, w) + } +} diff --git a/crates/gpu/src/dspark/dense.rs b/crates/gpu/src/dspark/dense.rs index e7edfcbad..3ed1efeb3 100644 --- a/crates/gpu/src/dspark/dense.rs +++ b/crates/gpu/src/dspark/dense.rs @@ -57,6 +57,7 @@ use model::arch::dspark::DraftHeld; use model::quant::quantize_q8_0; use super::{DraftLayers, OwnHeld, Q8Rows}; +use crate::flash::PlaneAppend; use crate::flash_gqa::{ FlashGqaKernels, GROUP, GqaArgs, HEAD_256, PACK_4, partials_ms_len, partials_v_len_256, }; @@ -407,6 +408,14 @@ fn zeros_f32(gpu: &Gpu, n: usize) -> Result, GpuError> { Ok(DeviceBuffer::zeroed(gpu.stream(), n.max(1))?) } +/// Key head `head`'s whole buffer of `bufs` as the view an append reads. +fn whole(bufs: &[DeviceBuffer], head: usize) -> Result, GpuError> { + let buf = bufs + .get(head) + .ok_or_else(|| shape_err(format!("key head {head} of {}", bufs.len())))?; + Window::of(buf, 0, buf.len()) +} + impl DenseLayers { /// Load `spec.layers` layers from `src`: every matrix quantized to Q8_0 on the host (the bf16 is kept /// nowhere), the query and key rows permuted as the module doc says, the caches zeroed. @@ -809,25 +818,17 @@ impl DraftLayers for DenseLayers { &mut s.kn[head], )?; elem.enqueue_rope(stream, &s.kn[head], &s.ctx_cs, hd, 1, n, &mut s.kr[head])?; - for second in 0..2 { - let word = Window::::of(&s.ctx_words, 4 * (second * n_kv + head), 1)?; - flash.enqueue_kv_append_pos_buf( - stream, - &s.kr[head], - &word, - &mut layer.kc, - n, - fault, - )?; - flash.enqueue_kv_append_pos_buf( - stream, - &s.vb[head], - &word, - &mut layer.vc, - n, - fault, - )?; - } + } + for second in 0..2 { + let at = PlaneAppend { + words: &s.ctx_words, + first_word: second * n_kv, + heads: n_kv, + m: n, + fault, + }; + flash.enqueue_kv_append_planes(stream, &at, |h| whole(&s.kr, h), &mut layer.kc)?; + flash.enqueue_kv_append_planes(stream, &at, |h| whole(&s.vb, h), &mut layer.vc)?; } } Ok(()) @@ -945,24 +946,16 @@ impl DraftLayers for DenseLayers { &mut s.kn[head], )?; elem.enqueue_rope(stream, &s.kn[head], &s.blk_cs, hd, 1, m, &mut s.kr[head])?; - let word = Window::::of(&s.blk_words, 4 * head, 1)?; - flash.enqueue_kv_append_pos_buf( - stream, - &s.kr[head], - &word, - &mut layer.kc, - m, - fault, - )?; - flash.enqueue_kv_append_pos_buf( - stream, - &s.vb[head], - &word, - &mut layer.vc, - m, - fault, - )?; } + let at = PlaneAppend { + words: &s.blk_words, + first_word: 0, + heads: n_kv, + m, + fault, + }; + flash.enqueue_kv_append_planes(stream, &at, |h| whole(&s.kr, h), &mut layer.kc)?; + flash.enqueue_kv_append_planes(stream, &at, |h| whole(&s.vb, h), &mut layer.vc)?; let plane_bytes = 2 * GUARD * hd; let plane_len = n_kv * planes * hd; let kc = Window::::of(layer.kc.buf(), plane_bytes, plane_len)?; diff --git a/crates/gpu/src/elem.rs b/crates/gpu/src/elem.rs index 54a8d80dd..0bd337945 100644 --- a/crates/gpu/src/elem.rs +++ b/crates/gpu/src/elem.rs @@ -1127,6 +1127,37 @@ mod elem_kernels { e.map(|g, u| mul_rn_f32(gelu_tanh_dev(g), u)); } + /// `y[i] = max(x[i], 0)^2` over `n` values. A non-finite input is copied + /// as it is: `max` of a non-finite is not this rule, and the square of + /// `−inf` is not 0. + #[kernel] + #[launch_bounds(256)] + #[launch_contract( + domain = 1, + block = (256, 1, 1), + requires = (x.len() >= n, y.len() >= n) + )] + pub fn relu_sqr(x: &[f32], n: u32, mut y: DisjointSlice) { + let i = thread::index_1d().get(); + if i >= n as usize { + return; + } + // SAFETY: i < n <= x.len() and y.len() by the launch contract; thread + // i owns entry i. + let v = unsafe { *x.get_unchecked(i) }; + let out = if !v.is_finite() { + v + } else { + let r = if v > 0.0 { v } else { 0.0 }; + r * r + }; + // SAFETY: i < n <= y.len() by the launch contract; thread i owns + // entry i. + unsafe { + *y.get_unchecked_mut(i) = out; + } + } + /// `y[i] = a[i] + b[i]`, elementwise over `n` values — exact; the body /// is one add, so nothing is extracted into a core. #[kernel] @@ -2201,6 +2232,29 @@ impl ElemKernels { Ok(()) } + /// Enqueue `y = max(x, 0)^2` over `n` values (flat). A non-finite input + /// stays that value. Asynchronous, allocation-free, capturable. + pub fn enqueue_relu_sqr( + &self, + stream: &CudaStream, + x: &DeviceBuffer, + n: usize, + y: &mut DeviceBuffer, + ) -> Result<(), GpuError> { + if n == 0 || x.len() < n || y.len() < n { + return Err(GpuError::shape( + "enqueue_relu_sqr", + format!("n={n}, x.len() {}, y.len() {}", x.len(), y.len()), + )); + } + let n = launch_u32("enqueue_relu_sqr", "n", n)?; + let prep = self + .module + .prepare_relu_sqr(LaunchConfig1D::new(n.div_ceil(256), 256, 0))?; + self.module.relu_sqr(stream, &prep, x, n, y)?; + Ok(()) + } + /// Enqueue `y = a + b` over `n` values (flat). Exact. Asynchronous, /// allocation-free, capturable. pub fn enqueue_add( diff --git a/crates/gpu/src/flash.rs b/crates/gpu/src/flash.rs index 1170147d2..03a3fa358 100644 --- a/crates/gpu/src/flash.rs +++ b/crates/gpu/src/flash.rs @@ -75,7 +75,7 @@ use crate::fault::{FaultSink, FaultSite}; use crate::flash_gqa::{SEGMENTS, seg_span}; use crate::launch_u32; use crate::q8_1_quant_vals; -use crate::tensor::{DeviceTensor, Q8Act}; +use crate::tensor::{DeviceTensor, Q8Act, Window}; use cuda_core::{CudaContext, CudaStream, DeviceBuffer, LaunchConfig1D}; use cuda_device::convert::{cvt_f16x2_f32, cvt_f32_f16x2_lo}; use cuda_device::{ @@ -1747,6 +1747,18 @@ mod flash_kernels { } } +/// [`FlashKernels::enqueue_kv_append_planes`]'s placement: the `heads` row +/// words that follow word `first_word` of `words` (head `h` at +/// `first_word + h`), the `m` rows each head appends, and the sink a row past +/// the cache raises on. +pub struct PlaneAppend<'a> { + pub words: &'a DeviceBuffer, + pub first_word: usize, + pub heads: usize, + pub m: usize, + pub fault: FaultSink, +} + /// The loaded P5 device module: `kv_append`, `kv_append_pos_buf`, /// `flash_latent`, the tensor-core segment pass `flash_latent_mma`, /// `flash_merge` and the q8 twins. Owns no context and no stream — every @@ -1926,6 +1938,40 @@ impl FlashKernels { Ok(()) } + /// [`FlashKernels::enqueue_kv_append_pos_buf`] for every key head of one + /// layer's head-major planes: head `h`'s `m` rows come from `src(h)` and + /// land at the row word `at.first_word + h` of `at.words` names, so a + /// plane of `ctx` rows a head is addressed by words `p + h · ctx`. One + /// launch per head. Asynchronous, allocation-free, capturable; `src` + /// builds views over buffers that stay in place for a captured graph's + /// life, and `at.words` stays allocated and in place likewise. + pub fn enqueue_kv_append_planes<'s>( + &self, + stream: &CudaStream, + at: &PlaneAppend<'_>, + mut src: impl FnMut(usize) -> Result, GpuError>, + cache: &mut DeviceTensor, + ) -> Result<(), GpuError> { + let what = "enqueue_kv_append_planes"; + if at.heads == 0 || at.words.len() < at.first_word + at.heads { + return Err(GpuError::shape( + what, + format!( + "{} heads from word {} of a {}-word buffer", + at.heads, + at.first_word, + at.words.len() + ), + )); + } + for head in 0..at.heads { + let rows = src(head)?; + let word = Window::::of(at.words, 4 * (at.first_word + head), 1)?; + self.enqueue_kv_append_pos_buf(stream, &rows, &word, cache, at.m, at.fault)?; + } + Ok(()) + } + /// Enqueue the latent flash attention: `q` holds `tokens * heads` rows /// of `rope_dims + latent_dims` f32 (row `t * heads + h`, content /// `[q_rope | q_nope2]`), `kv` is the `[rows x width]` u16 cache, and diff --git a/crates/gpu/src/head.rs b/crates/gpu/src/head.rs index e88e50509..91c60dcaf 100644 --- a/crates/gpu/src/head.rs +++ b/crates/gpu/src/head.rs @@ -63,6 +63,14 @@ impl OutW<'_> { } } +/// The head's q8_1 form, refused by name on a head that holds none (its +/// projection reads the f32 rows). +fn held(act: &mut Option) -> Result<&mut Q8Act, GpuError> { + act.as_mut().ok_or_else(|| { + GpuError::state("Head::enqueue", "a q8_1 form (a K-quant lm_head holds one)") + }) +} + /// The lm_head weight (`output.weight`) and its row width. The type is /// checked, not assumed — a file whose lm_head is another format needs a /// different gemv, and a silent plane reuse would misread its bytes. @@ -180,7 +188,9 @@ pub struct Head { /// The chain's input — the last block's output vector. The assembly round /// writes here through `input_mut`; the gate through `set_input`. x: DeviceBuffer, - act: Q8Act, + /// The q8_1 form of the normed rows a K-quant projection reads; a Q8_0 + /// head reads the f32 rows and holds none (its width need not divide 256). + act: Option, logits: DeviceBuffer, /// The `m` argmax tokens, then the fault's first-layer word and that /// layer's site mask as the argmax found them. Seeded with the (0, 0) @@ -281,7 +291,10 @@ impl Head { n_vocab, m, x, - act: Q8Act::with_k(stream, m, hidden)?, + act: match out_w { + OutW::Q8_0 { .. } => None, + OutW::Q6K(_) | OutW::Q4K(_) => Some(Q8Act::with_k(stream, m, hidden)?), + }, logits: DeviceBuffer::zeroed(stream, m * n_vocab)?, token_out: DeviceBuffer::from_host(stream, &vec![0u32; m + 2])?, rows_done, @@ -322,11 +335,7 @@ impl Head { + self.logits.num_bytes() + self.token_out.num_bytes() + self.rows_done.as_ref().map_or(0, DeviceBuffer::num_bytes) - + self.act.q3.num_bytes() - + self.act.q4.num_bytes() - + self.act.q6.num_bytes() - + self.act.s8.num_bytes() - + self.act.d8.num_bytes() + + self.act.as_ref().map_or(0, Q8Act::device_bytes) } /// Enqueue the whole head for the rows in `x`: rms_norm → quantize_q8_1 @@ -360,12 +369,14 @@ impl Head { let fault = gpu.fault_sink(LAYER_HEAD); match head_out_w(out)?.0 { OutW::Q6K(out_w) => { - gpu.enqueue_quantize_q8_1_head(rows, &mut self.act)?; - gpu.enqueue_gemv_q6k(out_w, &self.act, &mut self.logits)?; + let act = held(&mut self.act)?; + gpu.enqueue_quantize_q8_1_head(rows, act)?; + gpu.enqueue_gemv_q6k(out_w, &*act, &mut self.logits)?; } OutW::Q4K(out_w) => { - gpu.enqueue_quantize_q8_1_head(rows, &mut self.act)?; - gpu.enqueue_gemv_q4k(out_w, &self.act, &mut self.logits)?; + let act = held(&mut self.act)?; + gpu.enqueue_quantize_q8_1_head(rows, act)?; + gpu.enqueue_gemv_q4k(out_w, &*act, &mut self.logits)?; } OutW::Q8_0 { qs, d } => { gpu.q8f32() @@ -437,9 +448,10 @@ impl Head { { let shape = (self.eps, self.hidden, self.m); let rows = norm_rows(gpu, (w, self.gain.as_str()), &mut self.fin, shape, &self.x)?; - gpu.enqueue_quantize_q8_1_head(rows, &mut self.act)?; + let act = held(&mut self.act)?; + gpu.enqueue_quantize_q8_1_head(rows, act)?; tail( - &self.act, + &*act, head_q6k_w(w)?, gpu.fault_sink(LAYER_HEAD), &mut self.logits, diff --git a/crates/gpu/src/lib.rs b/crates/gpu/src/lib.rs index e45aa5f92..eccb990ed 100644 --- a/crates/gpu/src/lib.rs +++ b/crates/gpu/src/lib.rs @@ -102,6 +102,8 @@ pub use watchdog::Busy as EngineBusy; pub type Deepseek2Model = GpuModel; /// The qwen3moe engine: the whole model on one card. pub type Qwen3moeModel = GpuModel; +/// The nemotron_h engine: the whole model on one card. +pub type NemotronHModel = GpuModel; pub use tensor::{DeviceTensor, PartedBuffer, Q8Act, TensorWindow, Window, WindowMut, window}; /// Host-side failure: context creation, module loading, device allocation, diff --git a/crates/gpu/src/model.rs b/crates/gpu/src/model.rs index 52cabaebf..93dc85a0c 100644 --- a/crates/gpu/src/model.rs +++ b/crates/gpu/src/model.rs @@ -846,6 +846,12 @@ impl GpuModel { self.body.layers() } + /// The positions the resident caches were sized for: the most a + /// sequence holds. + pub fn ctx_max(&self) -> usize { + self.ctx_max + } + /// Device bytes of everything this model holds resident: the weights and /// the body, plus the heads' scratch — a multi-row pass's heads once one /// has run — and every parked slot's sequence diff --git a/crates/gpu/src/site.rs b/crates/gpu/src/site.rs index df0b8584d..c8511b979 100644 --- a/crates/gpu/src/site.rs +++ b/crates/gpu/src/site.rs @@ -16,7 +16,9 @@ //! bit `f32_gemv` on every row and column). A K-quant site's gemv is //! [`kgemv`]. A Q5_0 site's is [`blocks32_gemv`]. //! - [`kgemv`], a K-quant site at most eight columns over the q8_1 rows: -//! Q3_K, Q4_K and Q6_K their gemvs, row-major (`m` outputs a row); Q5_K +//! Q3_K, Q4_K and Q6_K their gemvs, row-major (`m` outputs a row; a Q6_K +//! row of an odd super-block count the down `_sel` over one expert, one +//! column); Q5_K //! the K-quant down `_sel` over a stack of one expert, token-major (a slot //! a column). [`SiteTy::kgemv_order`] is this family's layout and is //! `None` for Q5_0. Where a body copies a row-major output token-major, @@ -46,6 +48,7 @@ use crate::gemm::{ }; use crate::kquant::{Act, KquantKernels, SelDown}; use crate::q5::Q8Blocks32; +use crate::q6k_sel::Q6kSelKernels; use crate::q8f32::{GemvOut, Q8_0GemvMcolArgs}; use crate::tensor::Q8Act; use crate::weights::{DevWeight, Weights}; @@ -323,10 +326,12 @@ const KGEMV_COLS: usize = 8; /// The kernels a K-quant site's [`kgemv`] launches beyond the `Gpu`'s own: /// the K-quant `_sel` family, whose Q5_K down entry over a stack of one -/// expert is the Q5_K gemv, and that one expert's id for each of +/// expert is the Q5_K gemv, the Q6_K `_sel` entry, which a Q6_K row of an odd +/// super-block count runs the same way, and that one expert's id for each of /// [`KGEMV_COLS`] slots. pub struct KGemvKernels { kq: KquantKernels, + q6: Q6kSelKernels, one: DeviceBuffer, } @@ -336,6 +341,7 @@ impl KGemvKernels { pub fn load(gpu: &Gpu) -> Result { Ok(KGemvKernels { kq: KquantKernels::load(gpu.context(), gpu.fault_word())?, + q6: Q6kSelKernels::load(gpu.context(), gpu.fault_word())?, one: DeviceBuffer::zeroed(gpu.stream(), KGEMV_COLS)?, }) } @@ -358,6 +364,25 @@ pub fn kgemv( const WHAT: &str = "site::kgemv"; match ty { SiteTy::Q4K => gpu.enqueue_gemv_q4k(kq(w, WHAT, name)?, act, y), + // An odd super-block count leaves the rows unaligned to the words the + // plain gemv reads: the `_sel` entry addresses rows by byte, so the + // site runs as a stack of one expert, one column (where its layout + // is the row-major one). + SiteTy::Q6K if act.n_sb() % 2 == 1 => { + if act.m() != 1 { + return Err(GpuError::shape( + WHAT, + format!( + "{name} is Q6_K at an odd {} super-blocks a row: one column, got {}", + act.n_sb(), + act.m() + ), + )); + } + let wt = kq(w, WHAT, name)?; + kk.q6 + .enqueue_gemv_q6k_sel(gpu.stream(), wt, act, &kk.one, 1, wt.rows(), y) + } SiteTy::Q6K => gpu.enqueue_gemv_q6k(kq(w, WHAT, name)?, act, y), SiteTy::Q3K => gpu.enqueue_gemv_q3k(kq(w, WHAT, name)?, act, y), SiteTy::Q5K => { diff --git a/crates/gpu/src/weights.rs b/crates/gpu/src/weights.rs index 81d848ddf..0aea526c3 100644 --- a/crates/gpu/src/weights.rs +++ b/crates/gpu/src/weights.rs @@ -298,6 +298,7 @@ impl Weights { let budget = kept.iter().map(|&(_, t)| upload_budget(t)).sum(); let mut ring = UploadRing::new(stream, budget)?; for (shard, t) in kept { + file_slot(&t.name)?; let gguf = split.shard(shard).ok_or_else(|| { GpuError::shape( "Weights::load_where", @@ -720,6 +721,19 @@ fn derived_slot_free(by_name: &BTreeMap, name: &str) -> Result<(), Ok(()) } +/// A file tensor's name, refused in the [`DERIVED_PREFIX`] namespace: a file +/// tensor never stands where a derived weight is filed, so a lookup of a +/// derived name finds only what `insert_derived` filed. +fn file_slot(name: &str) -> Result<(), GpuError> { + if name.starts_with(DERIVED_PREFIX) { + return Err(GpuError::shape( + "Weights::load", + format!("file tensor {name:?} has a {DERIVED_PREFIX:?} name, which the engine derives"), + )); + } + Ok(()) +} + /// `Some(L)` for a `blk.L.*` tensor name, `None` for a non-block name; a /// `blk.` name whose layer does not parse is an error naming the tensor. fn block_index(name: &str) -> Result, GpuError> { @@ -795,6 +809,7 @@ fn load_segments( let t = model.tensors.get(row.tensor).ok_or_else(|| { GpuError::shape(PLACED, format!("placement row names tensor {}", row.tensor)) })?; + file_slot(&t.name)?; for seg in row .segments .iter() @@ -1175,7 +1190,7 @@ pub fn q8_0_planes(blocks: &[Q8Block]) -> (Vec, Vec) { #[cfg(test)] mod tests { - use super::{HEAD_TENSOR, derived_slot_free, head_tensor}; + use super::{HEAD_TENSOR, derived_slot_free, file_slot, head_tensor}; use gguf::Split; use gguf::write::{Layout, TensorDecl, Writer}; use std::collections::BTreeMap; @@ -1235,4 +1250,16 @@ mod tests { assert!(derived_slot_free(&by_name, "output_norm.weight").is_err()); assert!(derived_slot_free(&by_name, "derived.a").is_err()); } + + /// A file tensor in the derived namespace is refused by name, so it never + /// stands in for a weight the engine derives (a nemotron_h zero conv + /// bias); every other file name is taken. + #[test] + fn a_file_tensor_never_takes_a_derived_name() { + let err = file_slot("derived.nemotron_h.zero_conv_bias") + .expect_err("a derived name in the file") + .to_string(); + assert!(err.contains("derived.nemotron_h.zero_conv_bias"), "{err}"); + assert!(file_slot("output.weight").is_ok()); + } } diff --git a/crates/jinja/src/lib.rs b/crates/jinja/src/lib.rs index 5502c54df..2e5fdd3e6 100644 --- a/crates/jinja/src/lib.rs +++ b/crates/jinja/src/lib.rs @@ -256,7 +256,12 @@ fn py_space(c: char) -> bool { c.is_whitespace() || ('\u{1c}'..='\u{1f}').contains(&c) } +/// The byte at which the tag whose body starts at `from` closes. A comment ends at the first `#}`; the +/// closer of an expression or a statement is the first outside a string. fn find_close(src: &str, from: usize, close: &str) -> Option { + if close == "#}" { + return src[from..].find(close).map(|at| from + at); + } let b = src.as_bytes(); let mut i = from; let mut quote: Option = None; @@ -2719,9 +2724,25 @@ mod tests { assert_eq!(render("{{ 'a' }} {{ 'b' }}", json!({})), "a b"); } + #[test] + fn a_comment_ends_at_its_first_closer_whatever_it_quotes() { + // The Nemotron 3 Nano template's comment, `{# Assistant message doesn't have tool calls. #}`, + // holds one apostrophe. + assert_eq!(render("a{# doesn't #}b", json!({})), "ab"); + assert_eq!(render("a{# 'x #}b", json!({})), "ab"); + assert_eq!(render("a{# \"#}b", json!({})), "ab"); + let source = "한🙂 '#}' x"; + assert_eq!(find_close(source, 0, "#}"), source.find("#}")); + // Quotes still guard the closer of an expression. + assert_eq!(render("{{ '}}' }}", json!({})), "}}"); + } + + // PIN(2026-10-15): the `#}` round leaves this loop. A comment's text holds no expression, so a + // quote in it opens no string (Jinja2's lexer reads it as `(.*?)` up to the first closer); + // `a_comment_ends_at_its_first_closer_whatever_it_quotes` holds that closer. #[test] fn tag_closers_scan_unicode_without_slicing_inside_a_character() { - for close in ["}}", "%}", "#}"] { + for close in ["}}", "%}"] { let source = format!("한🙂 '{close}' \"{close}\" \\\u{00e9}{close}"); assert_eq!(find_close(&source, 0, close), Some(source.len() - 2)); } diff --git a/crates/model/src/arch/coverage.rs b/crates/model/src/arch/coverage.rs index 8bc1b26ce..72741fccd 100644 --- a/crates/model/src/arch/coverage.rs +++ b/crates/model/src/arch/coverage.rs @@ -95,6 +95,27 @@ pub const AVAILABLE: &[Available] = &[ at: "gpu-deepseek41/src/engram_gate.rs ROW", runs: |n| matches!(n, Need::Engram { row: 5120 }), }, + Available { + programs: &[Program::NemotronHBody], + at: "gpu/src/mamba2.rs Mamba2Kernels::enqueue_step (STATE 128, CONV_TAPS 4)", + runs: |n| { + matches!( + n, + Need::Mamba2 { + d_state: 128, + d_conv: 4, + n_heads, + n_groups, + .. + } if *n_groups > 0 && n_heads % n_groups == 0 + ) + }, + }, + Available { + programs: &[Program::NemotronHBody], + at: "gpu/src/elem.rs enqueue_relu_sqr", + runs: |n| matches!(n, Need::ReluSqr { .. }), + }, Available { programs: &[Program::Qwen3moeBody, Program::Qwen35Body], at: "models/src/shape.rs GQA", @@ -114,6 +135,23 @@ pub const AVAILABLE: &[Available] = &[ ) }, }, + Available { + programs: &[Program::NemotronHBody], + at: "gpu/src/flash_gqa.rs gqa_flash_seg_g5", + runs: |n| { + matches!( + n, + Need::Gqa { + head, + value, + group, + window: None, + sinks: false, + scaled: false, + } if value == head && gqa_row(*head, *group).is_some_and(|r| r.pack == 5) + ) + }, + }, Available { programs: &[Program::Qwen3moeBody], at: "gpu/src/rope_neox.rs HEAD", @@ -547,6 +585,15 @@ const GATED_BLOCK: &[GgmlType] = &[ GgmlType::Q6_K, ]; +/// The types a nemotron_h body site reads (`gpu/src/arch/nemotron_h/step.rs` +/// `SITE_TYPES`, each a launch of `gpu/src/site.rs`). +const NEMOTRON_SITES: &[GgmlType] = &[ + GgmlType::Q5_0, + GgmlType::Q8_0, + GgmlType::Q4_K, + GgmlType::Q6_K, +]; + /// The per-tensor type pins of the programs, as lists (the qwen3moe body's /// dense FFN; `Body35`'s /// site types, its `PROJ`, `EMBED` and `HEAD_TY` in @@ -748,6 +795,87 @@ const TYPE_PINS: &[TypePin] = &[ what: "output head (the head reads q8_0)", reads: &[GgmlType::Q8_0], }, + TypePin { + program: Program::NemotronHBody, + role: Role::TokenEmbedding, + matrices: false, + names: None, + what: "token embedding (the card reads q5_0 rows)", + reads: &[GgmlType::Q5_0], + }, + TypePin { + program: Program::NemotronHBody, + role: Role::Head, + matrices: false, + names: Some(|n| n == "output.weight"), + what: "output head (the head reads q6_K, q4_K and q8_0)", + reads: &[GgmlType::Q6_K, GgmlType::Q4_K, GgmlType::Q8_0], + }, + TypePin { + program: Program::NemotronHBody, + role: Role::Attention, + matrices: false, + names: Some(|n| { + use super::nemotron_h::names as nm; + [ + nm::SSM_IN, + nm::SSM_OUT, + nm::ATTN_Q, + nm::ATTN_K, + nm::ATTN_V, + nm::ATTN_OUTPUT, + ] + .into_iter() + .any(|stem| n.ends_with(stem)) + }), + what: "Mamba-2 and attention projections (the body reads q5_0, q8_0, q4_K and q6_K)", + reads: NEMOTRON_SITES, + }, + TypePin { + program: Program::NemotronHBody, + role: Role::DenseFfn, + matrices: false, + names: None, + what: "dense FFN (the body reads q5_0, q8_0, q4_K and q6_K)", + reads: NEMOTRON_SITES, + }, + TypePin { + program: Program::NemotronHBody, + role: Role::Attention, + matrices: false, + names: Some(|n| { + use super::nemotron_h::names as nm; + [ + nm::ATTN_NORM, + nm::SSM_A, + nm::SSM_D, + nm::SSM_DT_BIAS, + nm::SSM_CONV, + nm::SSM_CONV_BIAS, + nm::SSM_NORM, + ] + .into_iter() + .any(|stem| n.ends_with(stem)) + }), + what: "Mamba-2 parameters and normalization gains (the body reads f32 and bf16)", + reads: &[GgmlType::F32, GgmlType::BF16], + }, + TypePin { + program: Program::NemotronHBody, + role: Role::FfnNorm, + matrices: false, + names: None, + what: "normalization gains (the body reads f32 and bf16)", + reads: &[GgmlType::F32, GgmlType::BF16], + }, + TypePin { + program: Program::NemotronHBody, + role: Role::Head, + matrices: false, + names: Some(|n| n == "output_norm.weight"), + what: "output normalization gain (the head reads f32 and bf16)", + reads: &[GgmlType::F32, GgmlType::BF16], + }, ]; /// A tensor family whose type a program pins by the card kernel table's @@ -1051,6 +1179,8 @@ pub fn check_with( // by name and part (the default expert rule's card formats // would pass q3_K and q6_K gate·up to the load's refusal). Some(Program::Qwen3moeBody | Program::Qwen35Body) => {} + // The nemotron_h body is dense: a routed stack is an item. + Some(Program::NemotronHBody) => at(t.layer, Need::RoutedFormat(t.ty)), } } for need in weight_formats(program, t) { @@ -1877,6 +2007,112 @@ mod tests { ); } + /// A nemotron_h file's token table and head are items unless their type + /// is one the body reads: q5_0 rows for the table, the common head's + /// q8_0, q6_K or q4_K for `output`. + #[test] + fn the_nemotron_embedding_and_head_pins_read_the_bodys_types() { + let whole = |name: &str, role: Role, ty: GgmlType, needle: &str| { + let spec = ModelSpec { + arch: Arch::NemotronH, + hidden: 3136, + vocab: 131_072, + ctx_train: 262_144, + rms_eps: 1e-5, + layers: Vec::new(), + mtp: Vec::new(), + hc: None, + engram: None, + encoder: None, + chat: ChatSpec { + pre: String::new(), + template: None, + tools: None, + reasoning: None, + }, + }; + let model = ModelTensors { + tensors: vec![ModelTensor { + name: name.to_string(), + shard: 0, + layer: None, + role, + ty, + dims: vec![3136, 131_072], + file_bytes: 0, + gathered_rows: None, + }], + layers: 42, + experts: 0, + experts_used: 0, + }; + super::check(&spec, &model) + .into_iter() + .filter(|u| u.feature.contains(needle)) + .map(|u| u.feature) + .collect::>() + }; + let embed = |ty| { + whole( + "token_embd.weight", + Role::TokenEmbedding, + ty, + "token embedding", + ) + }; + assert_eq!(embed(GgmlType::Q5_0), Vec::::new()); + assert_eq!( + embed(GgmlType::Q8_0), + ["q8_0 token embedding (the card reads q5_0 rows)".to_string()] + ); + let head = |ty| whole("output.weight", Role::Head, ty, "output head"); + for ty in [GgmlType::Q8_0, GgmlType::Q6_K, GgmlType::Q4_K] { + assert_eq!(head(ty), Vec::::new(), "{ty}"); + } + assert_eq!( + head(GgmlType::Q5_K), + ["q5_K output head (the head reads q6_K, q4_K and q8_0)".to_string()] + ); + assert_eq!( + embed(GgmlType::F32), + ["f32 token embedding (the card reads q5_0 rows)".to_string()] + ); + assert_eq!( + head(GgmlType::F32), + ["f32 output head (the head reads q6_K, q4_K and q8_0)".to_string()] + ); + let proj = |name: &str, role: Role| whole(name, role, GgmlType::F32, "(the body reads"); + assert_eq!( + proj("blk.0.ssm_in.weight", Role::Attention), + [ + "f32 Mamba-2 and attention projections (the body reads q5_0, q8_0, q4_K and q6_K)" + .to_string() + ] + ); + assert_eq!( + proj("blk.2.ffn_up.weight", Role::DenseFfn), + ["f32 dense FFN (the body reads q5_0, q8_0, q4_K and q6_K)".to_string()] + ); + let vector = + |name: &str, role: Role, ty: GgmlType| whole(name, role, ty, "reads f32 and bf16"); + for (name, role) in [ + ("blk.0.attn_norm.weight", Role::Attention), + ("blk.0.ssm_a", Role::Attention), + ("blk.0.ssm_d", Role::Attention), + ("blk.0.ssm_dt.bias", Role::Attention), + ("blk.0.ssm_conv1d.weight", Role::Attention), + ("blk.0.ssm_conv1d.bias", Role::Attention), + ("blk.0.ssm_norm.weight", Role::Attention), + ("blk.2.attn_norm.weight", Role::FfnNorm), + ("output_norm.weight", Role::Head), + ] { + for ty in [GgmlType::F32, GgmlType::BF16] { + assert_eq!(vector(name, role, ty), Vec::::new(), "{name} {ty}"); + } + assert_eq!(vector(name, role, GgmlType::Q8_0).len(), 1, "{name} q8_0"); + } + } + /// The dense block's pin (mimo2, glm5next) and glm5next's shared-expert pin /// read the gated block's types (q8_0 and the K-quants `enqueue_block` /// dispatches); any other type is an item naming the type. diff --git a/crates/model/src/arch/names.rs b/crates/model/src/arch/names.rs index 9764bc267..d3e0e0430 100644 --- a/crates/model/src/arch/names.rs +++ b/crates/model/src/arch/names.rs @@ -4,7 +4,7 @@ /// `blk.{l}.{stem}`. #[must_use] -pub(crate) fn blk(l: usize, stem: &str) -> String { +pub fn blk(l: usize, stem: &str) -> String { format!("blk.{l}.{stem}") } diff --git a/crates/model/src/arch/nemotron_h/mod.rs b/crates/model/src/arch/nemotron_h/mod.rs index cb8ff1019..b5ac11505 100644 --- a/crates/model/src/arch/nemotron_h/mod.rs +++ b/crates/model/src/arch/nemotron_h/mod.rs @@ -1,11 +1,12 @@ //! `nemotron_h`: a dense hybrid of Mamba-2, squared-ReLU MLP and no-position //! attention. The file's keys (`hparams`), every tensor's role (`roles`), -//! the tensor names (`names`), the typed description (`spec`) and the gate -//! fixture's spec (`fixture`). No program runs it. The dialect's authority is -//! llama.cpp's `src/models/nemotron-h.cpp`. +//! the tensor names (`names`), the typed description (`spec`), the whole-card +//! plan (`place`) and the gate fixture's spec (`fixture`). The card body is `gpu::arch::nemotron_h`. +//! The dialect's authority is llama.cpp's `src/models/nemotron-h.cpp`. pub mod fixture; pub mod hparams; pub mod names; +pub mod place; pub mod roles; pub mod spec; diff --git a/crates/model/src/arch/nemotron_h/place.rs b/crates/model/src/arch/nemotron_h/place.rs new file mode 100644 index 000000000..7cb3a4913 --- /dev/null +++ b/crates/model/src/arch/nemotron_h/place.rs @@ -0,0 +1,328 @@ +//! A `nemotron_h` file planned onto a machine: the one path from the file's +//! headers to a whole-card plan. The hyperparameters, every tensor's role, +//! the typed description and each layer's store bytes ([`KvLayout`]) are read +//! once ([`PlanInputs::read`], which refuses a file whose features no program +//! runs, every one listed); then the placement ([`PlanInputs::plan`]) puts +//! the whole model on one card (the file holds no routed expert, so no tier +//! and no host leg has anything to hold), and refuses by name a machine of +//! more than one card or with a tier card, a context of no position, and a +//! context whose stores the card cannot hold. +//! +//! A sequence's stores are an attention layer's K and V planes, which grow +//! with the context, and a Mamba-2 layer's scan state and conv ring, which do +//! not; an MLP layer holds nothing ([`KvLayout::layer_bytes`]). + +use gguf::Split; +use models::ModelSpec; +use runtime::stores::{dense_kv_bytes, mamba2_bytes}; + +use super::hparams::{Hparams, Kind}; +use super::spec; +use crate::arch::coverage; +use crate::placement::accepts::{Program, routed_rule}; +use crate::placement::ctx_ask::PlanRefusal; +use crate::placement::{ + self, KvBytes, Machine, ModelTensors, PlacementError, Plan, PlanLevers, Unimplemented, + Violation, checked, joined, +}; + +/// What a plan of a `nemotron_h` file is made from, read from its headers. +#[derive(Clone, Debug)] +pub struct PlanInputs { + /// Every tensor with its role, and the model's layer count. + pub model: ModelTensors, + /// The typed model description the coverage check reads. + pub spec: ModelSpec, + /// Each layer's store bytes. + pub kv: KvLayout, +} + +/// Why a plan was refused. +#[derive(Debug, thiserror::Error)] +pub enum PlaceError { + /// The placement could not be built. + #[error(transparent)] + Placement(#[from] PlacementError), + /// A machine with an expert tier card: the file has no routed expert, so + /// the tier would hold nothing. + #[error( + "the machine has the expert tier card {card} (tier {tier}); the nemotron_h file has no routed expert for a tier to hold" + )] + Tier { + /// The tier card's name. + card: String, + /// Its index among the machine's tiers. + tier: usize, + }, + /// A machine of other than one card: the body runs its chain on one. + #[error("the placement puts the layers on {0} cards; the nemotron_h chain runs on one")] + Cards(usize), + /// The plan was built and breaks these invariants, every one of them. + #[error("the plan breaks its invariants: {}", joined(.0))] + Broken(Vec), + /// A context of no position: no plane, nothing to serve. + #[error("ctx_max 0: a plan serves at least one position")] + CtxZero, +} + +impl PlanRefusal for PlaceError { + fn placement(&self) -> Option<&PlacementError> { + match self { + Self::Placement(e) => Some(e), + _ => None, + } + } + + fn violations(&self) -> Option<&[Violation]> { + match self { + Self::Broken(v) => Some(v), + _ => None, + } + } +} + +impl PlanInputs { + /// `split`'s hyperparameters, then its tensors' roles, then its + /// description, the first that fails being the error; then a file with a + /// feature the engine does not run ([`PlacementError::Unimplemented`], + /// every one listed). + pub fn read(split: &Split) -> Result { + let inputs = PlanInputs::describe(split)?; + let missing = inputs.unimplemented(); + if missing.is_empty() { + Ok(inputs) + } else { + Err(PlacementError::Unimplemented(missing)) + } + } + + /// [`PlanInputs::read`] without the refusals: the file as it is, for a + /// gate that plans or describes a file whatever the coverage list holds. + pub fn describe(split: &Split) -> Result { + let hp = Hparams::read(split)?; + let read = spec::read(split)?; + let kv = KvLayout::of(&hp); + Ok(PlanInputs { + model: read.tensors, + spec: read.spec, + kv, + }) + } + + /// Every feature of the file the engine does not run, as + /// [`coverage::check`] lists them. Empty for a file the program runs. + pub fn unimplemented(&self) -> Vec { + coverage::check(&self.spec, &self.model) + } + + /// The placement of the file on `machine` at `ctx_max` positions under + /// the placement's `levers`: the whole model on the machine's one card. + pub fn plan<'a>( + &'a self, + machine: &'a Machine, + ctx_max: u64, + levers: &PlanLevers, + ) -> Result, PlaceError> { + if ctx_max == 0 { + return Err(PlaceError::CtxZero); + } + if let Some((tier, card)) = machine.tiers.iter().enumerate().next() { + return Err(PlaceError::Tier { + card: card.name.clone(), + tier, + }); + } + if machine.cards.len() != 1 { + return Err(PlaceError::Cards(machine.cards.len())); + } + let plan = placement::plan_routed( + &self.model, + machine, + ctx_max, + &self.kv, + levers, + routed_rule(Program::NemotronHBody), + )?; + checked(plan).map_err(PlaceError::Broken) + } +} + +/// What one layer holds per sequence. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Store { + /// An attention layer: K and V of this many bytes a position. + Rows(u64), + /// A Mamba-2 layer: this many bytes of scan state and conv ring, whatever + /// the context. + State(u64), + /// An MLP layer. + Nothing, +} + +/// Each layer's store bytes of a `nemotron_h` file: the bytes the body's +/// allocations hold ([`runtime::stores`] is their owner). +#[derive(Clone, Debug)] +pub struct KvLayout { + layers: Vec, +} + +impl KvLayout { + /// The layout of the layers `hp` names. + #[must_use] + pub fn of(hp: &Hparams) -> KvLayout { + let ssm = hp.ssm; + let state = mamba2_bytes(ssm.d_inner, ssm.d_state, ssm.d_conv, ssm.conv_dim); + let layers = hp + .kinds + .iter() + .zip(&hp.kv) + .map(|(kind, &kv_heads)| match kind { + Kind::Mamba => Store::State(state), + Kind::Attention => Store::Rows(dense_kv_bytes(kv_heads, hp.key_len, 1)), + Kind::Mlp => Store::Nothing, + }) + .collect(); + KvLayout { layers } + } +} + +impl KvBytes for KvLayout { + /// An attention layer's K and V of `ctx_max` positions, a Mamba-2 + /// layer's state, an MLP layer's nothing; 0 past the last layer. + fn layer_bytes(&self, layer: usize, ctx_max: u64) -> u64 { + match self.layers.get(layer) { + Some(Store::Rows(row)) => row.saturating_mul(ctx_max), + Some(Store::State(bytes)) => *bytes, + Some(Store::Nothing) | None => 0, + } + } +} + +#[cfg(test)] +mod tests { + use gguf::Split; + + use super::super::spec::tests::{CONV, HEAD, INNER, KV, geo, keys, tensors}; + use super::{KvBytes, PlaceError, PlanInputs, PlanLevers}; + use crate::arch::synthetic::{V, header_shaped}; + use crate::placement::workstation; + + /// The synthetic header's layers, in order: Mamba, MLP, attention, and + /// the same again. + const KINDS: &str = "mpampa"; + + fn inputs(tag: &str) -> PlanInputs { + let g = geo(); + let path = header_shaped( + tag, + "nemotron_h", + &keys(&g), + &[ + ("tokenizer.ggml.pre", V::Str("qwen2")), + ("tokenizer.chat_template", V::Str("{{ messages }}")), + ], + &tensors(&g, true), + ); + let split = Split::open(&path).expect("the synthetic header opens"); + let inputs = PlanInputs::describe(&split).expect("the synthetic header describes"); + let _ = std::fs::remove_file(&path); + inputs + } + + /// The conv channels and the scan state the synthetic layers hold. + fn mamba_bytes() -> u64 { + let g = geo(); + let conv_dim = INNER + 2 * g.n_groups * g.d_state; + (INNER * g.d_state + (CONV - 1 + 8) * conv_dim) * 4 + } + + /// An attention layer's bytes grow with the context by K and V of its + /// heads; a Mamba-2 layer's are one number at every context; an MLP layer + /// and a layer past the last hold none. + #[test] + fn each_kind_holds_its_own_bytes() { + let kv = inputs("nemotron-place-bytes").kv; + let row = 2 * (KV as u64 * HEAD) * 2; + assert_eq!(row, 64); + assert_eq!(mamba_bytes(), 62_848); + for (layer, kind) in KINDS.chars().enumerate() { + for ctx in [1, 9, 4096] { + let want = match kind { + 'm' => mamba_bytes(), + 'a' => row * ctx, + _ => 0, + }; + assert_eq!(kv.layer_bytes(layer, ctx), want, "layer {layer} at {ctx}"); + } + } + assert_eq!(kv.layer_bytes(KINDS.len(), 9), 0); + let sum: u64 = (0..KINDS.len()).map(|l| kv.layer_bytes(l, 9)).sum(); + assert_eq!(sum, 2 * mamba_bytes() + 2 * row * 9); + } + + /// Two plans of one file at two contexts differ in the cache term by the + /// attention layers' bytes of the extra positions and by nothing else: + /// the Mamba-2 state is not counted a position. + #[test] + fn the_context_moves_only_the_attention_term() { + let inputs = inputs("nemotron-place-ctx"); + let machine = workstation::plan_gate(inputs.model.layers); + let levers = PlanLevers::default(); + let (a, b) = (1_024u64, 4_096u64); + let small = inputs.plan(&machine, a, &levers).expect("a short plan"); + let large = inputs.plan(&machine, b, &levers).expect("a long plan"); + let row = 2 * (KV as u64 * HEAD) * 2; + let attn_layers = 2; + assert_eq!( + large.cards[0].kv_bytes - small.cards[0].kv_bytes, + attn_layers * row * (b - a) + ); + assert_eq!( + small.cards[0].kv_bytes, + 2 * mamba_bytes() + attn_layers * row * a + ); + assert_eq!(small.cards[0].dense_bytes, large.cards[0].dense_bytes); + assert_eq!(small.host.experts + small.host.expert_bytes, 0); + assert_eq!(small.nvme_bytes, 0); + } + + /// A context of no position, a tier card and a second card are refused by + /// name. + #[test] + fn a_machine_or_context_the_body_does_not_run_is_refused() { + let inputs = inputs("nemotron-place-refuse"); + let levers = PlanLevers::default(); + let machine = workstation::plan_gate(inputs.model.layers); + assert!(matches!( + inputs.plan(&machine, 0, &levers), + Err(PlaceError::CtxZero) + )); + let mut tiered = machine.clone(); + let mut tier = tiered.cards[0].clone(); + tier.layers = 0..0; + tier.head = false; + tier.token_embedding = false; + tiered.tiers.push(tier); + match inputs.plan(&tiered, 4, &levers) { + Err(e @ PlaceError::Tier { tier: 0, .. }) => assert!( + e.to_string() + .contains("no routed expert for a tier to hold"), + "{e}" + ), + other => panic!("{other:?}, not the tier refusal"), + } + let mut two = machine; + let mut second = two.cards[0].clone(); + second.layers = 0..0; + second.head = false; + second.token_embedding = false; + two.cards.push(second); + match inputs.plan(&two, 4, &levers) { + Err(e @ PlaceError::Cards(2)) => assert_eq!( + e.to_string(), + "the placement puts the layers on 2 cards; the nemotron_h chain runs on one" + ), + other => panic!("{other:?}, not the card-count refusal"), + } + } +} diff --git a/crates/model/src/arch/nemotron_h/spec.rs b/crates/model/src/arch/nemotron_h/spec.rs index 950b9916c..d13bb6014 100644 --- a/crates/model/src/arch/nemotron_h/spec.rs +++ b/crates/model/src/arch/nemotron_h/spec.rs @@ -105,7 +105,7 @@ pub fn spec_of(hp: &Hparams, chat: models::ChatSpec) -> Result Geo { + pub(in crate::arch::nemotron_h) fn geo() -> Geo { Geo { d_state: 128, n_groups: 4, @@ -148,7 +148,7 @@ mod tests { ) } - fn keys(g: &Geo) -> Vec<(&'static str, V)> { + pub(in crate::arch::nemotron_h) fn keys(g: &Geo) -> Vec<(&'static str, V)> { vec![ ("block_count", V::U32(N as u32)), ("embedding_length", V::U32(EMBD as u32)), @@ -221,7 +221,7 @@ mod tests { } } - fn tensors(g: &Geo, bias: bool) -> Vec<(String, Vec)> { + pub(in crate::arch::nemotron_h) fn tensors(g: &Geo, bias: bool) -> Vec<(String, Vec)> { let mut out = Vec::new(); one(names::token_embd(), vec![EMBD, EMBD], &mut out); one(names::output(), vec![EMBD, EMBD], &mut out); @@ -339,23 +339,22 @@ mod tests { needs.iter().all(|(n, _)| !matches!(n, Need::QkRope { .. })), "{needs:?}" ); - let items = coverage::check(&read.spec, &read.tensors); - assert!( - items - .iter() - .any(|u| u.feature == "a layer program for nemotron_h"), - "{items:?}" - ); - assert!( - items.iter().any(|u| u.feature.starts_with("Mamba-2 mixer")), - "{items:?}" - ); + // PIN(2026-10-15): the nemotron_h body now runs the program, the + // Mamba-2 and the squared-ReLU layers, so the clauses that listed them + // as missing are gone; this header's attention (head 8, group 2) has + // no flash row. Its f32 matrices are type items the body's pins name + // (`coverage::TYPE_PINS`), set aside here: this test is about needs. + let items: Vec<_> = coverage::check(&read.spec, &read.tensors) + .into_iter() + .filter(|u| !u.feature.starts_with("f32 ")) + .collect(); assert!( items .iter() - .any(|u| u.feature.starts_with("squared-ReLU dense layer")), + .all(|u| u.feature.starts_with("GQA flash, head 8, group 2")), "{items:?}" ); + assert!(!items.is_empty(), "{items:?}"); assert!( items.iter().all(|u| !u.feature.contains("rope")), "{items:?}" diff --git a/crates/model/tests/nemotron_meta.rs b/crates/model/tests/nemotron_meta.rs index b69f14d81..ef539a7a4 100644 --- a/crates/model/tests/nemotron_meta.rs +++ b/crates/model/tests/nemotron_meta.rs @@ -109,11 +109,10 @@ fn hw_nemotron_h_header() { bad.push("an attention layer emits a rope need".to_string()); } let items = model::arch::coverage::check(s, &read.tensors); - if !items - .iter() - .any(|u| u.feature == "a layer program for nemotron_h") - { - bad.push(format!("coverage has no program item: {items:?}")); + // PIN(2026-10-15): the nemotron_h body now runs the program, the Mamba-2 and the squared-ReLU + // layers, so the clause that listed them as missing is gone; the real header is fully covered. + if !items.is_empty() { + bad.push(format!("coverage has uncovered items: {items:?}")); } if read.tensors.tensors.len() != 263 { bad.push(format!("tensors {}", read.tensors.tensors.len())); diff --git a/crates/placement/src/accepts.rs b/crates/placement/src/accepts.rs index a4dc7eebb..d35fff461 100644 --- a/crates/placement/src/accepts.rs +++ b/crates/placement/src/accepts.rs @@ -32,6 +32,10 @@ pub enum Program { /// on a card (MXFP4 stacks, the common pair) and the rest on the host /// tier. Mimo2Body, + /// The nemotron_h body (`crates/gpu/src/arch/nemotron_h`): whole-card, + /// dense, its Mamba-2, NoPE GQA and squared-ReLU layers over the common + /// kernels; no routed experts. + NemotronHBody, } /// The program that runs `arch`'s layers; `None` when none does. @@ -44,7 +48,8 @@ pub fn program_of(arch: Arch) -> Option { Arch::Glm5Next => Some(Program::Glm5nextBody), Arch::Qwen4Exp => Some(Program::Qwen38Body), Arch::MiMo2 => Some(Program::Mimo2Body), - Arch::NemotronH | Arch::GemmaEmbedding2 => None, + Arch::NemotronH => Some(Program::NemotronHBody), + Arch::GemmaEmbedding2 => None, } } @@ -180,6 +185,7 @@ pub fn leg_arms(p: Program) -> Option { gate_up: |ty| ty == DS41_LEG[0], down: |ty| ty == DS41_LEG[2], }), + Program::NemotronHBody => None, } } @@ -217,6 +223,8 @@ pub fn routed_rule(p: Program) -> RoutedRule { gate_up: |ty| (ty == DS41_LEG[0]).then_some(CardFormat::KQuant), down: |ty| (ty == DS41_LEG[2]).then_some(CardFormat::KQuant), }, + // No routed stack: every one is refused by name. + Program::NemotronHBody => RoutedRule::PerType(|_| None), } } @@ -230,13 +238,14 @@ mod tests { use super::*; use crate::placement::{ModelTensor, ModelTensors, Role}; - const PROGRAMS: [Program; 6] = [ + const PROGRAMS: [Program; 7] = [ Program::Deepseek41Chain, Program::Qwen3moeBody, Program::Qwen35Body, Program::Glm5nextBody, Program::Qwen38Body, Program::Mimo2Body, + Program::NemotronHBody, ]; /// Every type the file format names. @@ -289,18 +298,21 @@ mod tests { /// read per type over the union of its parts (a Q6_K, a Q8_0 gate), the /// common leg's per type (a gate and an up of two types), V4.1's read per /// type in its own format (a Q6_K down) — each names its program and - /// triples. + /// triples. A program with no card leg (the dense nemotron_h body) cards + /// none. #[test] fn a_layer_the_plan_cards_is_one_its_load_runs() { let (model, layers) = triples(&types()); let mut broken = Vec::new(); for p in PROGRAMS { - assert!(leg_arms(p).is_some(), "{p:?} has a card leg"); let rule = routed_rule(p); for (l, &tys) in layers.iter().enumerate() { let stacks = [3 * l, 3 * l + 1, 3 * l + 2]; let cards = rule.layer_runs(&stacks, &model).expect("named stacks"); - if cards && leg_runs(p, tys) == Some(false) { + // PIN(2026-10-15): the dense nemotron_h body has no card leg, so + // the assert that every program has one is gone; a program with + // no card leg cards no layer. + if cards && leg_runs(p, tys) != Some(true) { broken.push(format!("{p:?} {tys:?}")); } } diff --git a/crates/refset/src/arch/nemotron_h/mod.rs b/crates/refset/src/arch/nemotron_h/mod.rs index 572e23309..28fefc053 100644 --- a/crates/refset/src/arch/nemotron_h/mod.rs +++ b/crates/refset/src/arch/nemotron_h/mod.rs @@ -22,7 +22,7 @@ pub static LOGITS: Family = Family { build: Some(Build::Is(super::tokenizer::NEMOTRON_BUILD)), runs: Some(super::tokenizer::nemotron_model), draft_runs: None, - consumers: &[], + consumers: &["gate-gpu-nemotron-e2e"], }; /// The architecture's families, in the order `refset-check` lists them. @@ -101,6 +101,10 @@ mod tests { .logits(&set.prompts[0]) .unwrap_or_else(|e| panic!("{e}")); assert_eq!(row[1], 1.5); + assert_eq!( + set.ids(&set.prompts[0]).unwrap_or_else(|e| panic!("{e}")), + [0, 1, 2, 3, 0] + ); std::fs::remove_dir_all(&d).unwrap_or_else(|e| panic!("{e}")); } diff --git a/crates/refset/src/nemotron.rs b/crates/refset/src/nemotron.rs index b1979cc94..ce4582cde 100644 --- a/crates/refset/src/nemotron.rs +++ b/crates/refset/src/nemotron.rs @@ -210,10 +210,8 @@ impl LogitsSet { )); } let mut out = Vec::with_capacity(p.n_vocab as usize); - for chunk in bytes.chunks_exact(4) { - out.push(f32::from_le_bytes( - chunk.try_into().expect("chunks_exact(4)"), - )); + for chunk in bytes.as_chunks::<4>().0 { + out.push(f32::from_le_bytes(*chunk)); } if let Some(i) = out.iter().position(|v| !v.is_finite()) { return Err(RefError::malformed( @@ -224,40 +222,53 @@ impl LogitsSet { Ok(out) } - fn check_prompt(&self, p: &Prompt) -> Result<(), RefError> { - let ids = self.dir.join(&p.ids_file); - let text = std::fs::read_to_string(&ids).map_err(|e| { + /// The prompt's token ids, in order: `ids_file`, one id a line, as many as `n_ids` names, each + /// inside the vocabulary. + pub fn ids(&self, p: &Prompt) -> Result, RefError> { + let path = self.dir.join(&p.ids_file); + let text = std::fs::read_to_string(&path).map_err(|e| { RefError::missing( - &ids, - format!("nemotron: cannot read {}: {e}", ids.display()), + &path, + format!("nemotron: cannot read {}: {e}", path.display()), ) })?; if p.n_ids == 0 || p.n_vocab == 0 { return Err(RefError::malformed( - ids.display(), + path.display(), "empty prompt or vocabulary", )); } - let mut n = 0usize; + let mut ids = Vec::with_capacity(p.n_ids as usize); for (i, line) in text.lines().enumerate() { if line.is_empty() { - return Err(RefError::malformed(At(&ids, i + 1), "empty token id")); + return Err(RefError::malformed(At(&path, i + 1), "empty token id")); } - n += 1; - let id: u32 = crate::columns::parse_field(line, "token id", &At(&ids, i + 1))?; + let id: u32 = crate::columns::parse_field(line, "token id", &At(&path, i + 1))?; if id >= p.n_vocab { return Err(RefError::malformed( - At(&ids, i + 1), + At(&path, i + 1), format!("token id {id} outside vocabulary {}", p.n_vocab), )); } + ids.push(id); } - if n != p.n_ids as usize { + if ids.len() != p.n_ids as usize { return Err(RefError::malformed( - ids.display(), - format!("{} holds {n} ids, {} wants {}", p.ids_file, p.name, p.n_ids), + path.display(), + format!( + "{} holds {} ids, {} wants {}", + p.ids_file, + ids.len(), + p.name, + p.n_ids + ), )); } + Ok(ids) + } + + fn check_prompt(&self, p: &Prompt) -> Result<(), RefError> { + self.ids(p)?; self.logits(p)?; Ok(()) } diff --git a/crates/runtime/src/stores.rs b/crates/runtime/src/stores.rs index 25f20ed1b..fa7eb918a 100644 --- a/crates/runtime/src/stores.rs +++ b/crates/runtime/src/stores.rs @@ -60,6 +60,29 @@ pub const fn recurrent_bytes(v_heads: usize, k_heads: usize, state: usize, conv: v * d * d * F32_BYTES + conv_ring_rows(conv) as u64 * channels * F32_BYTES } +/// A Mamba-2 layer's stores in f32 values, `[state, ring]`: `d_inner` channels +/// of `d_state` scan state, and the conv ring's rows +/// ([`conv_ring_rows`] of the `conv`-tap conv) of `conv_dim` channels. The +/// plan counts a layer's store by these lengths and the card body allocates +/// it by them. +#[must_use] +pub const fn mamba2_lens( + d_inner: usize, + d_state: usize, + conv: usize, + conv_dim: usize, +) -> [usize; 2] { + [d_inner * d_state, conv_ring_rows(conv) * conv_dim] +} + +/// A Mamba-2 layer's stores in bytes ([`mamba2_lens`], f32): what it holds +/// whatever the context, one sequence's. +#[must_use] +pub const fn mamba2_bytes(d_inner: usize, d_state: usize, conv: usize, conv_dim: usize) -> u64 { + let [state, ring] = mamba2_lens(d_inner, d_state, conv, conv_dim); + (state + ring) as u64 * F32_BYTES +} + /// Lanes of a gated-delta-rule layer's state on a model that verifies /// drafted rows (Qwen3.8's `Body38`): a verify of up to this many rows keeps /// the state after each of its rows in a lane of its own, and a commit moves @@ -131,10 +154,18 @@ pub const fn selecting_bytes( #[cfg(test)] mod tests { use super::{ - DELTA_LANES, conv_ring_rows, delta_lane_bytes, dense_kv_bytes, kv_row_bytes, - ple_ring_bytes, ple_ring_rows, pooled_rows, recurrent_bytes, selecting_bytes, + DELTA_LANES, conv_ring_rows, delta_lane_bytes, dense_kv_bytes, kv_row_bytes, mamba2_bytes, + mamba2_lens, ple_ring_bytes, ple_ring_rows, pooled_rows, recurrent_bytes, selecting_bytes, }; + /// A Mamba-2 layer of 32 channels at state 128 with a 4-tap conv over + /// 1,056 conv channels: 4,096 f32 of scan state and eleven conv rows. + #[test] + fn mamba2_sizes() { + assert_eq!(mamba2_lens(32, 128, 4, 1056), [4096, 11 * 1056]); + assert_eq!(mamba2_bytes(32, 128, 4, 1056), (4096 + 11 * 1056) * 4); + } + /// Qwen3.8's sizes: a GDN layer holds 48 heads of 128 × 128 f32 and /// eleven conv rows of 10,240 f32 channels; the PLE ring 17 rows of four /// 2,560-value streams; an attention position 2 × 2 × 256 f16 of K and V diff --git a/justfile b/justfile index 585badfc2..e4e566a36 100644 --- a/justfile +++ b/justfile @@ -252,6 +252,24 @@ gate-gpu-hc-gated: gate-gpu-mamba2: ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features gpu --release --bin gate_mamba2 && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_mamba2' +# nemotron_h (Nemotron 3 Nano 4B Q4_K_M) end-to-end gate, one card: the captured step's entries against the layer kinds the header derives +# (the scans, the squared ReLUs, the group-5 flash passes, the appends, the Q5_0 embedding row; no group-8 flash entry, no host node), eager = +# captured replay bit for bit over 16 tokens, reset rewinds the ids and the logits bits, the scan state carries (layer 0 nonzero, the last +# token's logits differ from the token alone), the fault word clean. A step that raises the fault word fails the clause naming it. Against +# mainline's last-position logits (ref-nemotron-logits, `just dump-ref-nemotron`): the body's argmax is the set's on every prompt, and +# each prompt's cosine keeps the floor with the same top-5 ids; the max |delta| and cosine print per prompt. A set the box lacks fails by name. +gate-gpu-nemotron-e2e: + BLOOMERY_MODEL=nemotron_h ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features gpu --release --bin gate_nemotron_e2e && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_nemotron_e2e' + +# The nemotron seat of bloomery-serve on Nemotron 3 Nano 4B Q4_K_M (gate_nemotron_serve's doc): `--plan` with no flag names its +# default context (the ladder's rung, held by the ctx line and the rung above refused) and exits before any load, `--parallel 2` is +# refused by name (the body holds one sequence); one server with no flag listens at the context its ctx line named, answers +# /completion at temperature 0 with the ids NemotronHModel gives stepped directly on the same ids (prose, and the chat turn the +# server's template renders), serves that turn on /v1/chat/completions (usage counts its ids), resends an edited prompt as a +# fresh run that keeps nothing and an extended one that keeps the held positions. One card. +gate-gpu-nemotron-serve: + BLOOMERY_MODEL=nemotron_h ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,vision --release --bin bloomery-serve --bin gate_nemotron_serve && D=target/nemotron-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_nemotron_serve --dir $D' + # GLM-5.3-Flash 라우터(288개 중 8개, sigmoid, ×2.5 — `router::glm5next`)를 모델 없이 합성 입력으로 호스트 규칙과 맞춘다. # 규칙마다 절이 하나다: logits·ids·weights·batch는 비트, scores는 sigmoid 밴드, ties·guard·fault·graph는 명시한 값. gate-gpu-glm-router: diff --git a/tools/check-arch.sh b/tools/check-arch.sh index 1fe58a253..fbd1834cd 100644 --- a/tools/check-arch.sh +++ b/tools/check-arch.sh @@ -202,6 +202,8 @@ dispatch=( 'crates/gpu/src/lib.rs type Deepseek2Model' # 같은 모양의 qwen3moe 별칭 — AnyEngine 은 이 별칭으로만 qwen3moe 를 부른다. 'crates/gpu/src/lib.rs type Qwen3moeModel' + # The same shape for nemotron_h — the nemotron seat and its gates name the body through this alias only. + 'crates/gpu/src/lib.rs type NemotronHModel' # gpu-vision 은 deepseek41v 탑 자신의 크레이트다(이름이 gpu- 가 아닐 뿐) — 인코더가 그 탑의 이름표를 읽는다. 'crates/gpu-vision/src/encoder.rs use' # The same crate's second tower (the Qwen3-VL merger of Clef, Qwen3.6 and Qwen3.8) reads its file's diff --git a/tools/gate-paths.tsv b/tools/gate-paths.tsv index 7875e5a2b..c83ad4f7d 100644 --- a/tools/gate-paths.tsv +++ b/tools/gate-paths.tsv @@ -23,27 +23,27 @@ crates/levers/src/lib.rs * at_main, the parser every binary calls first # V4.1: crates/gpu-deepseek41. The engine opens through body::open; app's V4.1 session impls (Open, Prompt, Keep for # the V4.1 body, Session) run only where the V4.1 body is the session's type, so a GLM session never runs them. -crates/gpu-deepseek41/src/lib.rs gate-ds41-load gate-gpu-ds41-attn gate-gpu-ds41-callstream gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-comp gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-moe gate-gpu-ds41-prefill gate-gpu-ds41-residency-a gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-woa gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-glm-mla gate-gpu-glm-router gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-mtp gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-mcol gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs the module list only: the gates that use any module of the crate (every row below) except gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve: bloomery-serve opens one seat a run (--model, or the file's architecture); these run its decide and qwen3 seats, and only another seat's module reaches this file -crates/gpu-deepseek41/src/{body.rs,body/**} gate-ds41-load gate-gpu-clef-serve gate-gpu-ds41-callstream gate-gpu-ds41-chain-attn gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-prefill gate-gpu-ds41-residency-a gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-glm5next-serve gate-gpu-qwen3-serve gate-gpu-lev-serve gate-gpu-ds41-nvtier gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs the engine's assembly, prefill, CED and program: every gate that opens the V4.1 model (body::open, app's V4.1 session) or names body's constants -crates/gpu-deepseek41/src/swap.rs gate-ds41-load gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-chain-attn gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-long gate-gpu-ds41-prefill gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-glm5next-serve gate-gpu-qwen3-serve gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-ds41-media gate-gpu-ds41-nvtier gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs the card-expert swap under the body, and the r8 unpack gate -crates/gpu-deepseek41/src/{chain/**,engram_gate.rs,span.rs,transpose.rs} gate-ds41-load gate-gpu-ds41-callstream gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-moe gate-gpu-ds41-prefill gate-gpu-ds41-residency-a gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-dspark-graph gate-gpu-dspark-kv gate-gpu-glm5next-e2e gate-gpu-glm5next-mtp gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-glm5next-serve gate-gpu-glm5next-residency gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs the layer pieces: the V4.1 body, the chain gates, DSpark's graph and KV, and the GLM body, which runs the V4.1 chain except gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve: bloomery-serve opens one seat a run (--model, or the file's architecture); these run its decide and qwen3 seats, and only another seat's module reaches this file -crates/gpu-deepseek41/src/attn.rs gate-ds41-load gate-gpu-ds41-attn gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-index gate-gpu-ds41-long gate-gpu-ds41-prefill gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-dspark-graph gate-gpu-dspark-kv gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-ds41-media gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs the chain's users and the attention, indexer and GLM MLA/selector gates (gate-gpu-ds41-attn gate-gpu-ds41-index gate-gpu-glm-mla gate-gpu-glm-sel) except gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve: bloomery-serve opens one seat a run (--model, or the file's architecture); these run its decide and qwen3 seats, and only another seat's module reaches this file -crates/gpu-deepseek41/src/rope.rs gate-ds41-load gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-comp gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-index gate-gpu-ds41-long gate-gpu-ds41-prefill gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-dspark-graph gate-gpu-dspark-kv gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-ds41-media gate-gpu-ds41-nvtier gate-gpu-glm5next-media gate-gpu-serve-start-ds41 gate-gpu-routed-legs the chain's users and the rope, compressor and indexer gates (gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-rope) except gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve: bloomery-serve opens one seat a run (--model, or the file's architecture); these run its decide and glm and qwen3 seats, and only another seat's module reaches this file | except gate-gpu-serve-start-glm: bloomery-serve opens one seat a run (--model, or the file's architecture); this recipe runs its glm seat, and only another seat's module reaches this file -crates/gpu-deepseek41/src/router.rs gate-ds41-load gate-gpu-ds41-callstream gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-moe gate-gpu-ds41-prefill gate-gpu-ds41-residency-a gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-dspark-graph gate-gpu-dspark-kv gate-gpu-glm-router gate-gpu-glm5next-e2e gate-gpu-glm5next-mtp gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs the chain's users and the MoE and GLM router gates (gate-gpu-ds41-moe gate-gpu-glm-router) except gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve: bloomery-serve opens one seat a run (--model, or the file's architecture); these run its decide and qwen3 seats, and only another seat's module reaches this file -crates/gpu-deepseek41/src/{params.rs,indexer.rs} gate-ds41-load gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-index gate-gpu-ds41-long gate-gpu-ds41-prefill gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-dspark-graph gate-gpu-dspark-kv gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-ds41-media gate-gpu-ds41-nvtier gate-gpu-glm5next-media gate-gpu-serve-start-ds41 gate-gpu-routed-legs the chain's users and the indexer gate (gate-gpu-ds41-index) except gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve: bloomery-serve opens one seat a run (--model, or the file's architecture); these run its decide and glm and qwen3 seats, and only another seat's module reaches this file | except gate-gpu-serve-start-glm: bloomery-serve opens one seat a run (--model, or the file's architecture); this recipe runs its glm seat, and only another seat's module reaches this file -crates/gpu-deepseek41/src/hc.rs gate-ds41-load gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-long gate-gpu-ds41-prefill gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-ds41-moe gate-gpu-ds41-media gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs the chain's users and the HC gates (gate-gpu-ds41-hc gate-gpu-dspark-hc) except gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve: bloomery-serve opens one seat a run (--model, or the file's architecture); these run its decide and qwen3 seats, and only another seat's module reaches this file -crates/gpu-deepseek41/src/block.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-glm5next-gemm gate-gpu-glm5next-e2e gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-stagger gate-gpu-glm5next-nvtier gate-gpu-glm5next-serve gate-gpu-ds41-lib gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs the gated block (dense layer, shared expert) by weight type: the MiMo and GLM bodies run it, so every serve gate that links MiMo and every GLM gate does; its plan tests are the crate lib gate's -crates/gpu-deepseek41/src/{compress.rs,index_key.rs} gate-ds41-load gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-comp gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-index gate-gpu-ds41-long gate-gpu-ds41-prefill gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-dspark-graph gate-gpu-dspark-kv gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-ds41-media gate-gpu-ds41-nvtier gate-gpu-glm5next-media gate-gpu-serve-start-ds41 gate-gpu-routed-legs the chain's users and the compressor and indexer gates (gate-gpu-ds41-comp gate-gpu-ds41-index) except gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve: bloomery-serve opens one seat a run (--model, or the file's architecture); these run its decide and glm and qwen3 seats, and only another seat's module reaches this file | except gate-gpu-serve-start-glm: bloomery-serve opens one seat a run (--model, or the file's architecture); this recipe runs its glm seat, and only another seat's module reaches this file -crates/gpu-deepseek41/src/dense.rs gate-ds41-load gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-long gate-gpu-ds41-moe gate-gpu-ds41-prefill gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-woa gate-gpu-dspark-graph gate-gpu-dspark-kv gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-mcol gate-gpu-ds41-media gate-gpu-ds41-nvtier gate-gpu-glm5next-media gate-gpu-serve-start-ds41 gate-gpu-routed-legs the chain's users and the MoE, output-projection and m-column gates (gate-gpu-ds41-moe gate-gpu-ds41-woa gate-gpu-mcol) except gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve: bloomery-serve opens one seat a run (--model, or the file's architecture); these run its decide and glm and qwen3 seats, and only another seat's module reaches this file | except gate-gpu-serve-start-glm: bloomery-serve opens one seat a run (--model, or the file's architecture); this recipe runs its glm seat, and only another seat's module reaches this file -crates/gpu-deepseek41/src/experts.rs gate-ds41-load gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-long gate-gpu-ds41-moe gate-gpu-ds41-prefill gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-woa gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-kv gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-mcol gate-gpu-ds41-media gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs the chain's users and the MoE, output-projection, DSpark expert and m-column gates (gate-gpu-ds41-moe gate-gpu-ds41-woa gate-gpu-dspark-experts gate-gpu-mcol) except gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve: bloomery-serve opens one seat a run (--model, or the file's architecture); these run its decide and qwen3 seats, and only another seat's module reaches this file -crates/gpu-deepseek41/src/{draft/**,experts_mxfp4.rs,markov.rs,hc_f32.rs} gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-twocard gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-glm5next-serve gate-gpu-qwen3-serve gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 the DSpark draft: app's V4.1 session (the draft it carries) and the DSpark gates +crates/gpu-deepseek41/src/lib.rs gate-ds41-load gate-gpu-ds41-attn gate-gpu-ds41-callstream gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-comp gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-moe gate-gpu-ds41-prefill gate-gpu-ds41-residency-a gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-woa gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-glm-mla gate-gpu-glm-router gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-mtp gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-mcol gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-nemotron-serve the module list only: the gates that use any module of the crate (every row below) except gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve: bloomery-serve opens one seat a run (--model, or the file's architecture); these run its decide and qwen3 seats, and only another seat's module reaches this file +crates/gpu-deepseek41/src/{body.rs,body/**} gate-ds41-load gate-gpu-clef-serve gate-gpu-ds41-callstream gate-gpu-ds41-chain-attn gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-prefill gate-gpu-ds41-residency-a gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-glm5next-serve gate-gpu-qwen3-serve gate-gpu-lev-serve gate-gpu-ds41-nvtier gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-nemotron-serve the engine's assembly, prefill, CED and program: every gate that opens the V4.1 model (body::open, app's V4.1 session) or names body's constants +crates/gpu-deepseek41/src/swap.rs gate-ds41-load gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-chain-attn gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-long gate-gpu-ds41-prefill gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-glm5next-serve gate-gpu-qwen3-serve gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-ds41-media gate-gpu-ds41-nvtier gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-nemotron-serve the card-expert swap under the body, and the r8 unpack gate +crates/gpu-deepseek41/src/{chain/**,engram_gate.rs,span.rs,transpose.rs} gate-ds41-load gate-gpu-ds41-callstream gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-moe gate-gpu-ds41-prefill gate-gpu-ds41-residency-a gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-dspark-graph gate-gpu-dspark-kv gate-gpu-glm5next-e2e gate-gpu-glm5next-mtp gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-glm5next-serve gate-gpu-glm5next-residency gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-nemotron-serve the layer pieces: the V4.1 body, the chain gates, DSpark's graph and KV, and the GLM body, which runs the V4.1 chain except gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve: bloomery-serve opens one seat a run (--model, or the file's architecture); these run its decide and qwen3 seats, and only another seat's module reaches this file +crates/gpu-deepseek41/src/attn.rs gate-ds41-load gate-gpu-ds41-attn gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-index gate-gpu-ds41-long gate-gpu-ds41-prefill gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-dspark-graph gate-gpu-dspark-kv gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-ds41-media gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-nemotron-serve the chain's users and the attention, indexer and GLM MLA/selector gates (gate-gpu-ds41-attn gate-gpu-ds41-index gate-gpu-glm-mla gate-gpu-glm-sel) except gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve: bloomery-serve opens one seat a run (--model, or the file's architecture); these run its decide and qwen3 seats, and only another seat's module reaches this file +crates/gpu-deepseek41/src/rope.rs gate-ds41-load gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-comp gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-index gate-gpu-ds41-long gate-gpu-ds41-prefill gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-dspark-graph gate-gpu-dspark-kv gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-ds41-media gate-gpu-ds41-nvtier gate-gpu-glm5next-media gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-nemotron-serve the chain's users and the rope, compressor and indexer gates (gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-rope) except gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve: bloomery-serve opens one seat a run (--model, or the file's architecture); these run its decide and glm and qwen3 seats, and only another seat's module reaches this file | except gate-gpu-serve-start-glm: bloomery-serve opens one seat a run (--model, or the file's architecture); this recipe runs its glm seat, and only another seat's module reaches this file +crates/gpu-deepseek41/src/router.rs gate-ds41-load gate-gpu-ds41-callstream gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-moe gate-gpu-ds41-prefill gate-gpu-ds41-residency-a gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-dspark-graph gate-gpu-dspark-kv gate-gpu-glm-router gate-gpu-glm5next-e2e gate-gpu-glm5next-mtp gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-nemotron-serve the chain's users and the MoE and GLM router gates (gate-gpu-ds41-moe gate-gpu-glm-router) except gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve: bloomery-serve opens one seat a run (--model, or the file's architecture); these run its decide and qwen3 seats, and only another seat's module reaches this file +crates/gpu-deepseek41/src/{params.rs,indexer.rs} gate-ds41-load gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-index gate-gpu-ds41-long gate-gpu-ds41-prefill gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-dspark-graph gate-gpu-dspark-kv gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-ds41-media gate-gpu-ds41-nvtier gate-gpu-glm5next-media gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-nemotron-serve the chain's users and the indexer gate (gate-gpu-ds41-index) except gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve: bloomery-serve opens one seat a run (--model, or the file's architecture); these run its decide and glm and qwen3 seats, and only another seat's module reaches this file | except gate-gpu-serve-start-glm: bloomery-serve opens one seat a run (--model, or the file's architecture); this recipe runs its glm seat, and only another seat's module reaches this file +crates/gpu-deepseek41/src/hc.rs gate-ds41-load gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-long gate-gpu-ds41-prefill gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-ds41-moe gate-gpu-ds41-media gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-nemotron-serve the chain's users and the HC gates (gate-gpu-ds41-hc gate-gpu-dspark-hc) except gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve: bloomery-serve opens one seat a run (--model, or the file's architecture); these run its decide and qwen3 seats, and only another seat's module reaches this file +crates/gpu-deepseek41/src/block.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-glm5next-gemm gate-gpu-glm5next-e2e gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-stagger gate-gpu-glm5next-nvtier gate-gpu-glm5next-serve gate-gpu-ds41-lib gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-nemotron-serve the gated block (dense layer, shared expert) by weight type: the MiMo and GLM bodies run it, so every serve gate that links MiMo and every GLM gate does; its plan tests are the crate lib gate's +crates/gpu-deepseek41/src/{compress.rs,index_key.rs} gate-ds41-load gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-comp gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-index gate-gpu-ds41-long gate-gpu-ds41-prefill gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-dspark-graph gate-gpu-dspark-kv gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-ds41-media gate-gpu-ds41-nvtier gate-gpu-glm5next-media gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-nemotron-serve the chain's users and the compressor and indexer gates (gate-gpu-ds41-comp gate-gpu-ds41-index) except gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve: bloomery-serve opens one seat a run (--model, or the file's architecture); these run its decide and glm and qwen3 seats, and only another seat's module reaches this file | except gate-gpu-serve-start-glm: bloomery-serve opens one seat a run (--model, or the file's architecture); this recipe runs its glm seat, and only another seat's module reaches this file +crates/gpu-deepseek41/src/dense.rs gate-ds41-load gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-long gate-gpu-ds41-moe gate-gpu-ds41-prefill gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-woa gate-gpu-dspark-graph gate-gpu-dspark-kv gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-mcol gate-gpu-ds41-media gate-gpu-ds41-nvtier gate-gpu-glm5next-media gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-nemotron-serve the chain's users and the MoE, output-projection and m-column gates (gate-gpu-ds41-moe gate-gpu-ds41-woa gate-gpu-mcol) except gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve: bloomery-serve opens one seat a run (--model, or the file's architecture); these run its decide and glm and qwen3 seats, and only another seat's module reaches this file | except gate-gpu-serve-start-glm: bloomery-serve opens one seat a run (--model, or the file's architecture); this recipe runs its glm seat, and only another seat's module reaches this file +crates/gpu-deepseek41/src/experts.rs gate-ds41-load gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-long gate-gpu-ds41-moe gate-gpu-ds41-prefill gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-woa gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-kv gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-mcol gate-gpu-ds41-media gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-nemotron-serve the chain's users and the MoE, output-projection, DSpark expert and m-column gates (gate-gpu-ds41-moe gate-gpu-ds41-woa gate-gpu-dspark-experts gate-gpu-mcol) except gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve: bloomery-serve opens one seat a run (--model, or the file's architecture); these run its decide and qwen3 seats, and only another seat's module reaches this file +crates/gpu-deepseek41/src/{draft/**,experts_mxfp4.rs,markov.rs,hc_f32.rs} gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-twocard gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-glm5next-serve gate-gpu-qwen3-serve gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-nemotron-serve the DSpark draft: app's V4.1 session (the draft it carries) and the DSpark gates # crates/gpu-gates' lib, below its bins. -crates/gpu-gates/src/record.rs gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-swap gate-gpu-qwen35moe-e2e gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3moe-e2e gate-q38-serve-rules gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media-seat gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text weekly-gpu-ds41-flowcounts the record kinds: the bins that print records (generate_ds41, bloomery-chat, bloomery-serve-ds41, the prefill, residency, callstream and swap gates) and generate.rs's users -crates/gpu-gates/src/generate.rs gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text the V4.1 generate loop shared by generate_ds41, the chat and serve bins and the tier and two-card gates -crates/gpu-gates/src/{bind.rs,serve_client.rs} gate-ds41-bind gate-gpu-clef-serve gate-gpu-lev-serve gate-q38-serve-rules gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs the V4.1 server's engine binding and its HTTP client: gpu-gates' bind tests; bloomery-serve-ds41 and its gate are weekly-gpu-ds41-serve (a trigger below) -crates/gpu-gates/src/vision_seat.rs gate-ds41-bind gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve weekly-gpu-ds41-serve gate-gpu-glm5next-media-seat gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-ds41-media gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-routed-legs gate-gpu-ds41-chat gate-gpu-glm5next-twocard weekly-gpu-glm5next-media-serve the vision seat every serve seat shares: gate-ds41-bind's unit tests (the vision_seat:: run), and the servers that bind it (the V4.1 seat's images) -crates/gpu-gates/src/image_plan.rs gate-gpu-clef-serve gate-gpu-glm5next-serve gate-gpu-lev-serve gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3-serve gate-gpu-qwen3moe-e2e gate-gpu-clef-hidden gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-chat gate-gpu-glm5next-twocard the plan's side of an image input (the tower's reserve in a placed plan and a whole-fit verdict): the qwen3 placement every Qwen3 bin includes, and the servers that bind the seat that names the reserve +crates/gpu-gates/src/record.rs gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-swap gate-gpu-qwen35moe-e2e gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3moe-e2e gate-q38-serve-rules gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media-seat gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text weekly-gpu-ds41-flowcounts gate-gpu-nemotron-serve the record kinds: the bins that print records (generate_ds41, bloomery-chat, bloomery-serve-ds41, the prefill, residency, callstream and swap gates) and generate.rs's users +crates/gpu-gates/src/generate.rs gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-nemotron-serve the V4.1 generate loop shared by generate_ds41, the chat and serve bins and the tier and two-card gates +crates/gpu-gates/src/{bind.rs,serve_client.rs} gate-ds41-bind gate-gpu-clef-serve gate-gpu-lev-serve gate-q38-serve-rules gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-nemotron-serve the V4.1 server's engine binding and its HTTP client: gpu-gates' bind tests; bloomery-serve-ds41 and its gate are weekly-gpu-ds41-serve (a trigger below) +crates/gpu-gates/src/vision_seat.rs gate-ds41-bind gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve weekly-gpu-ds41-serve gate-gpu-glm5next-media-seat gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-ds41-media gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-routed-legs gate-gpu-ds41-chat gate-gpu-glm5next-twocard weekly-gpu-glm5next-media-serve gate-gpu-nemotron-serve the vision seat every serve seat shares: gate-ds41-bind's unit tests (the vision_seat:: run), and the servers that bind it (the V4.1 seat's images) +crates/gpu-gates/src/image_plan.rs gate-gpu-clef-serve gate-gpu-glm5next-serve gate-gpu-lev-serve gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3-serve gate-gpu-qwen3moe-e2e gate-gpu-clef-hidden gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-chat gate-gpu-glm5next-twocard gate-gpu-nemotron-serve the plan's side of an image input (the tower's reserve in a placed plan and a whole-fit verdict): the qwen3 placement every Qwen3 bin includes, and the servers that bind the seat that names the reserve crates/gpu-gates/src/engine.rs gate-gpu-e2e AnyEngine, the V2-Lite and Qwen3 engine behind gate_e2e crates/gpu-gates/src/prompts.rs gate-gpu-ds41-long gate-gpu-e2e gate-gpu-hybrid gate-gpu-qwen3moe-e2e the prompt and greedy-file readers of the long, e2e, hybrid and qwen3moe e2e gates crates/gpu-gates/src/qwen3moe.rs gate-gpu-qwen3moe-down gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-attn gate-gpu-mimo2-flash the Qwen3 kernel gates' shared sets @@ -51,39 +51,39 @@ crates/gpu-gates/src/gemm32.rs gate-gpu-gemm gate-gpu-glm5next-gemm gate-gpu-p1 crates/gpu-gates/src/flash_band.rs gate-gpu-qwen35moe-attn gate-gpu-qwen3moe-flash gate-gpu-gemma-embedding2-head512 the exact attention and the prefill flash's band: the flash gates that judge against them crates/gpu-gates/src/rope_gate.rs gate-gpu-qwen35moe-attn gate-gpu-gemma-embedding2-head512 the shared NeoX host expectation, launch and readback used by these gates crates/gpu-gates/src/prefill_gate.rs gate-gpu-qwen35moe-attn gate-gpu-qwen3moe-flash gate-gpu-gemma-embedding2-head512 the shared prefill cache fixtures, window equivalence and refusal checks -crates/gpu-glm5next/src/gemm.rs gate-gpu-glm5next-gemm gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-routed-legs the GLM prompt front on the GEMM: no engine caller yet, its gate alone except gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve: bloomery-serve opens one seat a run (--model, or the file's architecture); these run its decide and qwen3 seats, and only another seat's module reaches this file | except gate-gpu-serve-start-ds41: bloomery-serve opens one seat a run (--model, or the file's architecture); this recipe runs its ds41 seat, and only another seat's module reaches this file -crates/gpu-glm5next/src/{lib.rs,body.rs,ffn.rs,forced.rs,host.rs,kda.rs,media.rs,mla.rs,nextn.rs,pair.rs,prefill.rs,program.rs,seq.rs,swap.rs,tensors.rs,tier.rs} gate-gpu-glm5next-e2e gate-gpu-glm5next-mtp gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-glm5next-gemm gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-routed-legs the GLM engine: the gates that drive the GLM model (gate_glm5next_e2e/_mtp/_twocard/_residency, generate_glm5next, and the GLM seat's bloomery-serve, which gate-gpu-glm5next-serve builds). The clef and qwen3 serve gates build bloomery-serve for their own seats and never run GLM code; gemm.rs keeps its row above (no engine caller) except gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve: bloomery-serve opens one seat a run (--model, or the file's architecture); these run its decide and qwen3 seats, and only another seat's module reaches this file | except gate-gpu-serve-start-ds41: bloomery-serve opens one seat a run (--model, or the file's architecture); this recipe runs its ds41 seat, and only another seat's module reaches this file +crates/gpu-glm5next/src/gemm.rs gate-gpu-glm5next-gemm gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-routed-legs gate-gpu-nemotron-serve the GLM prompt front on the GEMM: no engine caller yet, its gate alone except gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve: bloomery-serve opens one seat a run (--model, or the file's architecture); these run its decide and qwen3 seats, and only another seat's module reaches this file | except gate-gpu-serve-start-ds41: bloomery-serve opens one seat a run (--model, or the file's architecture); this recipe runs its ds41 seat, and only another seat's module reaches this file +crates/gpu-glm5next/src/{lib.rs,body.rs,ffn.rs,forced.rs,host.rs,kda.rs,media.rs,mla.rs,nextn.rs,pair.rs,prefill.rs,program.rs,seq.rs,swap.rs,tensors.rs,tier.rs} gate-gpu-glm5next-e2e gate-gpu-glm5next-mtp gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-glm5next-gemm gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-routed-legs gate-gpu-nemotron-serve the GLM engine: the gates that drive the GLM model (gate_glm5next_e2e/_mtp/_twocard/_residency, generate_glm5next, and the GLM seat's bloomery-serve, which gate-gpu-glm5next-serve builds). The clef and qwen3 serve gates build bloomery-serve for their own seats and never run GLM code; gemm.rs keeps its row above (no engine caller) except gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve: bloomery-serve opens one seat a run (--model, or the file's architecture); these run its decide and qwen3 seats, and only another seat's module reaches this file | except gate-gpu-serve-start-ds41: bloomery-serve opens one seat a run (--model, or the file's architecture); this recipe runs its ds41 seat, and only another seat's module reaches this file # crates/app: the session. Session is generic; each arch module holds the impls for its own body. -crates/app/src/{lib.rs,arch/mod.rs} gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-step gate-gpu-ds41-twocard gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-qwen3moe-e2e gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-mtp gate-q38-serve-rules gate-gpu-ds41-media gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-minicpm-text the generic session every app user runs: the V4.1 gates and bins, the GLM gates, the qwen3moe e2e gate and the Qwen3.6 draft gate -crates/app/src/arch/deepseek41/** gate-gpu-clef-serve gate-gpu-ds41-callstream gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-media gate-gpu-ds41-residency-a gate-gpu-ds41-twocard gate-gpu-glm5next-serve gate-gpu-qwen3-serve gate-gpu-lev-serve gate-gpu-ds41-step gate-gpu-ds41-prefill gate-gpu-ds41-nvtier gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs the V4.1 session impls and its draft: generate_ds41, the chat and serve bins, and the gates that drive them -crates/app/src/arch/glm5next/** gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen3-serve gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs the GLM session impls: the GLM e2e gate -crates/app/src/{mtp.rs,mtp/**,arch/qwen3moe/**} gate-app gate-q38-serve-rules gate-gpu-qwen3moe-e2e gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen4exp-mtp gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-glm5next-twocard gate-gpu-qwen3-serve gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-minicpm-text the shared MTP window and the qwen3moe session impls: the app tests, the qwen3moe e2e gate, the Qwen3.8 serve seat and MTP gates, and the Qwen3.6 draft gate (its Session taps) -crates/runtime/src/** gate-ops gate-placement gate-union gate-fixture gate-app gate-deepseek4-meta gate-ds41-bind gate-ds41-host gate-ds41-kld gate-ds41-load gate-ds41-meta gate-ds41-oracle gate-ds41-plan gate-dspark-read gate-glm5next-meta gate-qwen35moe-meta gate-qwen3moe-meta gate-qwen4exp-meta gate-ptx-spill gate-gpu-e2e gate-gpu-gemm gate-gpu-lib gate-gpu-gates-lib gate-gpu-head gate-gpu-hc-gated gate-gpu-kquant gate-gpu-mcol gate-gpu-moe gate-gpu-p10 gate-gpu-swap gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-gemm gate-gpu-glm5next-mtp gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-qwen3-serve gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-head gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-attn gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35moe-moe gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-ds41-callstream gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-comp gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-index gate-gpu-ds41-lib gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-moe gate-gpu-ds41-prefill gate-gpu-ds41-residency-a gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-woa gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-q38-serve-rules gate-gpu-nvtier gate-mimo2-meta gate-gpu-hybrid gate-gpu-p8 gate-gpu-linear gate-ds41-place gate-gpu-p8b gate-gpu-iq-sel gate-gpu-xstream gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-hf gate-gpu-minicpm-text gate-nemotron-meta gate-gpu-p1 gate-gpu-mamba2 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp the generation loop and its stores: every gate that names a runtime path in its own code (runtime::combine in gate_kquant and the chain-ffn/prefill bins, hc_gated, qsa, seqstate, swaprule, stores), loads weights through bloomery_gpu::weights (whose every loader entry funnels into upload_rows → runtime::words::stream_words), or opens a model or session (arch place.rs → runtime::stores; clef-hidden opens the Qwen3.5 body, whose body35.rs reads runtime::seqstate). The gates left out load no model and name no runtime path: the synthetic kernel gates upload through DeviceTensor::upload/from_host and read reference values host-side, the CPU oracle gates read the model host-side, and gate-runtime is picked as its own lib's tests -crates/hf/src/{safetensors.rs,dspark.rs} gate-gpu-qwen35-dspark gate-gpu-clef-serve gate-gpu-lev-serve gate-decision-clef gate-decision-lev gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 hf::safetensors is the Clef head reader (decision re-exports its errors through decision::Error, so the serve seats that open the decide row reach it) and the DSpark draft file reader; hf::dspark is read only by the Qwen3.6 draft gate +crates/app/src/{lib.rs,arch/mod.rs} gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-step gate-gpu-ds41-twocard gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-qwen3moe-e2e gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-mtp gate-q38-serve-rules gate-gpu-ds41-media gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-nemotron-serve the generic session every app user runs: the V4.1 gates and bins, the GLM gates, the qwen3moe e2e gate and the Qwen3.6 draft gate +crates/app/src/arch/deepseek41/** gate-gpu-clef-serve gate-gpu-ds41-callstream gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-media gate-gpu-ds41-residency-a gate-gpu-ds41-twocard gate-gpu-glm5next-serve gate-gpu-qwen3-serve gate-gpu-lev-serve gate-gpu-ds41-step gate-gpu-ds41-prefill gate-gpu-ds41-nvtier gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-nemotron-serve the V4.1 session impls and its draft: generate_ds41, the chat and serve bins, and the gates that drive them +crates/app/src/arch/glm5next/** gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen3-serve gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-nemotron-serve the GLM session impls: the GLM e2e gate +crates/app/src/{mtp.rs,mtp/**,arch/qwen3moe/**} gate-app gate-q38-serve-rules gate-gpu-qwen3moe-e2e gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen4exp-mtp gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-glm5next-twocard gate-gpu-qwen3-serve gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-nemotron-serve the shared MTP window and the qwen3moe session impls: the app tests, the qwen3moe e2e gate, the Qwen3.8 serve seat and MTP gates, and the Qwen3.6 draft gate (its Session taps) +crates/runtime/src/** gate-ops gate-placement gate-union gate-fixture gate-app gate-deepseek4-meta gate-ds41-bind gate-ds41-host gate-ds41-kld gate-ds41-load gate-ds41-meta gate-ds41-oracle gate-ds41-plan gate-dspark-read gate-glm5next-meta gate-qwen35moe-meta gate-qwen3moe-meta gate-qwen4exp-meta gate-ptx-spill gate-gpu-e2e gate-gpu-gemm gate-gpu-lib gate-gpu-gates-lib gate-gpu-head gate-gpu-hc-gated gate-gpu-kquant gate-gpu-mcol gate-gpu-moe gate-gpu-p10 gate-gpu-swap gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-gemm gate-gpu-glm5next-mtp gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-qwen3-serve gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-head gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-attn gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35moe-moe gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-ds41-callstream gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-comp gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-index gate-gpu-ds41-lib gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-moe gate-gpu-ds41-prefill gate-gpu-ds41-residency-a gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-woa gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-q38-serve-rules gate-gpu-nvtier gate-mimo2-meta gate-gpu-hybrid gate-gpu-p8 gate-gpu-linear gate-ds41-place gate-gpu-p8b gate-gpu-iq-sel gate-gpu-xstream gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-hf gate-gpu-minicpm-text gate-nemotron-meta gate-gpu-p1 gate-gpu-mamba2 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve the generation loop and its stores: every gate that names a runtime path in its own code (runtime::combine in gate_kquant and the chain-ffn/prefill bins, hc_gated, qsa, seqstate, swaprule, stores), loads weights through bloomery_gpu::weights (whose every loader entry funnels into upload_rows → runtime::words::stream_words), or opens a model or session (arch place.rs → runtime::stores; clef-hidden opens the Qwen3.5 body, whose body35.rs reads runtime::seqstate). The gates left out load no model and name no runtime path: the synthetic kernel gates upload through DeviceTensor::upload/from_host and read reference values host-side, the CPU oracle gates read the model host-side, and gate-runtime is picked as its own lib's tests +crates/hf/src/{safetensors.rs,dspark.rs} gate-gpu-qwen35-dspark gate-gpu-clef-serve gate-gpu-lev-serve gate-decision-clef gate-decision-lev gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-nemotron-serve hf::safetensors is the Clef head reader (decision re-exports its errors through decision::Error, so the serve seats that open the decide row reach it) and the DSpark draft file reader; hf::dspark is read only by the Qwen3.6 draft gate -crates/serve/{src/**,build.rs} gate-ds41-bind gate-gpu-clef-serve gate-gpu-lev-serve gate-serve gate-q38-serve-rules gate-ds41-place gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs the llama-server-compatible API: gpu-gates' bind tests (bloomery-serve-ds41's gate is weekly-gpu-ds41-serve, a trigger below); gate-serve is its own -crates/jinja/src/** gate-jinja gate-serve gate-ds41-bind gate-decision-clef gate-decision-lev gate-gpu-clef-serve gate-gpu-lev-serve gate-q38-serve-rules gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 the template engine: its own tests, serve's chat templates (the serve row's readers) and the decision models' templates (Clef's and lev's request renders) -crates/tokenizer/src/** gate-app gate-decision-clef gate-deepseek4-meta gate-ds41-bind gate-ds41-host gate-ds41-kld gate-ds41-load gate-ds41-meta gate-ds41-oracle gate-ds41-plan gate-dspark-read gate-fixture gate-glm5next-meta gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-comp gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-index gate-gpu-ds41-lib gate-gpu-ds41-long gate-gpu-ds41-prefill gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-e2e gate-gpu-gates-lib gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-lib gate-gpu-mcol gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35moe-moe gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-head gate-gpu-qwen3moe-router gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-ops gate-placement gate-qwen35moe-meta gate-qwen3moe-meta gate-qwen4exp-meta gate-union gate-gpu-clef-serve gate-gpu-lev-serve gate-decision-lev gate-q38-serve-rules gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-iq-sel gate-gpu-clef-hidden gate-gpu-qwen4exp-iqleg gate-gpu-glm5next-gemm gate-gpu-ds41-media gate-gpu-nvtier gate-gpu-qwen38-residency gate-mimo2-meta gate-gpu-glm5next-residency gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-ds41-place gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-nemotron-meta gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-qwen35-dspark the encoder and decoder, and runs_pre_tokenizer, which coverage::check asks at every placement plan (model::arch::*::place): every gate that plans a placement or tokenizes text +crates/serve/{src/**,build.rs} gate-ds41-bind gate-gpu-clef-serve gate-gpu-lev-serve gate-serve gate-q38-serve-rules gate-ds41-place gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-nemotron-serve the llama-server-compatible API: gpu-gates' bind tests (bloomery-serve-ds41's gate is weekly-gpu-ds41-serve, a trigger below); gate-serve is its own +crates/jinja/src/** gate-jinja gate-serve gate-ds41-bind gate-decision-clef gate-decision-lev gate-gpu-clef-serve gate-gpu-lev-serve gate-q38-serve-rules gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-nemotron-serve the template engine: its own tests, serve's chat templates (the serve row's readers) and the decision models' templates (Clef's and lev's request renders) +crates/tokenizer/src/** gate-app gate-decision-clef gate-deepseek4-meta gate-ds41-bind gate-ds41-host gate-ds41-kld gate-ds41-load gate-ds41-meta gate-ds41-oracle gate-ds41-plan gate-dspark-read gate-fixture gate-glm5next-meta gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-comp gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-index gate-gpu-ds41-lib gate-gpu-ds41-long gate-gpu-ds41-prefill gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-e2e gate-gpu-gates-lib gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-lib gate-gpu-mcol gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35moe-moe gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-head gate-gpu-qwen3moe-router gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-ops gate-placement gate-qwen35moe-meta gate-qwen3moe-meta gate-qwen4exp-meta gate-union gate-gpu-clef-serve gate-gpu-lev-serve gate-decision-lev gate-q38-serve-rules gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-iq-sel gate-gpu-clef-hidden gate-gpu-qwen4exp-iqleg gate-gpu-glm5next-gemm gate-gpu-ds41-media gate-gpu-nvtier gate-gpu-qwen38-residency gate-mimo2-meta gate-gpu-glm5next-residency gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-ds41-place gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-nemotron-meta gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-qwen35-dspark gate-gpu-nemotron-serve the encoder and decoder, and runs_pre_tokenizer, which coverage::check asks at every placement plan (model::arch::*::place): every gate that plans a placement or tokenizes text # crates/refset: the reference-set readers. Every oracle read goes through refset::arch's dispatch (node_dumps, # families) and a family's check; which family a gate asks for is its run-time arch, so a family's row keeps every # dispatch user. -crates/refset/src/{lib,family,mtpref,dsref,greedy,kld,vision,ik,columns}.rs gate-attn gate-derived gate-ds41-bind gate-ds41-host gate-ds41-kld gate-ds41-load gate-ds41-meta gate-ds41-oracle gate-ds41-plan gate-ffn gate-forward gate-gpu-block gate-gpu-clef-hidden gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-gates-lib gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-mtp gate-gpu-glm5next-residency gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-head gate-gpu-linear gate-gpu-mcol gate-gpu-moe gate-gpu-p0b gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p8 gate-gpu-p8b gate-gpu-qwen35moe-attn gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-qwen35moe-moe gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-vision gate-head gate-kv gate-moe gate-mt gate-ops gate-profile gate-qwen3moe-meta gate-union gate-vision gate-1-1 gate-tokenizer gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-ds41-place gate-q38-serve-rules gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-iq gate-gpu-iq-sel gate-gpu-e2e gate-gpu-hybrid gate-gpu-gemm gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-dspark-read gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-clef-serve gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-minicpmv-tower gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp the readers and the family check every set goes through: every gate that reads a reference set -crates/refset/src/arch/mod.rs gate-ds41-bind gate-ds41-kld gate-ds41-load gate-ds41-oracle gate-ds41-plan gate-gpu-block gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-media gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-gates-lib gate-gpu-head gate-gpu-linear gate-gpu-moe gate-gpu-p0b gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p8 gate-gpu-p8b gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-1-1 gate-tokenizer gate-ds41-place gate-gpu-qwen4exp-mtp gate-q38-serve-rules gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-media-seat gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media the per-arch dispatch (node_dumps, families, named): every oracle user except gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-glm5next-serve: bloomery-serve opens one seat a run (--model, or the file's architecture); these run its decide and glm seats, and only the qwen38 seat's module (its DraftFrom) reaches this file | except gate-gpu-serve-start-glm: bloomery-serve opens one seat a run (--model, or the file's architecture); this recipe runs its glm seat, and only another seat's module reaches this file | except gate-gpu-serve-start-ds41: bloomery-serve opens one seat a run (--model, or the file's architecture); this recipe runs its ds41 seat, and only another seat's module reaches this file -crates/refset/src/arch/deepseek41/** gate-ds41-bind gate-ds41-host gate-ds41-kld gate-ds41-load gate-ds41-meta gate-ds41-oracle gate-ds41-plan gate-gpu-block gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-long gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-gates-lib gate-gpu-glm5next-serve gate-gpu-head gate-gpu-linear gate-gpu-mcol gate-gpu-moe gate-gpu-p0b gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p8 gate-gpu-p8b gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-union gate-gpu-qwen3-serve gate-gpu-clef-serve gate-gpu-lev-serve gate-ds41-place gate-tokenizer gate-1-1 gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 the deepseek41 family's sets: the dispatch users and the gates that name the family +crates/refset/src/{lib,family,mtpref,dsref,greedy,kld,vision,ik,columns}.rs gate-attn gate-derived gate-ds41-bind gate-ds41-host gate-ds41-kld gate-ds41-load gate-ds41-meta gate-ds41-oracle gate-ds41-plan gate-ffn gate-forward gate-gpu-block gate-gpu-clef-hidden gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-gates-lib gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-mtp gate-gpu-glm5next-residency gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-head gate-gpu-linear gate-gpu-mcol gate-gpu-moe gate-gpu-p0b gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p8 gate-gpu-p8b gate-gpu-qwen35moe-attn gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-qwen35moe-moe gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-vision gate-head gate-kv gate-moe gate-mt gate-ops gate-profile gate-qwen3moe-meta gate-union gate-vision gate-1-1 gate-tokenizer gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-ds41-place gate-q38-serve-rules gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-iq gate-gpu-iq-sel gate-gpu-e2e gate-gpu-hybrid gate-gpu-gemm gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-dspark-read gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-clef-serve gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-minicpmv-tower gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e the readers and the family check every set goes through: every gate that reads a reference set +crates/refset/src/arch/mod.rs gate-ds41-bind gate-ds41-kld gate-ds41-load gate-ds41-oracle gate-ds41-plan gate-gpu-block gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-media gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-gates-lib gate-gpu-head gate-gpu-linear gate-gpu-moe gate-gpu-p0b gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p8 gate-gpu-p8b gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-1-1 gate-tokenizer gate-ds41-place gate-gpu-qwen4exp-mtp gate-q38-serve-rules gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-media-seat gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-nemotron-serve the per-arch dispatch (node_dumps, families, named): every oracle user except gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-glm5next-serve: bloomery-serve opens one seat a run (--model, or the file's architecture); these run its decide and glm seats, and only the qwen38 seat's module (its DraftFrom) reaches this file | except gate-gpu-serve-start-glm: bloomery-serve opens one seat a run (--model, or the file's architecture); this recipe runs its glm seat, and only another seat's module reaches this file | except gate-gpu-serve-start-ds41: bloomery-serve opens one seat a run (--model, or the file's architecture); this recipe runs its ds41 seat, and only another seat's module reaches this file +crates/refset/src/arch/deepseek41/** gate-ds41-bind gate-ds41-host gate-ds41-kld gate-ds41-load gate-ds41-meta gate-ds41-oracle gate-ds41-plan gate-gpu-block gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-long gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-gates-lib gate-gpu-glm5next-serve gate-gpu-head gate-gpu-linear gate-gpu-mcol gate-gpu-moe gate-gpu-p0b gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p8 gate-gpu-p8b gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-union gate-gpu-qwen3-serve gate-gpu-clef-serve gate-gpu-lev-serve gate-ds41-place gate-tokenizer gate-1-1 gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-nemotron-serve the deepseek41 family's sets: the dispatch users and the gates that name the family crates/refset/src/arch/deepseek41v/** gate-ds41-bind gate-ds41-kld gate-ds41-load gate-ds41-oracle gate-ds41-plan gate-gpu-block gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-media gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-gates-lib gate-gpu-head gate-gpu-linear gate-gpu-moe gate-gpu-p0b gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p8 gate-gpu-p8b gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-vision gate-vision gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 the deepseek41v family's sets: the dispatch users and the gates that name the family except gate-ds41-place: they read no deepseek41v set: only refset::arch's table (BY_ARCH), which picks a set's family by the file it names at run time, reaches it crates/refset/src/arch/glm5next/** gate-ds41-bind gate-ds41-kld gate-ds41-load gate-ds41-oracle gate-ds41-plan gate-gpu-block gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-gates-lib gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-head gate-gpu-linear gate-gpu-moe gate-gpu-p0b gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p8 gate-gpu-p8b gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-glm5next-gemm gate-tokenizer gate-ds41-place gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 the glm5next family's sets: the dispatch users and the gates that name the family crates/refset/src/arch/mimo2/** gate-ds41-bind gate-ds41-kld gate-ds41-load gate-ds41-oracle gate-ds41-plan gate-gpu-block gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-gates-lib gate-gpu-head gate-gpu-linear gate-gpu-mimo2-flash gate-gpu-moe gate-gpu-p0b gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p8 gate-gpu-p8b gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router weekly-gpu-mimo2-e2e the mimo2 family's sets: the dispatch users and the gates that name the family except gate-ds41-place: they read no mimo2 set: only refset::arch's table (BY_ARCH), which picks a set's family by the file it names at run time, reaches it crates/refset/src/arch/qwen35moe/** gate-ds41-bind gate-ds41-kld gate-ds41-load gate-ds41-oracle gate-ds41-plan gate-gpu-block gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-gates-lib gate-gpu-head gate-gpu-linear gate-gpu-moe gate-gpu-p0b gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p8 gate-gpu-p8b gate-gpu-qwen35moe-attn gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-qwen35moe-moe gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-qwen3-serve gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp the qwen35moe family's sets: the dispatch users and the gates that name the family except gate-ds41-place: they read no qwen35moe set: only refset::arch's table (BY_ARCH), which picks a set's family by the file it names at run time, reaches it -crates/refset/src/arch/qwen4exp/** gate-ds41-bind gate-ds41-kld gate-ds41-load gate-ds41-oracle gate-ds41-plan gate-gpu-block gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-gates-lib gate-gpu-glm5next-serve gate-gpu-head gate-gpu-linear gate-gpu-moe gate-gpu-p0b gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p8 gate-gpu-p8b gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3-serve gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-nvtier gate-q38-serve-rules gate-tokenizer gate-ds41-place gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media the qwen4exp family's sets: the dispatch users and the gates that name the family -crates/refset/src/{clefvis.rs,clefvis/**} gate-refset gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-ds41-hc gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-derived gate-mt gate-profile gate-ds41-meta gate-qwen3moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-tokenizer gate-1-1 gate-gpu-head gate-gpu-mcol gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-vision gate-gpu-vision gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-minicpmv-tower gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp the Clef image-input sets' reader: no gate reads a clefvis set yet, gate-refset runs its tests and walks every set in place +crates/refset/src/arch/qwen4exp/** gate-ds41-bind gate-ds41-kld gate-ds41-load gate-ds41-oracle gate-ds41-plan gate-gpu-block gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-gates-lib gate-gpu-glm5next-serve gate-gpu-head gate-gpu-linear gate-gpu-moe gate-gpu-p0b gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p8 gate-gpu-p8b gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3-serve gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-nvtier gate-q38-serve-rules gate-tokenizer gate-ds41-place gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-nemotron-serve the qwen4exp family's sets: the dispatch users and the gates that name the family +crates/refset/src/{clefvis.rs,clefvis/**} gate-refset gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-ds41-hc gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-derived gate-mt gate-profile gate-ds41-meta gate-qwen3moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-tokenizer gate-1-1 gate-gpu-head gate-gpu-mcol gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-vision gate-gpu-vision gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-minicpmv-tower gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e the Clef image-input sets' reader: no gate reads a clefvis set yet, gate-refset runs its tests and walks every set in place crates/refset/src/arch/minicpmv4_7/** gate-ds41-bind gate-ds41-kld gate-ds41-load gate-ds41-oracle gate-ds41-plan gate-gpu-block gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-gates-lib gate-gpu-linear gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-ds41-place gate-gpu-minicpmv-tower gate-refset the MiniCPM-V 4.7 image sets' reader (the tower per tile, the hidden states and four position lanes of an image prompt, the chat ids): the dispatch users, the tower gate that reads the tower sets, and gate-refset, which runs the reader's tests and walks every set in place crates/refset/src/arch/qwen35/** gate-refset gate-gpu-clef-hidden gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media the qwen35 families' sets: the hidden-state gate that names the families, and the table walk except gate-gpu-ds41-hc gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen3moe-down gate-gpu-linear gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-ds41-bind gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-ds41-load: they read no qwen35 set: only refset::arch's table (BY_ARCH), which picks a set's family by the file it names at run time, reaches it -crates/refset/src/emb2.rs gate-refset gate-gpu-qwen35moe-mtp the EmbeddingGemma 2 oracle sets: gate-refset checks these families except gate-1-1 gate-attn gate-derived gate-ds41-bind gate-ds41-host gate-ds41-kld gate-ds41-load gate-ds41-meta gate-ds41-oracle gate-ds41-place gate-ds41-plan gate-ffn gate-forward gate-gpu-block gate-gpu-clef-hidden gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-gates-lib gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-gemm gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-residency gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-glm5v-tower gate-gpu-head gate-gpu-linear gate-gpu-mcol gate-gpu-mimo2-flash gate-gpu-minicpm-text gate-gpu-minicpmv-tower gate-gpu-moe gate-gpu-p0b gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p8 gate-gpu-p8b gate-gpu-qwen3-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-attn gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-media gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-verify gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen3vl-tower gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-serve-start-ds41 gate-gpu-serve-start-glm gate-gpu-vision gate-head gate-kv gate-moe gate-mt gate-ops gate-profile gate-qwen3moe-meta gate-tokenizer gate-union gate-vision: they read no EmbeddingGemma 2 set: only the runtime family dispatch reaches this reader and these families +crates/refset/src/emb2.rs gate-refset gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e the EmbeddingGemma 2 oracle sets: gate-refset checks these families except gate-1-1 gate-attn gate-derived gate-ds41-bind gate-ds41-host gate-ds41-kld gate-ds41-load gate-ds41-meta gate-ds41-oracle gate-ds41-place gate-ds41-plan gate-ffn gate-forward gate-gpu-block gate-gpu-clef-hidden gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-gates-lib gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-gemm gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-residency gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-glm5v-tower gate-gpu-head gate-gpu-linear gate-gpu-mcol gate-gpu-mimo2-flash gate-gpu-minicpm-text gate-gpu-minicpmv-tower gate-gpu-moe gate-gpu-p0b gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p8 gate-gpu-p8b gate-gpu-qwen3-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-attn gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-media gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-verify gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen3vl-tower gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-serve-start-ds41 gate-gpu-serve-start-glm gate-gpu-vision gate-head gate-kv gate-moe gate-mt gate-ops gate-profile gate-qwen3moe-meta gate-tokenizer gate-union gate-vision: they read no EmbeddingGemma 2 set: only the runtime family dispatch reaches this reader and these families crates/refset/src/arch/gemma_embedding2/** gate-refset the EmbeddingGemma 2 oracle sets: gate-refset checks these families except gate-ds41-bind gate-ds41-kld gate-ds41-load gate-ds41-oracle gate-ds41-place gate-ds41-plan gate-gpu-block gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-gates-lib gate-gpu-linear gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router: they read no EmbeddingGemma 2 set: only the runtime family dispatch reaches this reader and these families crates/refset/src/{md5,tokenizer,dequant}.rs gate-tokenizer gate-1-1 gate-gpu-ds41-hc gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-attn gate-gpu-qwen35moe-e2e gate-gpu-minicpm-text the tokenizer and dequant set readers and the md5 they check by: the two gates that open those sets (gate-refset runs the crate's own tests) except gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen3moe-down gate-gpu-linear gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-ds41-bind gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-ds41-load: they read no dequant and tokenizer set: only refset::arch's table (BY_ARCH), which picks a set's family by the file it names at run time, reaches it | except gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency: they open their own families' sets through refset::family::Family::check_set, which reaches DequantSet::open only in its DequantManifest arm, an identity no set of theirs carries: they read no dequant set -crates/refset/src/arch/{tokenizer,dequant}.rs gate-tokenizer gate-1-1 the tokenizer and dequant families' rows and pins: the two gates that open those sets (gate-refset runs the crate's own tests) except gate-gpu-ds41-hc gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen3moe-down gate-gpu-linear gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-ds41-bind gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-ds41-load: they read no dequant and tokenizer set: only refset::arch's table (BY_ARCH), which picks a set's family by the file it names at run time, reaches it +crates/refset/src/arch/{tokenizer,dequant}.rs gate-tokenizer gate-1-1 gate-gpu-nemotron-e2e the tokenizer and dequant families' rows and pins: the two gates that open those sets (gate-refset runs the crate's own tests) except gate-gpu-ds41-hc gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen3moe-down gate-gpu-linear gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-ds41-bind gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-ds41-load: they read no dequant and tokenizer set: only refset::arch's table (BY_ARCH), which picks a set's family by the file it names at run time, reaches it crates/refset/src/census.rs gate-gpu-routed-legs the census stub reader (the manifest and the digest check): the routed-legs gate plans every set through it # The census manifest lands with tools/census (the writer session's branch); the row holds the gate's # selection for it ahead of the merge, and matches no file until that branch is in. @@ -98,85 +98,85 @@ tools/census/stubs.tsv gate-gpu-routed-legs the census manifest (refset::census: # files a gate opens. One row: every gate that links the modules, less the gates whose bins only read tensors # through gguf::Gguf or build synthetic blocks and call no arch module (their own code, and the bloomery-gpu # kernel modules and bloomery-gpu-gates helpers they call, name no model::arch:: path). the fixture generator, arch/coverage.rs and arch/qwen35moe/mtp.rs are carved out to their own rows below: their readers are bounded (a leaf the family's fixture module names; the place/engine/plan modules that call them). -crates/model/src/arch/{mod.rs,dspark.rs,classify.rs,stems.rs,deepseek2/**,qwen3moe/**,glm5next/**} gate-gpu-p4 gate-gpu-p6 gate-gpu-kquant gate-gpu-iq-sel gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-ptx-spill gate-dspark-read gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-q38-serve-rules gate-gpu-block gate-gpu-nvtier gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p5 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-cand gate-gpu-iq gate-gpu-q6k-ids gate-gpu-xstream gate-gpu-mimo2-flash gate-gpu-ple gate-gpu-vision gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 every gate that links an arch module, less the gguf-only kernel gates (above): a family's code runs wherever any family's file is opened -crates/model/src/arch/mimo2/** gate-ops gate-mimo2-meta gate-gpu-mimo2-flash gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-ds41-bind gate-ds41-place gate-gpu-routed-legs gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-minicpm-text weekly-gpu-mimo2-e2e weekly-gpu-mimo2-serve gate-gpu-qwen35moe-mtp MiMo-V2.6-Flash's reader and plan: its code runs only where a mimo2 file opens (its meta test, its flash gate) in bloomery-model's own unit tests (gate-ops runs --lib) and in bloomery-serve's mimo2 seat, which every serve gate's bin links except gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-clef-hidden gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen3moe-e2e gate-fixture gate-gpu-head gate-gpu-lib gate-gpu-ds41-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency: they open no mimo2 file: only model::arch's dispatch by the file's general.architecture (arch/mod.rs) reaches it -crates/model/src/arch/nemotron_h/** gate-ops gate-nemotron-meta gate-fixture gate-gpu-qwen35moe-mtp Nemotron 3 Nano 4B's header reader and fixture spec: no program runs the model, so its code runs where a nemotron_h file is described (its meta test), where its fixture is planned and verified (the fixture generator's gate), in bloomery-model's own unit tests, and behind the arch dispatch except gate-gpu-e2e gate-gpu-hybrid gate-gpu-p6 gate-gpu-ds41-moe gate-gpu-moe gate-gpu-p8 gate-gpu-block gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen35moe-moe gate-gpu-qwen3moe-router gate-qwen35moe-meta gate-gpu-qwen35-dspark gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-minicpm-text gate-gpu-qwen35-media gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen35moe-media gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-types gate-gpu-head gate-gpu-routed-legs gate-gpu-ds41-lib gate-gpu-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-nvtier gate-gpu-ds41-residency-a gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-long gate-gpu-ds41-chat gate-gpu-ds41-media gate-gpu-ds41-prefill gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-ds41-load gate-gpu-qwen3-serve gate-gpu-qwen38-serve gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-p8b gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-residency gate-gpu-glm5next-twocard gate-gpu-glm5next-serve gate-gpu-serve-start-ds41 gate-gpu-serve-start-glm: they link the arch dispatch and never open a nemotron_h file -crates/model/src/arch/gemma_embedding2/** gate-ops gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp EmbeddingGemma 2's header reader: no program runs it, so its code runs where a gemma-embedding2 file is described (its meta test), in bloomery-model's own unit tests, and behind the arch dispatch except gate-gpu-e2e gate-gpu-hybrid gate-gpu-p6 gate-gpu-ds41-moe gate-gpu-moe gate-gpu-p8 gate-gpu-block gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen35moe-moe gate-gpu-qwen3moe-router gate-qwen35moe-meta gate-gpu-qwen35-dspark gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-minicpm-text gate-gpu-qwen35-media gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen35moe-media gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-types gate-fixture gate-gpu-head gate-gpu-routed-legs gate-gpu-ds41-lib gate-gpu-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-nvtier gate-gpu-ds41-residency-a gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-long gate-gpu-ds41-chat gate-gpu-ds41-media gate-gpu-ds41-prefill gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-ds41-load gate-gpu-qwen3-serve gate-gpu-qwen38-serve gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-p8b gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-residency gate-gpu-glm5next-twocard gate-gpu-glm5next-serve gate-gpu-serve-start-ds41 gate-gpu-serve-start-glm: they link the arch dispatch and never open a gemma-embedding2 file -crates/model/src/arch/deepseek41/{hparams.rs,host.rs,kv.rs,mod.rs,names.rs,place.rs,plan.rs,roles.rs,spec.rs} gate-gpu-p4 gate-gpu-p6 gate-gpu-kquant gate-gpu-iq-sel gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-ptx-spill gate-dspark-read gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp every gate that links an arch module, less the gguf-only kernel gates (above): a family's code runs wherever any family's file is opened except gate-gpu-block gate-q38-serve-rules gate-gpu-nvtier: they open no deepseek41 file: only model::arch's dispatch by the file's general.architecture (arch/mod.rs) reaches it -crates/model/src/arch/qwen35moe/{head_list.rs,hparams.rs,host.rs,mod.rs,names.rs,place.rs,roles.rs,spec.rs} gate-gpu-p4 gate-gpu-p6 gate-gpu-kquant gate-gpu-iq-sel gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-ptx-spill gate-dspark-read gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-gpu-nvtier gate-q38-serve-rules gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp every gate that links an arch module, less the gguf-only kernel gates (above): a family's code runs wherever any family's file is opened except gate-gpu-block: they open no qwen35moe file: only model::arch's dispatch by the file's general.architecture (arch/mod.rs) reaches it +crates/model/src/arch/{mod.rs,dspark.rs,classify.rs,stems.rs,deepseek2/**,qwen3moe/**,glm5next/**} gate-gpu-p4 gate-gpu-p6 gate-gpu-kquant gate-gpu-iq-sel gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-ptx-spill gate-dspark-read gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-q38-serve-rules gate-gpu-block gate-gpu-nvtier gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p5 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-cand gate-gpu-iq gate-gpu-q6k-ids gate-gpu-xstream gate-gpu-mimo2-flash gate-gpu-ple gate-gpu-vision gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve every gate that links an arch module, less the gguf-only kernel gates (above): a family's code runs wherever any family's file is opened +crates/model/src/arch/mimo2/** gate-ops gate-mimo2-meta gate-gpu-mimo2-flash gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-ds41-bind gate-ds41-place gate-gpu-routed-legs gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-minicpm-text weekly-gpu-mimo2-e2e weekly-gpu-mimo2-serve gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve MiMo-V2.6-Flash's reader and plan: its code runs only where a mimo2 file opens (its meta test, its flash gate) in bloomery-model's own unit tests (gate-ops runs --lib) and in bloomery-serve's mimo2 seat, which every serve gate's bin links except gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-clef-hidden gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen3moe-e2e gate-fixture gate-gpu-head gate-gpu-lib gate-gpu-ds41-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency: they open no mimo2 file: only model::arch's dispatch by the file's general.architecture (arch/mod.rs) reaches it +crates/model/src/arch/nemotron_h/** gate-ops gate-nemotron-meta gate-fixture gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-iq-sel gate-placement gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-ds41-bind gate-ds41-place Nemotron 3 Nano 4B's header reader and fixture spec: its code runs where a nemotron_h file is described (its meta test), where its fixture is planned and verified (the fixture generator's gate), in bloomery-model's own unit tests, where the card body reads the layer kinds and widths at load (the end-to-end gate), and behind the arch dispatch and, through its plan (place.rs), in bloomery-serve's nemotron seat, which gate-gpu-nemotron-serve starts and every serve gate's bin links except gate-gpu-e2e gate-gpu-hybrid gate-gpu-p6 gate-gpu-ds41-moe gate-gpu-moe gate-gpu-p8 gate-gpu-block gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen35moe-moe gate-gpu-qwen3moe-router gate-qwen35moe-meta gate-gpu-qwen35-dspark gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-minicpm-text gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-types gate-gpu-head gate-gpu-routed-legs gate-gpu-ds41-lib gate-gpu-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-nvtier gate-gpu-ds41-residency-a gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-long gate-gpu-ds41-chat gate-gpu-ds41-media gate-gpu-ds41-prefill gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-ds41-load gate-gpu-qwen38-serve gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-p8b gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-residency gate-gpu-glm5next-twocard: they link the arch dispatch and never open a nemotron_h file +crates/model/src/arch/gemma_embedding2/** gate-ops gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve EmbeddingGemma 2's header reader: no program runs it, so its code runs where a gemma-embedding2 file is described (its meta test), in bloomery-model's own unit tests, and behind the arch dispatch except gate-gpu-e2e gate-gpu-hybrid gate-gpu-p6 gate-gpu-ds41-moe gate-gpu-moe gate-gpu-p8 gate-gpu-block gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen35moe-moe gate-gpu-qwen3moe-router gate-qwen35moe-meta gate-gpu-qwen35-dspark gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-minicpm-text gate-gpu-qwen35-media gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen35moe-media gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-types gate-fixture gate-gpu-head gate-gpu-routed-legs gate-gpu-ds41-lib gate-gpu-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-nvtier gate-gpu-ds41-residency-a gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-long gate-gpu-ds41-chat gate-gpu-ds41-media gate-gpu-ds41-prefill gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-ds41-load gate-gpu-qwen3-serve gate-gpu-qwen38-serve gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-p8b gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-residency gate-gpu-glm5next-twocard gate-gpu-glm5next-serve gate-gpu-serve-start-ds41 gate-gpu-serve-start-glm: they link the arch dispatch and never open a gemma-embedding2 file +crates/model/src/arch/deepseek41/{hparams.rs,host.rs,kv.rs,mod.rs,names.rs,place.rs,plan.rs,roles.rs,spec.rs} gate-gpu-p4 gate-gpu-p6 gate-gpu-kquant gate-gpu-iq-sel gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-ptx-spill gate-dspark-read gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve every gate that links an arch module, less the gguf-only kernel gates (above): a family's code runs wherever any family's file is opened except gate-gpu-block gate-q38-serve-rules gate-gpu-nvtier: they open no deepseek41 file: only model::arch's dispatch by the file's general.architecture (arch/mod.rs) reaches it +crates/model/src/arch/qwen35moe/{head_list.rs,hparams.rs,host.rs,mod.rs,names.rs,place.rs,roles.rs,spec.rs} gate-gpu-p4 gate-gpu-p6 gate-gpu-kquant gate-gpu-iq-sel gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-ptx-spill gate-dspark-read gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-gpu-nvtier gate-q38-serve-rules gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve every gate that links an arch module, less the gguf-only kernel gates (above): a family's code runs wherever any family's file is opened except gate-gpu-block: they open no qwen35moe file: only model::arch's dispatch by the file's general.architecture (arch/mod.rs) reaches it crates/model/src/{fixture/**,arch/deepseek41/fixture.rs} gate-ops gate-ds41-meta gate-fixture gate-deepseek4-meta gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-ds41-load gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-qwen35moe-e2e gate-gpu-minicpm-text the fixture generator and V4.1's FixtureSpec: no file but bin fixture, the fixture test and arch/deepseek41/fixture.rs names crate::fixture (grep), so the gates are the fixture gate and the --lib runs that execute arch::deepseek41::fixture's own unit tests (gate-ops unfiltered, gate-ds41-meta and gate-deepseek4-meta filtered arch::deepseek41) -crates/model/src/arch/{coverage.rs,qwen35moe/mtp.rs} gate-gpu-iq-sel gate-gpu-e2e gate-ops gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-glm5next-gemm gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-app gate-gpu-lib gate-gpu-gates-lib gate-ds41-plan gate-gpu-ds41-lib gate-ds41-bind gate-ds41-place gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-ptx-spill gate-deepseek4-meta gate-glm5next-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-gpu-glm-sel gate-fixture gate-q38-serve-rules gate-gpu-nvtier gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-p6 gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-gpu-head gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-qwen35moe-mtp coverage::check's and qwen35moe's mtp readers: deepseek41/glm5next/qwen35moe place.rs (unimplemented, super::mtp), gpu-gates' engine.rs, gpu's qwen3moe plan/plan38/body38 (turns_as_neox, .unimplemented()) and the meta tests that call coverage::check — every gate that opens or plans those families' files; the two modules' lists coincide -crates/model/src/arch/names.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-q38-serve-rules gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-lib gate-gpu-glm5next-nvtier gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-qwen3moe-e2e gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-iq-sel gate-gpu-ds41-moe gate-gpu-moe gate-gpu-p8 gate-gpu-block gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-router gate-placement gate-qwen35moe-meta gate-gpu-clef-hidden gate-gpu-qwen35-dspark gate-gpu-qwen35moe-moe gate-gpu-gemm gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-experts gate-ds41-host gate-fixture gate-union gate-gpu-ds41-comp gate-gpu-head gate-gpu-mcol gate-ds41-bind gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-place gate-gpu-ds41-chain-glue gate-gpu-ds41-engram gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-twocard gate-gpu-ds41-draft gate-gpu-ds41-long gate-gpu-ds41-nvtier gate-gpu-ds41-dspark-loop gate-gpu-ds41-media gate-gpu-ds41-prefill gate-ds41-load gate-gpu-ds41-chat gate-gpu-p8b gate-dspark-read gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-qwen35moe-mtp gate-app the routed stacks' common names (routed_stacks): the GLM, Qwen3.6 and Qwen3.8 stack tables (swap.rs, qwen3moe/swap.rs for swap35.rs and swap38.rs) and the GLM card check (glm5next_card.rs) call it — every gate that opens those bodies or runs that check -crates/model/src/arch/mod.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-ptx-spill gate-dspark-read gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-vision gate-deepseek4-meta gate-glm5next-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-q38-serve-rules gate-gpu-xstream gate-gpu-mimo2-flash gate-gpu-nvtier gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 the dispatch and Arch: every gate that links it, the gguf-only kernel gates too — bloomery-gpu's fault sites (fault.rs) and bloomery-gpu-gates' family check (lib.rs, the oracle) read Arch in them -crates/gpu/src/weights.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq-sel gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-qwen35moe-attn gate-gpu-glm5next-gemm gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-app gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-ds41-place gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-ptx-spill gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-q38-serve-rules gate-gpu-nvtier gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp the card weight loaders: the gates whose own targets name weights, or reach it through an open path (gpu's arch bodies -> model.rs -> host/swap_source.rs, gpu-deepseek41's body/chain/dense/experts/draft, gpu-glm5next's body/ffn; gate-gpu-xstream names GpuModel), less the synthetic kernel gates whose bins build blocks and name no weights path +crates/model/src/arch/{coverage.rs,qwen35moe/mtp.rs} gate-gpu-iq-sel gate-gpu-e2e gate-ops gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-glm5next-gemm gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-app gate-gpu-lib gate-gpu-gates-lib gate-ds41-plan gate-gpu-ds41-lib gate-ds41-bind gate-ds41-place gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-ptx-spill gate-deepseek4-meta gate-glm5next-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-gpu-glm-sel gate-fixture gate-q38-serve-rules gate-gpu-nvtier gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-p6 gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-gpu-head gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve coverage::check's and qwen35moe's mtp readers: deepseek41/glm5next/qwen35moe place.rs (unimplemented, super::mtp), gpu-gates' engine.rs, gpu's qwen3moe plan/plan38/body38 (turns_as_neox, .unimplemented()) and the meta tests that call coverage::check — every gate that opens or plans those families' files; the two modules' lists coincide +crates/model/src/arch/names.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-q38-serve-rules gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-lib gate-gpu-glm5next-nvtier gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-qwen3moe-e2e gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-iq-sel gate-gpu-ds41-moe gate-gpu-moe gate-gpu-p8 gate-gpu-block gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-router gate-placement gate-qwen35moe-meta gate-gpu-clef-hidden gate-gpu-qwen35-dspark gate-gpu-qwen35moe-moe gate-gpu-gemm gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-experts gate-ds41-host gate-fixture gate-union gate-gpu-ds41-comp gate-gpu-head gate-gpu-mcol gate-ds41-bind gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-place gate-gpu-ds41-chain-glue gate-gpu-ds41-engram gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-twocard gate-gpu-ds41-draft gate-gpu-ds41-long gate-gpu-ds41-nvtier gate-gpu-ds41-dspark-loop gate-gpu-ds41-media gate-gpu-ds41-prefill gate-ds41-load gate-gpu-ds41-chat gate-gpu-p8b gate-dspark-read gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-qwen35moe-mtp gate-app gate-gpu-nemotron-e2e gate-gpu-nemotron-serve the routed stacks' common names (routed_stacks): the GLM, Qwen3.6 and Qwen3.8 stack tables (swap.rs, qwen3moe/swap.rs for swap35.rs and swap38.rs) and the GLM card check (glm5next_card.rs) call it — every gate that opens those bodies or runs that check +crates/model/src/arch/mod.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-ptx-spill gate-dspark-read gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-vision gate-deepseek4-meta gate-glm5next-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-q38-serve-rules gate-gpu-xstream gate-gpu-mimo2-flash gate-gpu-nvtier gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve the dispatch and Arch: every gate that links it, the gguf-only kernel gates too — bloomery-gpu's fault sites (fault.rs) and bloomery-gpu-gates' family check (lib.rs, the oracle) read Arch in them +crates/gpu/src/weights.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq-sel gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-qwen35moe-attn gate-gpu-glm5next-gemm gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-app gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-ds41-place gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-ptx-spill gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-q38-serve-rules gate-gpu-nvtier gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve the card weight loaders: the gates whose own targets name weights, or reach it through an open path (gpu's arch bodies -> model.rs -> host/swap_source.rs, gpu-deepseek41's body/chain/dense/experts/draft, gpu-glm5next's body/ffn; gate-gpu-xstream names GpuModel), less the synthetic kernel gates whose bins build blocks and name no weights path # crates/gpu's per-arch bodies: GpuModel is generic; a body's impls run where B is that body. -crates/gpu/src/arch/qwen3moe/** gate-app gate-ds41-bind gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-e2e gate-gpu-gemm gate-gpu-glm5next-serve gate-gpu-lev-serve gate-gpu-nvtier gate-gpu-qwen3-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-qwen35moe-media gate-gpu-qwen35moe-moe gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-head gate-gpu-qwen3moe-router gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-q38-serve-rules gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp the Qwen3, Qwen3.5 and Qwen4 bodies: the bins that name them (the qwen gates, gate_gemm) and AnyEngine (gate_e2e; bind's tests build it under deepseek41) -crates/gpu/src/site.rs gate-ds41-bind gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-ds41-lib gate-gpu-e2e gate-gpu-gemm gate-gpu-glm5next-gemm gate-gpu-glm5next-e2e gate-gpu-glm5next-mtp gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-glm5next-stagger gate-gpu-glm5next-nvtier gate-gpu-glm5next-twocard gate-gpu-lev-serve gate-gpu-nvtier gate-gpu-qwen3-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-qwen35moe-media gate-gpu-qwen35moe-moe gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-head gate-gpu-qwen3moe-router gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-q38-serve-rules gate-gpu-p1 gate-gpu-qwen35moe-mtp the weight sites' launches by type: read by the Qwen bodies and the Clef hidden gate, the bodies that read a site in more than one type +crates/gpu/src/arch/qwen3moe/** gate-app gate-ds41-bind gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-e2e gate-gpu-gemm gate-gpu-glm5next-serve gate-gpu-lev-serve gate-gpu-nvtier gate-gpu-qwen3-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-qwen35moe-media gate-gpu-qwen35moe-moe gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-head gate-gpu-qwen3moe-router gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-q38-serve-rules gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve the Qwen3, Qwen3.5 and Qwen4 bodies: the bins that name them (the qwen gates, gate_gemm) and AnyEngine (gate_e2e; bind's tests build it under deepseek41) +crates/gpu/src/site.rs gate-ds41-bind gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-ds41-lib gate-gpu-e2e gate-gpu-gemm gate-gpu-glm5next-gemm gate-gpu-glm5next-e2e gate-gpu-glm5next-mtp gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-glm5next-stagger gate-gpu-glm5next-nvtier gate-gpu-glm5next-twocard gate-gpu-lev-serve gate-gpu-nvtier gate-gpu-qwen3-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-qwen35moe-media gate-gpu-qwen35moe-moe gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-head gate-gpu-qwen3moe-router gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-q38-serve-rules gate-gpu-p1 gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve the weight sites' launches by type: read by the Qwen bodies and the Clef hidden gate, the bodies that read a site in more than one type crates/gpu/src/arch/deepseek2/** gate-ds41-bind gate-gpu-e2e gate-gpu-hybrid gate-gpu-moe gate-gpu-p0b gate-gpu-p10 gate-gpu-p4 gate-gpu-p8 gate-gpu-p8b the V2-Lite body: the V2-Lite GPU gates and AnyEngine (gate_e2e) # Derived rows: each row's recipes are `python3 tools/recipes.py reach --row ` at the time of writing, the # gates whose targets can run code of the files through another lib (tools/recipes.py, Reach); files grouped only # where the derivation gives each the same gates. `just check-recipes` holds each to the derivation. -crates/gpu/src/{host/mod.rs,host/swap.rs,lib.rs} gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-app gate-gpu-head gate-gpu-mcol gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-vision gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 the Gpu and GpuError, whose Poison and Unresident variants hold host::Poison and host::swap::Unresident: every gate whose code names a bloomery_gpu error or the Gpu -crates/gpu/src/lane.rs gate-gpu-clef-serve gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-glm5next-serve gate-gpu-glm5next-twocard gate-gpu-lev-serve gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3-serve gate-q38-serve-rules gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 the lane probe a load prints beside its load line (probe_card, record::lane): the seats and generators that call it -crates/gpu/src/{host/lane.rs,watchdog.rs,flash_gqa.rs,elem.rs} gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-app gate-gpu-head gate-gpu-mcol gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 the Gpu's own parts: watchdog's wait_done and the swap lane's Shared, flash's seg_span and ElemKernels, which lib.rs's Gpu and host/mod.rs hold -crates/gpu/src/model.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-head gate-gpu-mcol gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp GpuModel, the ChainBody and Rollback traits and StepMode: the gates that open a model body or name its step mode -crates/gpu/src/host/swap_source.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp ResidencyGlue (host/served.rs) and DEADLINE (the qwen3moe bodies' swap38): the bodies that serve a residency -crates/gpu/src/model/slots.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-ds41-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp GpuModel's slot impls (slots, selected) and SlotRows: the gates that open a body with slots -crates/gpu/src/{host/nvtier.rs,host/nvtier/reader.rs,host/step.rs} gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp the host tier's step (hybrid::Chain in model.rs) and the NvTier arena (host/mod.rs): the gates that open a body with a host tier -crates/gpu/src/host/run.rs gate-gpu-e2e gate-gpu-ds41-chain-ffn gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-gpu-ds41-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp HostRun, the host expert leg: the V4.1, Qwen3.8 and GLM bodies that hold one -crates/gpu/src/host/xstream.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp XStream, the host tier's cross-stream lane (host/mod.rs), and XMode in the qwen38 bins -crates/gpu/src/host/prompt_stream.rs gate-gpu-clef-serve gate-gpu-glm5next-serve gate-gpu-lev-serve gate-gpu-nvtier gate-q38-serve-rules gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-qwen3moe-e2e gate-gpu-clef-hidden gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp the prompt call's stream state and its calls (host/mod.rs), reached through the bodies that hold it -crates/gpu/src/host/batch.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-linear gate-gpu-qwen3moe-e2e gate-app gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp BatchPort, the host tier's prompt batch (host/mod.rs), and the legs' ServeTimes -crates/gpu/src/head.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-head gate-gpu-mcol gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-chain-glue gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp Head and HeadNorm, which GpuModel holds (model.rs): the gates that open a model body -crates/gpu/src/rope_table.rs gate-gpu-e2e gate-gpu-qwen3moe-rope gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen35moe-attn gate-gpu-mimo2-flash gate-gpu-qwen3moe-e2e gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-q38-serve-rules gate-gpu-ds41-chain-glue gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 RopeTable (V4.1's params) and RopeRows (the qwen3moe bodies' scratch) -crates/gpu/src/media.rs gate-ds41-load gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-ds41-callstream gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-lib gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-prefill gate-gpu-ds41-residency-a gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-e2e gate-gpu-glm5next-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3moe-e2e gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-e2e gate-gpu-glm5next-residency gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-ds41-bind gate-ds41-place gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-qwen4exp-mtp gate-app gate-q38-serve-rules gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-nvtier gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp MediaSpan/ImageSpan/SeqRope and the MediaBody trait: the V4.1 and Qwen3.6/Clef media calls and the sessions that carry them -crates/gpu/src/iq_sel.rs gate-gpu-iq-sel gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-q38-serve-rules gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-mtp gate-gpu-glm5next-residency gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs IqSelKernels, the Qwen3.8 card's IQ selection (card38.rs), the common card leg's i-quant arms (card_leg.rs, which GLM's body and mimo2's reach), and its own gate -crates/gpu/src/checkpoint/{saved,staging}.rs gate-ds41-load gate-gpu-clef-serve gate-gpu-ds41-callstream gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-lib gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-prefill gate-gpu-ds41-residency-a gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-glm5next-e2e gate-gpu-glm5next-mtp gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-lev-serve gate-gpu-nvtier gate-gpu-qwen3-serve gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-q38-serve-rules gate-ds41-bind gate-ds41-place gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs the host copy of a sequence state (SeqState, the pinned staging pool) under the Qwen3.8 and GLM bodies' seq_save and seq_resume, the V4.1 serve seat and the gates that open those bodies -crates/gpu/src/checkpoint.rs gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-mtp gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-lev-serve gate-gpu-nvtier gate-gpu-qwen3-serve gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3moe-e2e gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-q38-serve-rules gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp the checkpoint ring (Checkpoints: the points a prompt call keeps, adopt, take, the save lane's point copy and its in-flight refusal) under the Qwen3.8 and GLM bodies and the gates that open them -crates/gpu/src/tensor.rs gate-app gate-ds41-bind gate-ds41-load gate-ds41-place gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-ds41-attn gate-gpu-ds41-callstream gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-comp gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-lib gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-moe gate-gpu-ds41-prefill gate-gpu-ds41-residency-a gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-woa gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-e2e gate-gpu-gemm gate-gpu-glm-mla gate-gpu-glm-router gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-gemm gate-gpu-glm5next-mtp gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-hc-gated gate-gpu-head gate-gpu-hybrid gate-gpu-iq-sel gate-gpu-kquant gate-gpu-lev-serve gate-gpu-mcol gate-gpu-moe gate-gpu-nvtier gate-gpu-p0b gate-gpu-p1 gate-gpu-p10 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p8 gate-gpu-p8b gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-q6k-ids gate-gpu-qwen3-serve gate-gpu-qwen35moe-attn gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35moe-moe gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-head gate-gpu-qwen3moe-router gate-gpu-qwen3vl-tower gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-swap gate-gpu-vision gate-q38-serve-rules gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 the device tensor and the 2-D copies (dtoh_2d, htod_2d) every gate that links the gpu crate through a body reads -crates/gpu-gates/src/lib.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-gpu-head gate-gpu-mcol gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-q38-serve-rules gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-dspark-read gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-vision gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 GateError and the family check (the oracle), which the bins' shared modules name -crates/gpu-gates/src/tier.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-gpu-head gate-gpu-mcol gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-q38-serve-rules gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-dspark-read gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-minicpm-text gate-gpu-qwen35moe-attn gate-gpu-qwen35moe-mtp Tier::from_env (lib.rs's family check) and tier::card, which shared/gate_card.rs runs in the card bins -crates/gpu-gates/src/residency.rs gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3moe-e2e gate-q38-serve-rules gate-gpu-nvtier gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-chat weekly-gpu-mimo2-e2e weekly-gpu-mimo2-serve gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 the unset residency rules' plan inputs (PlanRoom) and the pair rule's call (at_plan): the qwen38, GLM, V4.1 and MiMo residency, serve and placement bins -crates/gpu-gates/src/residency38.rs gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3moe-e2e gate-q38-serve-rules gate-gpu-nvtier gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-chat gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text the Qwen3.8 residency figures (checkpoint_bytes, CARD38, glm_seqs): the qwen38 and GLM residency, serve and placement bins +crates/gpu/src/{host/mod.rs,host/swap.rs,lib.rs} gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-app gate-gpu-head gate-gpu-mcol gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-vision gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve the Gpu and GpuError, whose Poison and Unresident variants hold host::Poison and host::swap::Unresident: every gate whose code names a bloomery_gpu error or the Gpu +crates/gpu/src/lane.rs gate-gpu-clef-serve gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-glm5next-serve gate-gpu-glm5next-twocard gate-gpu-lev-serve gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3-serve gate-q38-serve-rules gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-nemotron-serve the lane probe a load prints beside its load line (probe_card, record::lane): the seats and generators that call it +crates/gpu/src/{host/lane.rs,watchdog.rs,flash_gqa.rs,elem.rs} gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-app gate-gpu-head gate-gpu-mcol gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve the Gpu's own parts: watchdog's wait_done and the swap lane's Shared, flash's seg_span and ElemKernels, which lib.rs's Gpu and host/mod.rs hold +crates/gpu/src/model.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-head gate-gpu-mcol gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve GpuModel, the ChainBody and Rollback traits and StepMode: the gates that open a model body or name its step mode +crates/gpu/src/host/swap_source.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve ResidencyGlue (host/served.rs) and DEADLINE (the qwen3moe bodies' swap38): the bodies that serve a residency +crates/gpu/src/model/slots.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-ds41-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve GpuModel's slot impls (slots, selected) and SlotRows: the gates that open a body with slots +crates/gpu/src/{host/nvtier.rs,host/nvtier/reader.rs,host/step.rs} gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve the host tier's step (hybrid::Chain in model.rs) and the NvTier arena (host/mod.rs): the gates that open a body with a host tier +crates/gpu/src/host/run.rs gate-gpu-e2e gate-gpu-ds41-chain-ffn gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-gpu-ds41-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve HostRun, the host expert leg: the V4.1, Qwen3.8 and GLM bodies that hold one +crates/gpu/src/host/xstream.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve XStream, the host tier's cross-stream lane (host/mod.rs), and XMode in the qwen38 bins +crates/gpu/src/host/prompt_stream.rs gate-gpu-clef-serve gate-gpu-glm5next-serve gate-gpu-lev-serve gate-gpu-nvtier gate-q38-serve-rules gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-qwen3moe-e2e gate-gpu-clef-hidden gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve the prompt call's stream state and its calls (host/mod.rs), reached through the bodies that hold it +crates/gpu/src/host/batch.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-linear gate-gpu-qwen3moe-e2e gate-app gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve BatchPort, the host tier's prompt batch (host/mod.rs), and the legs' ServeTimes +crates/gpu/src/head.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-head gate-gpu-mcol gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-chain-glue gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve Head and HeadNorm, which GpuModel holds (model.rs): the gates that open a model body +crates/gpu/src/rope_table.rs gate-gpu-e2e gate-gpu-qwen3moe-rope gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen35moe-attn gate-gpu-mimo2-flash gate-gpu-qwen3moe-e2e gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-q38-serve-rules gate-gpu-ds41-chain-glue gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-serve RopeTable (V4.1's params) and RopeRows (the qwen3moe bodies' scratch) +crates/gpu/src/media.rs gate-ds41-load gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-ds41-callstream gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-lib gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-prefill gate-gpu-ds41-residency-a gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-e2e gate-gpu-glm5next-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3moe-e2e gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-e2e gate-gpu-glm5next-residency gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-ds41-bind gate-ds41-place gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-qwen4exp-mtp gate-app gate-q38-serve-rules gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-nvtier gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve MediaSpan/ImageSpan/SeqRope and the MediaBody trait: the V4.1 and Qwen3.6/Clef media calls and the sessions that carry them +crates/gpu/src/iq_sel.rs gate-gpu-iq-sel gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-q38-serve-rules gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-mtp gate-gpu-glm5next-residency gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-nemotron-serve IqSelKernels, the Qwen3.8 card's IQ selection (card38.rs), the common card leg's i-quant arms (card_leg.rs, which GLM's body and mimo2's reach), and its own gate +crates/gpu/src/checkpoint/{saved,staging}.rs gate-ds41-load gate-gpu-clef-serve gate-gpu-ds41-callstream gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-lib gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-prefill gate-gpu-ds41-residency-a gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-glm5next-e2e gate-gpu-glm5next-mtp gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-lev-serve gate-gpu-nvtier gate-gpu-qwen3-serve gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-q38-serve-rules gate-ds41-bind gate-ds41-place gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-nemotron-serve the host copy of a sequence state (SeqState, the pinned staging pool) under the Qwen3.8 and GLM bodies' seq_save and seq_resume, the V4.1 serve seat and the gates that open those bodies +crates/gpu/src/checkpoint.rs gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-mtp gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-lev-serve gate-gpu-nvtier gate-gpu-qwen3-serve gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3moe-e2e gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-q38-serve-rules gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve the checkpoint ring (Checkpoints: the points a prompt call keeps, adopt, take, the save lane's point copy and its in-flight refusal) under the Qwen3.8 and GLM bodies and the gates that open them +crates/gpu/src/tensor.rs gate-app gate-ds41-bind gate-ds41-load gate-ds41-place gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-ds41-attn gate-gpu-ds41-callstream gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-comp gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-lib gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-moe gate-gpu-ds41-prefill gate-gpu-ds41-residency-a gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-woa gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-e2e gate-gpu-gemm gate-gpu-glm-mla gate-gpu-glm-router gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-gemm gate-gpu-glm5next-mtp gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-hc-gated gate-gpu-head gate-gpu-hybrid gate-gpu-iq-sel gate-gpu-kquant gate-gpu-lev-serve gate-gpu-mcol gate-gpu-moe gate-gpu-nvtier gate-gpu-p0b gate-gpu-p1 gate-gpu-p10 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p8 gate-gpu-p8b gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-q6k-ids gate-gpu-qwen3-serve gate-gpu-qwen35moe-attn gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35moe-moe gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-head gate-gpu-qwen3moe-router gate-gpu-qwen3vl-tower gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-swap gate-gpu-vision gate-q38-serve-rules gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve the device tensor and the 2-D copies (dtoh_2d, htod_2d) every gate that links the gpu crate through a body reads +crates/gpu-gates/src/lib.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-gpu-head gate-gpu-mcol gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-q38-serve-rules gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-dspark-read gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-vision gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve GateError and the family check (the oracle), which the bins' shared modules name +crates/gpu-gates/src/tier.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-gpu-head gate-gpu-mcol gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-q38-serve-rules gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-dspark-read gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-minicpm-text gate-gpu-qwen35moe-attn gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve Tier::from_env (lib.rs's family check) and tier::card, which shared/gate_card.rs runs in the card bins +crates/gpu-gates/src/residency.rs gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3moe-e2e gate-q38-serve-rules gate-gpu-nvtier gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-chat weekly-gpu-mimo2-e2e weekly-gpu-mimo2-serve gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-nemotron-serve the unset residency rules' plan inputs (PlanRoom) and the pair rule's call (at_plan): the qwen38, GLM, V4.1 and MiMo residency, serve and placement bins +crates/gpu-gates/src/residency38.rs gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3moe-e2e gate-q38-serve-rules gate-gpu-nvtier gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-chat gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-nemotron-serve the Qwen3.8 residency figures (checkpoint_bytes, CARD38, glm_seqs): the qwen38 and GLM residency, serve and placement bins crates/gpu-gates/src/q38_fixture.rs gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen38-twocard the Qwen3.8 fixture's Shape, which shared/q38_arch.rs reads crates/gpu-gates/src/q36_fixture.rs gate-gpu-qwen35moe-e2e gate-gpu-minicpm-text the Qwen3.6 body's per-layer launch counts from the header, which gate_qwen35moe_e2e reads, and which the MiniCPM text recipe runs through that bin -crates/gpu-gates/src/act_rule.rs gate-gpu-iq gate-gpu-ds41-hc gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-minicpm-text gate-gpu-ds41-index gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-chain-attn gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-gpu-ds41-step gate-gpu-dspark-experts gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-qwen35moe-mtp the activation rules: the MoE, chain, index, comp, hc, iq, gemm and dspark experts bins that name them, and the fixture bands' readers of site_rel (shared/glm5next_tier.rs, the qwen4exp e2e and MTP bins, ds41-step's head; the qwen3moe experts, down and e2e bins and the qwen35moe e2e bin through ik_q8_2's non-finite check) -crates/gpu-gates/src/flip.rs gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-ds41-step gate-gpu-glm5next-mtp gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp Flip, which shared/e2e.rs, the qwen4exp MTP bin, ds41-step's head cap and the GLM MTP replay's margin cap read -crates/model/src/lib.rs gate-gpu-p4 gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp ModelError, which every model reader returns -crates/model/src/moe.rs gate-gpu-p4 gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp TierError in ModelError (lib.rs), and the host expert legs the bodies and tiers call -crates/model/src/ops.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-app gate-prompts gate-gpu-lib gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp DEFER_MAX_COLS (gpu's host/step.rs and host/tier.rs), Tensor2 and the matmul dispatch -crates/model/src/embed.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-ds41-lib gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen3-serve gate-ds41-load gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs RowLayout and EmbedTable: the V4.1 glue, the draft and the GLM body -crates/model/src/placement/workstation.rs gate-gpu-iq-sel gate-placement gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-fixture gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-app gate-gpu-ds41-chain-ffn gate-gpu-ds41-lib gate-gpu-e2e gate-gpu-hybrid gate-gpu-moe gate-gpu-p10 gate-gpu-p8 gate-gpu-p8b gate-gpu-swap gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp host_available, which GpuModel's placement (gpu's model.rs) and the qwen35moe plan read +crates/gpu-gates/src/act_rule.rs gate-gpu-iq gate-gpu-ds41-hc gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-minicpm-text gate-gpu-ds41-index gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-chain-attn gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-gpu-ds41-step gate-gpu-dspark-experts gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e the activation rules: the MoE, chain, index, comp, hc, iq, gemm and dspark experts bins that name them, and the fixture bands' readers of site_rel (shared/glm5next_tier.rs, the qwen4exp e2e and MTP bins, ds41-step's head; the qwen3moe experts, down and e2e bins and the qwen35moe e2e bin through ik_q8_2's non-finite check) +crates/gpu-gates/src/flip.rs gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-ds41-step gate-gpu-glm5next-mtp gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e Flip, which shared/e2e.rs, the qwen4exp MTP bin, ds41-step's head cap and the GLM MTP replay's margin cap read +crates/model/src/lib.rs gate-gpu-p4 gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve ModelError, which every model reader returns +crates/model/src/moe.rs gate-gpu-p4 gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve TierError in ModelError (lib.rs), and the host expert legs the bodies and tiers call +crates/model/src/ops.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-app gate-prompts gate-gpu-lib gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve DEFER_MAX_COLS (gpu's host/step.rs and host/tier.rs), Tensor2 and the matmul dispatch +crates/model/src/embed.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-ds41-lib gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen3-serve gate-ds41-load gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-nemotron-serve RowLayout and EmbedTable: the V4.1 glue, the draft and the GLM body +crates/model/src/placement/workstation.rs gate-gpu-iq-sel gate-placement gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-fixture gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-app gate-gpu-ds41-chain-ffn gate-gpu-ds41-lib gate-gpu-e2e gate-gpu-hybrid gate-gpu-moe gate-gpu-p10 gate-gpu-p8 gate-gpu-p8b gate-gpu-swap gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve host_available, which GpuModel's placement (gpu's model.rs) and the qwen35moe plan read crates/model/src/arch/qwen35moe/fixture.rs gate-fixture the Qwen3.8 fixture spec, which the fixture test reads -crates/model/data/mtp-head-rows-qwen38-*.txt gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-mtp gate-q38-serve-rules gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 Qwen3.8's shipped MTP head list, which arch/qwen35moe/head_list.rs includes (include_str!): the gates whose targets run its reader (head_rows_of, SHIPPED) +crates/model/data/mtp-head-rows-qwen38-*.txt gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-mtp gate-q38-serve-rules gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-nemotron-serve Qwen3.8's shipped MTP head list, which arch/qwen35moe/head_list.rs includes (include_str!): the gates whose targets run its reader (head_rows_of, SHIPPED) tools/fixture-variants.tsv gate-fixture the fixture variants' table, which fixture/variant.rs reads by path at run time: the fixture test -crates/models/src/lib.rs gate-gpu-p4 gate-gpu-p6 gate-gpu-iq-sel gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-runtime gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-dspark-read gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-nemotron-meta gate-gpu-qwen35moe-attn gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp its re-exports (Role, Need: every placement path names models::Role through them) and ModelSpec and Score, which the arch readers and models::shape read -crates/models/src/shape.rs gate-gpu-p6 gate-gpu-iq-sel gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-e2e gate-fixture gate-gpu-head gate-gpu-lib gate-ds41-plan gate-gpu-ds41-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-ds41-bind gate-ds41-place gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-placement gate-nemotron-meta gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp MoeShape and the router rows (RouterInst, router_row) the routers' kernels select by -crates/gguf/src/lib.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-qdot gate-engram gate-tokenizer gate-decision-clef gate-decision-lev gate-serve gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-vision gate-gpu-vision gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-refset gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-models gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-nemotron-meta gate-gpu-mamba2 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 LoadError (in GpuError and placement's errors) and Gguf: every gate that reads a file or names a load error -crates/gguf/src/quant.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-qdot gate-engram gate-tokenizer gate-decision-clef gate-decision-lev gate-serve gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-refset gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-vision gate-gpu-vision gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-dspark-read gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-models gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-hf gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-nemotron-meta gate-gpu-mamba2 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 GgmlType (gguf's lib.rs, placement) and the f16 conversion the card's fused kernels name; gate-dspark-read also runs gguf's own quant tests -crates/gguf/src/iq_tables.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-kquant gate-gpu-hc-gated gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-ds41-meta gate-qwen3moe-meta gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-gemm gate-gpu-glm5next-gemm gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-qdot gate-decision-clef gate-decision-lev gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-nemotron-meta gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp KVALUES_IQ4NL, which quant.rs's dequant and the card's IQ kernels read -crates/qdot/src/lib.rs gate-gpu-p4 gate-gpu-p6 gate-gpu-kquant gate-gpu-iq-sel gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen35moe-attn gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-qdot gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-nemotron-meta gate-gpu-mamba2 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp QdotError in ModelError, and the fused dots coverage and quant ask for -crates/threads/src/lib.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-ds41-meta gate-qwen3moe-meta gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-glm5next-gemm gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-threads gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-ds41-plan gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-nemotron-meta gate-gpu-p1 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp the pool: upload's memcpy_pool, ops' matmul and the host tier's legs -crates/engram/src/lib.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-lib gate-gpu-gates-lib gate-ds41-plan gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp EngramError (direct.rs) and Engram, which the V4.1 chain's glue holds -crates/models/src/headpick.rs gate-deepseek4-meta gate-ds41-meta gate-ds41-plan gate-fixture gate-glm5next-meta gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-twocard gate-gpu-glm5next-serve gate-gpu-iq-sel gate-gpu-lev-serve gate-gpu-nvtier gate-gpu-qwen3-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35moe-media gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3moe-e2e gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-mimo2-meta gate-ops gate-q38-serve-rules gate-qwen3moe-meta gate-qwen4exp-meta gate-runtime gate-ds41-bind gate-ds41-place gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35-dspark gate-gpu-gemm gate-gpu-head gate-gpu-lib gate-gpu-ds41-lib gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-nvtier gate-gpu-ds41-long gate-gpu-ds41-prefill gate-gpu-ds41-media gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-nemotron-meta gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp models::headpick, the row-list head's reader, digest and loaded set: every gate whose load reads a head list gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35-dspark gate-gpu-gemm gate-gpu-head gate-gpu-lib gate-gpu-ds41-lib gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-nvtier gate-gpu-ds41-long gate-gpu-ds41-prefill gate-gpu-ds41-media gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-twocard gate-gpu-glm5next-residency -crates/engram/src/rows.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gates-lib gate-q38-serve-rules gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 SplitTable, which the Qwen3.8 body opens -crates/hf/src/fetch.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-fixture gate-ds41-bind gate-ds41-place gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media the fetch Event that model_file.rs reports; the fixture fetch's range reads -crates/decision/src/lib.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-decision-lev gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 the decision crate's Error: the decision gates -crates/vision/src/lib.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-serve gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-vision gate-gpu-glm5next-serve gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-decision-clef gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-ds41-media gate-gpu-routed-legs VisionError, which the media and preprocess paths return -crates/gpu/src/gelu.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-mamba2 gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-gpu-clef-hidden gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-app gate-gpu-head gate-gpu-mcol gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-nvtier gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-vision gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-glm5v-tower gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 the shared tanh GELU rule: the registered elementwise module and image towers that use it -crates/gpu-vision/src/{aligner,attn,attn_group,chain,encoder,gelu_tanh,gemm_bf16,glm5v,layer_norm,mean_groups,mlp,norm,pos_bilinear,qwen3vl,rope2d}.rs gate-gpu-clef-serve gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-glm5v-tower gate-gpu-qwen3-serve gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-gpu-vision gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-lev-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media the image towers' card kernels: the tower, serve and media-seat gates that run them -crates/gpu-vision/src/lib.rs gate-gpu-clef-serve gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-glm5v-tower gate-gpu-qwen3-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-gpu-vision gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-lev-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media the tower crate's root, which the Qwen3.5 media gates also link -crates/vision/src/arch/glm5v/{card,hparams,mod,names,tensors}.rs gate-ds41-bind gate-ds41-place gate-gpu-clef-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-glm5next-stagger gate-gpu-glm5v-tower gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-gpu-vision gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 the GLM-5V family's hparams, tensor names and card homes -crates/vision/src/arch/glm5v/{media,size}.rs gate-ds41-bind gate-ds41-place gate-gpu-clef-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-glm5next-stagger gate-gpu-glm5v-tower gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 the GLM-5V media span and size rule the serve seats plan with -crates/vision/src/arch/glm5v/rules.rs gate-gpu-clef-serve gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-glm5v-tower gate-gpu-qwen3-serve gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-gpu-vision gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-lev-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media the GLM-5V preprocess rules the towers read -crates/vision/src/{arch/deepseek41v/mod.rs,arch/deepseek41v/tensors.rs} gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-vision gate-gpu-glm5next-serve gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-ds41-bind gate-ds41-place gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-ds41-media the V4.1 vision family's constants and its tensors' card homes -crates/vision/src/{image,preprocess}.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-decision-clef gate-serve gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-media gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-vision gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-gpu-glm5v-tower gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs the decoded image and its patch cut, which every image seat's prepare runs -crates/vision/src/resample.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-decision-clef gate-ds41-bind gate-ds41-place gate-gpu-ds41-media gate-gpu-qwen3-serve gate-gpu-vision gate-gpu-qwen3vl-tower gate-gpu-glm5v-tower gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-minicpmv-tower the resize and pad an image seat's prepare runs -crates/vision/src/arch/deepseek41v/grid.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-ds41-bind gate-ds41-place gate-gpu-ds41-media gate-gpu-qwen3-serve gate-gpu-vision gate-gpu-qwen3vl-tower gate-gpu-glm5v-tower gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-minicpmv-tower V4.1's grid plan, reached through the projector's arch dispatch -crates/vision/src/mrope/mod.rs gate-gpu-e2e gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-types gate-app gate-gpu-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-media gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-minicpmv-tower gate-gpu-minicpm-text gate-decision-clef gate-serve gate-gpu-qwen35moe-mtp the M-RoPE module's error and the public paths of its positions and rows -crates/vision/src/mrope/{seq,rows}.rs gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-types gate-gpu-lib gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp the M-RoPE positions of a sequence's images and the table rows they rewrite +crates/models/src/lib.rs gate-gpu-p4 gate-gpu-p6 gate-gpu-iq-sel gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-runtime gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-dspark-read gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-nemotron-meta gate-gpu-qwen35moe-attn gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve its re-exports (Role, Need: every placement path names models::Role through them) and ModelSpec and Score, which the arch readers and models::shape read +crates/models/src/shape.rs gate-gpu-p6 gate-gpu-iq-sel gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-e2e gate-fixture gate-gpu-head gate-gpu-lib gate-ds41-plan gate-gpu-ds41-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-ds41-bind gate-ds41-place gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-placement gate-nemotron-meta gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve MoeShape and the router rows (RouterInst, router_row) the routers' kernels select by +crates/gguf/src/lib.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-qdot gate-engram gate-tokenizer gate-decision-clef gate-decision-lev gate-serve gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-vision gate-gpu-vision gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-refset gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-models gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-nemotron-meta gate-gpu-mamba2 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve LoadError (in GpuError and placement's errors) and Gguf: every gate that reads a file or names a load error +crates/gguf/src/quant.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-qdot gate-engram gate-tokenizer gate-decision-clef gate-decision-lev gate-serve gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-refset gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-vision gate-gpu-vision gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-dspark-read gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-models gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-hf gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-nemotron-meta gate-gpu-mamba2 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve GgmlType (gguf's lib.rs, placement) and the f16 conversion the card's fused kernels name; gate-dspark-read also runs gguf's own quant tests +crates/gguf/src/iq_tables.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-kquant gate-gpu-hc-gated gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-ds41-meta gate-qwen3moe-meta gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-gemm gate-gpu-glm5next-gemm gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-qdot gate-decision-clef gate-decision-lev gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-nemotron-meta gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve KVALUES_IQ4NL, which quant.rs's dequant and the card's IQ kernels read +crates/qdot/src/lib.rs gate-gpu-p4 gate-gpu-p6 gate-gpu-kquant gate-gpu-iq-sel gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen35moe-attn gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-qdot gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-nemotron-meta gate-gpu-mamba2 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve QdotError in ModelError, and the fused dots coverage and quant ask for +crates/threads/src/lib.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-ds41-meta gate-qwen3moe-meta gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-glm5next-gemm gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-threads gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-ds41-plan gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-nemotron-meta gate-gpu-p1 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve the pool: upload's memcpy_pool, ops' matmul and the host tier's legs +crates/engram/src/lib.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-lib gate-gpu-gates-lib gate-ds41-plan gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve EngramError (direct.rs) and Engram, which the V4.1 chain's glue holds +crates/models/src/headpick.rs gate-deepseek4-meta gate-ds41-meta gate-ds41-plan gate-fixture gate-glm5next-meta gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-twocard gate-gpu-glm5next-serve gate-gpu-iq-sel gate-gpu-lev-serve gate-gpu-nvtier gate-gpu-qwen3-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35moe-media gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3moe-e2e gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-mimo2-meta gate-ops gate-q38-serve-rules gate-qwen3moe-meta gate-qwen4exp-meta gate-runtime gate-ds41-bind gate-ds41-place gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35-dspark gate-gpu-gemm gate-gpu-head gate-gpu-lib gate-gpu-ds41-lib gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-nvtier gate-gpu-ds41-long gate-gpu-ds41-prefill gate-gpu-ds41-media gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-nemotron-meta gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve models::headpick, the row-list head's reader, digest and loaded set: every gate whose load reads a head list gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35-dspark gate-gpu-gemm gate-gpu-head gate-gpu-lib gate-gpu-ds41-lib gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-nvtier gate-gpu-ds41-long gate-gpu-ds41-prefill gate-gpu-ds41-media gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-twocard gate-gpu-glm5next-residency +crates/engram/src/rows.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gates-lib gate-q38-serve-rules gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-nemotron-serve SplitTable, which the Qwen3.8 body opens +crates/hf/src/fetch.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-fixture gate-ds41-bind gate-ds41-place gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-nemotron-serve the fetch Event that model_file.rs reports; the fixture fetch's range reads +crates/decision/src/lib.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-decision-lev gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-nemotron-serve the decision crate's Error: the decision gates +crates/vision/src/lib.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-serve gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-vision gate-gpu-glm5next-serve gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-decision-clef gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-ds41-media gate-gpu-routed-legs gate-gpu-nemotron-serve VisionError, which the media and preprocess paths return +crates/gpu/src/gelu.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-mamba2 gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-gpu-clef-hidden gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-app gate-gpu-head gate-gpu-mcol gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-nvtier gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-vision gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-glm5v-tower gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve the shared tanh GELU rule: the registered elementwise module and image towers that use it +crates/gpu-vision/src/{aligner,attn,attn_group,chain,encoder,gelu_tanh,gemm_bf16,glm5v,layer_norm,mean_groups,mlp,norm,pos_bilinear,qwen3vl,rope2d}.rs gate-gpu-clef-serve gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-glm5v-tower gate-gpu-qwen3-serve gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-gpu-vision gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-lev-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-nemotron-serve the image towers' card kernels: the tower, serve and media-seat gates that run them +crates/gpu-vision/src/lib.rs gate-gpu-clef-serve gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-glm5v-tower gate-gpu-qwen3-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-gpu-vision gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-lev-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-nemotron-serve the tower crate's root, which the Qwen3.5 media gates also link +crates/vision/src/arch/glm5v/{card,hparams,mod,names,tensors}.rs gate-ds41-bind gate-ds41-place gate-gpu-clef-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-glm5next-stagger gate-gpu-glm5v-tower gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-gpu-vision gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-nemotron-serve the GLM-5V family's hparams, tensor names and card homes +crates/vision/src/arch/glm5v/{media,size}.rs gate-ds41-bind gate-ds41-place gate-gpu-clef-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-glm5next-stagger gate-gpu-glm5v-tower gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-nemotron-serve the GLM-5V media span and size rule the serve seats plan with +crates/vision/src/arch/glm5v/rules.rs gate-gpu-clef-serve gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-glm5v-tower gate-gpu-qwen3-serve gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-gpu-vision gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-lev-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-nemotron-serve the GLM-5V preprocess rules the towers read +crates/vision/src/{arch/deepseek41v/mod.rs,arch/deepseek41v/tensors.rs} gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-vision gate-gpu-glm5next-serve gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-ds41-bind gate-ds41-place gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-ds41-media gate-gpu-nemotron-serve the V4.1 vision family's constants and its tensors' card homes +crates/vision/src/{image,preprocess}.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-decision-clef gate-serve gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-media gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-vision gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-gpu-glm5v-tower gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-nemotron-serve the decoded image and its patch cut, which every image seat's prepare runs +crates/vision/src/resample.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-decision-clef gate-ds41-bind gate-ds41-place gate-gpu-ds41-media gate-gpu-qwen3-serve gate-gpu-vision gate-gpu-qwen3vl-tower gate-gpu-glm5v-tower gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-minicpmv-tower gate-gpu-nemotron-serve the resize and pad an image seat's prepare runs +crates/vision/src/arch/deepseek41v/grid.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-ds41-bind gate-ds41-place gate-gpu-ds41-media gate-gpu-qwen3-serve gate-gpu-vision gate-gpu-qwen3vl-tower gate-gpu-glm5v-tower gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-minicpmv-tower gate-gpu-nemotron-serve V4.1's grid plan, reached through the projector's arch dispatch +crates/vision/src/mrope/mod.rs gate-gpu-e2e gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-types gate-app gate-gpu-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-media gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-minicpmv-tower gate-gpu-minicpm-text gate-decision-clef gate-serve gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve the M-RoPE module's error and the public paths of its positions and rows +crates/vision/src/mrope/{seq,rows}.rs gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-types gate-gpu-lib gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve the M-RoPE positions of a sequence's images and the table rows they rewrite # The weekly tier's triggers (`just weekly` runs every weekly-* recipe). A weekly-* name in a row is a trigger: a # changed file the glob matches names that recipe in `just affected`, with or without --narrow, and a change of the @@ -213,4 +213,4 @@ crates/gpu/src/cand.rs weekly-gpu-ds41-cand the candidate mask's kernels, which crates/gpu-deepseek41/src/{indexer.rs,chain/attn.rs,chain/attn/batch.rs} weekly-gpu-ds41-cand the indexer's split launcher and the select path's candidate launches, per row of words in the step and per chunk in the prompt batch crates/gpu-deepseek41/src/{body.rs,body/ced.rs,body/prefill.rs} weekly-gpu-ds41-cand the decode and pair wiring, the triangle's candidate source, and the prompt batch's kept blocks per batch set crates/gpu-gates/src/bin/{gate_deepseek41_long.rs,gate_deepseek41_prefill.rs,shared/ds41_cand.rs} weekly-gpu-ds41-cand the weekly's two clauses and their candidate rule -crates/placement/src/** gate-app gate-deepseek4-meta gate-ds41-bind gate-ds41-host gate-ds41-kld gate-ds41-load gate-ds41-meta gate-ds41-oracle gate-ds41-plan gate-dspark-read gate-fixture gate-glm5next-meta gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-comp gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-index gate-gpu-ds41-lib gate-gpu-ds41-long gate-gpu-ds41-prefill gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-e2e gate-gpu-gates-lib gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-lib gate-gpu-mcol gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-qwen35moe-moe gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-head gate-gpu-qwen3moe-router gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-ops gate-placement gate-qwen35moe-meta gate-qwen3moe-meta gate-qwen4exp-meta gate-union gate-gpu-p4 gate-gpu-p6 gate-gpu-iq-sel gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-block gate-gpu-p10 gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-gpu-qwen3moe-rope gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-r8 gate-prompts gate-gpu-head gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-woa gate-gpu-ds41-media gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-p8b gate-mimo2-meta gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-nemotron-meta gate-gpu-qwen35moe-attn gate-gpu-p1 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp the placement planner, which bloomery-model re-exports as model::placement: every gate that plans a placement, reads a plan's figures or checks the r8 sidecar's refusal type +crates/placement/src/** gate-app gate-deepseek4-meta gate-ds41-bind gate-ds41-host gate-ds41-kld gate-ds41-load gate-ds41-meta gate-ds41-oracle gate-ds41-plan gate-dspark-read gate-fixture gate-glm5next-meta gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-comp gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-index gate-gpu-ds41-lib gate-gpu-ds41-long gate-gpu-ds41-prefill gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-e2e gate-gpu-gates-lib gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-lib gate-gpu-mcol gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-qwen35moe-moe gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-head gate-gpu-qwen3moe-router gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-ops gate-placement gate-qwen35moe-meta gate-qwen3moe-meta gate-qwen4exp-meta gate-union gate-gpu-p4 gate-gpu-p6 gate-gpu-iq-sel gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-block gate-gpu-p10 gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-gpu-qwen3moe-rope gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-r8 gate-prompts gate-gpu-head gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-woa gate-gpu-ds41-media gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-p8b gate-mimo2-meta gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-nemotron-meta gate-gpu-qwen35moe-attn gate-gpu-p1 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve the placement planner, which bloomery-model re-exports as model::placement: every gate that plans a placement, reads a plan's figures or checks the r8 sidecar's refusal type diff --git a/tools/hf-arch-check.py b/tools/hf-arch-check.py index 6a2df7e5e..267cf2455 100755 --- a/tools/hf-arch-check.py +++ b/tools/hf-arch-check.py @@ -100,10 +100,10 @@ def load_by_path(name, *rel): "qwen3": ("qwen3moe", "qwen35moe"), "mimo2": ("mimo2",), } -SERVES_VIA_FROM_NAME = {"glm": "Glm5next"} +SERVES_VIA_FROM_NAME = {"glm": "Glm5next", "nemotron": "NemotronH"} # crates/gpu-gates/src/bin/bloomery_serve.rs:Model::ALL and Model::word — the generative seats in the order # the refusal of an unserved file lists them; crates/serve/src/decide.rs:WORD is the decide seat's word. -SEAT_ORDER = ("ds41", "qwen38", "glm", "qwen3", "mimo2") +SEAT_ORDER = ("ds41", "qwen38", "glm", "qwen3", "mimo2", "nemotron") DECIDE_WORD = "decide" # crates/model/src/arch/mod.rs:Arch::from_name — variant -> the strings that name it. FROM_NAME = { @@ -1796,7 +1796,7 @@ def row_line(seat, arch, tensors=(), extra=(), vocab=0, quant="Q4_K_M", cards=No line, green, _ = row_line("glm", "glm5nextx", small) check("a glm5nextx file is refused by name", not green and "arch=glm5nextx route=REFUSED: a glm5nextx file, which no seat serves; the seats are " - "--model ds41, qwen38, glm, qwen3, mimo2, decide" in line, line) + "--model ds41, qwen38, glm, qwen3, mimo2, nemotron, decide" in line, line) line, green, _ = row_line("glm", "deepseek41", small) check("a file the router seats elsewhere is refused for the row's seat", not green and "in the ds41 seat, and the README row says glm" in line, line) diff --git a/tools/recipes.py b/tools/recipes.py index 4a7ededb0..7cdfc9ea2 100755 --- a/tools/recipes.py +++ b/tools/recipes.py @@ -10582,11 +10582,12 @@ def sel(f: str) -> set[str]: # a whole-string .rs literal of a workspace package is documentation, not a run-time read: levers' # registry names the in-place lever files, and only the targets that compile such a file select it dspark = sel("crates/gpu-gates/src/bin/shared/ds41_dspark.rs") - # the bloomery-serve builders: the four seat gates and the two serve-start gates (each builds the whole bin) + # the bloomery-serve builders: the five seat gates and the two serve-start gates (each builds the whole bin) + # PIN(2026-10-15): gate-gpu-nemotron-serve joins them; its recipe builds bloomery-serve beside its own bin expect(dspark == {"gate-gpu-clef-serve", "gate-gpu-ds41-chat", "gate-gpu-ds41-draft", "gate-gpu-ds41-dspark-loop", "gate-gpu-ds41-prefill", "gate-gpu-ds41-tier", "gate-gpu-ds41-twocard", "gate-gpu-glm5next-serve", - "gate-gpu-lev-serve", "gate-gpu-qwen3-serve", "gate-gpu-serve-start-ds41", "gate-gpu-serve-start-glm", - "gate-ptx-spill"}, + "gate-gpu-lev-serve", "gate-gpu-nemotron-serve", "gate-gpu-qwen3-serve", "gate-gpu-serve-start-ds41", + "gate-gpu-serve-start-glm", "gate-ptx-spill"}, f"shared/ds41_dspark.rs (a #[path] module of the bins that name it) selects {sorted(dspark)}") hybrid = sel("crates/gpu/src/hybrid.rs") expect("gate-gpu-e2e" in hybrid and "gate-ops" not in hybrid and len(hybrid) < len(gates) - 20, diff --git a/tools/ref/ptx-shapes.tsv b/tools/ref/ptx-shapes.tsv index 53a822135..e62bfa6b9 100644 --- a/tools/ref/ptx-shapes.tsv +++ b/tools/ref/ptx-shapes.tsv @@ -313,6 +313,7 @@ generate_ds41 qwen3moe_router_norm_m_prefetch 0 0 generate_ds41 qwen3moe_router_norm_prefetch 0 0 generate_ds41 qwen3moe_router_route 0 0 generate_ds41 qwen3moe_token_major 0 0 +generate_ds41 relu_sqr 0 0 generate_ds41 rms_norm 0 0 generate_ds41 rope 0 0 generate_ds41 router_topk 0 0 @@ -559,6 +560,7 @@ gate_e2e qwen3moe_router_norm_m_prefetch 0 0 gate_e2e qwen3moe_router_norm_prefetch 0 0 gate_e2e qwen3moe_router_route 0 0 gate_e2e qwen3moe_token_major 0 0 +gate_e2e relu_sqr 0 0 gate_e2e rms_norm 0 0 gate_e2e rope 0 0 gate_e2e router_topk 0 0 diff --git a/tools/unsafe-ratchet.txt b/tools/unsafe-ratchet.txt index 4fd372ab4..8933ae1a6 100644 --- a/tools/unsafe-ratchet.txt +++ b/tools/unsafe-ratchet.txt @@ -237,7 +237,9 @@ gguf 9 2 0 # prefill_512_p (one unsafe fn) has 23 blocks: query read/stage (2), key/value stage address/copy/zero/commit (4), # ldmatrix pointer formation and address conversion (6), cp.async waits (2), score loads and MMA calls (4), # exchange write/read (2), value load and MMA calls (3). Each block has its SAFETY line; shared macros count once. -gpu 1814 198 10 +# PIN(2026-10-15): gpu 1814 -> 1816 blocks, new code, not old code relaxed: relu_sqr returns with the nemotron_h +# body that calls it (its input and output through raw pointers), each block with its SAFETY line. +gpu 1816 198 10 # PIN(2026-10-01): gpu-deepseek41 459 -> 464 blocks, 51 -> 52 fns, a new kernel entry, not old code relaxed: # ds41_ffn_card_acc_8_tier (the tier form of the card sum at 8 slots, chain/ffn/batch.rs) adds its two blocks and # card_acc_tier_at (the one unsafe fn) with its three, each with its SAFETY line.