diff --git a/crates/decision/src/encode.rs b/crates/decision/src/encode.rs index ffa55db6e..622ee5f58 100644 --- a/crates/decision/src/encode.rs +++ b/crates/decision/src/encode.rs @@ -107,7 +107,9 @@ pub(crate) fn encode_media_with( placeholders: (0..image_tokens.len()) .map(|i| media.image_placeholder(i)) .collect(), - token: media.image_token(), + token: media + .image_token() + .ok_or_else(|| Error::Images("the projector has no chat image token".into()))?, counts: image_tokens, }; encode_inner(tokens, req, Some(images), max_length) @@ -380,8 +382,8 @@ mod tests { PLACEHOLDER } - fn image_token(&self) -> u32 { - PAD + fn image_token(&self) -> Option { + Some(PAD) } fn prepare(&self, _: &vision::Rgb8) -> Result { @@ -484,8 +486,8 @@ mod tests { "<|image_pad|>" } - fn image_token(&self) -> u32 { - PAD + fn image_token(&self) -> Option { + Some(PAD) } fn prepare(&self, _: &vision::Rgb8) -> Result { diff --git a/crates/gpu-gates/Cargo.toml b/crates/gpu-gates/Cargo.toml index dbb138857..767860a4f 100644 --- a/crates/gpu-gates/Cargo.toml +++ b/crates/gpu-gates/Cargo.toml @@ -122,7 +122,7 @@ vision = ["gpu", "dep:bloomery-gpu-vision", "dep:bloomery-vision"] clef = ["gpu", "dep:bloomery-decision", "dep:bloomery-serve", "dep:serde_json"] glm5next = ["deepseek41", "dep:bloomery-gpu-glm5next", "bloomery-app/glm5next"] mimo2 = ["gpu", "dep:bloomery-gpu-mimo2", "bloomery-app/mimo2"] -gemma-embedding2 = ["gpu", "dep:bloomery-gpu-gemma-embedding2"] +gemma-embedding2 = ["gpu", "dep:bloomery-gpu-gemma-embedding2", "dep:bloomery-serve"] # A bin whose `main` sits behind a feature requires that feature: cargo refuses to build it without the # feature before it links anything, instead of linking the feature-off stub `main` over the working @@ -541,12 +541,16 @@ required-features = ["gpu"] [[bin]] name = "gate_gemma_embedding2_e2e" -required-features = ["gemma-embedding2"] +required-features = ["gemma-embedding2", "vision", "deepseek41"] [[bin]] name = "gate_gemma_embedding2_serve" required-features = ["deepseek41", "gemma-embedding2"] +[[bin]] +name = "gate_gemma4v_tower" +required-features = ["vision"] + [[bin]] name = "gate_qwen3moe_router" required-features = ["gpu"] diff --git a/crates/gpu-gates/src/arch/embedding.rs b/crates/gpu-gates/src/arch/embedding.rs index b3fb0d920..79571e7fc 100644 --- a/crates/gpu-gates/src/arch/embedding.rs +++ b/crates/gpu-gates/src/arch/embedding.rs @@ -15,6 +15,8 @@ pub const ROWS: &[Row] = &[ ctx: model::arch::gemma_embedding2::hparams::CARD_CTX, bytes, open, + projector_file: Some("mmproj-embeddinggemma-2-BF16.gguf"), + projector_types: &[vision::arch::gemma4v::PROJECTOR_TYPE], }, ]; diff --git a/crates/gpu-gates/src/bin/bloomery_serve.rs b/crates/gpu-gates/src/bin/bloomery_serve.rs index 30b54ccc9..484242571 100644 --- a/crates/gpu-gates/src/bin/bloomery_serve.rs +++ b/crates/gpu-gates/src/bin/bloomery_serve.rs @@ -140,7 +140,9 @@ mod drive { const USAGE: &str = "usage: bloomery-serve [--model ds41|qwen38|glm|qwen3|mimo2|nemotron|decide] \ [-m PATH | --model-file PATH | --hf [:]] [--head \ [--head-config ]] [--embd-model FILE | --embd-hf \ - [:]] [:]] [--embd-ctx N] [--embd-mmproj FILE | \ + --embd-no-mmproj] [--embd-image-min-tokens N] [--embd-image-max-tokens N] \ + | --version"; diff --git a/crates/gpu-gates/src/bin/gate_gemma4v_tower.rs b/crates/gpu-gates/src/bin/gate_gemma4v_tower.rs new file mode 100644 index 000000000..3598fd86b --- /dev/null +++ b/crates/gpu-gates/src/bin/gate_gemma4v_tower.rs @@ -0,0 +1,152 @@ +//! The Gemma4 tower from pinned image bytes through every layer to projected rows. +//! Each numerical band is 2.5 times that row's CPU/card oracle distance. + +#[path = "shared/emb2_image.rs"] +mod image; + +#[allow( + dead_code, + reason = "the shared runner serves several gates; this one reads its distance helper" +)] +#[path = "shared/e2e.rs"] +mod e2e; + +use bloomery_gpu::Gpu; +use bloomery_gpu_gates::{GateError, checks_failed, exit_with, ref_model_path, verdict}; +use bloomery_gpu_vision::{bf16_f32, chain::TapSink, gemma4v::Encoder}; +use gguf::Gguf; +use refset::arch::gemma_embedding2::{TOWER, TOWER_CPU, TOWER_IMAGES}; +use std::collections::BTreeMap; +use vision::arch::TokenLimits; +use vision::arch::gemma4v::{Hparams, MERGE, conv_pixels, preprocess}; + +#[derive(Default)] +struct Taps(BTreeMap>); +impl TapSink for Taps { + fn wants(&self, _: &str) -> bool { + true + } + fn take(&mut self, name: &str, _: usize, bits: Vec) { + self.0 + .insert(name.to_string(), bits.into_iter().map(bf16_f32).collect()); + } + fn take_f32(&mut self, name: &str, _: usize, values: Vec) { + self.0.insert(name.to_string(), values); + } +} + +fn pooling(p: &vision::Patches, taps: &Taps, hp: &Hparams) -> Result { + use bloomery_gpu_vision::{f32_bf16, mean_groups::mean_groups_ref}; + let last = taps + .0 + .get(&format!("layer_out-{}", hp.n_layer - 1)) + .ok_or("no last layer for pooling")?; + let pooled = taps.0.get("pooled").ok_or("no pooled tap")?; + let mut gathered = Vec::with_capacity(last.len()); + for gy in 0..p.n_vit_h / MERGE { + for gx in 0..p.n_vit_w / MERGE { + for dy in 0..MERGE { + for dx in 0..MERGE { + let row = ((gy * MERGE + dy) * p.n_vit_w + gx * MERGE + dx) * hp.dim; + gathered.extend(last[row..row + hp.dim].iter().map(|&v| f32_bf16(v))); + } + } + } + } + let expected: Vec<_> = mean_groups_ref(&gathered, hp.dim, MERGE * MERGE) + .into_iter() + .map(|bits| bf16_f32(f32_bf16(bf16_f32(bits) * (hp.dim as f32).sqrt()))) + .collect(); + Ok(pooled + .iter() + .map(|v| v.to_bits()) + .eq(expected.iter().map(|v| v.to_bits()))) +} + +fn run() -> Result<(), GateError> { + bloomery_levers::at_main(&[])?; + let card = image::open(&TOWER)?; + let cpu = image::open(&TOWER_CPU)?; + let model = ref_model_path()?; + let file = Gguf::open(image::projector(&model)?)?; + let hp = Hparams::read(&file)?; + let limits = TokenLimits::new(70, 1120)?; + let rule = hp.size_rule(limits)?; + let gpu = Gpu::new()?; + let stream = gpu.stream_handle(); + let mut enc = Encoder::load(gpu.context(), &stream, &file, limits)?; + let mut ok = true; + for name in TOWER_IMAGES { + let (rgb, _) = image::image(&card, &cpu, name)?; + let p = preprocess(&rgb, &rule)?; + let n = p.n_vit_h * p.n_vit_w; + let tokens = n / (MERGE * MERGE); + let raw_shape = [p.plan.best_w as u64, p.plan.best_h as u64, 3, 1]; + let raw = image::row(&card, &format!("{name}/inp_raw_scaled"), raw_shape)?; + let raw_cpu = image::row(&cpu, &format!("{name}/inp_raw_scaled"), raw_shape)?; + let mut patches = Vec::with_capacity(raw.len()); + let mut pixels = Vec::with_capacity(raw.len()); + for py in 0..p.n_vit_h { + for px in 0..p.n_vit_w { + for c in 0..3 { + for y in 0..hp.patch { + for x in 0..hp.patch { + let at = (c * p.plan.best_h + py * hp.patch + y) * p.plan.best_w + + px * hp.patch + + x; + patches.push(bloomery_gpu_vision::f32_bf16(raw[at])); + pixels.push(raw[at]); + } + } + } + } + } + let exact = p.bf16 == patches + && e2e::same_bits(&raw, &raw_cpu) + && e2e::same_bits(&conv_pixels(&p.bf16)?, &pixels); + println!( + "(p) {name}: canvas and normalized patch bits {}", + verdict(exact) + ); + ok &= exact; + let mut taps = Taps::default(); + enc.encode_tapped(&p, &mut taps)?; + let pool_ok = pooling(&p, &taps, &hp)?; + println!( + "(o) {name}: 3x3 raster pooling and scale bits {}", + verdict(pool_ok) + ); + ok &= pool_ok; + for (node, ours) in &taps.0 { + let cols = if node == "projected" { + hp.out_dim + } else { + hp.dim + }; + let count = if node == "pos_embd" || node.starts_with("layer_out-") { + n + } else { + tokens + }; + let shape = [cols as u64, count as u64, 1, 1]; + let key = format!("{name}/{node}"); + let reference = image::row(&card, &key, shape)?; + let twin = image::row(&cpu, &key, shape)?; + ok &= image::compare(&format!("(t) {key}"), ours, &reference, &twin); + } + for node in ["pos_embd", "pooled", "projected"] + .into_iter() + .map(String::from) + .chain((0..hp.n_layer).map(|l| format!("layer_out-{l}"))) + { + if !taps.0.contains_key(&node) { + return Err(format!("{name}: no tap {node}").into()); + } + } + } + if ok { Ok(()) } else { Err(checks_failed()) } +} + +fn main() -> std::process::ExitCode { + exit_with("gate_gemma4v_tower", run()) +} diff --git a/crates/gpu-gates/src/bin/gate_gemma_embedding2_e2e.rs b/crates/gpu-gates/src/bin/gate_gemma_embedding2_e2e.rs index f2cbf939b..cf1146b0e 100644 --- a/crates/gpu-gates/src/bin/gate_gemma_embedding2_e2e.rs +++ b/crates/gpu-gates/src/bin/gate_gemma_embedding2_e2e.rs @@ -2,7 +2,7 @@ //! end to end on the Q8_0 file the sets are dumped from ([`MODEL`], held to //! its sha256), against llama.cpp mainline's text set dumped on the card //! ([`TEXT`]) and its CPU twin ([`TEXT_CPU`]), both read through -//! [`Emb2Set::open`]: each case the set names, its ids read from the +//! [`refset::emb2::Emb2Set::open`]: each case the set names, its ids read from the //! tokenizer oracle's gemma4 set ([`IDS_SET`]) and held to the set's count //! and digest, through one pass at the load's context with its taps read //! back. @@ -22,7 +22,9 @@ //! case holds the global layers. //! 2. (d) The longest case run again with no taps is bit for bit the first //! run's rows. -//! 3. (r) Refused by name, the load untouched after each: a pass of no ids, +//! 3. (i) Image-only and `mix_free-448x448` inputs assembled by the serving helper, +//! pooled and normalized against the server card set and its CPU twin. +//! 4. (r) Refused by name, the load untouched after each: a pass of no ids, //! of one id past the load's context and of an id past the vocabulary, //! and a load at one position past [`CARD_CTX`]. @@ -39,22 +41,26 @@ fn main() -> std::process::ExitCode { #[path = "shared/e2e.rs"] mod e2e; +#[path = "shared/emb2_image.rs"] +mod image; + #[cfg(feature = "gemma-embedding2")] mod gate { use std::path::Path; use std::time::Instant; use crate::e2e::{clause, elapsed, refused_by, rel, same_bits, tap}; + use crate::image::open; use bloomery_gpu::Gpu; - use bloomery_gpu_gates::{GateError, RefManifest, checks_failed, data_dir, sha256, verdict}; + use bloomery_gpu_gates::{ + GateError, RefManifest, checks_failed, data_dir, ref_model_path, sha256, verdict, + }; use bloomery_gpu_gemma_embedding2::{Encoder, Pass, card_bytes}; use gguf::Split; use model::arch::gemma_embedding2::hparams::{CARD_CTX, Hparams}; use refset::arch::gemma_embedding2::{ IDS_SET, MODEL, MODEL_SHA256, REL_FLOOR as FLOOR, TEXT, TEXT_CPU, TWIN, }; - use refset::emb2::Emb2Set; - use refset::family::Family; use refset::ik::RefHeader; use refset::tokenizer::read_ids; @@ -270,9 +276,95 @@ mod gate { Ok(ok && same) } - /// `family`'s set, read through its reader. - fn open(family: &Family) -> Result { - Ok(Emb2Set::open(&family.path(family.sets[0]), family)?.man) + fn images(enc: &mut Encoder, model: &Path) -> Result { + use crate::image; + use bloomery_gpu::media::MediaSpan; + use bloomery_gpu_gates::bind::Vocab; + use bloomery_gpu_vision::gemma4v::Encoder as Tower; + use refset::arch::gemma_embedding2::{SERVER, SERVER_CPU, TOWER, TOWER_CPU, TOWER_IMAGES}; + use serve::embeddings::{InputPart, assemble}; + use vision::arch::{TokenLimits, gemma4v}; + + let card = image::open(&TOWER)?; + let cpu = image::open(&TOWER_CPU)?; + let server = image::open(&SERVER)?; + let server_cpu = image::open(&SERVER_CPU)?; + let file = gguf::Gguf::open(image::projector(model)?)?; + let hp = gemma4v::Hparams::read(&file)?; + let limits = TokenLimits::new(70, 1120)?; + let rule = hp.size_rule(limits)?; + let stream = enc.context().new_stream_with_priority(0)?; + let mut tower = Tower::load(enc.context(), &stream, &file, limits)?; + let tok = Vocab::new(tokenizer::Tokenizer::from_gguf(model)?)?; + let mut ok = true; + for name in TOWER_IMAGES { + let (rgb, digest) = image::image(&card, &cpu, name)?; + let p = gemma4v::preprocess(&rgb, &rule)?; + let tokens = p.plan.n_llm_h * p.plan.n_llm_w; + let rows = tower.encode(&p)?.rows.buf().to_host_vec(&stream)?; + for mixed in [false, true] { + if mixed && *name != "free-448x448" { + continue; + } + let mut parts = Vec::new(); + if mixed { + parts.push(InputPart::Text("A picture of coloured rings and ramps: ")); + } + parts.push(InputPart::Image { + begin: gemma4v::IMAGE_BEGIN, + end: gemma4v::IMAGE_END, + positions: tokens, + }); + let assembled = assemble(parts.into_iter().map(Ok), &tok, CARD_CTX)?; + let ids = &assembled.ids; + let span = assembled + .spans + .first() + .ok_or("image assembly returned no span")?; + let key = format!("{}_{name}", if mixed { "mix" } else { "img" }); + // The server pair must have consumed these same bytes and position count. + if image::image(&server, &server_cpu, &key)?.1 != digest { + return Err(format!("{key}: server/tower image sha256 mismatch").into()); + } + for man in [&server, &server_cpu] { + let prefix = format!("# image\t{key}\t"); + let expected = format!("prompt_tokens {}", ids.len()); + if !man + .header + .other + .iter() + .any(|l| l.starts_with(&prefix) && l.split('\t').any(|f| f == expected)) + { + return Err(format!( + "{key}: server position count differs from {}", + ids.len() + ) + .into()); + } + } + let pass = enc.pass_media( + ids, + &[MediaSpan { + at: span.clone(), + rows: &rows, + }], + false, + )?; + let norm = serve::embed::normalize(&pass.pooled, serve::embed::Norm::default())?; + for (node, ours) in [("raw", &pass.pooled), ("embd", &norm)] { + let key = format!("{key}/{node}"); + let expected = image::row(&server, &key, [768, 1, 1, 1])?; + let twin = image::row(&server_cpu, &key, [768, 1, 1, 1])?; + ok &= image::compare( + &format!("(i) {key} pass_media {} positions", ids.len()), + ours, + &expected, + &twin, + ); + } + } + } + Ok(ok) } pub fn run() -> Result<(), GateError> { @@ -281,14 +373,15 @@ mod gate { if !args.is_empty() { return Err(format!("no arguments; got {args:?}").into()); } - let digest = sha256(Path::new(MODEL))?; + let model = ref_model_path()?; + let digest = sha256(&model)?; if digest != MODEL_SHA256 { return Err(format!( "{MODEL}: sha256 {digest}, the sets are dumped from {MODEL_SHA256}" ) .into()); } - let split = Split::open(MODEL).map_err(|e| format!("open {MODEL}: {e}"))?; + let split = Split::open(&model).map_err(|e| format!("open {}: {e}", model.display()))?; let hp = Hparams::read(&split).map_err(|e| format!("{MODEL}: {e}"))?; let read = model::arch::spec(&split).map_err(|e| format!("{MODEL}: {e}"))?; let encoder = read @@ -350,7 +443,8 @@ mod gate { refusals(&mut enc, c, &first.rows, hp.n_vocab), ); } - let past = Split::open(MODEL).map_err(|e| format!("open {MODEL}: {e}"))?; + ok &= clause("(i)", "image inputs", images(&mut enc, &model)); + let past = Split::open(&model).map_err(|e| format!("open {}: {e}", model.display()))?; let refused = refused_by( &Encoder::open(past, ctx + 1, Gpu::new()?), "Encoder::open", diff --git a/crates/gpu-gates/src/bin/gate_serve_start.rs b/crates/gpu-gates/src/bin/gate_serve_start.rs index d11f1d00d..9c9101ba3 100644 --- a/crates/gpu-gates/src/bin/gate_serve_start.rs +++ b/crates/gpu-gates/src/bin/gate_serve_start.rs @@ -238,7 +238,10 @@ fn open_images(family: Family) -> Result { let seat = vision_seat::encoder_seat(Place::A, proj.card.total())?; let png = read_image()?; let span = proj.media.prepare(&vision::Rgb8::from_png(&png)?)?.span_len; - let token = proj.media.image_token(); + let token = proj + .media + .image_token() + .ok_or("the projector has no chat image token")?; Ok(Images { file, proj, diff --git a/crates/gpu-gates/src/bin/shared/emb2_image.rs b/crates/gpu-gates/src/bin/shared/emb2_image.rs new file mode 100644 index 000000000..4153ab28b --- /dev/null +++ b/crates/gpu-gates/src/bin/shared/emb2_image.rs @@ -0,0 +1,87 @@ +//! The EmbeddingGemma image gates' pinned inputs and CPU-twin comparisons. + +use crate::e2e::rel; +use bloomery_gpu_gates::{GateError, RefManifest, sha256, verdict}; +use refset::arch::gemma_embedding2::{MMPROJ_BF16, MMPROJ_BF16_SHA256, TWIN}; +use refset::emb2::Emb2Set; +use refset::family::Family; +use refset::ik::ref_tensor_of_in; +use std::path::{Path, PathBuf}; +use vision::Rgb8; + +pub fn open(family: &Family) -> Result { + Ok(Emb2Set::open(&family.path(family.sets[0]), family)?.man) +} + +pub fn row(man: &RefManifest, name: &str, shape: [u64; 4]) -> Result, GateError> { + let r = man.tensor(name, 0)?; + if r.ty != "f32" || r.ne != shape { + return Err(format!("{name}: {} {:?}, expected f32 {shape:?}", r.ty, r.ne).into()); + } + let data = ref_tensor_of_in(&man.dir, r)?; + if data.iter().any(|v| !v.is_finite()) { + return Err(format!("{name}: non-finite oracle value").into()); + } + Ok(data) +} + +pub fn compare(name: &str, ours: &[f32], card: &[f32], cpu: &[f32]) -> bool { + let distance = rel(ours, card); + let twin = rel(cpu, card); + let band = TWIN * twin; + let ok = !card.is_empty() && distance.is_finite() && twin.is_finite() && distance <= band; + println!( + "{name}: rel {distance:.9e}, cpu-twin {twin:.9e}, band {band:.9e} ({TWIN} x twin) {}", + verdict(ok) + ); + ok +} + +pub fn projector(model: &Path) -> Result { + let path = model.with_file_name( + Path::new(MMPROJ_BF16) + .file_name() + .ok_or("no projector filename")?, + ); + let digest = sha256(&path)?; + if digest != MMPROJ_BF16_SHA256 { + return Err(format!( + "{}: projector sha256 {digest}, expected {MMPROJ_BF16_SHA256}", + path.display() + ) + .into()); + } + Ok(path) +} + +/// Both sets must name the same image bytes; the file is checked before decoding. +pub fn image( + card: &RefManifest, + cpu: &RefManifest, + name: &str, +) -> Result<(Rgb8, String), GateError> { + let header = |man: &RefManifest| -> Result<(PathBuf, String), GateError> { + let prefix = format!("# image\t{name}\t"); + let mut lines = man + .header + .other + .iter() + .filter_map(|l| l.strip_prefix(&prefix)); + let line = lines + .next() + .ok_or_else(|| format!("{name}: missing image identity"))?; + if lines.next().is_some() { + return Err(format!("{name}: duplicate image identity").into()); + } + let f: Vec<_> = line.split('\t').collect(); + if f.len() < 2 { + return Err(format!("{name}: malformed image identity").into()); + } + Ok((PathBuf::from(f[0]), f[1].to_string())) + }; + let (path, digest) = header(card)?; + if header(cpu)?.1 != digest || sha256(&path)? != digest { + return Err(format!("{name}: CPU/card image sha256 mismatch").into()); + } + Ok((Rgb8::from_png(&std::fs::read(path)?)?, digest)) +} diff --git a/crates/gpu-gates/src/bin/shared/glm5next_media.rs b/crates/gpu-gates/src/bin/shared/glm5next_media.rs index b9b7728e8..c0b5f17e4 100644 --- a/crates/gpu-gates/src/bin/shared/glm5next_media.rs +++ b/crates/gpu-gates/src/bin/shared/glm5next_media.rs @@ -328,7 +328,14 @@ pub mod seat { at.cap, at.card.total() ); - set_prefill_media(s.model_mut(), Some(at.media.image_token()))?; + set_prefill_media( + s.model_mut(), + Some( + at.media + .image_token() + .ok_or("the projector has no chat image token")?, + ), + )?; let ctx = s.model().gpu().context().clone(); let mut tower = Resident::open(&ctx, &at)?; let images: Vec = SIZES diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/glm.rs b/crates/gpu-gates/src/bin/shared/serve_seats/glm.rs index 01fbdaab5..850e39ed1 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/glm.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/glm.rs @@ -1317,7 +1317,14 @@ fn open_encoder( return Ok(None); }; let t = Instant::now(); - set_prefill_media(s.model_mut(), Some(v.media.image_token()))?; + set_prefill_media( + s.model_mut(), + Some( + v.media + .image_token() + .ok_or("the projector has no chat image token")?, + ), + )?; let resident = vision_seat::Resident::open(s.model().gpu().context(), &v)?; resident.announce(&v, vision_seat::ScratchHome::Own, t); Ok(Some(resident)) diff --git a/crates/gpu-gates/src/side_seat.rs b/crates/gpu-gates/src/side_seat.rs index c09d021fb..396e2f7cc 100644 --- a/crates/gpu-gates/src/side_seat.rs +++ b/crates/gpu-gates/src/side_seat.rs @@ -32,6 +32,8 @@ pub use architectures::ROWS; #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct Flags { model: source::Flags, + image: vision_seat::Flags, + ctx: Option, } impl Flags { @@ -39,15 +41,47 @@ impl Flags { /// file is read here. A flag with no value or given twice, and `--embd-model` beside /// `--embd-hf`, are refused by name. pub fn take(args: &[String]) -> Result<(Flags, Vec), GateError> { + let mut f = Flags::default(); let mut selected = Vec::new(); let mut rest = Vec::with_capacity(args.len()); + let mut image_seen = std::collections::HashSet::new(); let mut it = args.iter(); - while let Some(arg) = it.next() { - let flag = match arg.as_str() { + while let Some(a) = it.next() { + let flag = a.as_str(); + if let Some(image_flag) = match flag { + "--embd-mmproj" => Some("--mmproj"), + "--embd-no-mmproj" => Some("--no-mmproj"), + "--embd-image-min-tokens" => Some("--image-min-tokens"), + "--embd-image-max-tokens" => Some("--image-max-tokens"), + _ => None, + } { + if !image_seen.insert(flag) { + return Err(format!("{flag} is given twice").into()); + } + f.image + .take(image_flag, &mut it.by_ref().map(String::as_str)) + .map_err(|e| format!("{flag}: {e}"))?; + continue; + } + if flag == "--embd-ctx" { + let v = it.next().ok_or("--embd-ctx needs a value")?; + let ctx = v + .parse::() + .map_err(|e| format!("--embd-ctx {v:?}: {e}"))?; + if f.ctx.is_some() { + return Err("--embd-ctx is given twice".into()); + } + if ctx == 0 { + return Err("--embd-ctx needs one positive count".into()); + } + f.ctx = Some(ctx); + continue; + } + let flag = match flag { "--embd-model" => "--embd-model", "--embd-hf" => "--hf", _ => { - rest.push(arg.clone()); + rest.push(a.clone()); continue; } }; @@ -56,7 +90,11 @@ impl Flags { } let (model, _) = source::take(&selected, &["--embd-model"]) .map_err(|e| format!("embedding flags (--embd-model/--embd-hf): {e}"))?; - Ok((Flags { model }, rest)) + f.model = model; + if !f.names() && (!image_seen.is_empty() || f.ctx.is_some()) { + return Err("embedding image/context flags need --embd-model or --embd-hf".into()); + } + Ok((f, rest)) } /// Whether the flags name an embedding model. @@ -87,14 +125,68 @@ impl Flags { .arch_get_u64("context_length") .and_then(|n| usize::try_from(n).ok()) .ok_or_else(|| format!("{}: no {arch}.context_length", path.display()))?; - let ctx = row.ctx.min(train); - let bytes = (row.bytes)(&split, ctx) + let limit = row.ctx.min(train); + let ctx = self.ctx.unwrap_or(limit); + if ctx > limit { + return Err( + format!("--embd-ctx {ctx}: the side model supports at most {limit}").into(), + ); + } + let mut bytes = (row.bytes)(&split, ctx) .map_err(|e| format!("--embd-model {}: {e}", path.display()))?; + let projector_path = if let Some(p) = self.image.mmproj()? { + Some(p.to_path_buf()) + } else if let (Some(hf), Some(mmproj)) = (&self.model.hf, row.projector_file) { + if cfg!(feature = "vision") && !self.image.disabled() { + let repo = hf::RepoRef::parse(hf)?; + Some( + model_file::fetch_exact(&repo.repo, &[mmproj])? + .into_iter() + .next() + .ok_or("the projector fetch returned no file")?, + ) + } else { + None + } + } else { + None + }; + let projector = match projector_path.as_deref() { + Some(_) if !cfg!(feature = "vision") => { + return Err("--embd-mmproj needs the vision build feature".into()); + } + Some(p) => { + let p = vision_seat::Projector::open(p, &self.image, row.projector_types)?; + let n_embd = split + .arch_get_u64("embedding_length") + .and_then(|n| usize::try_from(n).ok()) + .ok_or("the embedding width is absent")?; + p.check_text( + &vision_seat::TextFacts { + n_embd, + imrope: None, + ple: vision_seat::PleImage::Absent, + }, + &|_| None, + )?; + bytes = bytes + .checked_add(p.card.total()) + .ok_or("embedding model and tower byte count overflows")?; + Some(p) + } + None if self.image.sets_limits() => { + return Err( + "--embd-image-min-tokens/--embd-image-max-tokens need --embd-mmproj".into(), + ); + } + None => None, + }; Ok(Some(Side { row, path, ctx, bytes, + projector, })) } } @@ -111,6 +203,10 @@ pub struct Row { pub bytes: fn(&Split, usize) -> Result, /// The model of `side`'s file opened on `seat`'s card, on the thread that runs its passes. pub open: Open, + /// The BF16 projector this row reads from an `--embd-hf` repository. + pub projector_file: Option<&'static str>, + /// The projector types this embedding architecture accepts. + pub projector_types: &'static [&'static str], } /// A row's open: the model of `side`'s file on `seat`'s card. @@ -148,6 +244,8 @@ pub struct Side { pub ctx: usize, /// The bytes it takes on its card ([`Row::bytes`]). pub bytes: u64, + /// The image tower sized before the chat plan; its bytes share this side seat's reserve. + pub projector: Option, } impl Side { @@ -177,7 +275,11 @@ impl Side { } /// One pass: the input's ids and where its answer goes. -type Job = (Vec, mpsc::Sender, EmbedError>>); +type Job = ( + Vec, + Vec, + mpsc::Sender, EmbedError>>, +); /// The model the server's embeddings endpoints call, held on a thread of its own: the facts read /// once at its open, and the channel each pass goes through. Dropping it closes the channel and @@ -189,6 +291,7 @@ pub struct SideEngine { name: String, ctx: usize, dims: Vec, + media: Option, } impl SideEngine { @@ -214,12 +317,24 @@ impl SideEngine { return; } }; - let facts = (model.tokenizer(), model.name(), model.ctx(), model.dims()); + let facts = ( + model.tokenizer(), + model.name(), + model.ctx(), + model.dims(), + model.media(), + ); if opened_tx.send(Ok(facts)).is_err() { return; } - for (ids, answer) in rx { - match catch_unwind(|| model.embed(&ids)) { + for (ids, images, answer) in rx { + match catch_unwind(|| { + if images.is_empty() { + model.embed(&ids) + } else { + model.embed_media(&ids, &images) + } + }) { Ok(v) => { let _ = answer.send(v); } @@ -233,7 +348,7 @@ impl SideEngine { } }) .map_err(|e| format!("starting the embedding model's thread: {e}"))?; - let (tokenizer, name, ctx, dims) = match opened.recv() { + let (tokenizer, name, ctx, dims, media) = match opened.recv() { Ok(Ok(facts)) => facts, refused => { // The thread ended with its open; nothing is left on its card. @@ -251,6 +366,7 @@ impl SideEngine { name, ctx, dims, + media, }) } } @@ -271,6 +387,9 @@ impl Drop for SideEngine { } impl Embed for SideEngine { + fn media(&self) -> Option { + self.media.clone() + } fn tokenizer(&self) -> Arc { Arc::clone(&self.tokenizer) } @@ -288,10 +407,19 @@ impl Embed for SideEngine { } fn embed(&mut self, ids: &[u32]) -> Result, EmbedError> { + self.embed_media(ids, &[]) + } + + fn embed_media( + &mut self, + ids: &[u32], + images: &[serve::media::MediaFeed], + ) -> Result, EmbedError> { let ended = || EmbedError("the embedding model's thread ended".to_owned()); let jobs = self.jobs.as_ref().ok_or_else(ended)?; let (answer, answered) = mpsc::channel(); - jobs.send((ids.to_vec(), answer)).map_err(|_| ended())?; + jobs.send((ids.to_vec(), images.to_vec(), answer)) + .map_err(|_| ended())?; answered.recv().map_err(|_| ended())? } } @@ -336,6 +464,49 @@ mod tests { let (none, _) = Flags::take(&args(&["--port", "1"])).expect("none"); assert_eq!(none, Flags::default()); assert!(!none.names()); + let (image, rest) = Flags::take(&args(&[ + "--embd-model", + "/e.gguf", + "--embd-ctx", + "8192", + "--embd-mmproj", + "/p.gguf", + "--embd-image-min-tokens", + "70", + "--embd-image-max-tokens", + "1120", + "--port", + "8080", + ])) + .expect("image flags"); + assert_eq!(rest, args(&["--port", "8080"])); + assert_eq!(image.ctx, Some(8192)); + assert_eq!( + image.image.mmproj().expect("projector"), + Some(std::path::Path::new("/p.gguf")) + ); + assert!(image.image.sets_limits()); + let default = crate::vision_seat::TokenLimits::new(1, 1).expect("limits"); + let limits = image.image.limits_over(default).expect("image limits"); + assert_eq!((limits.min(), limits.max()), (70, 1120)); + for flag in ["--embd-image-min-tokens", "--embd-image-max-tokens"] { + let (zero, _) = + Flags::take(&args(&["--embd-model", "/a", flag, "0"])).expect("count parsed"); + let e = zero + .image + .limits_over(default) + .expect_err("zero limit refused") + .to_string(); + assert!( + e.starts_with("--image-min-tokens/--image-max-tokens: image size limits: "), + "{flag}: {e}" + ); + assert!(e.contains("need 1 <= min <= max"), "{flag}: {e}"); + let (one, _) = + Flags::take(&args(&["--embd-model", "/a", flag, "1"])).expect("count parsed"); + let limits = one.image.limits_over(default).expect("one token limit"); + assert_eq!((limits.min(), limits.max()), (1, 1)); + } for (bad, why) in [ (&["--embd-model"][..], "--embd-model needs a value"), (&["--embd-hf"], "--hf needs a value"), @@ -343,6 +514,63 @@ mod tests { (&["--embd-hf", "x/y", "--embd-hf", "a/b"], "named twice"), (&["--embd-model", "/a", "--embd-hf", "x/y"], "named twice"), (&["--embd-hf", "x/y", "--embd-model", "/a"], "named twice"), + (&["--embd-mmproj", "/p"], "need --embd-model or --embd-hf"), + (&["--embd-no-mmproj"], "need --embd-model or --embd-hf"), + ( + &["--embd-image-min-tokens", "70"], + "need --embd-model or --embd-hf", + ), + ( + &["--embd-image-max-tokens", "1120"], + "need --embd-model or --embd-hf", + ), + (&["--embd-ctx", "8192"], "need --embd-model or --embd-hf"), + ( + &["--embd-model", "/a", "--embd-ctx", "1", "--embd-ctx", "2"], + "--embd-ctx is given twice", + ), + ( + &["--embd-model", "/a", "--embd-ctx", "0"], + "needs one positive count", + ), + ( + &[ + "--embd-model", + "/a", + "--embd-mmproj", + "/p", + "--embd-mmproj", + "/q", + ], + "--embd-mmproj is given twice", + ), + ( + &["--embd-model", "/a", "--embd-no-mmproj", "--embd-no-mmproj"], + "--embd-no-mmproj is given twice", + ), + ( + &[ + "--embd-model", + "/a", + "--embd-image-min-tokens", + "70", + "--embd-image-min-tokens", + "80", + ], + "--embd-image-min-tokens is given twice", + ), + ( + &[ + "--embd-model", + "/a", + "--embd-image-max-tokens", + "1120", + "--embd-image-max-tokens", + "1200", + ], + "--embd-image-max-tokens is given twice", + ), + (&["--embd-model", "/a", "--embd-mmproj"], "--embd-mmproj"), ] { let e = Flags::take(&args(bad)).expect_err("refused").to_string(); assert!(e.contains(why), "{bad:?}: {e}"); diff --git a/crates/gpu-gates/src/side_seat/gemma_embedding2.rs b/crates/gpu-gates/src/side_seat/gemma_embedding2.rs index 8a2f7f551..3d5a1c3ef 100644 --- a/crates/gpu-gates/src/side_seat/gemma_embedding2.rs +++ b/crates/gpu-gates/src/side_seat/gemma_embedding2.rs @@ -11,6 +11,8 @@ use serve::embed::{Embed, EmbedError}; use super::{Side, architectures}; use crate::bind::Vocab; use crate::vision_seat::EncoderSeat; +#[cfg(feature = "vision")] +use crate::vision_seat::{Image, Opened, Resident}; pub(super) fn bytes(split: &Split, ctx: usize) -> Result { card_bytes(split, ctx).map_err(|e| e.to_string()) @@ -20,7 +22,9 @@ pub(super) fn open(side: &Side, seat: &EncoderSeat) -> Result, St let load = || -> Result, crate::GateError> { let split = Split::open(&side.path)?; let dims = architectures::gemma_embedding2_dims(&split)?; - let actual = card_bytes(&split, side.ctx)?; + let actual = card_bytes(&split, side.ctx)? + .checked_add(side.projector.as_ref().map_or(0, |p| p.card.total())) + .ok_or("embedding model and tower byte count overflows")?; if actual != side.bytes || seat.reserve.is_some_and(|b| b != actual) { return Err(format!( "embedding model: load needs {actual} B, sized {} B, reserved {:?}", @@ -41,6 +45,22 @@ pub(super) fn open(side: &Side, seat: &EncoderSeat) -> Result, St .device(); let gpu = Gpu::open_card(&seat.name, Some(device))?; let encoder = Encoder::open(split, side.ctx, gpu)?; + #[cfg(feature = "vision")] + let vision = side + .projector + .as_ref() + .map(|p| { + let at = Opened { + path: p.path.clone(), + seat: seat.clone(), + media: p.media.clone(), + cap: p.cap, + card: p.card, + limits: p.limits, + }; + Resident::open(encoder.context(), &at) + }) + .transpose()?; let name = side .path .file_name() @@ -49,6 +69,8 @@ pub(super) fn open(side: &Side, seat: &EncoderSeat) -> Result, St .into_owned(); Ok(Box::new(Model { encoder, + #[cfg(feature = "vision")] + vision, tok, name, dims, @@ -59,12 +81,25 @@ pub(super) fn open(side: &Side, seat: &EncoderSeat) -> Result, St struct Model { encoder: Encoder, + #[cfg(feature = "vision")] + vision: Option, tok: Arc, name: String, dims: Vec, } impl Embed for Model { + fn media(&self) -> Option { + #[cfg(feature = "vision")] + { + self.vision.as_ref().map(Resident::media) + } + #[cfg(not(feature = "vision"))] + { + None + } + } + fn tokenizer(&self) -> Arc { self.tok.clone() } @@ -87,4 +122,27 @@ impl Embed for Model { .map(|pass| pass.pooled) .map_err(|e| EmbedError(e.to_string())) } + + #[cfg(feature = "vision")] + fn embed_media( + &mut self, + ids: &[u32], + images: &[serve::media::MediaFeed], + ) -> Result, EmbedError> { + let tower = self + .vision + .as_mut() + .ok_or_else(|| EmbedError("image input needs --embd-mmproj".into()))?; + let encoded = tower + .encode_with( + self.encoder.context(), + images.iter().map(|i| Image::of(i.at, &i.prepared)), + crate::vision_seat::merged_cells, + ) + .map_err(|e| EmbedError(e.to_string()))?; + self.encoder + .pass_media(ids, &encoded.rows().media_spans(), false) + .map(|p| p.pooled) + .map_err(|e| EmbedError(e.to_string())) + } } diff --git a/crates/gpu-gates/src/vision_seat.rs b/crates/gpu-gates/src/vision_seat.rs index 0ba68775e..07ec6cf25 100644 --- a/crates/gpu-gates/src/vision_seat.rs +++ b/crates/gpu-gates/src/vision_seat.rs @@ -56,8 +56,8 @@ use gguf::Gguf; use model::placement::workstation::{CardSpec, DeviceInfo}; use vision::arch::card::CardBytes; use vision::arch::minicpmv::{self, Group}; -use vision::arch::qwen3vl::TokenLimits; -use vision::arch::{deepseek41v, glm5v, qwen3vl}; +pub(crate) use vision::arch::qwen3vl::TokenLimits; +use vision::arch::{deepseek41v, gemma4v, glm5v, qwen3vl}; use vision::mrope::Canvas; use vision::{EncoderInput, MediaModel, Patches, Prepared, VisionError}; @@ -175,6 +175,12 @@ impl Flags { self.min_tokens.is_some() || self.max_tokens.is_some() } + /// Whether the caller explicitly disables a repository's projector. + #[must_use] + pub fn disabled(&self) -> bool { + self.no_mmproj + } + /// The image-token limits: the flags' values over `default`, the limits the projector starts /// from when no flag gives one ([`Rule`]), refused by name when `min` is 0 or above `max`. pub fn limits_over(&self, default: TokenLimits) -> Result { @@ -186,7 +192,7 @@ impl Flags { } } -/// A flag's count: a positive integer, or refused by name. +/// A flag's count: an unsigned integer, or refused by name. fn count(flag: &str, v: &str) -> Result { v.parse::() .map_err(|e| format!("{flag} {v:?}: {e}").into()) @@ -270,6 +276,15 @@ pub fn rules() -> &'static [Rule] { default_limits: |_| TokenLimits::new(1, MINICPM_TILE_TOKENS), cap: |_, limits| Ok(limits.max()), }, + Rule { + projector: gemma4v::PROJECTOR_TYPE, + ids: &[], + text_rope: None, + takes_limits: true, + out_dim: |f| Ok(gemma4v::Hparams::read(f)?.out_dim), + default_limits: |f| gemma4v::Hparams::read(f)?.default_limits(), + cap: |_, limits| Ok(limits.max()), + }, ]; RULES } @@ -323,10 +338,7 @@ impl Projector { pub fn open(path: &Path, flags: &Flags, takes: &[&str]) -> Result { let at = |e: &dyn Display| format!("--mmproj {}: {e}", path.display()); let file = Gguf::open(path).map_err(|e| at(&e))?; - let kind = file - .value("clip.projector_type") - .and_then(gguf::Value::as_str) - .ok_or_else(|| at(&"clip.projector_type is absent or not a string"))?; + let kind = vision::arch::projector_type(&file).map_err(|e| at(&e))?; if !takes.contains(&kind) { return Err(at(&format!( "projector type {kind:?}; this seat's text model takes {takes:?}" @@ -420,20 +432,23 @@ impl Projector { )) ))); } - let image = self.media.image_token(); match text.ple { PleImage::Absent => {} PleImage::NoKey => { return Err(at(&"the text model carries per-layer embeddings and no \ ple.image_token_id, which must be the projector's image token")); } - PleImage::Id(id) if id != image => { - return Err(at(&format!( - "the text model's ple.image_token_id is {id}, the projector's image token \ - is {image}" - ))); + PleImage::Id(id) => { + let image = self.media.image_token().ok_or_else(|| { + at(&"the projector has no chat image token for ple.image_token_id") + })?; + if id != image { + return Err(at(&format!( + "the text model's ple.image_token_id is {id}, the projector's image token \ + is {image}" + ))); + } } - PleImage::Id(_) => {} } Ok(()) } @@ -1459,8 +1474,8 @@ mod tests { fn image_marker(&self) -> &str { "" } - fn image_token(&self) -> u32 { - self.0 + fn image_token(&self) -> Option { + Some(self.0) } fn prepare(&self, _: &vision::Rgb8) -> Result { Err(VisionError::Limits { @@ -1853,7 +1868,7 @@ mod tests { assert_eq!( open(&bare, &none, &["deepseek41v"]), format!( - "--mmproj {}: clip.projector_type is absent or not a string", + "--mmproj {}: metadata clip.projector_type: is absent", bare.0.display() ) ); diff --git a/crates/gpu-gemma-embedding2/src/lib.rs b/crates/gpu-gemma-embedding2/src/lib.rs index 0241dc85f..fbf0deba2 100644 --- a/crates/gpu-gemma-embedding2/src/lib.rs +++ b/crates/gpu-gemma-embedding2/src/lib.rs @@ -512,6 +512,11 @@ impl Encoder { pub fn ctx(&self) -> usize { self.ctx } + + /// The context the encoder's pass owns; a tower call binds it back after its encode. + pub fn context(&self) -> &std::sync::Arc { + self.gpu.context() + } } /// Rows `rows` of `name` in `split`, `k` values each, decoded to f32 and diff --git a/crates/gpu-gemma-embedding2/src/pass.rs b/crates/gpu-gemma-embedding2/src/pass.rs index 9be7b1d05..8959fb9a2 100644 --- a/crates/gpu-gemma-embedding2/src/pass.rs +++ b/crates/gpu-gemma-embedding2/src/pass.rs @@ -398,6 +398,17 @@ impl Encoder { /// the error, and so is an output value that is not finite, by its /// position. pub fn pass(&mut self, ids: &[u32], taps: bool) -> Result { + self.pass_media(ids, &[], taps) + } + + /// One bidirectional pass with image rows already scaled by the tower, + /// replacing their spans after the text rows receive `sqrt(hidden)`. + pub fn pass_media( + &mut self, + ids: &[u32], + spans: &[bloomery_gpu::media::MediaSpan<'_>], + taps: bool, + ) -> Result { let n = ids.len(); if n == 0 || n > self.ctx { return Err(GpuError::Shape { @@ -405,7 +416,18 @@ impl Encoder { detail: format!("{n} positions; a pass takes 1 to {}", self.ctx), }); } - let inp_scaled = self.token_rows(ids)?; + bloomery_gpu::media::check_spans(PASS, spans, n, self.hp.n_embd)?; + let mut inp_scaled = self.token_rows(ids)?; + for span in spans { + let dst = &mut inp_scaled[span.at.start * self.hp.n_embd..span.at.end * self.hp.n_embd]; + bloomery_gpu::media::widen_bf16(span.rows, dst); + if dst.iter().any(|v| !v.is_finite()) { + return Err(GpuError::shape( + PASS, + "image row contains a non-finite value", + )); + } + } let Encoder { gpu, hp, @@ -522,6 +544,59 @@ fn mean(rows: &[f32], width: usize) -> Vec { mod tests { use super::*; + #[test] + #[ignore = "needs a CUDA card"] + fn hw_media_spans_are_checked_before_the_pass() { + use bloomery_gpu::media::MediaSpan; + + crate::tests::with_model(64, 2048, |split, hp| -> Result<(), GpuError> { + let mut enc = Encoder::open(split, hp.n_ctx, Gpu::new()?)?; + let rows = vec![0; hp.n_embd]; + for spans in [ + vec![MediaSpan { + at: 0..0, + rows: &[], + }], + vec![MediaSpan { + at: 1..2, + rows: &rows, + }], + vec![MediaSpan { + at: 0..1, + rows: &[], + }], + vec![ + MediaSpan { + at: 0..1, + rows: &rows, + }, + MediaSpan { + at: 0..1, + rows: &rows, + }, + ], + ] { + assert!(matches!( + enc.pass_media(&[0], &spans, false), + Err(GpuError::Shape { .. }) + )); + } + assert!( + enc.pass_media( + &[0], + &[MediaSpan { + at: 0..1, + rows: &rows + }], + false + ) + .is_ok() + ); + Ok(()) + }) + .unwrap(); + } + #[test] #[ignore = "needs a CUDA card"] fn hw_per_layer_projection_scale_near_epsilon() { diff --git a/crates/gpu-vision/src/arch/gemma4v.rs b/crates/gpu-vision/src/arch/gemma4v.rs new file mode 100644 index 000000000..11ebb7940 --- /dev/null +++ b/crates/gpu-vision/src/arch/gemma4v.rs @@ -0,0 +1,863 @@ +//! Gemma4 vision: F32 patch convolution and learned positions, BF16 ViT +//! linears, sandwich RMS norms, gainless V norm, scale-one non-causal +//! attention, the header's GELU GLU, three-by-three pooling and the input projection. +//! Audio tensors of the shared projector are never uploaded. +//! +//! Q/K rows and gains use [0..16, 32..48, 16..32, 48..64] in each head. +//! This expresses the two NEOX half-head turns through the shared whole-head +//! RoPE entry; V and the attention output keep their original channel order. +//! Residuals stay F32; the shared GEMMs and attention consume BF16 rows. + +use crate::{ + attn::{AttnArgs, AttnKernels, QkvLayout}, + chain::{Encoded, NoTaps, TapSink, read_bf16, read_f32, tap, tap_f32}, + gemm_bf16::{Epilogue, FloatGemmArgs, GemmArgs, GemmKernels}, + mean_groups::MeanGroupsKernels, + norm::{FloatNormArgs, HeadNormArgs, NormKernels}, + pointwise::{self, Op}, + rope2d::{RopeArgs, RopeKernels, RopeTable}, +}; +use bloomery_gpu::gemm::Gemm32Kernels; +use bloomery_gpu::{DeviceTensor, GpuError, WindowMut}; +use cuda_core::{CudaContext, CudaStream, DeviceBuffer}; +use gguf::Gguf; +use std::collections::BTreeMap; +use std::sync::Arc; +use vision::arch::gemma4v::{ + Hparams, MERGE, + card::{self, ScratchLens}, +}; +use vision::{Patches, arch::TokenLimits}; + +const WHAT: &str = "gemma4v tower"; + +struct Linear { + w: DeviceBuffer, + input: (f32, f32), + output: (f32, f32), +} +struct Weights { + patch: DeviceTensor, + pos: DeviceBuffer, + linear: BTreeMap, + norm: BTreeMap>, + std_bias: DeviceBuffer, + std_scale: DeviceBuffer, + ones: DeviceBuffer, + head_ones: DeviceBuffer, +} + +/// A resident Gemma4 vision tower and its bounded buffers, one image at a time. +pub struct Encoder { + hp: Hparams, + limits: TokenLimits, + standardize: bool, + w: Weights, + gemm: GemmKernels, + patch_gemm: Gemm32Kernels, + norm: NormKernels, + ops: pointwise::Kernels, + rope: RopeKernels, + attn: AttnKernels, + mean: MeanGroupsKernels, + s: Scratch, + weight_bytes: u64, + scratch_bytes: u64, + launches: usize, + stream: Arc, +} + +struct Scratch { + raw: DeviceBuffer, + patch: DeviceBuffer, + x: DeviceBuffer, + residual: DeviceBuffer, + h: DeviceBuffer, + h32: DeviceBuffer, + work: DeviceBuffer, + att: DeviceBuffer, + gathered: DeviceBuffer, + qkv: DeviceBuffer, + gate: DeviceBuffer, + up: DeviceBuffer, + act: DeviceBuffer, + tmp1: DeviceBuffer, + tmp2: DeviceBuffer, + pooled: DeviceBuffer, + scaled: DeviceBuffer, + pooled_norm: DeviceBuffer, + rows: DeviceBuffer, + cs: DeviceBuffer, +} + +impl Scratch { + fn new(stream: &CudaStream, l: ScratchLens) -> Result { + let b = |n| DeviceBuffer::::zeroed(stream, n); + Ok(Self { + raw: DeviceBuffer::zeroed(stream, l.raw)?, + patch: DeviceBuffer::zeroed(stream, l.hidden)?, + x: DeviceBuffer::zeroed(stream, l.hidden)?, + residual: DeviceBuffer::zeroed(stream, l.hidden)?, + h: b(l.hidden)?, + h32: DeviceBuffer::zeroed(stream, l.hidden)?, + work: b(l.hidden)?, + att: b(l.hidden)?, + gathered: b(l.hidden)?, + qkv: b(l.qkv)?, + gate: b(l.ff)?, + up: b(l.ff)?, + act: b(l.ff)?, + tmp1: b(l.ff)?, + tmp2: b(l.ff)?, + pooled: b(l.pooled)?, + scaled: b(l.pooled)?, + pooled_norm: b(l.pooled)?, + rows: b(l.rows)?, + cs: DeviceBuffer::zeroed(stream, 64 * l.patches)?, + }) + } + fn bytes(&self) -> u64 { + 4 * (self.raw.len() + + self.patch.len() + + self.cs.len() + + self.x.len() + + self.residual.len() + + self.h32.len()) as u64 + + 2 * [ + &self.h, + &self.work, + &self.att, + &self.gathered, + &self.qkv, + &self.gate, + &self.up, + &self.act, + &self.tmp1, + &self.tmp2, + &self.pooled, + &self.scaled, + &self.pooled_norm, + &self.rows, + ] + .iter() + .map(|b| b.len() as u64) + .sum::() + } +} + +fn permute(v: &[T], k: usize) -> Vec { + (0..v.len() / k) + .flat_map(|r| { + let h = r / 64 * 64; + let j = r % 64; + let src = h + [0, 32, 16, 48][j / 16] + j % 16; + v[src * k..(src + 1) * k].iter().copied() + }) + .collect() +} + +fn clamp(file: &Gguf, name: &str, side: &str) -> Result<(f32, f32), GpuError> { + let read = |suffix: &str, default| { + let key = name.replace(".weight", &format!(".{side}_{suffix}")); + if file.find(&key).is_none() { + return Ok(default); + } + let v = read_f32(file, &key, WHAT)?; + match v.as_slice() { + [x] if x.is_finite() => Ok(*x), + _ => Err(GpuError::shape( + WHAT, + format!("{key}: not one finite scalar"), + )), + } + }; + let bounds = (read("min", -f32::MAX)?, read("max", f32::MAX)?); + if bounds.0 > bounds.1 { + return Err(GpuError::shape( + WHAT, + format!("{name}: {side} clamp is reversed"), + )); + } + Ok(bounds) +} + +impl Encoder { + /// Load the BF16 vision half of `file` and buffers for `limits.max()` pooled tokens. + pub fn load( + ctx: &Arc, + stream: &Arc, + file: &Gguf, + limits: TokenLimits, + ) -> Result { + let hp = Hparams::read(file).map_err(|e| GpuError::plan(WHAT, e.to_string()))?; + let bytes = card::of(file, &hp, limits).map_err(|e| GpuError::plan(WHAT, e.to_string()))?; + let table = vision::arch::gemma4v::tensors::check(file, &hp) + .map_err(|e| GpuError::plan(WHAT, e.to_string()))?; + let mut linear = BTreeMap::new(); + let mut norm = BTreeMap::new(); + for t in &table { + if t.ty == gguf::GgmlType::BF16 { + let mut v = read_bf16(file, &t.name, WHAT)?; + if v.iter().any(|&bits| !crate::bf16_f32(bits).is_finite()) { + return Err(GpuError::shape( + WHAT, + format!("{} contains a non-finite weight", t.name), + )); + } + if t.name.ends_with(".attn_q.weight") || t.name.ends_with(".attn_k.weight") { + v = permute(&v, hp.dim); + } + linear.insert( + t.name.clone(), + Linear { + w: DeviceBuffer::from_host(stream, &v)?, + input: clamp(file, &t.name, "input")?, + output: clamp(file, &t.name, "output")?, + }, + ); + } else if t.name.starts_with("v.blk.") && t.name.ends_with(".weight") { + let mut v = read_f32(file, &t.name, WHAT)?; + if t.name.ends_with(".attn_q_norm.weight") + || t.name.ends_with(".attn_k_norm.weight") + { + v = permute(&v, 1); + } + if v.iter().any(|x| !x.is_finite()) { + return Err(GpuError::shape( + WHAT, + format!("{} contains a non-finite gain", t.name), + )); + } + norm.insert(t.name.clone(), DeviceBuffer::from_host(stream, &v)?); + } + } + let vector = |name, default| -> Result<_, GpuError> { + let v = if file.find(name).is_some() { + read_f32(file, name, WHAT)? + } else { + vec![default; hp.dim] + }; + if v.iter().any(|x| !x.is_finite()) { + return Err(GpuError::shape( + WHAT, + format!("{name} contains a non-finite value"), + )); + } + Ok(DeviceBuffer::from_host(stream, &v)?) + }; + let patch = read_f32(file, "v.patch_embd.weight", WHAT)?; + let pos = read_f32(file, "v.position_embd.weight", WHAT)?; + if patch.iter().chain(&pos).any(|x| !x.is_finite()) { + return Err(GpuError::shape(WHAT, "non-finite patch or position weight")); + } + let w = Weights { + patch: DeviceTensor::upload(stream, &patch, hp.dim, 3 * hp.patch * hp.patch)?, + pos: DeviceBuffer::from_host(stream, &pos)?, + linear, + norm, + std_bias: vector("v.std_bias", 0.0)?, + std_scale: vector("v.std_scale", 1.0)?, + ones: DeviceBuffer::from_host(stream, &vec![1.0; hp.dim])?, + head_ones: DeviceBuffer::from_host(stream, &[1.0f32; 64])?, + }; + let weight_bytes = 4 + * (w.patch.buf().len() + + w.pos.len() + + w.std_bias.len() + + w.std_scale.len() + + w.ones.len() + + w.head_ones.len() + + w.norm.values().map(DeviceBuffer::len).sum::()) as u64 + + 2 * w.linear.values().map(|l| l.w.len() as u64).sum::(); + let lens = + ScratchLens::new(&hp, limits.max()).map_err(|e| GpuError::plan(WHAT, e.to_string()))?; + let s = Scratch::new(stream, lens)?; + let gemm = GemmKernels::load(ctx, stream)?; + let scratch_bytes = s.bytes() + gemm.scratch_bytes(); + if (weight_bytes, scratch_bytes) != (bytes.weights, bytes.scratch) { + return Err(GpuError::shape( + WHAT, + format!( + "uploaded weights/scratch {weight_bytes}/{scratch_bytes}, reserved {}/{}", + bytes.weights, bytes.scratch + ), + )); + } + let standardize = file.find("v.std_bias").is_some(); + let clamps = w + .linear + .iter() + .map(|(name, w)| { + let post = name.ends_with(".attn_out.weight") || name.ends_with(".ffn_down.weight"); + usize::from(w.input != (-f32::MAX, f32::MAX)) + + if w.output == (-f32::MAX, f32::MAX) { + 0 + } else if post { + 1 + } else { + 2 + } + }) + .sum::(); + let launches = 7 + 22 * hp.n_layer + usize::from(standardize) + clamps; + Ok(Self { + hp, + limits, + standardize, + w, + s, + weight_bytes, + scratch_bytes, + launches, + stream: Arc::clone(stream), + gemm, + patch_gemm: Gemm32Kernels::load(ctx)?, + norm: NormKernels::load(ctx)?, + ops: pointwise::Kernels::load(ctx)?, + rope: RopeKernels::load(ctx)?, + attn: AttnKernels::load(ctx)?, + mean: MeanGroupsKernels::load(ctx)?, + }) + } + + /// Encode one raster patch grid with no intermediate readback. + pub fn encode(&mut self, patches: &Patches) -> Result, GpuError> { + self.encode_tapped(patches, &mut NoTaps) + } + + /// Encode one grid, handing requested graph-node rows to `taps`. + pub fn encode_tapped( + &mut self, + p: &Patches, + taps: &mut dyn TapSink, + ) -> Result, GpuError> { + let n = p + .n_vit_h + .checked_mul(p.n_vit_w) + .ok_or_else(|| GpuError::shape(WHAT, "patch count overflows"))?; + let raw = n + .checked_mul(p.patch_len) + .ok_or_else(|| GpuError::shape(WHAT, "patch values overflow"))?; + let hp = &self.hp; + let d = hp.dim; + if n == 0 + || n > 9 * self.limits.max() + || !p.n_vit_w.is_multiple_of(MERGE) + || !p.n_vit_h.is_multiple_of(MERGE) + || p.n_vit_w > hp.pos_rows + || p.n_vit_h > hp.pos_rows + || p.patch_len != 3 * hp.patch * hp.patch + || raw != p.bf16.len() + || p.plan.span_grid() != (p.n_vit_w / MERGE, p.n_vit_h / MERGE) + { + return Err(GpuError::shape( + WHAT, + "patch geometry, token budget or learned positions do not match", + )); + } + let stream = &self.stream; + let s = &mut self.s; + let input = vision::arch::gemma4v::conv_pixels(&p.bf16) + .map_err(|e| GpuError::shape(WHAT, e.to_string()))?; + WindowMut::of_mut(&mut s.raw, 0, raw)?.copy_from_host(stream, &input)?; + self.patch_gemm + .enqueue_f32_tile(stream, &self.w.patch, &s.raw, n, &mut s.patch)?; + self.ops.positions( + stream, + pointwise::PositionArgs { + x: &s.patch, + table: &self.w.pos, + rows: n, + width: d, + grid_w: p.n_vit_w, + pos_rows: hp.pos_rows, + out: &mut s.x, + }, + )?; + tap_f32(stream, taps, "pos_embd", &s.x, n, d)?; + let table = RopeTable::neox_halves(p.n_vit_h, p.n_vit_w, 100.0); + WindowMut::of_mut(&mut s.cs, 0, table.cs.len())?.copy_from_host(stream, &table.cs)?; + let c = Chain { + hp, + norm: &self.norm, + rope: &self.rope, + attn: &self.attn, + mean: &self.mean, + standardize: self.standardize, + gemm: &self.gemm, + ops: &self.ops, + w: &self.w, + }; + for l in 0..hp.n_layer { + c.attention(stream, s, n, l)?; + c.ffn(stream, s, n, l)?; + tap_f32(stream, taps, &format!("layer_out-{l}"), &s.x, n, d)?; + } + c.finish(stream, s, n, p.n_vit_w, taps)?; + let rows = n / 9; + Ok(Encoded { + rows: DeviceTensor::window_of(&s.rows, 0, rows, hp.out_dim)?, + launches: self.launches, + }) + } +} + +// Patch input and output buffers are free after embedding and hold the post-norm work. +struct PostScratch<'a> { + input: &'a mut DeviceBuffer, + raw: &'a mut DeviceBuffer, + clamped: &'a mut DeviceBuffer, + norm16: &'a mut DeviceBuffer, + norm32: &'a mut DeviceBuffer, +} + +struct Chain<'a> { + hp: &'a Hparams, + norm: &'a NormKernels, + rope: &'a RopeKernels, + attn: &'a AttnKernels, + mean: &'a MeanGroupsKernels, + standardize: bool, + gemm: &'a GemmKernels, + ops: &'a pointwise::Kernels, + w: &'a Weights, +} +impl Chain<'_> { + fn attention( + &self, + stream: &CudaStream, + s: &mut Scratch, + n: usize, + l: usize, + ) -> Result<(), GpuError> { + let hp = self.hp; + let d = hp.dim; + let name = |stem| format!("v.blk.{l}.{stem}.weight"); + self.norm.enqueue_f32( + stream, + FloatNormArgs { + x: &s.x, + gain: self.gain(&name("ln1"))?, + eps: hp.eps, + rows: n, + to_f32: false, + y16: &mut s.h, + y32: &mut s.h32, + }, + )?; + for (stem, col) in [("attn_q", 0), ("attn_k", d), ("attn_v", 2 * d)] { + self.linear( + stream, + &name(stem), + &s.h, + (n, d, d), + &mut s.work, + (&mut s.tmp1, &mut s.tmp2), + )?; + self.op( + stream, + Op::Copy, + &s.work, + &s.work, + (n, d, 3 * d, col), + &mut s.qkv, + )?; + } + for (stem, col) in [("attn_q_norm", 0), ("attn_k_norm", d)] { + self.norm.enqueue_heads( + stream, + HeadNormArgs { + x: &mut s.qkv, + gain: self.gain(&name(stem))?, + eps: hp.eps, + rows: n, + row_width: 3 * d, + col0: col, + n_heads: hp.n_head, + }, + )?; + } + self.norm.enqueue_heads( + stream, + HeadNormArgs { + x: &mut s.qkv, + gain: &self.w.head_ones, + eps: hp.eps, + rows: n, + row_width: 3 * d, + col0: 2 * d, + n_heads: hp.n_head, + }, + )?; + self.rope.enqueue( + stream, + RopeArgs { + cs: &s.cs, + n, + row_width: 3 * d, + q0: 0, + k0: d, + n_heads: hp.n_head, + x: &mut s.qkv, + }, + )?; + self.attn.enqueue( + stream, + AttnArgs { + qkv: &s.qkv, + layout: QkvLayout { + row_width: 3 * d, + q0: 0, + k0: d, + v0: 2 * d, + n_heads: hp.n_head, + }, + n, + scale: 1.0, + out_width: d, + out: &mut s.att, + }, + )?; + self.post( + stream, + (&name("attn_out"), &s.att, d), + &name("attn_post_norm"), + n, + PostScratch { + input: &mut s.tmp1, + raw: &mut s.raw, + clamped: &mut s.patch, + norm16: &mut s.h, + norm32: &mut s.h32, + }, + )?; + self.ops + .add_f32(stream, &s.x, &s.h32, n * d, &mut s.residual)?; + Ok(()) + } + fn ffn( + &self, + stream: &CudaStream, + s: &mut Scratch, + n: usize, + l: usize, + ) -> Result<(), GpuError> { + let hp = self.hp; + let (d, f) = (hp.dim, hp.ff); + let name = |stem| format!("v.blk.{l}.{stem}.weight"); + self.norm.enqueue_f32( + stream, + FloatNormArgs { + x: &s.residual, + gain: self.gain(&name("ln2"))?, + eps: hp.eps, + rows: n, + to_f32: false, + y16: &mut s.h, + y32: &mut s.h32, + }, + )?; + self.linear( + stream, + &name("ffn_gate"), + &s.h, + (n, f, d), + &mut s.gate, + (&mut s.tmp1, &mut s.tmp2), + )?; + self.linear( + stream, + &name("ffn_up"), + &s.h, + (n, f, d), + &mut s.up, + (&mut s.tmp1, &mut s.tmp2), + )?; + let op = if hp.gelu_tanh { + Op::GeGlu + } else { + Op::QuickGeGlu + }; + self.op(stream, op, &s.gate, &s.up, (n, f, f, 0), &mut s.act)?; + self.post( + stream, + (&name("ffn_down"), &s.act, f), + &name("ffn_post_norm"), + n, + PostScratch { + input: &mut s.tmp1, + raw: &mut s.raw, + clamped: &mut s.patch, + norm16: &mut s.h, + norm32: &mut s.h32, + }, + )?; + self.ops + .add_f32(stream, &s.residual, &s.h32, n * d, &mut s.x)?; + Ok(()) + } + fn finish( + &self, + stream: &CudaStream, + s: &mut Scratch, + n: usize, + grid_w: usize, + taps: &mut dyn TapSink, + ) -> Result<(), GpuError> { + let hp = self.hp; + let d = hp.dim; + let rows = n / 9; + self.ops.pool_f32( + stream, + pointwise::FloatPoolArgs { + x: &s.x, + rows: n, + width: d, + grid_w, + out: &mut s.gathered, + }, + )?; + self.mean + .enqueue(stream, n, 9, d, &s.gathered, &mut s.pooled)?; + + self.ops.enqueue( + stream, + pointwise::Args { + op: Op::Scale, + x: &s.pooled, + b: &s.pooled, + gain: &self.w.ones, + bias: &self.w.std_bias, + rows, + width: d, + out_width: d, + out_col: 0, + scale: (d as f32).sqrt(), + clamp: (-f32::MAX, f32::MAX), + out: &mut s.scaled, + }, + )?; + tap(stream, taps, "pooled", &s.scaled, rows, d)?; + if self.standardize { + self.ops.enqueue( + stream, + pointwise::Args { + op: Op::Affine, + x: &s.scaled, + b: &s.scaled, + gain: &self.w.std_scale, + bias: &self.w.std_bias, + rows, + width: d, + out_width: d, + out_col: 0, + scale: 1.0, + clamp: (-f32::MAX, f32::MAX), + out: &mut s.pooled, + }, + )?; + tap(stream, taps, "std_scaled", &s.pooled, rows, d)?; + } + let pooled = if self.standardize { + &s.pooled + } else { + &s.scaled + }; + self.norm.enqueue( + stream, + pooled, + &self.w.ones, + hp.eps, + rows, + &mut s.pooled_norm, + )?; + self.linear( + stream, + "mm.input_projection.weight", + &s.pooled_norm, + (rows, hp.out_dim, d), + &mut s.rows, + (&mut s.tmp1, &mut s.tmp2), + )?; + tap(stream, taps, "projected", &s.rows, rows, hp.out_dim)?; + Ok(()) + } + + fn gain(&self, name: &str) -> Result<&DeviceBuffer, GpuError> { + self.w + .norm + .get(name) + .ok_or_else(|| GpuError::tensor(WHAT, name, "a norm gain")) + } + fn op( + &self, + stream: &CudaStream, + op: Op, + x: &DeviceBuffer, + b: &DeviceBuffer, + shape: (usize, usize, usize, usize), + out: &mut DeviceBuffer, + ) -> Result<(), GpuError> { + let (rows, width, out_width, out_col) = shape; + self.ops.enqueue( + stream, + pointwise::Args { + op, + x, + b, + gain: &self.w.ones, + bias: &self.w.std_bias, + rows, + width, + out_width, + out_col, + scale: 1.0, + clamp: (-f32::MAX, f32::MAX), + out, + }, + ) + } + fn post( + &self, + stream: &CudaStream, + (name, x, k): (&str, &DeviceBuffer, usize), + gain: &str, + n: usize, + s: PostScratch<'_>, + ) -> Result<(), GpuError> { + let d = self.hp.dim; + let w = self + .w + .linear + .get(name) + .ok_or_else(|| GpuError::tensor(WHAT, name, "a BF16 linear weight"))?; + let input = if w.input != (-f32::MAX, f32::MAX) { + self.ops.enqueue( + stream, + pointwise::Args { + op: Op::Clamp, + x, + b: x, + gain: &self.w.ones, + bias: &self.w.std_bias, + rows: n, + width: k, + out_width: k, + out_col: 0, + scale: 1.0, + clamp: w.input, + out: s.input, + }, + )?; + &*s.input + } else { + x + }; + self.gemm.enqueue_f32( + stream, + FloatGemmArgs { + a: input, + b: &w.w, + m: n, + n: d, + k, + c: s.raw, + }, + )?; + let input = if w.output != (-f32::MAX, f32::MAX) { + self.ops + .clamp_f32(stream, s.raw, w.output, n * d, s.clamped)?; + &*s.clamped + } else { + &*s.raw + }; + self.norm.enqueue_f32( + stream, + FloatNormArgs { + x: input, + gain: self.gain(gain)?, + eps: self.hp.eps, + rows: n, + to_f32: true, + y16: s.norm16, + y32: s.norm32, + }, + ) + } + + fn linear( + &self, + stream: &CudaStream, + name: &str, + x: &DeviceBuffer, + shape: (usize, usize, usize), + y: &mut DeviceBuffer, + temp: (&mut DeviceBuffer, &mut DeviceBuffer), + ) -> Result<(), GpuError> { + let (n, out, k) = shape; + let w = self + .w + .linear + .get(name) + .ok_or_else(|| GpuError::tensor(WHAT, name, "a BF16 linear weight"))?; + let clamp_into = |x: &DeviceBuffer, width, bounds, out: &mut DeviceBuffer| { + self.ops.enqueue( + stream, + pointwise::Args { + op: Op::Clamp, + x, + b: x, + gain: &self.w.ones, + bias: &self.w.std_bias, + rows: n, + width, + out_width: width, + out_col: 0, + scale: 1.0, + clamp: bounds, + out, + }, + ) + }; + let src = if w.input != (-f32::MAX, f32::MAX) { + clamp_into(x, k, w.input, temp.0)?; + &*temp.0 + } else { + x + }; + self.gemm.enqueue( + stream, + GemmArgs { + a: src, + b: &w.w, + bias: None, + epilogue: Epilogue::None, + resid: None, + m: n, + n: out, + k, + c: y, + }, + )?; + if w.output != (-f32::MAX, f32::MAX) { + clamp_into(y, out, w.output, temp.1)?; + self.op(stream, Op::Copy, temp.1, temp.1, (n, out, out, 0), y)?; + } + Ok(()) + } +} + +impl crate::Tower for Encoder { + fn encode(&mut self, p: &Patches) -> Result, GpuError> { + Encoder::encode(self, p) + } + fn out_dim(&self) -> usize { + self.hp.out_dim + } + fn weight_bytes(&self) -> u64 { + self.weight_bytes + } + fn scratch_bytes(&self) -> u64 { + self.scratch_bytes + } +} diff --git a/crates/gpu-vision/src/chain.rs b/crates/gpu-vision/src/chain.rs index 7b2a0ac69..fd3e9b39f 100644 --- a/crates/gpu-vision/src/chain.rs +++ b/crates/gpu-vision/src/chain.rs @@ -11,11 +11,19 @@ use gguf::{GgmlType, Gguf}; /// Where an encode's intermediate tensors go when a caller asks for them (a gate). `wants` is /// asked before each named point; a `true` synchronizes the stream and hands the tensor over -/// as bf16 bits, `cols` values per row. The names are each tower's own (listed in its module +/// through `take` for BF16 or `take_f32` for F32, `cols` values per row. Every sink handles both +/// storage formats. The names are each tower's own (listed in its module /// doc), the oracle's file stems or graph node names a gate maps them to. pub trait TapSink { fn wants(&self, name: &str) -> bool; fn take(&mut self, name: &str, cols: usize, bits: Vec); + fn take_f32(&mut self, name: &str, cols: usize, values: Vec) { + self.take( + name, + cols, + values.into_iter().map(crate::f32_bf16).collect(), + ); + } } /// The sink of a plain encode. @@ -26,6 +34,7 @@ impl TapSink for NoTaps { false } fn take(&mut self, _: &str, _: usize, _: Vec) {} + fn take_f32(&mut self, _: &str, _: usize, _: Vec) {} } /// An encode's result: the rows the text model reads in place of an image's tokens (`out_dim` bf16 @@ -53,6 +62,26 @@ pub(crate) fn tap( Ok(()) } +/// Hand an F32 residual stream to a sink without changing its storage precision. +pub(crate) fn tap_f32( + stream: &CudaStream, + taps: &mut dyn TapSink, + name: &str, + buf: &DeviceBuffer, + rows: usize, + cols: usize, +) -> Result<(), GpuError> { + if taps.wants(name) { + stream.synchronize()?; + taps.take_f32( + name, + cols, + Window::::of(buf, 0, rows * cols)?.to_host_vec(stream)?, + ); + } + Ok(()) +} + /// The first `len` values of `buf`, after the stream's work so far. pub(crate) fn to_host( stream: &CudaStream, diff --git a/crates/gpu-vision/src/gemm_bf16.rs b/crates/gpu-vision/src/gemm_bf16.rs index 2790cec85..ae28c333e 100644 --- a/crates/gpu-vision/src/gemm_bf16.rs +++ b/crates/gpu-vision/src/gemm_bf16.rs @@ -2,6 +2,7 @@ //! accumulation on `mma.sync.m16n8k16`, bf16 output, with the epilogues the encoder's linear //! layers need. `B` is a linear layer's weight as the file stores it (one row per output), so no //! operand is transposed on the host. +//! The unbiased F32 store shares the same accumulator walk and keeps the result for a following F32 norm. //! //! Tiling: one block of [`THREADS`] threads computes a [`TILE_M`] × [`TILE_N`] tile of `C`; its four //! warps each own a 32 × 32 quarter (2 m16 × 4 n8 fragments, 32 f32 accumulators per lane). The K @@ -90,85 +91,48 @@ pub(crate) fn gelu_erf_dev(x: f32) -> f32 { mul_rn_f32(mul_rn_f32(x, 0.5), add_rn_f32(1.0, e)) } -// ---------------------------------------------------------------- kernels - -#[cuda_module] -mod gemm_kernels { - use super::*; - - /// `C = A·Bᵀ` with the module rule's bias and epilogue: `a` is `m` rows of `k` bf16, `b` is - /// `n` rows of `k` bf16, `c` (and `resid`, when read) `m` rows of `n` bf16, all row-major. - /// `use_bias` 1 adds `bias[j]` (f32) before the rounding; `gelu` 1 and `use_resid` 1 select - /// the epilogues (the launcher passes at most one). Block `b` computes tile row - /// `b / (n / TILE_N)`, tile column `b % (n / TILE_N)`. `n` a multiple of [`TILE_N`] and `k` - /// even and non-zero (host-checked). - #[allow( - clippy::too_many_arguments, - reason = "kernel entry: the device ABI takes the arguments flat (rust-quality R8)" - )] - #[kernel] - #[launch_bounds(128)] - #[launch_contract( - domain = 1, - block = (128, 1, 1), - requires = ( - a.len() >= m * k, - b.len() >= n * k, - bias.len() >= n * use_bias, - resid.len() >= m * n * use_resid, - c.len() >= m * n - ) - )] - pub fn vis_gemm_bf16( - a: &[u16], - b: &[u16], - bias: &[f32], - resid: &[u16], - m: u32, - n: u32, - k: u32, - use_bias: u32, - gelu: u32, - use_resid: u32, - mut c: DisjointSlice, - ) { - static mut AS: SharedArray = SharedArray::UNINIT; - static mut BS: SharedArray = SharedArray::UNINIT; +/// One tensor-core accumulator walk for both output storage formats. +macro_rules! bf16_accumulate { + ($a:ident, $b:ident, $m:ident, $n:ident, $k:ident; + $acc:ident, $rows:ident, $cols:ident, $tm:ident, $tn:ident, $lane:ident, + $wm:ident, $wn:ident, $AS:ident, $BS:ident) => { + static mut $AS: SharedArray = SharedArray::UNINIT; + static mut $BS: SharedArray = SharedArray::UNINIT; let tid = cuda_device::thread::threadIdx_x() as usize; let blk = cuda_device::thread::blockIdx_x() as usize; - let rows = m as usize; - let cols = n as usize; - let kw = (k / 2) as usize; // u32 words per operand row - let tiles_n = cols / TILE_N; - let tm = blk / tiles_n; - let tn = blk - tm * tiles_n; - if tm * TILE_M >= rows { + let $rows = $m as usize; + let $cols = $n as usize; + let kw = ($k / 2) as usize; // u32 words per operand row + let tiles_n = $cols / TILE_N; + let $tm = blk / tiles_n; + let $tn = blk - $tm * tiles_n; + if $tm * TILE_M >= $rows { return; // block-uniform: no barrier and no warp collective is skipped } // SAFETY: block-shared, TILE_WORDS words each, written only by the staging loop between // the barriers that publish it. let (as_, bs_) = unsafe { ( - SharedArray::as_raw_mut_ptr(&raw mut AS), - SharedArray::as_raw_mut_ptr(&raw mut BS), + SharedArray::as_raw_mut_ptr(&raw mut $AS), + SharedArray::as_raw_mut_ptr(&raw mut $BS), ) }; - let aw = a.as_ptr().cast::(); - let bw = b.as_ptr().cast::(); + let aw = $a.as_ptr().cast::(); + let bw = $b.as_ptr().cast::(); - let lane = cuda_device::warp::lane_id() as usize; + let $lane = cuda_device::warp::lane_id() as usize; let wid = tid / 32; - let wm = wid / 2; // this warp's 32-row half of the tile - let wn = wid % 2; // and its 32-column half + let $wm = wid / 2; // this warp's 32-row half of the tile + let $wn = wid % 2; // and its 32-column half // `ldmatrix.x4` lane roles. A: row `lane % 16` of an m16 fragment at k half `lane / 16`. // B: row `(lane % 8) + 8 * (lane / 16)` of an n16 pair at k half `(lane / 8) % 2`. - let a_row = lane % 16; - let a_half = lane / 16; - let b_row = (lane % 8) + 8 * (lane / 16); - let b_half = (lane / 8) % 2; + let a_row = $lane % 16; + let a_half = $lane / 16; + let b_row = ($lane % 8) + 8 * ($lane / 16); + let b_half = ($lane / 8) % 2; - let mut acc = [[0.0f32; 4]; 8]; + let mut $acc = [[0.0f32; 4]; 8]; let k_steps = kw.div_ceil(TILE_K / 2); let mut ks = 0usize; @@ -185,14 +149,14 @@ mod gemm_kernels { let r = i / (TILE_K / 2); let w = i - r * (TILE_K / 2); let gw = w0 + w; - let ar = tm * TILE_M + r; - if ar < rows && gw < kw { + let ar = $tm * TILE_M + r; + if ar < $rows && gw < kw { // SAFETY: ar < m and gw < k / 2, so the word is inside row `ar` of `a`, // whose m·k bf16 the launch contract bounds; device buffers are 4-byte // aligned and k is even, so the word read is aligned. ra[s] = unsafe { *aw.add(ar * kw + gw) }; } - let br = tn * TILE_N + r; + let br = $tn * TILE_N + r; if gw < kw { // SAFETY: br < n (tiles cover n exactly) and gw < k / 2 bound the word // inside `b`'s n·k bf16 (launch contract), aligned as above. @@ -223,7 +187,7 @@ mod gemm_kernels { let mut mi = 0usize; while mi < 2 { cuda_device::thread::__unroll_config::<0>(); - let row = wm * 32 + mi * 16 + a_row; + let row = $wm * 32 + mi * 16 + a_row; // SAFETY: row < TILE_M and word kk·8 + a_half·4 + 4 <= TILE_K / 2 keep the // 16-byte row inside the A tile, published by the barrier above. let p = unsafe { as_.add(row * ROW_WORDS + kk * 8 + a_half * 4) }; @@ -241,7 +205,7 @@ mod gemm_kernels { let mut nj = 0usize; while nj < 2 { cuda_device::thread::__unroll_config::<0>(); - let row = wn * 32 + nj * 16 + b_row; + let row = $wn * 32 + nj * 16 + b_row; // SAFETY: row < TILE_N and the same word bound keep the row inside the B // tile, published by the barrier above. let p = unsafe { bs_.add(row * ROW_WORDS + kk * 8 + b_half * 4) }; @@ -260,13 +224,13 @@ mod gemm_kernels { // SAFETY: the whole warp issues these `mma.sync` (no lane-dependent // branch encloses them) with the fragments it loaded. unsafe { - acc[mi * 4 + nj * 2] = cuda_device::wmma::mma_m16n8k16_f32_bf16( - acc[mi * 4 + nj * 2], + $acc[mi * 4 + nj * 2] = cuda_device::wmma::mma_m16n8k16_f32_bf16( + $acc[mi * 4 + nj * 2], af[mi], [bf[0], bf[1]], ); - acc[mi * 4 + nj * 2 + 1] = cuda_device::wmma::mma_m16n8k16_f32_bf16( - acc[mi * 4 + nj * 2 + 1], + $acc[mi * 4 + nj * 2 + 1] = cuda_device::wmma::mma_m16n8k16_f32_bf16( + $acc[mi * 4 + nj * 2 + 1], af[mi], [bf[2], bf[3]], ); @@ -279,6 +243,52 @@ mod gemm_kernels { } ks += 1; } + }; +} + +// ---------------------------------------------------------------- kernels + +#[cuda_module] +mod gemm_kernels { + use super::*; + + /// `C = A·Bᵀ` with the module rule's bias and epilogue: `a` is `m` rows of `k` bf16, `b` is + /// `n` rows of `k` bf16, `c` (and `resid`, when read) `m` rows of `n` bf16, all row-major. + /// `use_bias` 1 adds `bias[j]` (f32) before the rounding; `gelu` 1 and `use_resid` 1 select + /// the epilogues (the launcher passes at most one). Block `b` computes tile row + /// `b / (n / TILE_N)`, tile column `b % (n / TILE_N)`. `n` a multiple of [`TILE_N`] and `k` + /// even and non-zero (host-checked). + #[allow( + clippy::too_many_arguments, + reason = "kernel entry: the device ABI takes the arguments flat (rust-quality R8)" + )] + #[kernel] + #[launch_bounds(128)] + #[launch_contract( + domain = 1, + block = (128, 1, 1), + requires = ( + a.len() >= m * k, + b.len() >= n * k, + bias.len() >= n * use_bias, + resid.len() >= m * n * use_resid, + c.len() >= m * n + ) + )] + pub fn vis_gemm_bf16( + a: &[u16], + b: &[u16], + bias: &[f32], + resid: &[u16], + m: u32, + n: u32, + k: u32, + use_bias: u32, + gelu: u32, + use_resid: u32, + mut c: DisjointSlice, + ) { + bf16_accumulate!(a, b, m, n, k; acc, rows, cols, tm, tn, lane, wm, wn, AS, BS); // Epilogue: accumulator register j of fragment (mi, ni) is row `group + 8·(j / 2)`, // column `2·(lane % 4) + j % 2` of that fragment. @@ -321,6 +331,44 @@ mod gemm_kernels { f += 1; } } + /// Unbiased BF16 operands with the same tensor-core accumulation, stored in F32. + #[kernel] + #[launch_bounds(128)] + #[launch_contract(domain = 1, block = (128, 1, 1), + requires = (a.len() >= m * k, b.len() >= n * k, c.len() >= m * n))] + pub fn vis_gemm_bf16_f32( + a: &[u16], + b: &[u16], + m: u32, + n: u32, + k: u32, + mut c: DisjointSlice, + ) { + bf16_accumulate!(a, b, m, n, k; acc, rows, cols, tm, tn, lane, wm, wn, AS, BS); + + let group = lane / 4; + let t4 = lane % 4; + let mut f = 0usize; + while f < 8 { + cuda_device::thread::__unroll_config::<0>(); + let mi = f / 4; + let ni = f % 4; + let mut j = 0usize; + while j < 4 { + cuda_device::thread::__unroll_config::<0>(); + let row = tm * TILE_M + wm * 32 + mi * 16 + group + 8 * (j / 2); + let col = tn * TILE_N + wn * 32 + ni * 8 + 2 * t4 + j % 2; + if row < rows { + // SAFETY: the tile covers n columns exactly and each live cell has one owner lane. + unsafe { + *c.get_unchecked_mut(row * cols + col) = acc[f][j]; + } + } + j += 1; + } + f += 1; + } + } } // -------------------------------------------------------------- launchers @@ -339,6 +387,16 @@ pub struct GemmArgs<'a> { pub c: &'a mut DeviceBuffer, } +/// An unbiased BF16 GEMM whose accumulator is stored as F32. +pub(crate) struct FloatGemmArgs<'a> { + pub a: &'a DeviceBuffer, + pub b: &'a DeviceBuffer, + pub m: usize, + pub n: usize, + pub k: usize, + pub c: &'a mut DeviceBuffer, +} + /// The loaded GEMM module. Owns no stream: each enqueue takes the caller's stream. pub struct GemmKernels { module: gemm_kernels::LoadedModule, @@ -349,6 +407,10 @@ pub struct GemmKernels { } impl GemmKernels { + /// Bytes of the unused-operand stand-ins this module holds on the card. + pub(crate) fn scratch_bytes(&self) -> u64 { + 4 * self.dummy_f32.len() as u64 + 2 * self.dummy_u16.len() as u64 + } /// Load this file's device bundle into `ctx`. Load-time only. pub fn load(ctx: &Arc, stream: &CudaStream) -> Result { // SAFETY: this crate owns the embedded device bundle produced for the module above; the @@ -361,6 +423,51 @@ impl GemmKernels { }) } + /// Enqueue an unbiased BF16 GEMM into F32 storage, asynchronously and without allocation. + pub(crate) fn enqueue_f32( + &self, + stream: &CudaStream, + a: FloatGemmArgs<'_>, + ) -> Result<(), GpuError> { + let what = "GemmKernels::enqueue_f32"; + if a.m == 0 || a.n == 0 || !a.n.is_multiple_of(TILE_N) || a.k == 0 || !a.k.is_multiple_of(2) + { + return Err(GpuError::shape( + what, + "need positive m, n a TILE_N multiple and positive even k", + )); + } + let extent = |x: usize, y| { + x.checked_mul(y) + .ok_or_else(|| GpuError::shape(what, "extent overflows")) + }; + let (mk, nk, mn) = (extent(a.m, a.k)?, extent(a.n, a.k)?, extent(a.m, a.n)?); + for count in [mk, nk, mn] { + launch_u32(what, "element count", count)?; + } + if a.a.len() < mk || a.b.len() < nk || a.c.len() < mn { + return Err(GpuError::shape( + what, + "operands do not hold the matrix extents", + )); + } + let grid = launch_u32(what, "grid", a.m.div_ceil(TILE_M) * (a.n / TILE_N))?; + let prep = + self.module + .prepare_vis_gemm_bf16_f32(LaunchConfig1D::new(grid, THREADS_U32, 0))?; + self.module.vis_gemm_bf16_f32( + stream, + &prep, + a.a, + a.b, + launch_u32(what, "m", a.m)?, + launch_u32(what, "n", a.n)?, + launch_u32(what, "k", a.k)?, + a.c, + )?; + Ok(()) + } + /// Enqueue one GEMM ([`GemmArgs`]). `n` a multiple of [`TILE_N`], `k` even and non-zero, `m` /// non-zero; a `Residual` epilogue needs `resid`, and a `resid` without it is refused. /// Asynchronous, allocation-free. diff --git a/crates/gpu-vision/src/lib.rs b/crates/gpu-vision/src/lib.rs index 47d0969ed..584443b6b 100644 --- a/crates/gpu-vision/src/lib.rs +++ b/crates/gpu-vision/src/lib.rs @@ -57,12 +57,15 @@ pub mod chain; pub mod encoder; pub mod gelu_tanh; pub mod gemm_bf16; +#[path = "arch/gemma4v.rs"] +pub mod gemma4v; pub mod glm5v; pub mod layer_norm; pub mod mean_groups; pub mod minicpmv; pub mod mlp; pub mod norm; +pub(crate) mod pointwise; pub mod pos_bilinear; pub mod qwen3vl; pub mod rope2d; @@ -96,7 +99,7 @@ use vision::arch::{ /// One image tower on the card, whichever projector type its file declares. A call takes /// `&mut self`: one image at a time owns the tower's activations, which are allocated at load, and /// the rows it returns borrow them until the next call. -pub trait Tower { +pub trait Tower: Send { /// Encode one image's patches into the rows of its tokens, without waiting for the card; /// read them on the stream the tower was opened with. fn encode(&mut self, patches: &Patches) -> Result, GpuError>; @@ -140,10 +143,11 @@ pub fn open( file: &Gguf, limits: TokenLimits, ) -> Result, GpuError> { - let projector = file - .value("clip.projector_type") - .and_then(gguf::Value::as_str); + let projector = vision::arch::projector_type(file).ok(); match projector { + Some(vision::arch::gemma4v::PROJECTOR_TYPE) => { + Ok(Box::new(gemma4v::Encoder::load(ctx, stream, file, limits)?)) + } Some(deepseek41v::PROJECTOR_TYPE) => { Ok(Box::new(encoder::Encoder::load(ctx, stream, file)?)) } diff --git a/crates/gpu-vision/src/norm.rs b/crates/gpu-vision/src/norm.rs index e09dbbeab..36714a0f4 100644 --- a/crates/gpu-vision/src/norm.rs +++ b/crates/gpu-vision/src/norm.rs @@ -188,6 +188,118 @@ mod norm_kernels { } } + /// RMSNorm of an F32 residual stream, stored as F32 or narrowed for a BF16 GEMM. + #[allow( + clippy::too_many_arguments, + reason = "kernel entry uses the flat device ABI (rust-quality R8)" + )] + #[kernel] + #[launch_bounds(256)] + #[launch_contract( + domain = 1, + block = (256, 1, 1), + requires = (x.len() >= rows * dim, gain.len() >= dim, y16.len() >= rows * dim, y32.len() >= rows * dim) + )] + pub fn vis_rms_norm_f32( + x: &[f32], + gain: &[f32], + eps: f32, + dim: u32, + rows: u32, + to_f32: u32, + mut y16: DisjointSlice, + mut y32: DisjointSlice, + ) { + static mut WSUM: SharedArray = SharedArray::UNINIT; + + let r = thread::blockIdx_x() as usize; + if r >= rows as usize { + return; + } + let tid = thread::threadIdx_x() as usize; + let d = dim as usize; + let words = d / 2; + let base = r * d; + let mut acc = 0.0f32; + let mut w = tid; + while w < words { + let at = base + 2 * w; + // SAFETY: r < rows and 2w < dim keep at inside the input row. + let a = unsafe { *x.get_unchecked(at) }; + // SAFETY: r < rows and 2w + 1 < dim keep at + 1 inside the input row. + let b = unsafe { *x.get_unchecked(at + 1) }; + acc = fma_rn_f32(a, a, acc); + acc = fma_rn_f32(b, b, acc); + w += THREADS; + } + acc = add_rn_f32(acc, warp::shuffle_xor_f32(acc, 16)); + acc = add_rn_f32(acc, warp::shuffle_xor_f32(acc, 8)); + acc = add_rn_f32(acc, warp::shuffle_xor_f32(acc, 4)); + acc = add_rn_f32(acc, warp::shuffle_xor_f32(acc, 2)); + acc = add_rn_f32(acc, warp::shuffle_xor_f32(acc, 1)); + // SAFETY: block-shared, WARPS == blockDim.x / 32, written before the barrier that + // publishes it. + let ws = unsafe { SharedArray::as_raw_mut_ptr(&raw mut WSUM) }; + if warp::lane_id() == 0 { + // SAFETY: tid / 32 < WARPS bounds the slot; lane 0 is its one writer. + unsafe { + *ws.add(tid / 32) = acc; + } + } + thread::sync_threads(); + // SAFETY: the eight slots, published by the barrier above. + let sums = unsafe { + [ + *ws, + *ws.add(1), + *ws.add(2), + *ws.add(3), + *ws.add(4), + *ws.add(5), + *ws.add(6), + *ws.add(7), + ] + }; + let mean = div_rn_f32(warp_tree(sums), d as f32); + let scale = div_rn_f32(1.0, sqrt_rn_f32(add_rn_f32(mean, eps))); + let mut w = tid; + while w < words { + let at = base + 2 * w; + // SAFETY: r < rows and 2*w < d bound this input lane. + let a = unsafe { *x.get_unchecked(at) }; + // SAFETY: r < rows and 2*w + 1 < d bound this input lane. + let b = unsafe { *x.get_unchecked(at + 1) }; + // SAFETY: 2*w < d <= gain.len(). + let g0 = unsafe { *gain.get_unchecked(2 * w) }; + // SAFETY: 2*w + 1 < d <= gain.len(). + let g1 = unsafe { *gain.get_unchecked(2 * w + 1) }; + let y0 = mul_rn_f32(g0, mul_rn_f32(a, scale)); + let y1 = mul_rn_f32(g1, mul_rn_f32(b, scale)); + if to_f32 != 0 { + // SAFETY: at < rows*d <= y32.len(); this thread owns this lane. + unsafe { + *y32.get_unchecked_mut(at) = y0; + } + // SAFETY: at + 1 < rows*d <= y32.len(); this thread owns this lane. + unsafe { + *y32.get_unchecked_mut(at + 1) = y1; + } + } else { + let y0 = f32_to_bf16_rne(y0); + // SAFETY: at < rows*d <= y16.len(); this thread owns this lane. + unsafe { + *y16.get_unchecked_mut(at) = y0; + } + let y1 = f32_to_bf16_rne(y1); + // SAFETY: at + 1 < rows*d <= y16.len(); this thread owns this lane. + unsafe { + *y16.get_unchecked_mut(at + 1) = y1; + } + } + w += THREADS; + } + } + /// `x = RMSNorm(x)` in place over the `n_heads` heads of [`HEAD`] bf16 values at columns /// `col0 + h·64` of each of `rows` rows of `row_width`, every head with the gain of 64: the /// module rule per head. One warp per (row, head), eight to a block; lane `t` owns the pair at @@ -265,6 +377,17 @@ pub struct NormKernels { module: norm_kernels::LoadedModule, } +/// Both output buffers hold the row extent; only the selected storage format is written. +pub(crate) struct FloatNormArgs<'a> { + pub x: &'a DeviceBuffer, + pub gain: &'a DeviceBuffer, + pub eps: f32, + pub rows: usize, + pub to_f32: bool, + pub y16: &'a mut DeviceBuffer, + pub y32: &'a mut DeviceBuffer, +} + impl NormKernels { /// Load this file's device bundle into `ctx`. Load-time only. pub fn load(ctx: &Arc) -> Result { @@ -318,6 +441,50 @@ impl NormKernels { )?; Ok(()) } + pub(crate) fn enqueue_f32( + &self, + stream: &CudaStream, + a: FloatNormArgs<'_>, + ) -> Result<(), GpuError> { + let what = "NormKernels::enqueue_f32"; + let dim = a.gain.len(); + let n = a + .rows + .checked_mul(dim) + .ok_or_else(|| GpuError::shape(what, "element count overflows"))?; + launch_u32(what, "element count", n)?; + if dim == 0 + || !dim.is_multiple_of(2) + || a.rows == 0 + || a.x.len() < n + || a.y16.len() < n + || a.y32.len() < n + { + return Err(GpuError::shape( + what, + "the input and both output buffers must hold every even-width row", + )); + } + let prep = self.module.prepare_vis_rms_norm_f32(LaunchConfig1D::new( + launch_u32(what, "rows", a.rows)?, + THREADS_U32, + 0, + ))?; + self.module.vis_rms_norm_f32( + stream, + &prep, + a.x, + a.gain, + a.eps, + launch_u32(what, "dim", dim)?, + launch_u32(what, "rows", a.rows)?, + u32::from(a.to_f32), + a.y16, + a.y32, + )?; + Ok(()) + } + /// Enqueue the per-head norm ([`HeadNormArgs`]): in place, one launch per segment. The gain is /// [`HEAD`] values; the heads must lie inside the row. Asynchronous. pub fn enqueue_heads( diff --git a/crates/gpu-vision/src/pointwise.rs b/crates/gpu-vision/src/pointwise.rs new file mode 100644 index 000000000..205212059 --- /dev/null +++ b/crates/gpu-vision/src/pointwise.rs @@ -0,0 +1,482 @@ +//! Small row operations between a tower's shared GEMMs and norms. + +use bloomery_gpu::{GpuError, launch_u32}; +use cuda_core::{CudaContext, CudaStream, DeviceBuffer, LaunchConfig1D}; +use cuda_device::convert::{bf16_to_f32, f32_to_bf16_rne}; +use cuda_device::float::{add_rn_f32, div_rn_f32, mul_rn_f32}; +use cuda_device::{DisjointSlice, kernel, launch_bounds, launch_contract, thread}; +use cuda_host::cuda_module; +use std::sync::Arc; + +/// A row operation, each with a separate f32 rounding at its arithmetic boundaries. +#[derive(Clone, Copy)] +pub(crate) enum Op { + GeGlu, + QuickGeGlu, + Affine, + Clamp, + Copy, + Scale, +} + +#[cuda_module] +mod kernels { + use super::*; + + /// Apply one row operation, writing each destination lane once. + #[allow( + clippy::too_many_arguments, + reason = "kernel entry uses the flat device ABI (rust-quality R8)" + )] + #[kernel] + #[launch_bounds(256)] + #[launch_contract(domain = 1, block = (256, 1, 1), requires = (x.len() >= n, out.len() >= n))] + pub(crate) fn vis_row_op( + x: &[u16], + b: &[u16], + gain: &[f32], + bias: &[f32], + n: u32, + width: u32, + stride: u32, + col0: u32, + op: u32, + scale: f32, + lo: f32, + hi: f32, + mut out: DisjointSlice, + ) { + let i = thread::index_1d().get(); + if i >= n as usize { + return; + } + let w = width as usize; + let row = i / w; + let d = i % w; + // SAFETY: i < n <= x.len(). + let a = bf16_to_f32(unsafe { *x.get_unchecked(i) }); + let y = match op { + 1 | 7 => { + // SAFETY: both gated operations require b.len() >= n; i < n. + let other = bf16_to_f32(unsafe { *b.get_unchecked(i) }); + if op == 1 { + mul_rn_f32(bloomery_gpu::gelu::gelu_tanh_dev(a), other) + } else { + let e = mul_rn_f32(-1.702, a).exp(); + let gelu = mul_rn_f32(a, div_rn_f32(1.0, add_rn_f32(1.0, e))); + mul_rn_f32(gelu, other) + } + } + 2 => { + // SAFETY: Affine checks gain at width; d < width. + let gain = unsafe { *gain.get_unchecked(d) }; + // SAFETY: Affine checks bias at width; d < width. + let bias = unsafe { *bias.get_unchecked(d) }; + mul_rn_f32(add_rn_f32(mul_rn_f32(a, scale), -bias), gain) + } + 3 => { + if a < lo { + lo + } else if a > hi { + hi + } else { + a + } + } + 6 => mul_rn_f32(a, scale), + _ => a, + }; + let dst = row * stride as usize + col0 as usize + d; + // SAFETY: the host proves the row extent and col0 + width <= stride; + // each source lane has one distinct destination lane. + unsafe { + *out.get_unchecked_mut(dst) = f32_to_bf16_rne(y); + } + } + + /// Clamp F32 values without narrowing the result. + #[kernel] + #[launch_bounds(256)] + #[launch_contract(domain = 1, block = (256, 1, 1), requires = (x.len() >= n, out.len() >= n))] + pub(crate) fn vis_clamp_f32(x: &[f32], n: u32, lo: f32, hi: f32, mut out: DisjointSlice) { + let i = thread::index_1d().get(); + if i >= n as usize { + return; + } + // SAFETY: i < n <= x.len(). + let value = unsafe { *x.get_unchecked(i) }; + let value = if value < lo { + lo + } else if value > hi { + hi + } else { + value + }; + // SAFETY: i < n <= out.len(); thread i owns this output. + unsafe { + *out.get_unchecked_mut(i) = value; + } + } + + /// Add two F32 residual streams without an intervening narrowing. + #[kernel] + #[launch_bounds(256)] + #[launch_contract(domain = 1, block = (256, 1, 1), requires = (a.len() >= n, b.len() >= n, out.len() >= n))] + pub(crate) fn vis_add_f32(a: &[f32], b: &[f32], n: u32, mut out: DisjointSlice) { + let i = thread::index_1d().get(); + if i >= n as usize { + return; + } + // SAFETY: i < n <= a.len(). + let av = unsafe { *a.get_unchecked(i) }; + // SAFETY: i < n <= b.len(). + let bv = unsafe { *b.get_unchecked(i) }; + let value = add_rn_f32(av, bv); + // SAFETY: i < n <= out.len(); thread i owns this output. + unsafe { + *out.get_unchecked_mut(i) = value; + } + } + + /// Gather three-by-three raster cells and narrow each value for the shared group mean. + #[kernel] + #[launch_bounds(256)] + #[launch_contract(domain = 1, block = (256, 1, 1), requires = (x.len() >= n, out.len() >= n))] + pub(crate) fn vis_pool_order_f32( + x: &[f32], + n: u32, + width: u32, + grid_w: u32, + mut out: DisjointSlice, + ) { + let i = thread::index_1d().get(); + if i >= n as usize { + return; + } + let w = width as usize; + let row = i / w; + let group = row / 9; + let cell = row % 9; + let cols = grid_w as usize / 3; + let src = ((group / cols * 3 + cell / 3) * grid_w as usize + group % cols * 3 + cell % 3) + * w + + i % w; + // SAFETY: complete three-by-three groups permute n lanes, so src < n <= x.len(). + let value = unsafe { *x.get_unchecked(src) }; + let value = f32_to_bf16_rne(value); + // SAFETY: i < n <= out.len(); thread i owns this output. + unsafe { + *out.get_unchecked_mut(i) = value; + } + } + + /// Add the learned x and y rows to the F32 residual stream. + #[kernel] + #[launch_bounds(256)] + #[launch_contract(domain = 1, block = (256, 1, 1), requires = (x.len() >= n, out.len() >= n))] + pub(crate) fn vis_position_rows( + x: &[f32], + table: &[f32], + n: u32, + width: u32, + grid_w: u32, + pos_rows: u32, + mut out: DisjointSlice, + ) { + let i = thread::index_1d().get(); + if i >= n as usize { + return; + } + let w = width as usize; + let row = i / w; + let d = i % w; + let px = row % grid_w as usize; + let py = row / grid_w as usize; + // SAFETY: i < n <= x.len(). + let x = unsafe { *x.get_unchecked(i) }; + // SAFETY: px < pos_rows and d < width bound the first table plane. + let px = unsafe { *table.get_unchecked(px * w + d) }; + // SAFETY: py < pos_rows and d < width bound the second table plane. + let py = unsafe { *table.get_unchecked((pos_rows as usize + py) * w + d) }; + let a = add_rn_f32(x, px); + let y = add_rn_f32(a, py); + if let Some(dst) = out.get_mut(thread::index_1d()) { + *dst = y; + } + } +} + +/// Operands and extents of one row operation; `out_col` places a segment inside wider rows. +pub(crate) struct Args<'a> { + pub op: Op, + pub x: &'a DeviceBuffer, + pub b: &'a DeviceBuffer, + pub gain: &'a DeviceBuffer, + pub bias: &'a DeviceBuffer, + pub rows: usize, + pub width: usize, + pub out_width: usize, + pub out_col: usize, + pub scale: f32, + pub clamp: (f32, f32), + pub out: &'a mut DeviceBuffer, +} + +/// The learned position addition's input and output. +pub(crate) struct PositionArgs<'a> { + pub x: &'a DeviceBuffer, + pub table: &'a DeviceBuffer, + pub rows: usize, + pub width: usize, + pub grid_w: usize, + pub pos_rows: usize, + pub out: &'a mut DeviceBuffer, +} + +pub(crate) struct FloatPoolArgs<'a> { + pub x: &'a DeviceBuffer, + pub rows: usize, + pub width: usize, + pub grid_w: usize, + pub out: &'a mut DeviceBuffer, +} + +/// Loaded row-operation kernels. +pub(crate) struct Kernels { + module: kernels::LoadedModule, +} + +impl Kernels { + pub(crate) fn clamp_f32( + &self, + stream: &CudaStream, + x: &DeviceBuffer, + bounds: (f32, f32), + n: usize, + out: &mut DeviceBuffer, + ) -> Result<(), GpuError> { + let what = "vision F32 clamp"; + if n == 0 + || x.len() < n + || out.len() < n + || !bounds.0.is_finite() + || !bounds.1.is_finite() + || bounds.0 > bounds.1 + { + return Err(GpuError::shape(what, "invalid clamp bounds or buffers")); + } + let prep = self.module.prepare_vis_clamp_f32(LaunchConfig1D::new( + launch_u32(what, "grid", n.div_ceil(256))?, + 256, + 0, + ))?; + self.module.vis_clamp_f32( + stream, + &prep, + x, + launch_u32(what, "n", n)?, + bounds.0, + bounds.1, + out, + )?; + Ok(()) + } + + pub(crate) fn add_f32( + &self, + stream: &CudaStream, + a: &DeviceBuffer, + b: &DeviceBuffer, + n: usize, + out: &mut DeviceBuffer, + ) -> Result<(), GpuError> { + let what = "vision F32 residual add"; + if n == 0 || a.len() < n || b.len() < n || out.len() < n { + return Err(GpuError::shape( + what, + "residual operands do not hold the output extent", + )); + } + let prep = self.module.prepare_vis_add_f32(LaunchConfig1D::new( + launch_u32(what, "grid", n.div_ceil(256))?, + 256, + 0, + ))?; + self.module + .vis_add_f32(stream, &prep, a, b, launch_u32(what, "n", n)?, out)?; + Ok(()) + } + + pub(crate) fn pool_f32( + &self, + stream: &CudaStream, + a: FloatPoolArgs<'_>, + ) -> Result<(), GpuError> { + let FloatPoolArgs { + x, + rows, + width, + grid_w, + out, + } = a; + let what = "vision F32 pooling gather"; + let group_rows = grid_w + .checked_mul(3) + .ok_or_else(|| GpuError::shape(what, "pool grid overflows"))?; + let n = rows + .checked_mul(width) + .ok_or_else(|| GpuError::shape(what, "element count overflows"))?; + if n == 0 + || grid_w == 0 + || !grid_w.is_multiple_of(3) + || !rows.is_multiple_of(group_rows) + || x.len() < n + || out.len() < n + { + return Err(GpuError::shape( + what, + "pooling buffers must hold complete three-by-three groups", + )); + } + let u = |name, value| launch_u32(what, name, value); + let prep = self.module.prepare_vis_pool_order_f32(LaunchConfig1D::new( + u("grid", n.div_ceil(256))?, + 256, + 0, + ))?; + self.module.vis_pool_order_f32( + stream, + &prep, + x, + u("n", n)?, + u("width", width)?, + u("grid width", grid_w)?, + out, + )?; + Ok(()) + } + + /// Load this module's device bundle into the tower's context. + pub(crate) fn load(ctx: &Arc) -> Result { + // SAFETY: this crate owns the module's embedded device bundle. + let module = unsafe { bloomery_gpu::shared_module!(kernels, ctx)? }; + Ok(Self { module }) + } + + /// Apply one checked row operation asynchronously. + pub(crate) fn enqueue(&self, stream: &CudaStream, a: Args<'_>) -> Result<(), GpuError> { + let what = "vision row operation"; + let n = a + .rows + .checked_mul(a.width) + .ok_or_else(|| GpuError::shape(what, "element count overflows"))?; + let need = a + .rows + .checked_mul(a.out_width) + .ok_or_else(|| GpuError::shape(what, "output count overflows"))?; + if n == 0 + || a.out_col + .checked_add(a.width) + .is_none_or(|v| v > a.out_width) + || a.x.len() < n + || a.out.len() < need + || !a.scale.is_finite() + || !a.clamp.0.is_finite() + || !a.clamp.1.is_finite() + || a.clamp.0 > a.clamp.1 + { + return Err(GpuError::shape( + what, + "invalid row extent, buffers, scale or clamp", + )); + } + let op = match a.op { + Op::GeGlu | Op::QuickGeGlu if a.b.len() < n => { + return Err(GpuError::shape(what, "second operand is short")); + } + Op::Affine if a.gain.len() < a.width || a.bias.len() < a.width => { + return Err(GpuError::shape(what, "affine vectors are short")); + } + Op::GeGlu => 1, + Op::QuickGeGlu => 7, + Op::Affine => 2, + Op::Clamp => 3, + Op::Copy => 5, + Op::Scale => 6, + }; + let u = |name, v| launch_u32(what, name, v); + let prep = self.module.prepare_vis_row_op(LaunchConfig1D::new( + u("grid", n.div_ceil(256))?, + 256, + 0, + ))?; + self.module.vis_row_op( + stream, + &prep, + a.x, + a.b, + a.gain, + a.bias, + u("n", n)?, + u("width", a.width)?, + u("stride", a.out_width)?, + u("column", a.out_col)?, + op, + a.scale, + a.clamp.0, + a.clamp.1, + a.out, + )?; + Ok(()) + } + + /// Add learned x/y position rows to the patch embeddings asynchronously. + pub(crate) fn positions( + &self, + stream: &CudaStream, + a: PositionArgs<'_>, + ) -> Result<(), GpuError> { + let what = "vision learned position rows"; + let n = a + .rows + .checked_mul(a.width) + .ok_or_else(|| GpuError::shape(what, "element count overflows"))?; + let table = a + .pos_rows + .checked_mul(a.width) + .and_then(|n| n.checked_mul(2)) + .ok_or_else(|| GpuError::shape(what, "position table count overflows"))?; + if n == 0 + || a.grid_w == 0 + || !a.rows.is_multiple_of(a.grid_w) + || a.grid_w > a.pos_rows + || a.rows / a.grid_w > a.pos_rows + || a.x.len() < n + || a.out.len() < n + || a.table.len() < table + { + return Err(GpuError::shape( + what, + "patch or position extent does not fit its buffer", + )); + } + let u = |name, v| launch_u32(what, name, v); + let prep = self.module.prepare_vis_position_rows(LaunchConfig1D::new( + u("grid", n.div_ceil(256))?, + 256, + 0, + ))?; + self.module.vis_position_rows( + stream, + &prep, + a.x, + a.table, + u("n", n)?, + u("width", a.width)?, + u("grid width", a.grid_w)?, + u("position rows", a.pos_rows)?, + a.out, + )?; + Ok(()) + } +} diff --git a/crates/gpu-vision/src/rope2d.rs b/crates/gpu-vision/src/rope2d.rs index 9424724d0..465ab5533 100644 --- a/crates/gpu-vision/src/rope2d.rs +++ b/crates/gpu-vision/src/rope2d.rs @@ -55,6 +55,25 @@ pub struct RopeTable { } impl RopeTable { + /// Gemma4's two NEOX half-head turns, applied through this module's + /// whole-head pairing after the q/k rows and gains are permuted at load. + #[must_use] + pub(crate) fn neox_halves(n_h: usize, n_w: usize, theta: f32) -> RopeTable { + let step = f64::from(theta).powf(f64::from(-2.0f32 / PAIRS as f32)) as f32; + let mut cs = Vec::with_capacity(n_h * n_w * 2 * PAIRS); + for (y, x) in raster(n_h, n_w) { + for pos in [x, y] { + let mut angle = pos as f32; + for _ in 0..PER_AXIS { + cs.push(f64::from(angle).cos() as f32); + cs.push(f64::from(angle).sin() as f32); + angle *= step; + } + } + } + RopeTable { n_h, n_w, cs } + } + /// The table of an `n_h × n_w` patch grid at RoPE base `theta`, as the reference builds it /// in f32: `f_i = 1.0 / theta^(i / 16)` — the exponent `arange(0, 32, 2) / 32` is exact in /// f32, the power correctly rounded (the device's `powf` the reference calls is within an diff --git a/crates/gpu/src/lib.rs b/crates/gpu/src/lib.rs index 6832eb58d..a7f451cac 100644 --- a/crates/gpu/src/lib.rs +++ b/crates/gpu/src/lib.rs @@ -239,7 +239,7 @@ pub enum GpuError { impl GpuError { /// A geometry contract broken by the caller of `what`. - pub(crate) fn shape(what: &'static str, detail: impl Into) -> GpuError { + pub fn shape(what: &'static str, detail: impl Into) -> GpuError { GpuError::Shape { what, detail: detail.into(), @@ -248,11 +248,7 @@ impl GpuError { /// A tensor `what` looked up that is absent or in the wrong format; /// `need` completes "is not …". - pub(crate) fn tensor( - what: &'static str, - name: impl Into, - need: &'static str, - ) -> GpuError { + pub fn tensor(what: &'static str, name: impl Into, need: &'static str) -> GpuError { GpuError::Tensor { what, name: name.into(), diff --git a/crates/refset/src/arch/gemma_embedding2/mod.rs b/crates/refset/src/arch/gemma_embedding2/mod.rs index 5f5413212..d25343b92 100644 --- a/crates/refset/src/arch/gemma_embedding2/mod.rs +++ b/crates/refset/src/arch/gemma_embedding2/mod.rs @@ -14,7 +14,7 @@ //! its `# model` is the projector file and its `# arch` the projector's //! (`clip`). [`TOWER_CPU`] is its CPU twin. //! - [`SERVER`]: llama-server's `/v1/embeddings` vectors of every text case and -//! image, normalized and raw (`tools/ref/emb2/server_vectors.py`). +//! image, normalized and raw (`tools/ref/emb2/server_vectors.py`). [`SERVER_CPU`] is its CPU twin. //! - [`AUDIO_MEL`]: the pinned host preprocessor's mel from synthetic decoded //! mono PCM (`tools/ref/emb2/mel.sh`), with no model or device opened. //! @@ -61,8 +61,23 @@ pub const TEXT_CPU_SET: &str = "ref_gemma_embedding2_text.cpu"; pub const TOWER_SET: &str = "ref_gemma_embedding2_tower"; pub const TOWER_CPU_SET: &str = "ref_gemma_embedding2_tower.cpu"; +/// The image inputs the mainline tower dump supplies, in the profile's order. +pub const TOWER_IMAGES: &[&str] = &[ + "small-200x200", + "min-96x64", + "free-448x448", + "free-640x480", + "pad-777x513", + "strip-1000x1", + "free-1024x768", +]; + +/// The gate that reads the tower sets. +pub const TOWER_GATE: &str = "gate-gpu-gemma4v-tower"; + /// llama-server's vectors. pub const SERVER_SET: &str = "ref_gemma_embedding2_server"; +pub const SERVER_CPU_SET: &str = "ref_gemma_embedding2_server.cpu"; /// A row's band in the encoder and serving gates: this many times the CPU /// twin's distance from the card set (`shared/glm5next_media.rs`'s `TWIN`, @@ -97,6 +112,7 @@ fn mmproj() -> Result { /// The gate that reads the text sets. const E2E: &[&str] = &["gate-gpu-gemma-embedding2-e2e"]; +const IMAGE: &[&str] = &[TOWER_GATE, "gate-gpu-gemma-embedding2-e2e"]; /// The text cases' token rows, pooled vectors and layer taps on the card. pub static TEXT: Family = Family { @@ -128,28 +144,37 @@ pub static TEXT_CPU: Family = Family { ) }; /// The image cases' tower taps and rows on the card. -pub static TOWER: Family = family( - "tower-gemma4v", - &[TOWER_SET], - Kind::Tower, - &PROFILE, - crate::clefvis::TOWER_ARCH, - mmproj, - RECIPE, -); +pub static TOWER: Family = Family { + consumers: IMAGE, + ..family( + "tower-gemma4v", + &[TOWER_SET], + Kind::Tower, + &PROFILE, + crate::clefvis::TOWER_ARCH, + mmproj, + RECIPE, + ) +}; /// The same, with no card in view. -pub static TOWER_CPU: Family = family( - "tower-gemma4v-cpu", - &[TOWER_CPU_SET], - Kind::TowerCpu, - &PROFILE, - crate::clefvis::TOWER_ARCH, - mmproj, - RECIPE, -); +pub static TOWER_CPU: Family = Family { + consumers: IMAGE, + ..family( + "tower-gemma4v-cpu", + &[TOWER_CPU_SET], + Kind::TowerCpu, + &PROFILE, + crate::clefvis::TOWER_ARCH, + mmproj, + RECIPE, + ) +}; /// llama-server's `/v1/embeddings` vectors. pub static SERVER: Family = Family { - consumers: &["gate-gpu-gemma-embedding2-serve"], + consumers: &[ + "gate-gpu-gemma-embedding2-e2e", + "gate-gpu-gemma-embedding2-serve", + ], ..family( "server-gemma-embedding2", &[SERVER_SET], @@ -183,8 +208,30 @@ pub static AUDIO_MEL: Family = Family { consumers: &["gate-emb2-mel"], }; +/// The same requests with the text model and projector on the CPU. +pub static SERVER_CPU: Family = Family { + consumers: E2E, + ..family( + "server-gemma-embedding2-cpu", + &[SERVER_CPU_SET], + Kind::ServerCpu, + &PROFILE, + ARCH, + model, + RECIPE, + ) +}; + /// The architecture's families, in the order `refset-check` lists them. -pub static FAMILIES: &[&Family] = &[&TEXT, &TEXT_CPU, &TOWER, &TOWER_CPU, &SERVER, &AUDIO_MEL]; +pub static FAMILIES: &[&Family] = &[ + &TEXT, + &TEXT_CPU, + &TOWER, + &TOWER_CPU, + &SERVER, + &SERVER_CPU, + &AUDIO_MEL, +]; #[cfg(test)] mod tests; diff --git a/crates/refset/src/arch/gemma_embedding2/tests.rs b/crates/refset/src/arch/gemma_embedding2/tests.rs index 899c44a44..e8ce7239e 100644 --- a/crates/refset/src/arch/gemma_embedding2/tests.rs +++ b/crates/refset/src/arch/gemma_embedding2/tests.rs @@ -1,6 +1,6 @@ use super::{ - ARCH, MMPROJ_BF16, MMPROJ_BF16_SHA256, MODEL, MODEL_SHA256, PROFILE, SERVER, TEXT, TEXT_CPU, - TOWER, TOWER_CPU, + ARCH, MMPROJ_BF16, MMPROJ_BF16_SHA256, MODEL, MODEL_SHA256, PROFILE, SERVER, SERVER_CPU, TEXT, + TEXT_CPU, TOWER, TOWER_CPU, }; use crate::RefError; use crate::emb2::tests::{check, header, remove, set_dir}; @@ -40,12 +40,13 @@ type FileCase<'a> = (Kind, &'a Family, Vec<(&'a str, &'a str)>, &'a str); /// the trailer is written, and the text the error names. type RefusedCase<'a> = (Kind, &'a Family, (&'a str, &'a str), bool, &'a str); -const ALL: [(Kind, &Family); 5] = [ +const ALL: [(Kind, &Family); 6] = [ (Kind::Text, &TEXT), (Kind::TextCpu, &TEXT_CPU), (Kind::Tower, &TOWER), (Kind::TowerCpu, &TOWER_CPU), (Kind::Server, &SERVER), + (Kind::ServerCpu, &SERVER_CPU), ]; /// Each family takes a set of its own kind and device, of the pinned files, @@ -75,12 +76,14 @@ fn each_family_takes_a_set_of_its_kind() -> Result<(), RefError> { #[test] fn a_set_of_another_kind_or_device_is_foreign() -> Result<(), RefError> { let dir = set_dir("kind")?; - let cases: [(Kind, &Family, &str); 5] = [ + let cases: [(Kind, &Family, &str); 7] = [ (Kind::TextCpu, &TEXT, "device \"cpu\""), (Kind::Text, &TEXT_CPU, "device \"cuda\""), (Kind::TowerCpu, &TOWER, "device \"cpu\""), (Kind::Tower, &TOWER_CPU, "device \"cuda\""), (Kind::Server, &TEXT, "emb2 \"server\""), + (Kind::ServerCpu, &SERVER, "device \"cpu\""), + (Kind::Server, &SERVER_CPU, "device \"cuda\""), ]; for (kind, family, want) in cases { match check(&dir, family, &header(&PROFILE, ARCH, kind, &[]), true) { diff --git a/crates/refset/src/emb2.rs b/crates/refset/src/emb2.rs index 1dda4d98e..cfbd12c5c 100644 --- a/crates/refset/src/emb2.rs +++ b/crates/refset/src/emb2.rs @@ -18,6 +18,7 @@ pub enum Kind { Tower, TowerCpu, Server, + ServerCpu, } impl Kind { @@ -27,7 +28,7 @@ impl Kind { match self { Kind::Text | Kind::TextCpu => "text", Kind::Tower | Kind::TowerCpu => "tower", - Kind::Server => "server", + Kind::Server | Kind::ServerCpu => "server", } } @@ -35,7 +36,7 @@ impl Kind { #[must_use] pub const fn device(self) -> &'static str { match self { - Kind::TextCpu | Kind::TowerCpu => "cpu", + Kind::TextCpu | Kind::TowerCpu | Kind::ServerCpu => "cpu", Kind::Text | Kind::Tower | Kind::Server => "cuda", } } @@ -114,7 +115,7 @@ impl Emb2Set { )); } }, - Kind::Text | Kind::TextCpu | Kind::Server => { + Kind::Text | Kind::TextCpu | Kind::Server | Kind::ServerCpu => { let model = one(&man, "model")?; let [path] = model[..] else { return Err(RefError::malformed( diff --git a/crates/refset/src/emb2/tests.rs b/crates/refset/src/emb2/tests.rs index 689f658cc..e1f55c3a1 100644 --- a/crates/refset/src/emb2/tests.rs +++ b/crates/refset/src/emb2/tests.rs @@ -27,7 +27,11 @@ pub(crate) fn header( let device = format!( "{}\tthreads\t{}\tcard\t{}", kind.device(), - if kind == Kind::Server { "-" } else { "16" }, + if matches!(kind, Kind::Server | Kind::ServerCpu) { + "-" + } else { + "16" + }, if kind.device() == "cpu" { "cpu" } else { diff --git a/crates/serve/src/api.rs b/crates/serve/src/api.rs index 3db8eca11..9ea005d57 100644 --- a/crates/serve/src/api.rs +++ b/crates/serve/src/api.rs @@ -93,7 +93,7 @@ mod anthropic; /// The embeddings endpoints over the side model, a child of this module so /// they read a prompt as the completion path does. #[path = "embeddings.rs"] -mod embeddings; +pub mod embeddings; /// Per-token log-probabilities, which the generation loop collects and the /// completion and chat answers render. #[path = "logprobs.rs"] diff --git a/crates/serve/src/arch/gemma4v.rs b/crates/serve/src/arch/gemma4v.rs new file mode 100644 index 000000000..0d938ab12 --- /dev/null +++ b/crates/serve/src/arch/gemma4v.rs @@ -0,0 +1,59 @@ +//! The embedding input contract with the Gemma4 image model. + +use super::*; +use crate::mock::MockEmbed; + +#[test] +fn image_parts_preserve_delimiters_and_place_spans_in_one_input() { + let mut side = Side::new(Box::new(MockEmbed::new(128))); + let rule = + vision::size::SizeRule::new(16, 3, vision::arch::TokenLimits::new(1, 4).unwrap()).unwrap(); + side.media = Some(Arc::new(vision::arch::gemma4v::Media::new(rule))); + let image = json!({"type":"image_url","image_url":{"url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAIAAAABCAIAAAB7QOjdAAAADUlEQVR4nGM4kWIERAAKxQK9KV6Y1QAAAABJRU5ErkJggg=="}}); + let body = json!({"content":[{"type":"text","text":"a<|image>"},image.clone(),{"type":"text","text":"b"},image]}); + let parsed = inputs(&side, "input", &json!([body])).unwrap_or_else(|e| panic!("{}", e.message)); + assert_eq!(parsed.len(), 1); + let got = &parsed[0]; + assert_eq!(got.images.len(), 2); + let mut want = vec![side.tok.bos()]; + want.extend(side.tok.encode("a<|image>")); + for (i, after) in [(0, "b"), (1, "")] { + want.extend(side.tok.encode("<|image>")); + assert_eq!(got.images[i].at, want.len()); + assert_eq!(got.images[i].prepared.span_len, 1); + want.extend([side.tok.bos(); 1]); + want.extend(side.tok.encode("")); + want.extend(side.tok.encode(after)); + } + if side.tok.add_eos() { + want.push(side.tok.eos()); + } + assert_eq!(got.ids, want); + let model = side.media.as_ref().expect("image preparer"); + assert_eq!(model.image_token(), None); + assert!(matches!( + media::expand_prompt(&[side.tok.bos()], &[], &**model), + Err(media::MediaError::NoChatImageToken) + )); + side.ctx = got.ids.len(); + assert!(inputs(&side, "input", &json!([body.clone()])).is_ok()); + side.ctx = got.images[1].at; + let mut over = body.clone(); + let parts = over["content"].as_array_mut().unwrap(); + parts.extend(std::iter::repeat_n(parts[1].clone(), 64)); + parts.push(json!({"type":"image_url","image_url":"https://example.com/not-prepared.png"})); + let error = inputs(&side, "input", &json!([over])) + .err() + .expect("context refusal"); + assert_eq!(error.message, "image input exceeds the embedding context"); + side.media = None; + assert!(inputs(&side, "input", &json!([body])).is_err()); + assert!( + inputs( + &side, + "input", + &json!({"content":[{"type":"image_url","image_url":"https://example.com/x.png"}]}) + ) + .is_err() + ); +} diff --git a/crates/serve/src/embed.rs b/crates/serve/src/embed.rs index d21dbb742..f8f4a4e62 100644 --- a/crates/serve/src/embed.rs +++ b/crates/serve/src/embed.rs @@ -28,9 +28,23 @@ pub trait Embed: Send { /// The widths a request's `dimensions` may cut the pooled vector to, ascending; the last is /// the pooled vector's own width. fn dims(&self) -> Vec; + /// The side model's image preparer, present when a projector is loaded. + fn media(&self) -> Option { + None + } /// One pass over one input: at least one id and at most [`Embed::ctx`], each below the /// vocabulary's size. The pooled vector, the full width (the last of [`Embed::dims`]). fn embed(&mut self, ids: &[u32]) -> Result, EmbedError>; + /// One pass with prepared images replacing the positions they occupy. + fn embed_media( + &mut self, + _ids: &[u32], + _images: &[crate::media::MediaFeed], + ) -> Result, EmbedError> { + Err(EmbedError( + "the embedding model has no image encoder (--embd-mmproj)".into(), + )) + } } /// A normalization llama.cpp's `common_embd_normalize` names (`embd_normalize`): `-1` none, `0` diff --git a/crates/serve/src/embeddings.rs b/crates/serve/src/embeddings.rs index 64ed12fcd..0c43ada2b 100644 --- a/crates/serve/src/embeddings.rs +++ b/crates/serve/src/embeddings.rs @@ -4,7 +4,7 @@ //! `6868a52c`), with these differences: a field of another type than it takes is a 400 naming it //! (optional fields treat `null` as unset); `/v1/embeddings` refuses a `model` of another type //! than a string or a name other than the side model's, while the native routes ignore it; an object -//! input (`{"prompt_string": …}`, or a `content` array of parts) is a 400, an input past the model's +//! input with `prompt_string` is a 400, an input past the model's //! context is a 400 with no batch limit before it, an `embd_normalize` below `-1` or one whose //! norm f32 cannot scale by is a 400 ([`crate::embed::normalize`]), and `dimensions` cuts the //! pooled vector to one of the model's widths before it is normalized. @@ -13,6 +13,8 @@ //! array of token ids, an array of strings and token ids, or an array of these with no id at its //! top level, one vector each. Text is tokenized with the side model's vocabulary, the first text //! element of each input with its special tokens, as llama-server's `tokenize_mixed` puts them. +//! An object with a `content` array prepares image parts through the side model's media rule. +//! Delimiter text and every image span share one pass, with BOS and EOS around the whole input. //! //! The side model runs one request at a time. A failure in it ends the server as an engine's does: //! the request gets a 500, `/health` and every later request a 503, and the server exits after its @@ -21,6 +23,7 @@ use std::io; use std::net::TcpStream; +use std::ops::Range; use std::panic::{self, AssertUnwindSafe}; use std::sync::{Arc, Mutex}; @@ -46,6 +49,7 @@ pub(super) struct Side { pub(super) name: String, pub(super) ctx: usize, dims: Vec, + media: Option, } impl Side { @@ -58,6 +62,7 @@ impl Side { "an embedding model names its width, the last of its dims: {dims:?}" ); Side { + media: embed.media(), tok: embed.tokenizer(), name: embed.name(), ctx: embed.ctx(), @@ -131,7 +136,7 @@ fn answer(state: &State, req: &Request, oai: bool) -> Result { .collect(), )); } - let n_tokens: usize = inputs.iter().map(Vec::len).sum(); + let n_tokens: usize = inputs.iter().map(|i| i.ids.len()).sum(); let data: Vec = vectors .iter() .enumerate() @@ -202,22 +207,141 @@ fn cut(side: &Side, b: &Map) -> Result { }) } -/// The request's inputs as ids: an array with no id at its top level is one input an element, +/// The request's inputs as ids and prepared images: an array with no id at its top level is one input an element, /// anything else one input, each read as a `/completion` prompt with the special tokens on. No /// input, an input of no id and one of more ids than the model's context are refused by name. -fn inputs(side: &Side, key: &str, v: &Value) -> Result>, ApiError> { +struct Input { + ids: Vec, + images: Vec, +} + +/// One ordered embedding input part, after an image has been prepared and sized. +pub enum InputPart<'a> { + Text(&'a str), + /// Tokenized delimiters around `positions` image rows. + Image { + begin: &'a str, + end: &'a str, + positions: usize, + }, +} + +/// Token ids and the image spans that replace their filler rows, in part order. +pub struct Assembled { + pub ids: Vec, + pub spans: Vec>, +} + +/// Tokenize each part separately, with BOS and EOS around the whole input. Image spans are +/// bounded by `ctx` before allocation; the caller checks the total length, including text and EOS. +pub fn assemble<'a>( + parts: impl Iterator, String>>, + tok: &dyn Tokenizer, + ctx: usize, +) -> Result { + let mut parts = parts.peekable(); + if parts.peek().is_none() { + return Err("Input content cannot be empty".into()); + } + let mut ids = Vec::new(); + let mut spans = Vec::new(); + if tok.add_bos() { + ids.push(tok.bos()); + } + for part in parts { + match part? { + InputPart::Text(text) => ids.extend(tok.encode(text)), + InputPart::Image { + begin, + end, + positions, + } => { + if positions == 0 { + return Err("an embedding image has an empty span".into()); + } + ids.extend(tok.encode(begin)); + let at = ids.len(); + let last = at + .checked_add(positions) + .filter(|&n| n <= ctx) + .ok_or("image input exceeds the embedding context")?; + ids.resize(last, tok.bos()); + spans.push(at..last); + ids.extend(tok.encode(end)); + } + } + } + if tok.add_eos() { + ids.push(tok.eos()); + } + if ids.is_empty() { + return Err("Input content cannot be empty".into()); + } + Ok(Assembled { ids, spans }) +} + +fn input(side: &Side, key: &str, v: &Value) -> Result { + if let Some(parts) = v + .as_object() + .and_then(|o| o.get("content")) + .and_then(Value::as_array) + { + let mut images = Vec::new(); + let ordered = parts.iter().map(|part| -> Result, String> { + match media::parse_part(part).map_err(|e| e.to_string())? { + media::Part::Text(text) => Ok(InputPart::Text(text)), + media::Part::Image(url) => { + let model = side + .media + .as_ref() + .ok_or("image input needs --embd-mmproj")?; + let (begin, end) = model + .embedding_delimiters() + .ok_or("the side projector has no embedding delimiters")?; + let file = media::load(url).map_err(|e| e.to_string())?; + let prepared = + Arc::new(media::prepare(&file, &**model).map_err(|e| e.to_string())?); + let positions = prepared.span_len; + images.push(media::MediaFeed { + at: 0, + key: file.key, + prepared, + }); + Ok(InputPart::Image { + begin, + end, + positions, + }) + } + } + }); + let Assembled { ids, spans } = assemble(ordered, &*side.tok, side.ctx).map_err(invalid)?; + for (image, span) in images.iter_mut().zip(spans) { + image.at = span.start; + } + Ok(Input { ids, images }) + } else { + Ok(Input { + ids: prompt_ids(&*side.tok, key, v, true)?, + images: Vec::new(), + }) + } +} + +fn inputs(side: &Side, key: &str, v: &Value) -> Result, ApiError> { let is_id = |x: &Value| x.is_u64() || x.is_i64(); let inputs = match v { Value::Array(a) if !a.iter().any(is_id) => a .iter() - .map(|x| prompt_ids(&*side.tok, key, x, true)) + .map(|x| input(side, key, x)) .collect::, _>>()?, - v => vec![prompt_ids(&*side.tok, key, v, true)?], + v => vec![input(side, key, v)?], }; if inputs.is_empty() { return Err(invalid("\"input\" must not be empty")); } - for ids in &inputs { + for input in &inputs { + let ids = &input.ids; if ids.is_empty() { return Err(invalid("Input content cannot be empty")); } @@ -240,7 +364,7 @@ fn inputs(side: &Side, key: &str, v: &Value) -> Result>, ApiError> /// Every input's pooled vector, through the side model under its lock, each pass after /// [`open`] holds. A model failure, a panic in it, its lock poisoned, and a vector that is not its /// width or not finite are fatal ([`fail`]). -fn run(state: &State, side: &Side, inputs: &[Vec]) -> Result>, ApiError> { +fn run(state: &State, side: &Side, inputs: &[Input]) -> Result>, ApiError> { let Ok(mut model) = side.embed.lock() else { return Err(fail( state, @@ -250,9 +374,15 @@ fn run(state: &State, side: &Side, inputs: &[Vec]) -> Result>, }; let width = side.width(); let mut pooled = Vec::with_capacity(inputs.len()); - for ids in inputs { + for input in inputs { open(state)?; - let v = match panic::catch_unwind(AssertUnwindSafe(|| model.embed(ids))) { + let v = match panic::catch_unwind(AssertUnwindSafe(|| { + if input.images.is_empty() { + model.embed(&input.ids) + } else { + model.embed_media(&input.ids, &input.images) + } + })) { Ok(Ok(v)) => v, Ok(Err(e)) => return Err(fail(state, side, e.0)), Err(p) => { @@ -301,6 +431,10 @@ fn fail(state: &State, side: &Side, error: String) -> ApiError { } } +#[cfg(test)] +#[path = "arch/gemma4v.rs"] +mod image_tests; + #[cfg(test)] mod tests { use std::net::SocketAddr; @@ -584,7 +718,14 @@ mod tests { assert!(open(&waiting_state).is_ok()); entered.send(()).expect("entered"); let side = waiting_state.side.as_ref().expect("a side model"); - run(&waiting_state, side, &[vec![id("a")]]) + run( + &waiting_state, + side, + &[Input { + ids: vec![id("a")], + images: Vec::new(), + }], + ) }); ready .recv_timeout(Duration::from_secs(5)) diff --git a/crates/serve/src/lib.rs b/crates/serve/src/lib.rs index aced69594..e9228cf5c 100644 --- a/crates/serve/src/lib.rs +++ b/crates/serve/src/lib.rs @@ -20,6 +20,7 @@ //! reads at a terminal. mod api; +pub use api::embeddings; pub mod decide; pub mod dsml; pub mod embed; diff --git a/crates/serve/src/media.rs b/crates/serve/src/media.rs index 48d491f91..a19dc1cf5 100644 --- a/crates/serve/src/media.rs +++ b/crates/serve/src/media.rs @@ -48,8 +48,8 @@ impl MediaModel for NumberedMedia { Some(vec![100, 200 + index as u32, 300]) } - fn image_token(&self) -> u32 { - 7 + fn image_token(&self) -> Option { + Some(7) } fn prepare(&self, _: &Rgb8) -> Result { @@ -243,6 +243,9 @@ impl From> for Prompt { /// are the request's fault. #[derive(Debug, thiserror::Error)] pub enum MediaError { + /// The projector supports embedding delimiters rather than a chat placeholder. + #[error("the projector has no chat image token; use its embedding input")] + NoChatImageToken, /// A content part that is not a JSON object. #[error("a content part must be an object")] PartNotObject, @@ -462,7 +465,7 @@ pub fn expand_prompt( urls: &[&str], model: &dyn MediaModel, ) -> Result { - let token = model.image_token(); + let token = model.image_token().ok_or(MediaError::NoChatImageToken)?; let placeholders = ids.iter().filter(|&&t| t == token).count(); if placeholders != urls.len() { return Err(MediaError::Expand(ExpandError::PlaceholderCount { @@ -686,8 +689,8 @@ mod tests { "" } - fn image_token(&self) -> u32 { - 7 + fn image_token(&self) -> Option { + Some(7) } fn part_join(&self) -> PartJoin { diff --git a/crates/serve/src/mock.rs b/crates/serve/src/mock.rs index 175bafa00..d7ce67cc3 100644 --- a/crates/serve/src/mock.rs +++ b/crates/serve/src/mock.rs @@ -1003,8 +1003,8 @@ impl MediaModel for MockMedia { IMAGE_SPECIAL } - fn image_token(&self) -> u32 { - IMAGE_ID + fn image_token(&self) -> Option { + Some(IMAGE_ID) } fn prepare(&self, image: &Rgb8) -> Result { diff --git a/crates/vision/src/arch/deepseek41v/media.rs b/crates/vision/src/arch/deepseek41v/media.rs index c86e33d43..d5def122a 100644 --- a/crates/vision/src/arch/deepseek41v/media.rs +++ b/crates/vision/src/arch/deepseek41v/media.rs @@ -61,8 +61,8 @@ impl MediaModel for Media { IMAGE_PLACEHOLDER } - fn image_token(&self) -> u32 { - IMAGE_TOKEN_ID + fn image_token(&self) -> Option { + Some(IMAGE_TOKEN_ID) } fn part_separator(&self) -> &str { @@ -111,7 +111,7 @@ mod tests { Cow::Borrowed(IMAGE_PLACEHOLDER) )); assert_eq!(m.image_marker(), IMAGE_PLACEHOLDER); - assert_eq!(m.image_token(), 129_264); + assert_eq!(m.image_token(), Some(129_264)); assert_eq!(m.part_separator(), "\n\n"); assert_eq!(m.part_join(), PartJoin::Uniform("\n\n")); } @@ -131,7 +131,9 @@ mod tests { assert_eq!((patches.n_vit_h, patches.n_vit_w), (39, 39)); assert_eq!(patches.bf16.len(), 39 * 39 * 3 * 14 * 14); assert_eq!( - image_span(&patches.plan, m.image_token()).ids.len(), + image_span(&patches.plan, m.image_token().expect("chat image token")) + .ids + .len(), p.span_len ); } diff --git a/crates/vision/src/arch/gemma4v/card.rs b/crates/vision/src/arch/gemma4v/card.rs new file mode 100644 index 000000000..715432ff5 --- /dev/null +++ b/crates/vision/src/arch/gemma4v/card.rs @@ -0,0 +1,88 @@ +//! The resident vision weights and the bounded activation buffers. + +use super::{Hparams, MERGE, tensors}; +use crate::{ + VisionError, + arch::{TokenLimits, card::CardBytes}, +}; +use gguf::Gguf; + +/// Element counts of the tower's buffers at the largest supported canvas. +#[derive(Clone, Copy)] +pub struct ScratchLens { + pub patches: usize, + pub raw: usize, + pub hidden: usize, + pub qkv: usize, + pub ff: usize, + pub rows: usize, + pub pooled: usize, +} + +impl ScratchLens { + /// Buffer extents for `tokens` pooled rows; refuse overflow before allocation. + pub fn new(hp: &Hparams, tokens: usize) -> Result { + if tokens == 0 || hp.patch == 0 || hp.dim == 0 || hp.ff == 0 || hp.out_dim == 0 { + return Err(VisionError::Limits { + detail: "gemma4v scratch extents must be positive".into(), + }); + } + let mul = |a: usize, b: usize| { + a.checked_mul(b) + .filter(|&v| v <= u32::MAX as usize) + .ok_or_else(|| VisionError::Limits { + detail: format!("gemma4v scratch {a} times {b} does not fit u32"), + }) + }; + let patches = mul(tokens, MERGE * MERGE)?; + let patch_values = mul(mul(hp.patch, hp.patch)?, 3)?; + let qkv_width = mul(hp.dim, 3)?; + Ok(Self { + patches, + raw: mul(patches, patch_values)?, + hidden: mul(patches, hp.dim)?, + qkv: mul(patches, qkv_width)?, + ff: mul(patches, hp.ff)?, + rows: mul(tokens, hp.out_dim)?, + pooled: mul(tokens, hp.dim)?, + }) + } + + /// Bytes of the buffers allocated by the tower. + #[must_use] + pub fn bytes(&self) -> u64 { + // F32 patches, patch output, residuals and post norms; BF16 linears, attention, + // pooling gather, gate/up/activation, output rows; f32 position angles. + 4 * (self.raw as u64 + 4 * self.hidden as u64) + + 2 * (4 * self.hidden as u64 + + self.qkv as u64 + + 5 * self.ff as u64 + + 3 * self.pooled as u64 + + self.rows as u64) + + 4 * 64 * self.patches as u64 + } +} + +/// The vision weights and scratch held on the side seat's card. +pub fn of(file: &Gguf, hp: &Hparams, limits: TokenLimits) -> Result { + hp.size_rule(limits)?; + let weights: u64 = tensors::check(file, hp)? + .iter() + .filter(|r| { + ![".input_min", ".input_max", ".output_min", ".output_max"] + .iter() + .any(|s| r.name.ends_with(s)) + }) + .map(|r| r.bytes()) + .sum(); + let weights = weights + + 4 * (hp.dim + 64) as u64 + + if file.find("v.std_bias").is_none() { + 8 * hp.dim as u64 + } else { + 0 + }; + // The shared BF16 GEMM holds one f32 bias and one bf16 residual stand-in. + let scratch = ScratchLens::new(hp, limits.max())?.bytes() + 6; + Ok(CardBytes { weights, scratch }) +} diff --git a/crates/vision/src/arch/gemma4v/hparams.rs b/crates/vision/src/arch/gemma4v/hparams.rs new file mode 100644 index 000000000..4bb42f5bf --- /dev/null +++ b/crates/vision/src/arch/gemma4v/hparams.rs @@ -0,0 +1,138 @@ +//! The vision half of the Gemma4 projector header. + +use super::{MERGE, PROJECTOR_TYPE}; +use crate::VisionError; +use crate::arch::{ + TokenLimits, + header::{ + KEY_HAS_VISION, KEY_MEAN, KEY_STD, check_architecture, check_eps, check_projector, + check_true, counts, need, refusal, + }, +}; +use crate::size::SizeRule; +use gguf::Gguf; + +/// Dimensions of the EmbeddingGemma 2 vision tower. +#[derive(Clone, Debug)] +pub struct Hparams { + pub n_layer: usize, + pub dim: usize, + pub ff: usize, + pub n_head: usize, + pub patch: usize, + pub out_dim: usize, + pub eps: f32, + pub pos_rows: usize, + /// The header chooses tanh GELU; without that key mainline uses Quick GELU. + pub gelu_tanh: bool, +} + +impl Hparams { + /// Read the vision header and refuse a geometry this tower does not run. + pub fn read(file: &Gguf) -> Result { + check_architecture(file)?; + check_projector(file, PROJECTOR_TYPE)?; + check_true(file, KEY_HAS_VISION, "the vision tower")?; + if let Some(v) = file.value("clip.vision.projector.scale_factor") + && v.as_u64() != Some(MERGE as u64) + { + return Err(refusal( + "clip.vision.projector.scale_factor", + format!("is {v:?}; gemma4v pools {MERGE} by {MERGE} patches"), + )); + } + let [n_layer, dim, ff, n_head, patch, out_dim] = counts( + file, + PROJECTOR_TYPE, + [ + ( + "clip.vision.block_count", + 16, + "the EmbeddingGemma 2 projector", + ), + ( + "clip.vision.embedding_length", + 768, + "the EmbeddingGemma 2 projector", + ), + ( + "clip.vision.feed_forward_length", + 3072, + "the EmbeddingGemma 2 projector", + ), + ( + "clip.vision.attention.head_count", + 12, + "the EmbeddingGemma 2 projector", + ), + ( + "clip.vision.patch_size", + 16, + "the EmbeddingGemma 2 projector", + ), + ( + "clip.vision.projection_dim", + 512, + "the EmbeddingGemma 2 projector", + ), + ], + )?; + for (key, want) in [(KEY_MEAN, 0.0f32), (KEY_STD, 1.0)] { + let v = need(file, key)?; + if !matches!(v, gguf::Value::Array(a) if a.len() == 3 && a.iter().all(|x| x.as_f32() == Some(want))) + { + return Err(refusal(key, format!("is {v:?}; gemma4v uses [{want}; 3]"))); + } + } + let eps = check_eps(file, 1e-6, "the Gemma4 vision RMS norm")?; + let gelu_tanh = activation(file)?; + Ok(Self { + n_layer, + dim, + ff, + n_head, + patch, + out_dim, + eps, + pos_rows: 10240, + gelu_tanh, + }) + } + + /// The mainline image budget when neither flag supplies a bound. + pub fn default_limits(&self) -> Result { + TokenLimits::new(70, 1120) + } + + /// The shared mtmd dynamic resize rule at `limits`. + pub fn size_rule(&self, limits: TokenLimits) -> Result { + SizeRule::new(self.patch, MERGE, limits) + } +} + +fn activation(m: &impl crate::arch::header::Meta) -> Result { + let flag = |key| match m.value(key) { + None => Ok(false), + Some(gguf::Value::Bool(v)) => Ok(*v), + Some(v) => Err(refusal(key, format!("is {v:?}, not a boolean"))), + }; + if flag("clip.use_silu")? { + return Err(refusal("clip.use_silu", "gemma4v does not run SiLU")); + } + flag("clip.use_gelu") +} + +#[cfg(test)] +mod tests { + use super::activation; + use crate::arch::header::testing::Table; + use gguf::Value; + #[test] + fn absent_activation_flags_select_mainline_quick_gelu() { + let table = Table(vec![]); + assert!(!activation(&table).unwrap()); + assert!(activation(&Table(vec![]).with("clip.use_gelu", Value::Bool(true))).unwrap()); + assert!(activation(&Table(vec![]).with("clip.use_silu", Value::Bool(true))).is_err()); + assert!(activation(&table.with("clip.use_gelu", Value::U32(1))).is_err()); + } +} diff --git a/crates/vision/src/arch/gemma4v/media.rs b/crates/vision/src/arch/gemma4v/media.rs new file mode 100644 index 000000000..3089dd27f --- /dev/null +++ b/crates/vision/src/arch/gemma4v/media.rs @@ -0,0 +1,92 @@ +//! Raster patches of the mtmd canvas. + +use super::{IMAGE_BEGIN, IMAGE_END}; +use crate::size::SizeRule; +use crate::{GridPlan, MediaModel, Patches, Prepared, Rgb8, VisionError}; +use std::borrow::Cow; + +/// Resize onto the mtmd canvas and cut raster patches after the graph's `2*x - 1`. +pub fn preprocess(image: &Rgb8, rule: &SizeRule) -> Result { + image.check()?; + let (best_w, best_h) = rule.aligned(image.width, image.height)?; + let (nx, ny) = rule.grid(best_w, best_h); + if nx.checked_mul(ny).is_none_or(|n| n > rule.max_tokens()) { + return Err(VisionError::Limits { + detail: "gemma4v aligned canvas exceeds the image token budget".into(), + }); + } + let plan = GridPlan { + n_llm_h: ny, + n_llm_w: nx, + best_h, + best_w, + }; + let resized = crate::resample::pad_ceil(image, best_w, best_h, [0; 3])?; + let patches = crate::preprocess::patchify( + &resized, + plan, + crate::preprocess::PatchLayout::row_major(rule.patch()), + )?; + if patches.n_vit_h > 10240 || patches.n_vit_w > 10240 { + return Err(VisionError::Limits { + detail: "gemma4v canvas exceeds the learned position table".into(), + }); + } + Ok(patches) +} + +/// Restore the original normalized bytes for the F32 convolution: their bf16 encodings are distinct. +pub fn conv_pixels(encoded: &[u16]) -> Result, VisionError> { + encoded + .iter() + .map(|&bits| { + let value = f32::from_bits(u32::from(bits) << 16); + let byte = ((value + 1.0) * 127.5).round(); + if !value.is_finite() + || !(0.0..=255.0).contains(&byte) + || crate::preprocess::normalize(byte as u8) != bits + { + return Err(VisionError::Format( + "gemma4v convolution input is not a normalized RGB byte".into(), + )); + } + Ok((byte / 255.0) * 2.0 - 1.0) + }) + .collect() +} + +/// The image preparation rule; the side seat inserts delimiters through its tokenizer. +#[derive(Clone, Copy, Debug)] +pub struct Media { + rule: SizeRule, +} + +impl Media { + /// An image preparer at the file's patch size and the request's token budget. + #[must_use] + pub fn new(rule: SizeRule) -> Self { + Self { rule } + } +} + +impl MediaModel for Media { + fn embedding_delimiters(&self) -> Option<(&str, &str)> { + Some((IMAGE_BEGIN, IMAGE_END)) + } + fn image_placeholder(&self, _: usize) -> Cow<'_, str> { + Cow::Owned(format!("{IMAGE_BEGIN}{IMAGE_END}")) + } + fn image_marker(&self) -> &str { + IMAGE_BEGIN + } + fn image_token(&self) -> Option { + None + } + fn prepare(&self, image: &Rgb8) -> Result { + let patches = preprocess(image, &self.rule)?; + Ok(Prepared::of_grid( + patches.plan.n_llm_h * patches.plan.n_llm_w, + patches, + )) + } +} diff --git a/crates/vision/src/arch/gemma4v/mod.rs b/crates/vision/src/arch/gemma4v/mod.rs new file mode 100644 index 000000000..646da8ae1 --- /dev/null +++ b/crates/vision/src/arch/gemma4v/mod.rs @@ -0,0 +1,61 @@ +//! Gemma4's vision input: dynamic aspect-preserving canvases, raster patches, +//! learned x/y positions and three-by-three pooled rows. + +pub mod card; +mod hparams; +mod media; +pub mod tensors; + +pub use hparams::Hparams; +pub use media::{Media, conv_pixels, preprocess}; + +/// The projector type of the vision half of a Gemma4 multimodal file. +pub const PROJECTOR_TYPE: &str = "gemma4v"; +/// Patches pooled along each axis. +pub const MERGE: usize = 3; +/// The tokenizer spelling before an image's rows. +pub const IMAGE_BEGIN: &str = "<|image>"; +/// The tokenizer spelling after an image's rows. +pub const IMAGE_END: &str = ""; + +#[cfg(test)] +mod tests { + use super::*; + use crate::size::SizeRule; + use crate::{Rgb8, arch::TokenLimits}; + + #[test] + fn gemma4v_canvas_and_patch_order() { + let rule = SizeRule::new(16, MERGE, TokenLimits::new(70, 1120).unwrap()).unwrap(); + for ((w, h), expected) in [ + ((224, 224), (432, 432)), + ((1000, 700), (1008, 720)), + ((2600, 1800), (1920, 1296)), + ((1, 1), (432, 432)), + ] { + assert_eq!(rule.aligned(w, h).unwrap(), expected); + } + let small = SizeRule::new(16, MERGE, TokenLimits::new(1, 4).unwrap()).unwrap(); + let p = preprocess(&Rgb8::filled(96, 48, [0, 128, 255]), &small).unwrap(); + assert_eq!((p.n_vit_w, p.n_vit_h, p.plan.span_grid()), (6, 3, (2, 1))); + assert_eq!(p.patch_len, 768); + assert_eq!(p.bf16[0], crate::preprocess::f32_to_bf16(-1.0)); + assert_eq!( + p.bf16[256], + crate::preprocess::f32_to_bf16(2.0 * (128.0f32 / 255.0) - 1.0) + ); + assert_eq!(p.bf16[512], crate::preprocess::f32_to_bf16(1.0)); + assert!(preprocess(&Rgb8::filled(0, 1, [0; 3]), &small).is_err()); + let encoded: Vec<_> = (0..=255).map(crate::preprocess::normalize).collect(); + let pixels = conv_pixels(&encoded).unwrap(); + for (byte, value) in pixels.into_iter().enumerate() { + assert_eq!( + value.to_bits(), + (2.0 * (byte as f32 / 255.0) - 1.0).to_bits() + ); + } + for invalid in [0, 0x7fc0, 0x7f80, 0xff80] { + assert!(conv_pixels(&[invalid]).is_err()); + } + } +} diff --git a/crates/vision/src/arch/gemma4v/tensors.rs b/crates/vision/src/arch/gemma4v/tensors.rs new file mode 100644 index 000000000..5aec37f58 --- /dev/null +++ b/crates/vision/src/arch/gemma4v/tensors.rs @@ -0,0 +1,122 @@ +//! The vision tensors of a multimodal Gemma4 projector; audio tensors stay on the host. + +use super::{Hparams, PROJECTOR_TYPE}; +use crate::{ + VisionError, + arch::table::{self, Expected}, +}; +use gguf::{GgmlType, Gguf}; + +/// Names and ggml shapes of every required vision tensor. +#[must_use] +pub fn expected(hp: &Hparams) -> Vec> { + let (d, f, out, p) = ( + hp.dim as u64, + hp.ff as u64, + hp.out_dim as u64, + hp.patch as u64, + ); + let mut rows = vec![ + Expected::row( + "v.patch_embd.weight".into(), + &[p, p, 3, d], + GgmlType::F32, + (), + ), + Expected::row( + "v.position_embd.weight".into(), + &[d, hp.pos_rows as u64, 2], + GgmlType::F32, + (), + ), + Expected::row( + "mm.input_projection.weight".into(), + &[d, out], + GgmlType::BF16, + (), + ), + ]; + for l in 0..hp.n_layer { + for stem in [ + "ln1", + "ln2", + "attn_post_norm", + "ffn_post_norm", + "attn_q_norm", + "attn_k_norm", + ] { + let n = if stem.ends_with("_q_norm") || stem.ends_with("_k_norm") { + 64 + } else { + d + }; + rows.push(Expected::row( + format!("v.blk.{l}.{stem}.weight"), + &[n], + GgmlType::F32, + (), + )); + } + for (stem, dims) in [ + ("attn_q", [d, d]), + ("attn_k", [d, d]), + ("attn_v", [d, d]), + ("attn_out", [d, d]), + ("ffn_gate", [d, f]), + ("ffn_up", [d, f]), + ("ffn_down", [f, d]), + ] { + rows.push(Expected::row( + format!("v.blk.{l}.{stem}.weight"), + &dims, + GgmlType::BF16, + (), + )); + } + } + rows +} + +/// Check the complete vision half, including optional standardization and clamp scalars. +pub fn check(file: &Gguf, hp: &Hparams) -> Result>, VisionError> { + let mut rows = expected(hp); + let has_bias = file.find("v.std_bias").is_some(); + let has_scale = file.find("v.std_scale").is_some(); + if has_bias != has_scale { + return Err(VisionError::Tensor { + name: "v.std_bias/v.std_scale".into(), + detail: "standardization needs both vectors".into(), + }); + } + if has_bias { + for name in ["v.std_bias", "v.std_scale"] { + rows.push(Expected::row( + name.into(), + &[hp.dim as u64], + GgmlType::F32, + (), + )); + } + } + let matrices: Vec = rows + .iter() + .filter(|r| r.dims.len() == 2) + .map(|r| r.name.clone()) + .collect(); + for name in matrices { + for suffix in ["input_min", "input_max", "output_min", "output_max"] { + let n = name.replace(".weight", &format!(".{suffix}")); + if file.find(&n).is_some() { + rows.push(Expected::row(n, &[1], GgmlType::F32, ())); + } + } + } + table::check( + PROJECTOR_TYPE, + &rows, + file.iter_tensors() + .filter(|t| !t.name.starts_with("a.") && !t.name.starts_with("mm.a.")) + .map(|t| (t.name.as_str(), t.dims.as_slice(), t.ty)), + )?; + Ok(rows) +} diff --git a/crates/vision/src/arch/glm5v/media.rs b/crates/vision/src/arch/glm5v/media.rs index 475cc583a..cc102cbec 100644 --- a/crates/vision/src/arch/glm5v/media.rs +++ b/crates/vision/src/arch/glm5v/media.rs @@ -90,8 +90,8 @@ impl MediaModel for Media { IMAGE_PLACEHOLDER } - fn image_token(&self) -> u32 { - IMAGE_TOKEN_ID + fn image_token(&self) -> Option { + Some(IMAGE_TOKEN_ID) } fn part_join(&self) -> PartJoin { @@ -262,7 +262,7 @@ mod tests { Cow::Borrowed(IMAGE_PLACEHOLDER) )); assert_eq!(m.image_marker(), IMAGE_PLACEHOLDER); - assert_eq!(m.image_token(), IMAGE_TOKEN_ID); + assert_eq!(m.image_token(), Some(IMAGE_TOKEN_ID)); assert_eq!(m.part_join(), PartJoin::LlamaServer); assert_eq!(m.part_separator(), "\n"); } diff --git a/crates/vision/src/arch/header.rs b/crates/vision/src/arch/header.rs index d513f0a63..244614669 100644 --- a/crates/vision/src/arch/header.rs +++ b/crates/vision/src/arch/header.rs @@ -61,7 +61,7 @@ pub(crate) fn check_architecture(gguf: &Gguf) -> Result<(), VisionError> { /// The file names `projector` as its projector type. pub(crate) fn check_projector(m: &impl Meta, projector: &str) -> Result<(), VisionError> { - match need(m, KEY_PROJECTOR)?.as_str() { + match projector_value(m)?.as_str() { Some(p) if p == projector => Ok(()), Some(other) => Err(refusal( KEY_PROJECTOR, @@ -71,6 +71,15 @@ pub(crate) fn check_projector(m: &impl Meta, projector: &str) -> Result<(), Visi } } +pub(crate) fn projector_value(m: &impl Meta) -> Result<&Value, VisionError> { + match m.value(KEY_PROJECTOR) { + Some(v) => Ok(v), + None => m + .value("clip.vision.projector_type") + .ok_or_else(|| refusal(KEY_PROJECTOR, "is absent")), + } +} + /// A boolean key that must be true; `why` says what needs it. pub(crate) fn check_true(m: &impl Meta, key: &str, why: &str) -> Result<(), VisionError> { match need(m, key)?.as_bool() { diff --git a/crates/vision/src/arch/minicpmv/media.rs b/crates/vision/src/arch/minicpmv/media.rs index 5e12868dc..ef89d1b12 100644 --- a/crates/vision/src/arch/minicpmv/media.rs +++ b/crates/vision/src/arch/minicpmv/media.rs @@ -61,8 +61,8 @@ impl MediaModel for Media { &self.marker } - fn image_token(&self) -> u32 { - self.pad + fn image_token(&self) -> Option { + Some(self.pad) } fn part_join(&self) -> PartJoin { @@ -108,7 +108,7 @@ mod tests { "12" ); assert_eq!(m.image_marker(), ""); - assert_eq!(m.image_token(), 248_077); + assert_eq!(m.image_token(), Some(248_077)); assert_eq!(m.part_join(), PartJoin::LlamaServer); // ``, the digits, ``, ``. let head = m.image_head(12).expect("a numbered model has a head"); @@ -135,8 +135,12 @@ mod tests { panic!("{w}x{h}: a MiniCPM-V image is a group of tiles"); }; assert_eq!((group.tiles.len(), group.rows()), (tiles, span), "{w}x{h}"); - let ids = - expand_spans(&[5, 248_077, 6], m.image_token(), &[p.span_len]).expect("expand"); + let ids = expand_spans( + &[5, 248_077, 6], + m.image_token().expect("chat image token"), + &[p.span_len], + ) + .expect("expand"); // One span, the range itself. `[1..1 + span]` is a one-element array of a // range, which is the lint's single-range init, not the positions inside it. assert_eq!(ids.spans.len(), 1, "{w}x{h}"); diff --git a/crates/vision/src/arch/mod.rs b/crates/vision/src/arch/mod.rs index bfdf092e1..f5d5667bd 100644 --- a/crates/vision/src/arch/mod.rs +++ b/crates/vision/src/arch/mod.rs @@ -14,6 +14,7 @@ pub mod card; pub mod deepseek41v; pub mod gemma4a; +pub mod gemma4v; pub mod glm5v; pub(crate) mod header; pub mod minicpmv; @@ -27,12 +28,12 @@ use std::sync::Arc; use gguf::Gguf; use self::card::CardBytes; -use self::header::{KEY_PROJECTOR, Meta, check_architecture, need, refusal}; +use self::header::{KEY_PROJECTOR, Meta, check_architecture, projector_value, refusal}; use crate::{MediaModel, VisionError}; /// The bounds on how many tokens an image takes: [`open`]'s limits, which every projector that /// plans from them reads in this one type. -pub use self::qwen3vl::TokenLimits; +pub use crate::size::TokenLimits; /// What an encoder file opens to: the tower's bytes on its card, and the model's image input. pub type Opened = (CardBytes, Arc); @@ -54,6 +55,11 @@ pub type Opened = (CardBytes, Arc); pub fn open(file: &Gguf, limits: TokenLimits) -> Result { check_architecture(file)?; match projector(file)? { + Projector::Gemma4v => { + let hp = gemma4v::Hparams::read(file)?; + let media = gemma4v::Media::new(hp.size_rule(limits)?); + Ok((gemma4v::card::of(file, &hp, limits)?, Arc::new(media))) + } Projector::Deepseek41v => { let hp = deepseek41v::Hparams::read(file)?; Ok(( @@ -92,6 +98,7 @@ pub fn open(file: &Gguf, limits: TokenLimits) -> Result { /// The projector types this crate runs. #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum Projector { + Gemma4v, Deepseek41v, Glm5v, Qwen3vl, @@ -100,8 +107,9 @@ enum Projector { /// The file's `clip.projector_type`, refused by name when it is a tower this crate does not run. fn projector(m: &impl Meta) -> Result { - let v = need(m, KEY_PROJECTOR)?; + let v = projector_value(m)?; match v.as_str() { + Some(gemma4v::PROJECTOR_TYPE) => Ok(Projector::Gemma4v), Some(deepseek41v::PROJECTOR_TYPE) => Ok(Projector::Deepseek41v), Some(glm5v::PROJECTOR_TYPE) => Ok(Projector::Glm5v), Some(qwen3vl::PROJECTOR_TYPE) => Ok(Projector::Qwen3vl), @@ -109,8 +117,9 @@ fn projector(m: &impl Meta) -> Result { Some(other) => Err(refusal( KEY_PROJECTOR, format!( - "is \"{other}\"; this crate reads \"{}\", \"{}\", \"{}\" and \"{}\"", + "is \"{other}\"; this crate reads \"{}\", \"{}\", \"{}\", \"{}\" and \"{}\"", deepseek41v::PROJECTOR_TYPE, + gemma4v::PROJECTOR_TYPE, glm5v::PROJECTOR_TYPE, minicpmv::PROJECTOR_TYPE, qwen3vl::PROJECTOR_TYPE @@ -120,6 +129,13 @@ fn projector(m: &impl Meta) -> Result { } } +/// The vision projector type: the common key, or the vision key of a multimodal file. +pub fn projector_type(file: &Gguf) -> Result<&str, VisionError> { + projector_value(file)? + .as_str() + .ok_or_else(|| refusal(KEY_PROJECTOR, "is not a string")) +} + #[cfg(test)] mod tests { use gguf::{Gguf, Value}; @@ -135,8 +151,8 @@ mod tests { clef().with("clip.projector_type", Value::String(projector.into())) } - /// The projector type picks the module; a stranger is refused by its name, which lists the - /// four this crate reads. + /// The projector type picks the module; a stranger is refused by its name, which lists + /// the types this crate reads. #[test] fn the_projector_type_picks_the_tower() { assert_eq!( @@ -155,7 +171,7 @@ mod tests { assert_eq!( projector(&table("qwen2vl_merger")).unwrap_err().to_string(), "metadata clip.projector_type: is \"qwen2vl_merger\"; this crate reads \ - \"deepseek41v\", \"glm5v\", \"minicpmv4_7\" and \"qwen3vl_merger\"" + \"deepseek41v\", \"gemma4v\", \"glm5v\", \"minicpmv4_7\" and \"qwen3vl_merger\"" ); assert_eq!( projector(&clef().with("clip.projector_type", Value::U32(7))) @@ -197,7 +213,7 @@ mod tests { let t = table("qwen3vl_merger").with("clip.vision.projection_dim", Value::U32(out)); let (card, media) = open_table(&t, "clip").expect("open"); assert_eq!((card.weights, card.scratch), (weights, scratch), "{out}"); - assert_eq!(media.image_token(), 248_056); + assert_eq!(media.image_token(), Some(248_056)); assert_eq!(media.part_join(), PartJoin::LlamaServer); } } @@ -216,7 +232,7 @@ mod tests { (1_128_026_112, scratch), "{max}" ); - assert_eq!(media.image_token(), 154_854); + assert_eq!(media.image_token(), Some(154_854)); assert_eq!(media.part_join(), PartJoin::LlamaServer); } let image = crate::Rgb8::filled(1000, 600, [90, 120, 30]); @@ -257,7 +273,7 @@ mod tests { assert_eq!( refused(&table("qwen2vl_merger"), "clip"), "metadata clip.projector_type: is \"qwen2vl_merger\"; this crate reads \ - \"deepseek41v\", \"glm5v\", \"minicpmv4_7\" and \"qwen3vl_merger\"" + \"deepseek41v\", \"gemma4v\", \"glm5v\", \"minicpmv4_7\" and \"qwen3vl_merger\"" ); assert!(refused(&table("qwen3vl_merger"), "llama").starts_with("metadata architecture:")); let huge = TokenLimits::new(16, 3_000_000).expect("limits"); diff --git a/crates/vision/src/arch/qwen3vl/card.rs b/crates/vision/src/arch/qwen3vl/card.rs index 441e1da26..69aa7f2b8 100644 --- a/crates/vision/src/arch/qwen3vl/card.rs +++ b/crates/vision/src/arch/qwen3vl/card.rs @@ -12,11 +12,11 @@ use std::collections::HashSet; use super::hparams::TILE_N; -use super::size::TokenLimits; use super::tensors::{Export, expected}; use super::{Hparams, TEMPORAL_FRAMES, names}; pub use crate::arch::card::CardBytes; use crate::arch::card::activation_bytes; +use crate::size::TokenLimits; /// The tower of a file of `hp` at `limits` image tokens (module doc). #[must_use] diff --git a/crates/vision/src/arch/qwen3vl/hparams.rs b/crates/vision/src/arch/qwen3vl/hparams.rs index 7e0561f08..f3a156326 100644 --- a/crates/vision/src/arch/qwen3vl/hparams.rs +++ b/crates/vision/src/arch/qwen3vl/hparams.rs @@ -9,12 +9,12 @@ use gguf::{Gguf, Value}; use super::PROJECTOR_TYPE; -use super::size::{SizeRule, TokenLimits}; use crate::VisionError; use crate::arch::header::{ KEY_HAS_VISION, Meta, check_architecture, check_eps, check_mean_std, check_projector, check_true, count_multiple, counts, need, refusal, }; +use crate::size::{SizeRule, TokenLimits}; const KEY_GELU: &str = "clip.use_gelu"; const KEY_DEEPSTACK: &str = "clip.vision.is_deepstack_layers"; diff --git a/crates/vision/src/arch/qwen3vl/media.rs b/crates/vision/src/arch/qwen3vl/media.rs index 198e432ae..68dc71c8d 100644 --- a/crates/vision/src/arch/qwen3vl/media.rs +++ b/crates/vision/src/arch/qwen3vl/media.rs @@ -7,11 +7,11 @@ use std::borrow::Cow; use super::TEMPORAL_FRAMES; -use super::size::SizeRule; use crate::GridPlan; use crate::media::{MediaModel, PartJoin, Prepared}; use crate::preprocess::{PatchLayout, Patches, patchify}; use crate::resample::pad_ceil; +use crate::size::SizeRule; use crate::{Rgb8, VisionError}; /// The placeholder Clef's prompt carries for one image. @@ -81,8 +81,8 @@ impl MediaModel for Media { IMAGE_PLACEHOLDER } - fn image_token(&self) -> u32 { - IMAGE_TOKEN_ID + fn image_token(&self) -> Option { + Some(IMAGE_TOKEN_ID) } fn part_join(&self) -> PartJoin { @@ -105,9 +105,9 @@ mod tests { use super::{IMAGE_PLACEHOLDER, Media, PAD_BLACK, VISION_END_ID, VISION_START_ID}; use crate::MropeSeq; use crate::Rgb8; - use crate::arch::qwen3vl::size::{SizeRule, TokenLimits}; use crate::media::{MediaModel, PartJoin}; use crate::preprocess::{f32_to_bf16, normalize}; + use crate::size::{SizeRule, TokenLimits}; fn media() -> Media { Media::new(SizeRule::new(16, 2, TokenLimits::DEFAULT).expect("rule")) @@ -126,7 +126,7 @@ mod tests { Cow::Borrowed(IMAGE_PLACEHOLDER) )); assert_eq!(m.image_marker(), IMAGE_PLACEHOLDER); - assert_eq!(m.image_token(), 248_056); + assert_eq!(m.image_token(), Some(248_056)); assert_eq!((VISION_START_ID, VISION_END_ID), (248_053, 248_054)); assert_eq!(PAD_BLACK, [0, 0, 0]); assert_eq!(m.part_separator(), "\n"); diff --git a/crates/vision/src/arch/qwen3vl/mod.rs b/crates/vision/src/arch/qwen3vl/mod.rs index e58c66be5..1c2cb82e5 100644 --- a/crates/vision/src/arch/qwen3vl/mod.rs +++ b/crates/vision/src/arch/qwen3vl/mod.rs @@ -3,7 +3,7 @@ //! MLP, a learned 48×48 position table, a Conv3d patch embedding stored as two 16×16 kernels — //! and the merger (a 2×2 merge into a 4608-wide MLP). //! -//! The host side of an image is llama.cpp's mtmd rule for this projector: [`size`] plans the +//! The host side of an image is llama.cpp's mtmd rule for this projector: [`crate::size`] plans the //! pixel size, [`media`] pads the image onto a black canvas, normalizes it and cuts it in merge //! order with each patch repeated over the two temporal frames. @@ -11,12 +11,12 @@ pub mod card; pub mod hparams; pub mod media; pub mod names; -pub mod size; pub mod tensors; +pub use crate::size; +pub use crate::size::{SizeRule, TokenLimits}; pub use hparams::Hparams; pub use media::{Media, preprocess}; -pub use size::{SizeRule, TokenLimits}; /// `projector_type` of the files this module reads. pub const PROJECTOR_TYPE: &str = "qwen3vl_merger"; diff --git a/crates/vision/src/lib.rs b/crates/vision/src/lib.rs index 15c6ac850..1f33133a4 100644 --- a/crates/vision/src/lib.rs +++ b/crates/vision/src/lib.rs @@ -18,7 +18,7 @@ //! GGUF), each read once and refused by name when this crate does not run it, and what the //! projector's own image rule needs: V4.1's resize plan and span layout (`deepseek41v::grid`, //! `deepseek41v::span`), its reference `load_image` and its bytes on its card; Clef's size rule -//! and pad colour (`qwen3vl::size`, `qwen3vl::media`). +//! and pad colour (`size`, `qwen3vl::media`). //! //! The reference of V4.1 is deepseek-ai/DeepSeek-V4.1-Flash `inference/image_processor.py` and //! `inference/vision.py`, and Pillow's `libImaging/Resample.c` and `ImageOps.py` of the version @@ -31,6 +31,7 @@ pub mod media; pub mod mrope; pub mod preprocess; pub mod resample; +pub mod size; pub use image::{FileKind, Rgb8}; pub use media::{EncoderInput, MediaModel, Prepared}; diff --git a/crates/vision/src/media.rs b/crates/vision/src/media.rs index 3a355daec..4d5deda38 100644 --- a/crates/vision/src/media.rs +++ b/crates/vision/src/media.rs @@ -78,6 +78,10 @@ impl EncoderInput { /// A model's image input, as data plus one prepare. pub trait MediaModel { + /// Ordinary text around image rows in an embedding input; no placeholder token is needed. + fn embedding_delimiters(&self) -> Option<(&str, &str)> { + None + } /// The placeholder text the chat template carries for the image at `index`: its position /// among all the images of the prompt, from 0, in the order of the images. A model that /// numbers its images writes the number into the text; one that does not ignores it. @@ -96,10 +100,11 @@ pub trait MediaModel { None } - /// The id [`expand_spans`] finds in the tokenized prompt and expands to one copy + /// The optional id [`expand_spans`] finds in a tokenized chat prompt and expands to one copy /// per position of the image ([`Prepared::span_len`]): the one id V4.1's placeholder /// tokenizes to, and the `<|image_pad|>` between the two delimiter ids Clef's tokenizes to. - fn image_token(&self) -> u32; + /// `None` when images use embedding delimiters instead of a chat placeholder. + fn image_token(&self) -> Option; /// The separator between a message's content parts when they are flattened into one text: /// the separator of a [`PartJoin::Uniform`] join. diff --git a/crates/vision/src/arch/qwen3vl/size.rs b/crates/vision/src/size.rs similarity index 98% rename from crates/vision/src/arch/qwen3vl/size.rs rename to crates/vision/src/size.rs index 8533b56a5..9adc9f690 100644 --- a/crates/vision/src/arch/qwen3vl/size.rs +++ b/crates/vision/src/size.rs @@ -1,5 +1,5 @@ //! The pixel size an image is brought to before it is cut into patches: llama.cpp's mtmd rule -//! for this projector (`img_tool::calc_size_preserved_ratio`, the dyn-size preprocessor), ported +//! shared by dynamic-size projectors (`img_tool::calc_size_preserved_ratio`), ported //! operation for operation. //! //! Each side is rounded to a multiple of `align = patch * merge`; a size over the pixel limit is @@ -108,6 +108,10 @@ impl SizeRule { self.merge } + pub(crate) fn max_tokens(&self) -> usize { + self.limits.max() + } + /// Pixels a token covers on a side: `patch * merge`. #[must_use] pub fn align(&self) -> usize { diff --git a/docs/serving.md b/docs/serving.md index 874cd3f8b..a082f9e46 100644 --- a/docs/serving.md +++ b/docs/serving.md @@ -37,8 +37,8 @@ Your OpenAI client works unchanged. bloomery speaks llama-server's HTTP API and other than the embedding model's is a 400 `invalid_request_error` and runs no pass; `null` is unset. `/embedding` and `/embeddings` ignore `model`. An input past the model's context is a 400 `exceed_context_size_error` naming both lengths. llama-server checks its physical batch first and answers an input - past it with a 500; this server has no batch limit there. An object input (`prompt_string`, or a `content` array - of parts) is a 400. + past it with a 500; this server has no batch limit there. An object with a `content` array takes text and image + parts as described below. An object with `prompt_string` is a 400. - **Tool calls on every generative model**: DeepSeek-V4.1-Flash (DSML), GLM-5.3-Flash (its XML), Qwen3-30B (Hermes JSON), Qwen3.6-35B and Qwen3.8-Flash-Next (the `` markup) parse into `tool_calls` and `tool_use` blocks, so agent clients work against each of them. @@ -256,3 +256,74 @@ binary's `--help`. - A pinned nightly with a pinned cuda-oxide revision from our fork ([`BUILD.md`](BUILD.md)). In progress: serving on N cards for every model; Clef from the IQ and Q2_K files; DeepSeek-V4-Flash-0731. + +## EmbeddingGemma 2 images + +A side seat serves the `gemma-embedding2` encoder beside a generative model. Build `bloomery-serve` with +`--features glm5next,clef,mimo2,gemma-embedding2,vision` through `cargo oxide`, and add these flags to the chat +model's command: + +```text +--embd-model embeddinggemma-2-Q8_0.gguf +--embd-mmproj mmproj-embeddinggemma-2-BF16.gguf +--embd-ctx 8192 +``` + +`--embd-hf [:]` selects the text file and the row's BF16 projector from that repository. +`--embd-no-mmproj` disables that automatic projector. A local `--embd-model` loads images only with an explicit +`--embd-mmproj`. The projector's vision matrices must be BF16; a Q8_0 projector is refused by tensor name. +The F32 patch convolution and learned position tables retain their file precision. With no activation keys, +the tower follows mainline's Quick GELU; `clip.use_gelu=true` selects its tanh GELU and SiLU is refused. Audio tensors of a combined +vision/audio file stay on the host and are not counted in the card reserve. + +The encoder and tower share the embedding side seat's card and thread, with streams of their own. Before the chat placement +is made, their weight and activation bytes are counted together. On a card outside the chat plan this costs the +plan no bytes; otherwise the plan holds them in its named `embedding model` reserve, reducing the chat expert +budget. The side context bounds the complete input, including image rows and delimiter text. + +Send one object per input inside OpenAI's `input` array: + +```json +{ + "input": [ + { + "content": [ + {"type": "text", "text": "A description of this picture:"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,..."}} + ] + } + ], + "dimensions": 256 +} +``` + +An image URL is a base64 `data:` URL for PNG or JPEG, as on the chat path; HTTP URLs are refused. Multiple image +parts remain in input order. Each image is resized with mtmd's aspect-preserving dynamic canvas, bicubic filter +and centered black padding. The default budget is 70 to 1120 pooled image tokens. Set +`--embd-image-min-tokens` and `--embd-image-max-tokens` to change it independently of the chat model's image flags. +A canvas exceeding the reserved token budget or learned position table is a request error before a tower call. + +`<|image>` and `` are tokenized around each image's rows with the server's vocabulary rule, +included in the single bidirectional pass and included in mean pooling, as in the pinned mainline server. +Only text embeddings receive the text encoder's `sqrt(hidden)` scale: image rows enter already projected. +The entire input runs once; image chunks are not decoded separately. A request without a loaded image tower, +with an unsupported image part or beyond the side context is refused with a 400. + +The reference families in `crates/refset/src/arch/gemma_embedding2` own the projector path, its digest and +llama.cpp build pin. `just gate-gpu-gemma4v-tower` reads the mainline card and CPU tower sets and every required +layer tensor, comparing the device to the card with each band derived from the CPU twin. +`just gate-gpu-gemma-embedding2-e2e` also runs image-only and `mix_free-448x448` text-first inputs through +`pass_media`, using `serve::embeddings::assemble` for the same part ordering and token spans as the HTTP route. +It compares raw and normalized vectors with the server and its CPU twin (`just dump-ref-emb2 server.cpu`). + +To compare a running HTTP server directly with saved llama-server vectors: + +```sh +python3 tools/ref/emb2/server_vectors.py compare --url http://127.0.0.1:8080 \ + --set "$BLOOMERY_DATA/ref_gemma_embedding2_server" \ + --image free-448x448=/path/to/free-448x448.png --mix-image free-448x448 +``` + +Use the exact image bytes named by the reference set. Add `--ids-dir "$BLOOMERY_DATA/ref-tokenizer-gemma4"` +and `--case NAME` for text cases. Each selected `raw` and `embd` row prints cosine and maximum absolute +difference. Exit 0 means the comparison completed; it applies no numerical acceptance band. diff --git a/justfile b/justfile index 97079c8eb..419b21e5e 100644 --- a/justfile +++ b/justfile @@ -785,7 +785,7 @@ build-emb1-ref: (build-embedding-ref "gemma-embedding") build-embedding-ref MODEL: BLOOMERY_MODEL={{MODEL}} ./tools/box.sh 'bash tools/ref/clefvis/build-clefvis.sh' -# EmbeddingGemma 2, the oracle sets (tools/ref/emb2/emb2.sh; refset families in crates/refset/src/arch/gemma_embedding2) into $BLOOMERY_DATA: every text case's token rows, pooled vector and layer taps (ref_gemma_embedding2_text), the image cases' tower taps and rows (ref_gemma_embedding2_tower), llama-server's /v1/embeddings vectors of both (ref_gemma_embedding2_server), and the CPU twins of the first two (.cpu). The text cases and their ids are the tokenizer oracle's gemma4 set: `just dump-ref-tokenizer gemma4` first. STEPS names some of text, tower, server, text.cpu and tower.cpu (none: all five). The card steps go through the card's gate lock (EMB2_BOUND is the call's and each step's bound); no lease. +# EmbeddingGemma 2, the oracle sets (tools/ref/emb2/emb2.sh; refset families in crates/refset/src/arch/gemma_embedding2) into $BLOOMERY_DATA: every text case's token rows, pooled vector and layer taps (ref_gemma_embedding2_text), the image cases' tower taps and rows (ref_gemma_embedding2_tower), llama-server's /v1/embeddings vectors of both (ref_gemma_embedding2_server), and the CPU twins of all three (.cpu). The text cases and their ids are the tokenizer oracle's gemma4 set: `just dump-ref-tokenizer gemma4` first. STEPS names some of text, tower, server, text.cpu, tower.cpu and server.cpu (none: all six). The card steps go through the card's gate lock (EMB2_BOUND is the call's and each step's bound); no lease. dump-ref-emb2 *STEPS: (dump-ref-embedding "gemma-embedding2" STEPS) # Gemma4a host mel from synthetic decoded PCM, using the existing pinned mainline tree. @@ -1264,8 +1264,13 @@ gate-gpu-gemma-embedding2-serve: gate-gpu-gemma-embedding2-e2e: ./tools/box.sh 'bash tools/gate.sh --oxide --release -p bloomery-gpu-gemma-embedding2 --lib -- --include-ignored' ./tools/box.sh 'bash tools/gate.sh --release -p bloomery-model --lib -- arch::gemma_embedding2::spec::tests::key_bounds_are_the_symmetric_mask --exact' - ./tools/box.sh 'bash tools/gate.sh --oxide --release -p bloomery-gpu-gates --features gemma-embedding2 --bin gate_gemma_embedding2_e2e' - ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features gemma-embedding2 --release --bin gate_gemma_embedding2_e2e && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_gemma_embedding2_e2e' + ./tools/box.sh 'bash tools/gate.sh --oxide --release -p bloomery-gpu-gates --features gemma-embedding2,vision,deepseek41 --bin gate_gemma_embedding2_e2e' + BLOOMERY_MODEL=gemma-embedding2 ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features gemma-embedding2,vision,deepseek41 --release --bin gate_gemma_embedding2_e2e && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_gemma_embedding2_e2e' + +# Gemma4 vision's mainline card and CPU sets, including every block's rows. +# Every layer and projected row is held to its CPU/card twin distance. +gate-gpu-gemma4v-tower: + BLOOMERY_MODEL=gemma-embedding2 ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features vision --release --bin gate_gemma4v_tower && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_gemma4v_tower' # GLM-5.3-Flash MLA 층의 카드 부분 — 잠재 RMS + f16 추가, 인덱서 키 LayerNorm + [키; 게이트] 추가(`latent`), # 창 0·싱크 −∞·스케일 1/16으로 다시 쓰는 V4.1 어텐션 — 을 모델 없이 합성 입력으로 호스트 규칙에 맞춘다. diff --git a/tools/gate-paths.tsv b/tools/gate-paths.tsv index 4a7cf368e..e91a5c7ba 100644 --- a/tools/gate-paths.tsv +++ b/tools/gate-paths.tsv @@ -41,7 +41,7 @@ crates/gpu-deepseek41/src/{draft/**,experts_mxfp4.rs,markov.rs,hc_f32.rs} gate-g # crates/gpu-gates' lib, below its bins. crates/gpu-gates/src/record.rs gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-swap gate-gpu-qwen35moe-e2e gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3moe-e2e gate-q38-serve-rules gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media-seat gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text weekly-gpu-ds41-flowcounts gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve gate-gpu-qwen38-nodes the record kinds: the bins that print records (generate_ds41, bloomery-chat, bloomery-serve-ds41, the prefill, residency, callstream and swap gates) and generate.rs's users crates/gpu-gates/src/generate.rs gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-serve the V4.1 generate loop shared by generate_ds41, the chat and serve bins and the tier and two-card gates -crates/gpu-gates/src/{bind.rs,serve_client.rs} gate-ds41-bind gate-gpu-clef-serve gate-gpu-lev-serve gate-q38-serve-rules gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve the V4.1 server's engine binding and its HTTP client: gpu-gates' bind tests; bloomery-serve-ds41 and its gate are weekly-gpu-ds41-serve (a trigger below) +crates/gpu-gates/src/{bind.rs,serve_client.rs} gate-ds41-bind gate-gpu-clef-serve gate-gpu-lev-serve gate-q38-serve-rules gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve gate-gpu-gemma-embedding2-e2e the V4.1 server's engine binding and its HTTP client: gpu-gates' bind tests; bloomery-serve-ds41 and its gate are weekly-gpu-ds41-serve (a trigger below) crates/gpu-gates/src/vision_seat.rs gate-ds41-bind gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve weekly-gpu-ds41-serve gate-gpu-glm5next-media-seat gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-ds41-media gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-routed-legs gate-gpu-ds41-chat gate-gpu-glm5next-twocard weekly-gpu-glm5next-media-serve gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-q38-serve-rules gate-gpu-qwen38-serve gate-gpu-gemma-embedding2-serve the vision seat every serve seat shares: gate-ds41-bind's unit tests (the vision_seat:: run), and the servers that bind it (the V4.1 seat's images) crates/gpu-gates/src/image_plan.rs gate-gpu-clef-serve gate-gpu-glm5next-serve gate-gpu-lev-serve gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3-serve gate-gpu-qwen3moe-e2e gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve the plan's side of an image input (a sequence's rotation table a position): the qwen3 placement every Qwen3 bin includes, and the servers that bind its seat crates/gpu-gates/src/engine.rs gate-gpu-e2e AnyEngine, the V2-Lite and Qwen3 engine behind gate_e2e @@ -59,17 +59,17 @@ crates/app/src/{lib.rs,arch/mod.rs} gate-gpu-ds41-callstream gate-gpu-ds41-resid crates/app/src/arch/deepseek41/** gate-gpu-clef-serve gate-gpu-ds41-callstream gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-media gate-gpu-ds41-residency-a gate-gpu-ds41-twocard gate-gpu-glm5next-serve gate-gpu-qwen3-serve gate-gpu-lev-serve gate-gpu-ds41-step gate-gpu-ds41-prefill gate-gpu-ds41-nvtier gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve the V4.1 session impls and its draft: generate_ds41, the chat and serve bins, and the gates that drive them crates/app/src/arch/glm5next/** gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen3-serve gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve the GLM session impls: the GLM e2e gate crates/app/src/{mtp.rs,mtp/**,arch/qwen3moe/**} gate-app gate-q38-serve-rules gate-gpu-qwen3moe-e2e gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen4exp-mtp gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-glm5next-twocard gate-gpu-qwen3-serve gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-serve the shared MTP window and the qwen3moe session impls: the app tests, the qwen3moe e2e gate, the Qwen3.8 serve seat and MTP gates, and the Qwen3.6 draft gate (its Session taps) -crates/runtime/src/** gate-ops gate-placement gate-union gate-fixture gate-app gate-deepseek4-meta gate-ds41-bind gate-ds41-host gate-ds41-kld gate-ds41-load gate-ds41-meta gate-ds41-oracle gate-ds41-plan gate-dspark-read gate-glm5next-meta gate-qwen35moe-meta gate-qwen3moe-meta gate-qwen4exp-meta gate-ptx-spill gate-gpu-e2e gate-gpu-gemm gate-gpu-lib gate-gpu-gates-lib gate-gpu-head gate-gpu-hc-gated gate-gpu-kquant gate-gpu-mcol gate-gpu-moe gate-gpu-p10 gate-gpu-swap gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-gemm gate-gpu-glm5next-mtp gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-qwen3-serve gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-head gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-attn gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35moe-moe gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-ds41-callstream gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-comp gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-index gate-gpu-ds41-lib gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-moe gate-gpu-ds41-prefill gate-gpu-ds41-residency-a gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-woa gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-q38-serve-rules gate-gpu-nvtier gate-mimo2-meta gate-gpu-hybrid gate-gpu-p8 gate-gpu-linear gate-ds41-place gate-gpu-p8b gate-gpu-iq-sel gate-gpu-xstream gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-hf gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gpu-p1 gate-gpu-mamba2 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gemma-embedding-meta gate-gpu-qwen38-nodes gate-gpu-nemotron-h-moe-e2e gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve the generation loop and its stores: every gate that names a runtime path in its own code (runtime::combine in gate_kquant and the chain-ffn/prefill bins, hc_gated, qsa, seqstate, swaprule, stores), loads weights through bloomery_gpu::weights (whose every loader entry funnels into upload_rows → runtime::words::stream_words), or opens a model or session (arch place.rs → runtime::stores; clef-hidden opens the Qwen3.5 body, whose body35.rs reads runtime::seqstate). The gates left out load no model and name no runtime path: the synthetic kernel gates upload through DeviceTensor::upload/from_host and read reference values host-side, the CPU oracle gates read the model host-side, and gate-runtime is picked as its own lib's tests +crates/runtime/src/** gate-ops gate-placement gate-union gate-fixture gate-app gate-deepseek4-meta gate-ds41-bind gate-ds41-host gate-ds41-kld gate-ds41-load gate-ds41-meta gate-ds41-oracle gate-ds41-plan gate-dspark-read gate-glm5next-meta gate-qwen35moe-meta gate-qwen3moe-meta gate-qwen4exp-meta gate-ptx-spill gate-gpu-e2e gate-gpu-gemm gate-gpu-lib gate-gpu-gates-lib gate-gpu-head gate-gpu-hc-gated gate-gpu-kquant gate-gpu-mcol gate-gpu-moe gate-gpu-p10 gate-gpu-swap gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-gemm gate-gpu-glm5next-mtp gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-qwen3-serve gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-head gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-attn gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35moe-moe gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-ds41-callstream gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-comp gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-index gate-gpu-ds41-lib gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-moe gate-gpu-ds41-prefill gate-gpu-ds41-residency-a gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-woa gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-q38-serve-rules gate-gpu-nvtier gate-mimo2-meta gate-gpu-hybrid gate-gpu-p8 gate-gpu-linear gate-ds41-place gate-gpu-p8b gate-gpu-iq-sel gate-gpu-xstream gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-hf gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gpu-p1 gate-gpu-mamba2 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gemma-embedding-meta gate-gpu-qwen38-nodes gate-gpu-nemotron-h-moe-e2e gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower the generation loop and its stores: every gate that names a runtime path in its own code (runtime::combine in gate_kquant and the chain-ffn/prefill bins, hc_gated, qsa, seqstate, swaprule, stores), loads weights through bloomery_gpu::weights (whose every loader entry funnels into upload_rows → runtime::words::stream_words), or opens a model or session (arch place.rs → runtime::stores; clef-hidden opens the Qwen3.5 body, whose body35.rs reads runtime::seqstate). The gates left out load no model and name no runtime path: the synthetic kernel gates upload through DeviceTensor::upload/from_host and read reference values host-side, the CPU oracle gates read the model host-side, and gate-runtime is picked as its own lib's tests crates/hf/src/{safetensors.rs,dspark.rs} gate-gpu-qwen35-dspark gate-gpu-clef-serve gate-gpu-lev-serve gate-decision-clef gate-decision-lev gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve hf::safetensors is the Clef head reader (decision re-exports its errors through decision::Error, so the serve seats that open the decide row reach it) and the DSpark draft file reader; hf::dspark is read only by the Qwen3.6 draft gate -crates/serve/{src/**,build.rs} gate-ds41-bind gate-gpu-clef-serve gate-gpu-lev-serve gate-serve gate-q38-serve-rules gate-ds41-place gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve the llama-server-compatible API: gpu-gates' bind tests (bloomery-serve-ds41's gate is weekly-gpu-ds41-serve, a trigger below); gate-serve is its own +crates/serve/{src/**,build.rs} gate-ds41-bind gate-gpu-clef-serve gate-gpu-lev-serve gate-serve gate-q38-serve-rules gate-ds41-place gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve gate-gpu-gemma-embedding2-e2e the llama-server-compatible API: gpu-gates' bind tests (bloomery-serve-ds41's gate is weekly-gpu-ds41-serve, a trigger below); gate-serve is its own crates/jinja/src/** gate-jinja gate-serve gate-ds41-bind gate-decision-clef gate-decision-lev gate-gpu-clef-serve gate-gpu-lev-serve gate-q38-serve-rules gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve the template engine: its own tests, serve's chat templates (the serve row's readers) and the decision models' templates (Clef's and lev's request renders) crates/tokenizer/src/** gate-app gate-decision-clef gate-deepseek4-meta gate-ds41-bind gate-ds41-host gate-ds41-kld gate-ds41-load gate-ds41-meta gate-ds41-oracle gate-ds41-plan gate-dspark-read gate-fixture gate-glm5next-meta gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-comp gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-index gate-gpu-ds41-lib gate-gpu-ds41-long gate-gpu-ds41-prefill gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-e2e gate-gpu-gates-lib gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-lib gate-gpu-mcol gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35moe-moe gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-head gate-gpu-qwen3moe-router gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-ops gate-placement gate-qwen35moe-meta gate-qwen3moe-meta gate-qwen4exp-meta gate-union gate-gpu-clef-serve gate-gpu-lev-serve gate-decision-lev gate-q38-serve-rules gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-iq-sel gate-gpu-clef-hidden gate-gpu-qwen4exp-iqleg gate-gpu-glm5next-gemm gate-gpu-ds41-media gate-gpu-nvtier gate-gpu-qwen38-residency gate-mimo2-meta gate-gpu-glm5next-residency gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-ds41-place gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-qwen35-dspark gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve the encoder and decoder, and runs_pre_tokenizer, which coverage::check asks at every placement plan (model::arch::*::place): every gate that plans a placement or tokenizes text # crates/refset: the reference-set readers. Every oracle read goes through refset::arch's dispatch (node_dumps, # families) and a family's check; which family a gate asks for is its run-time arch, so a family's row keeps every # dispatch user. -crates/refset/src/{lib,family,mtpref,dsref,greedy,kld,vision,ik,columns}.rs gate-attn gate-derived gate-ds41-bind gate-ds41-host gate-ds41-kld gate-ds41-load gate-ds41-meta gate-ds41-oracle gate-ds41-plan gate-ffn gate-forward gate-gpu-block gate-gpu-clef-hidden gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-gates-lib gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-mtp gate-gpu-glm5next-residency gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-head gate-gpu-linear gate-gpu-mcol gate-gpu-moe gate-gpu-p0b gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p8 gate-gpu-p8b gate-gpu-qwen35moe-attn gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-qwen35moe-moe gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-vision gate-head gate-kv gate-moe gate-mt gate-ops gate-profile gate-qwen3moe-meta gate-union gate-vision gate-1-1 gate-tokenizer gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-ds41-place gate-q38-serve-rules gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-iq gate-gpu-iq-sel gate-gpu-e2e gate-gpu-hybrid gate-gpu-gemm gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-dspark-read gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-clef-serve gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-minicpmv-tower gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-emb2-mel the readers and the family check every set goes through: every gate that reads a reference set +crates/refset/src/{lib,family,mtpref,dsref,greedy,kld,vision,ik,columns}.rs gate-attn gate-derived gate-ds41-bind gate-ds41-host gate-ds41-kld gate-ds41-load gate-ds41-meta gate-ds41-oracle gate-ds41-plan gate-ffn gate-forward gate-gpu-block gate-gpu-clef-hidden gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-gates-lib gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-mtp gate-gpu-glm5next-residency gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-head gate-gpu-linear gate-gpu-mcol gate-gpu-moe gate-gpu-p0b gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p8 gate-gpu-p8b gate-gpu-qwen35moe-attn gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-qwen35moe-moe gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-vision gate-head gate-kv gate-moe gate-mt gate-ops gate-profile gate-qwen3moe-meta gate-union gate-vision gate-1-1 gate-tokenizer gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-ds41-place gate-q38-serve-rules gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-iq gate-gpu-iq-sel gate-gpu-e2e gate-gpu-hybrid gate-gpu-gemm gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-dspark-read gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-clef-serve gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-minicpmv-tower gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-emb2-mel gate-gpu-gemma4v-tower the readers and the family check every set goes through: every gate that reads a reference set crates/refset/src/arch/mod.rs gate-ds41-bind gate-ds41-kld gate-ds41-load gate-ds41-oracle gate-ds41-plan gate-gpu-block gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-media gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-gates-lib gate-gpu-head gate-gpu-linear gate-gpu-moe gate-gpu-p0b gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p8 gate-gpu-p8b gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-1-1 gate-tokenizer gate-ds41-place gate-gpu-qwen4exp-mtp gate-q38-serve-rules gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-media-seat gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve the per-arch dispatch (node_dumps, families, named): every oracle user except gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-glm5next-serve: bloomery-serve opens one seat a run (--model, or the file's architecture); these run its decide and glm seats, and only the qwen38 seat's module (its DraftFrom) reaches this file | except gate-gpu-serve-start-glm: bloomery-serve opens one seat a run (--model, or the file's architecture); this recipe runs its glm seat, and only another seat's module reaches this file | except gate-gpu-serve-start-ds41: bloomery-serve opens one seat a run (--model, or the file's architecture); this recipe runs its ds41 seat, and only another seat's module reaches this file crates/refset/src/arch/deepseek41/** gate-ds41-bind gate-ds41-host gate-ds41-kld gate-ds41-load gate-ds41-meta gate-ds41-oracle gate-ds41-plan gate-gpu-block gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-long gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-gates-lib gate-gpu-glm5next-serve gate-gpu-head gate-gpu-linear gate-gpu-mcol gate-gpu-moe gate-gpu-p0b gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p8 gate-gpu-p8b gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-union gate-gpu-qwen3-serve gate-gpu-clef-serve gate-gpu-lev-serve gate-ds41-place gate-tokenizer gate-1-1 gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve the deepseek41 family's sets: the dispatch users and the gates that name the family crates/refset/src/arch/deepseek41v/** gate-ds41-bind gate-ds41-kld gate-ds41-load gate-ds41-oracle gate-ds41-plan gate-gpu-block gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-media gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-gates-lib gate-gpu-head gate-gpu-linear gate-gpu-moe gate-gpu-p0b gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p8 gate-gpu-p8b gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-vision gate-vision gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 the deepseek41v family's sets: the dispatch users and the gates that name the family except gate-ds41-place: they read no deepseek41v set: only refset::arch's table (BY_ARCH), which picks a set's family by the file it names at run time, reaches it @@ -77,11 +77,11 @@ crates/refset/src/arch/glm5next/** gate-ds41-bind gate-ds41-kld gate-ds41-load g crates/refset/src/arch/mimo2/** gate-ds41-bind gate-ds41-kld gate-ds41-load gate-ds41-oracle gate-ds41-plan gate-gpu-block gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-gates-lib gate-gpu-head gate-gpu-linear gate-gpu-mimo2-flash gate-gpu-moe gate-gpu-p0b gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p8 gate-gpu-p8b gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router weekly-gpu-mimo2-e2e the mimo2 family's sets: the dispatch users and the gates that name the family except gate-ds41-place: they read no mimo2 set: only refset::arch's table (BY_ARCH), which picks a set's family by the file it names at run time, reaches it crates/refset/src/arch/qwen35moe/** gate-ds41-bind gate-ds41-kld gate-ds41-load gate-ds41-oracle gate-ds41-plan gate-gpu-block gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-gates-lib gate-gpu-head gate-gpu-linear gate-gpu-moe gate-gpu-p0b gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p8 gate-gpu-p8b gate-gpu-qwen35moe-attn gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-qwen35moe-moe gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-qwen3-serve gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp the qwen35moe family's sets: the dispatch users and the gates that name the family except gate-ds41-place: they read no qwen35moe set: only refset::arch's table (BY_ARCH), which picks a set's family by the file it names at run time, reaches it crates/refset/src/arch/qwen4exp/** gate-ds41-bind gate-ds41-kld gate-ds41-load gate-ds41-oracle gate-ds41-plan gate-gpu-block gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-gates-lib gate-gpu-glm5next-serve gate-gpu-head gate-gpu-linear gate-gpu-moe gate-gpu-p0b gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p8 gate-gpu-p8b gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3-serve gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-nvtier gate-q38-serve-rules gate-tokenizer gate-ds41-place gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve the qwen4exp family's sets: the dispatch users and the gates that name the family -crates/refset/src/{clefvis.rs,clefvis/**} gate-refset gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-ds41-hc gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-derived gate-mt gate-profile gate-ds41-meta gate-qwen3moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-tokenizer gate-1-1 gate-gpu-head gate-gpu-mcol gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-vision gate-gpu-vision gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-minicpmv-tower gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-h-moe-e2e gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-emb2-mel the Clef image-input sets' reader: no gate reads a clefvis set yet, gate-refset runs its tests and walks every set in place +crates/refset/src/{clefvis.rs,clefvis/**} gate-refset gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-ds41-hc gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-derived gate-mt gate-profile gate-ds41-meta gate-qwen3moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-tokenizer gate-1-1 gate-gpu-head gate-gpu-mcol gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-vision gate-gpu-vision gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-minicpmv-tower gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-h-moe-e2e gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-emb2-mel gate-gpu-gemma4v-tower the Clef image-input sets' reader: no gate reads a clefvis set yet, gate-refset runs its tests and walks every set in place crates/refset/src/arch/minicpmv4_7/** gate-ds41-bind gate-ds41-kld gate-ds41-load gate-ds41-oracle gate-ds41-plan gate-gpu-block gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-gates-lib gate-gpu-linear gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-ds41-place gate-gpu-minicpmv-tower gate-refset the MiniCPM-V 4.7 image sets' reader (the tower per tile, the hidden states and four position lanes of an image prompt, the chat ids): the dispatch users, the tower gate that reads the tower sets, and gate-refset, which runs the reader's tests and walks every set in place crates/refset/src/arch/qwen35/** gate-refset gate-gpu-clef-hidden gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media the qwen35 families' sets: the hidden-state gate that names the families, and the table walk except gate-gpu-ds41-hc gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen3moe-down gate-gpu-linear gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-ds41-bind gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-ds41-load: they read no qwen35 set: only refset::arch's table (BY_ARCH), which picks a set's family by the file it names at run time, reaches it -crates/refset/src/emb2.rs gate-refset gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-h-moe-e2e gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-emb2-mel the EmbeddingGemma 2 oracle sets: gate-refset checks these families except gate-1-1 gate-attn gate-derived gate-ds41-bind gate-ds41-host gate-ds41-kld gate-ds41-load gate-ds41-meta gate-ds41-oracle gate-ds41-place gate-ds41-plan gate-ffn gate-forward gate-gpu-block gate-gpu-clef-hidden gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-gates-lib gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-gemm gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-residency gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-glm5v-tower gate-gpu-head gate-gpu-linear gate-gpu-mcol gate-gpu-mimo2-flash gate-gpu-minicpm-text gate-gpu-minicpmv-tower gate-gpu-moe gate-gpu-p0b gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p8 gate-gpu-p8b gate-gpu-qwen3-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-attn gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-media gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-verify gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen3vl-tower gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-serve-start-ds41 gate-gpu-serve-start-glm gate-gpu-vision gate-head gate-kv gate-moe gate-mt gate-ops gate-profile gate-qwen3moe-meta gate-tokenizer gate-union gate-vision: they read no EmbeddingGemma 2 set: only the runtime family dispatch reaches this reader and these families -crates/refset/src/arch/gemma_embedding2/** gate-refset gate-gpu-gemma-embedding2-e2e gate-gemma-embedding2-meta gate-gpu-gemma-embedding2-serve gate-emb2-mel the EmbeddingGemma 2 oracle sets: gate-refset checks these families except gate-ds41-bind gate-ds41-kld gate-ds41-load gate-ds41-oracle gate-ds41-place gate-ds41-plan gate-gpu-block gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-gates-lib gate-gpu-linear gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router: they read no EmbeddingGemma 2 set: only the runtime family dispatch reaches this reader and these families +crates/refset/src/emb2.rs gate-refset gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-h-moe-e2e gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-emb2-mel gate-gpu-gemma4v-tower the EmbeddingGemma 2 oracle sets: gate-refset checks these families except gate-1-1 gate-attn gate-derived gate-ds41-bind gate-ds41-host gate-ds41-kld gate-ds41-load gate-ds41-meta gate-ds41-oracle gate-ds41-place gate-ds41-plan gate-ffn gate-forward gate-gpu-block gate-gpu-clef-hidden gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-gates-lib gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-gemm gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-residency gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-glm5v-tower gate-gpu-head gate-gpu-linear gate-gpu-mcol gate-gpu-mimo2-flash gate-gpu-minicpm-text gate-gpu-minicpmv-tower gate-gpu-moe gate-gpu-p0b gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p8 gate-gpu-p8b gate-gpu-qwen3-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-attn gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-media gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-verify gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen3vl-tower gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-serve-start-ds41 gate-gpu-serve-start-glm gate-gpu-vision gate-head gate-kv gate-moe gate-mt gate-ops gate-profile gate-qwen3moe-meta gate-tokenizer gate-union gate-vision: they read no EmbeddingGemma 2 set: only the runtime family dispatch reaches this reader and these families +crates/refset/src/arch/gemma_embedding2/** gate-refset gate-gpu-gemma-embedding2-e2e gate-gemma-embedding2-meta gate-gpu-gemma-embedding2-serve gate-emb2-mel gate-gpu-gemma4v-tower the EmbeddingGemma 2 oracle sets: gate-refset checks these families except gate-ds41-bind gate-ds41-kld gate-ds41-load gate-ds41-oracle gate-ds41-place gate-ds41-plan gate-gpu-block gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-gates-lib gate-gpu-linear gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router: they read no EmbeddingGemma 2 set: only the runtime family dispatch reaches this reader and these families crates/refset/src/{md5,tokenizer,dequant}.rs gate-tokenizer gate-1-1 gate-gpu-ds41-hc gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-attn gate-gpu-qwen35moe-e2e gate-gpu-minicpm-text gate-gpu-nemotron-h-moe-e2e gate-gpu-gemma-embedding2-e2e the tokenizer and dequant set readers and the md5 they check by: the two gates that open those sets (gate-refset runs the crate's own tests) except gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen3moe-down gate-gpu-linear gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-ds41-bind gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-ds41-load: they read no dequant and tokenizer set: only refset::arch's table (BY_ARCH), which picks a set's family by the file it names at run time, reaches it | except gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency: they open their own families' sets through refset::family::Family::check_set, which reaches DequantSet::open only in its DequantManifest arm, an identity no set of theirs carries: they read no dequant set crates/refset/src/arch/{tokenizer,dequant}.rs gate-tokenizer gate-1-1 gate-gpu-nemotron-e2e the tokenizer and dequant families' rows and pins: the two gates that open those sets (gate-refset runs the crate's own tests) except gate-gpu-ds41-hc gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen3moe-down gate-gpu-linear gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-ds41-bind gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-ds41-load: they read no dequant and tokenizer set: only refset::arch's table (BY_ARCH), which picks a set's family by the file it names at run time, reaches it crates/refset/src/census.rs gate-gpu-routed-legs the census stub reader (the manifest and the digest check): the routed-legs gate plans every set through it @@ -98,7 +98,7 @@ tools/census/stubs.tsv gate-gpu-routed-legs the census manifest (refset::census: # files a gate opens. One row: every gate that links the modules, less the gates whose bins only read tensors # through gguf::Gguf or build synthetic blocks and call no arch module (their own code, and the bloomery-gpu # kernel modules and bloomery-gpu-gates helpers they call, name no model::arch:: path). the fixture generator, arch/coverage.rs and arch/qwen35moe/mtp.rs are carved out to their own rows below: their readers are bounded (a leaf the family's fixture module names; the place/engine/plan modules that call them). -crates/model/src/arch/{mod.rs,dspark.rs,classify.rs,stems.rs,deepseek2/**,qwen3moe/**,glm5next/**} gate-gpu-p4 gate-gpu-p6 gate-gpu-kquant gate-gpu-iq-sel gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-ptx-spill gate-dspark-read gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-q38-serve-rules gate-gpu-block gate-gpu-nvtier gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p5 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-cand gate-gpu-iq gate-gpu-q6k-ids gate-gpu-xstream gate-gpu-mimo2-flash gate-gpu-ple gate-gpu-vision gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve every gate that links an arch module, less the gguf-only kernel gates (above): a family's code runs wherever any family's file is opened +crates/model/src/arch/{mod.rs,dspark.rs,classify.rs,stems.rs,deepseek2/**,qwen3moe/**,glm5next/**} gate-gpu-p4 gate-gpu-p6 gate-gpu-kquant gate-gpu-iq-sel gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-ptx-spill gate-dspark-read gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-q38-serve-rules gate-gpu-block gate-gpu-nvtier gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p5 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-cand gate-gpu-iq gate-gpu-q6k-ids gate-gpu-xstream gate-gpu-mimo2-flash gate-gpu-ple gate-gpu-vision gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower every gate that links an arch module, less the gguf-only kernel gates (above): a family's code runs wherever any family's file is opened crates/model/src/arch/mimo2/** gate-ops gate-mimo2-meta gate-gpu-mimo2-flash gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-ds41-bind gate-ds41-place gate-gpu-routed-legs gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-minicpm-text weekly-gpu-mimo2-e2e weekly-gpu-mimo2-serve gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve MiMo-V2.6-Flash's reader and plan: its code runs only where a mimo2 file opens (its meta test, its flash gate) in bloomery-model's own unit tests (gate-ops runs --lib) and in bloomery-serve's mimo2 seat, which every serve gate's bin links except gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-clef-hidden gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen3moe-e2e gate-fixture gate-gpu-head gate-gpu-lib gate-gpu-ds41-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency: they open no mimo2 file: only model::arch's dispatch by the file's general.architecture (arch/mod.rs) reaches it crates/model/src/arch/nemotron_h/** gate-ops gate-nemotron-meta gate-fixture gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-iq-sel gate-placement gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-ds41-bind gate-ds41-place gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve Nemotron 3 Nano 4B's header reader and fixture spec: its code runs where a nemotron_h file is described (its meta test), where its fixture is planned and verified (the fixture generator's gate), in bloomery-model's own unit tests, where the card body reads the layer kinds and widths at load (the end-to-end gate), and behind the arch dispatch and, through its plan (place.rs), in bloomery-serve's nemotron seat, which gate-gpu-nemotron-serve starts and every serve gate's bin links except gate-gpu-e2e gate-gpu-hybrid gate-gpu-p6 gate-gpu-ds41-moe gate-gpu-moe gate-gpu-p8 gate-gpu-block gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen35moe-moe gate-gpu-qwen3moe-router gate-qwen35moe-meta gate-gpu-qwen35-dspark gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-minicpm-text gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-types gate-gpu-head gate-gpu-routed-legs gate-gpu-ds41-lib gate-gpu-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-nvtier gate-gpu-ds41-residency-a gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-long gate-gpu-ds41-chat gate-gpu-ds41-media gate-gpu-ds41-prefill gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-ds41-load gate-gpu-qwen38-serve gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-p8b gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-residency gate-gpu-glm5next-twocard: they link the arch dispatch and never open a nemotron_h file crates/model/src/arch/gemma_embedding2/** gate-ops gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-iq-sel gate-placement gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-ds41-bind gate-ds41-place gate-gpu-gemma-embedding2-serve EmbeddingGemma 2's header reader: no program runs it, so its code runs where a gemma-embedding2 file is described (its meta test), in bloomery-model's own unit tests, and behind the arch dispatch except gate-gpu-e2e gate-gpu-hybrid gate-gpu-p6 gate-gpu-ds41-moe gate-gpu-moe gate-gpu-p8 gate-gpu-block gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen35moe-moe gate-gpu-qwen3moe-router gate-qwen35moe-meta gate-gpu-qwen35-dspark gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-minicpm-text gate-gpu-qwen35-media gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen35moe-media gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-types gate-fixture gate-gpu-head gate-gpu-routed-legs gate-gpu-ds41-lib gate-gpu-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-nvtier gate-gpu-ds41-residency-a gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-long gate-gpu-ds41-chat gate-gpu-ds41-media gate-gpu-ds41-prefill gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-ds41-load gate-gpu-qwen3-serve gate-gpu-qwen38-serve gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-p8b gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-residency gate-gpu-glm5next-twocard gate-gpu-glm5next-serve gate-gpu-serve-start-ds41 gate-gpu-serve-start-glm: they link the arch dispatch and never open a gemma-embedding2 file @@ -107,9 +107,9 @@ crates/model/src/arch/deepseek41/{hparams.rs,host.rs,kv.rs,mod.rs,names.rs,place crates/model/src/arch/qwen35moe/{head_list.rs,hparams.rs,host.rs,mod.rs,names.rs,place.rs,roles.rs,spec.rs} gate-gpu-p4 gate-gpu-p6 gate-gpu-kquant gate-gpu-iq-sel gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-ptx-spill gate-dspark-read gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-gpu-nvtier gate-q38-serve-rules gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-qwen38-nodes gate-gpu-nemotron-h-moe-e2e gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve every gate that links an arch module, less the gguf-only kernel gates (above): a family's code runs wherever any family's file is opened except gate-gpu-block: they open no qwen35moe file: only model::arch's dispatch by the file's general.architecture (arch/mod.rs) reaches it crates/model/src/{fixture/**,arch/deepseek41/fixture.rs} gate-ops gate-ds41-meta gate-fixture gate-deepseek4-meta gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-ds41-load gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-qwen35moe-e2e gate-gpu-minicpm-text gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e the fixture generator and V4.1's FixtureSpec: no file but bin fixture, the fixture test and arch/deepseek41/fixture.rs names crate::fixture (grep), so the gates are the fixture gate and the --lib runs that execute arch::deepseek41::fixture's own unit tests (gate-ops unfiltered, gate-ds41-meta and gate-deepseek4-meta filtered arch::deepseek41) crates/model/src/arch/{coverage.rs,qwen35moe/mtp.rs} gate-gpu-iq-sel gate-gpu-e2e gate-ops gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-glm5next-gemm gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-app gate-gpu-lib gate-gpu-gates-lib gate-ds41-plan gate-gpu-ds41-lib gate-ds41-bind gate-ds41-place gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-ptx-spill gate-deepseek4-meta gate-glm5next-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-gpu-glm-sel gate-fixture gate-q38-serve-rules gate-gpu-nvtier gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-p6 gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-gpu-head gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-qwen38-nodes gate-gpu-nemotron-h-moe-e2e gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve coverage::check's and qwen35moe's mtp readers: deepseek41/glm5next/qwen35moe place.rs (unimplemented, super::mtp), gpu-gates' engine.rs, gpu's qwen3moe plan/plan38/body38 (turns_as_neox, .unimplemented()) and the meta tests that call coverage::check — every gate that opens or plans those families' files; the two modules' lists coincide -crates/model/src/arch/names.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-q38-serve-rules gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-lib gate-gpu-glm5next-nvtier gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-qwen3moe-e2e gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-iq-sel gate-gpu-ds41-moe gate-gpu-moe gate-gpu-p8 gate-gpu-block gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-router gate-placement gate-qwen35moe-meta gate-gpu-clef-hidden gate-gpu-qwen35-dspark gate-gpu-qwen35moe-moe gate-gpu-gemm gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-experts gate-ds41-host gate-fixture gate-union gate-gpu-ds41-comp gate-gpu-head gate-gpu-mcol gate-ds41-bind gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-place gate-gpu-ds41-chain-glue gate-gpu-ds41-engram gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-twocard gate-gpu-ds41-draft gate-gpu-ds41-long gate-gpu-ds41-nvtier gate-gpu-ds41-dspark-loop gate-gpu-ds41-media gate-gpu-ds41-prefill gate-ds41-load gate-gpu-ds41-chat gate-gpu-p8b gate-dspark-read gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-qwen35moe-mtp gate-app gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve the routed stacks' common names (routed_stacks): the GLM, Qwen3.6 and Qwen3.8 stack tables (swap.rs, qwen3moe/swap.rs for swap35.rs and swap38.rs) and the GLM card check (glm5next_card.rs) call it — every gate that opens those bodies or runs that check -crates/model/src/arch/mod.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-ptx-spill gate-dspark-read gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-vision gate-deepseek4-meta gate-glm5next-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-q38-serve-rules gate-gpu-xstream gate-gpu-mimo2-flash gate-gpu-nvtier gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve the dispatch and Arch: every gate that links it, the gguf-only kernel gates too — bloomery-gpu's fault sites (fault.rs) and bloomery-gpu-gates' family check (lib.rs, the oracle) read Arch in them -crates/gpu/src/weights.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq-sel gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-qwen35moe-attn gate-gpu-glm5next-gemm gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-app gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-ds41-place gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-ptx-spill gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-q38-serve-rules gate-gpu-nvtier gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve the card weight loaders: the gates whose own targets name weights, or reach it through an open path (gpu's arch bodies -> model.rs -> host/swap_source.rs, gpu-deepseek41's body/chain/dense/experts/draft, gpu-glm5next's body/ffn; gate-gpu-xstream names GpuModel), less the synthetic kernel gates whose bins build blocks and name no weights path +crates/model/src/arch/names.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-q38-serve-rules gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-lib gate-gpu-glm5next-nvtier gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-qwen3moe-e2e gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-iq-sel gate-gpu-ds41-moe gate-gpu-moe gate-gpu-p8 gate-gpu-block gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-router gate-placement gate-qwen35moe-meta gate-gpu-clef-hidden gate-gpu-qwen35-dspark gate-gpu-qwen35moe-moe gate-gpu-gemm gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-experts gate-ds41-host gate-fixture gate-union gate-gpu-ds41-comp gate-gpu-head gate-gpu-mcol gate-ds41-bind gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-place gate-gpu-ds41-chain-glue gate-gpu-ds41-engram gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-twocard gate-gpu-ds41-draft gate-gpu-ds41-long gate-gpu-ds41-nvtier gate-gpu-ds41-dspark-loop gate-gpu-ds41-media gate-gpu-ds41-prefill gate-ds41-load gate-gpu-ds41-chat gate-gpu-p8b gate-dspark-read gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-qwen35moe-mtp gate-app gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower the routed stacks' common names (routed_stacks): the GLM, Qwen3.6 and Qwen3.8 stack tables (swap.rs, qwen3moe/swap.rs for swap35.rs and swap38.rs) and the GLM card check (glm5next_card.rs) call it — every gate that opens those bodies or runs that check +crates/model/src/arch/mod.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-ptx-spill gate-dspark-read gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-vision gate-deepseek4-meta gate-glm5next-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-q38-serve-rules gate-gpu-xstream gate-gpu-mimo2-flash gate-gpu-nvtier gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower the dispatch and Arch: every gate that links it, the gguf-only kernel gates too — bloomery-gpu's fault sites (fault.rs) and bloomery-gpu-gates' family check (lib.rs, the oracle) read Arch in them +crates/gpu/src/weights.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq-sel gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-qwen35moe-attn gate-gpu-glm5next-gemm gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-app gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-ds41-place gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-ptx-spill gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-q38-serve-rules gate-gpu-nvtier gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower the card weight loaders: the gates whose own targets name weights, or reach it through an open path (gpu's arch bodies -> model.rs -> host/swap_source.rs, gpu-deepseek41's body/chain/dense/experts/draft, gpu-glm5next's body/ffn; gate-gpu-xstream names GpuModel), less the synthetic kernel gates whose bins build blocks and name no weights path # crates/gpu's per-arch bodies: GpuModel is generic; a body's impls run where B is that body. crates/gpu/src/arch/qwen3moe/** gate-app gate-ds41-bind gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-e2e gate-gpu-gemm gate-gpu-glm5next-serve gate-gpu-lev-serve gate-gpu-nvtier gate-gpu-qwen3-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-qwen35moe-media gate-gpu-qwen35moe-moe gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-head gate-gpu-qwen3moe-router gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-q38-serve-rules gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-serve the Qwen3, Qwen3.5 and Qwen4 bodies: the bins that name them (the qwen gates, gate_gemm) and AnyEngine (gate_e2e; bind's tests build it under deepseek41) @@ -119,64 +119,64 @@ crates/gpu/src/arch/deepseek2/** gate-ds41-bind gate-gpu-e2e gate-gpu-hybrid gat # Derived rows: each row's recipes are `python3 tools/recipes.py reach --row ` at the time of writing, the # gates whose targets can run code of the files through another lib (tools/recipes.py, Reach); files grouped only # where the derivation gives each the same gates. `just check-recipes` holds each to the derivation. -crates/gpu/src/{host/mod.rs,host/swap.rs,lib.rs} gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-app gate-gpu-head gate-gpu-mcol gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-vision gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve the Gpu and GpuError, whose Poison and Unresident variants hold host::Poison and host::swap::Unresident: every gate whose code names a bloomery_gpu error or the Gpu +crates/gpu/src/{host/mod.rs,host/swap.rs,lib.rs} gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-app gate-gpu-head gate-gpu-mcol gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-vision gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower the Gpu and GpuError, whose Poison and Unresident variants hold host::Poison and host::swap::Unresident: every gate whose code names a bloomery_gpu error or the Gpu crates/gpu/src/lane.rs gate-gpu-clef-serve gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-glm5next-serve gate-gpu-glm5next-twocard gate-gpu-lev-serve gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3-serve gate-q38-serve-rules gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve the lane probe a load prints beside its load line (probe_card, record::lane): the seats and generators that call it -crates/gpu/src/{host/lane.rs,watchdog.rs,flash_gqa.rs,elem.rs} gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-app gate-gpu-head gate-gpu-mcol gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve the Gpu's own parts: watchdog's wait_done and the swap lane's Shared, flash's seg_span and ElemKernels, which lib.rs's Gpu and host/mod.rs hold -crates/gpu/src/model.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-head gate-gpu-mcol gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-p2 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve GpuModel, the ChainBody and Rollback traits and StepMode: the gates that open a model body or name its step mode -crates/gpu/src/host/swap_source.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve ResidencyGlue (host/served.rs) and DEADLINE (the qwen3moe bodies' swap38): the bodies that serve a residency -crates/gpu/src/model/slots.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-ds41-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve GpuModel's slot impls (slots, selected) and SlotRows: the gates that open a body with slots -crates/gpu/src/{host/nvtier.rs,host/nvtier/reader.rs,host/step.rs} gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve the host tier's step (hybrid::Chain in model.rs) and the NvTier arena (host/mod.rs): the gates that open a body with a host tier +crates/gpu/src/{host/lane.rs,watchdog.rs,flash_gqa.rs,elem.rs} gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-app gate-gpu-head gate-gpu-mcol gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower gate-gpu-vision gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-gpu-glm5v-tower the Gpu's own parts: watchdog's wait_done and the swap lane's Shared, flash's seg_span and ElemKernels, which lib.rs's Gpu and host/mod.rs hold +crates/gpu/src/model.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-head gate-gpu-mcol gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-p2 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower GpuModel, the ChainBody and Rollback traits and StepMode: the gates that open a model body or name its step mode +crates/gpu/src/host/swap_source.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower ResidencyGlue (host/served.rs) and DEADLINE (the qwen3moe bodies' swap38): the bodies that serve a residency +crates/gpu/src/model/slots.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-ds41-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower GpuModel's slot impls (slots, selected) and SlotRows: the gates that open a body with slots +crates/gpu/src/{host/nvtier.rs,host/nvtier/reader.rs,host/step.rs} gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower the host tier's step (hybrid::Chain in model.rs) and the NvTier arena (host/mod.rs): the gates that open a body with a host tier crates/gpu/src/host/run.rs gate-gpu-e2e gate-gpu-ds41-chain-ffn gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-gpu-ds41-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-serve HostRun, the host expert leg: the V4.1, Qwen3.8 and GLM bodies that hold one -crates/gpu/src/host/xstream.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve XStream, the host tier's cross-stream lane (host/mod.rs), and XMode in the qwen38 bins +crates/gpu/src/host/xstream.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower XStream, the host tier's cross-stream lane (host/mod.rs), and XMode in the qwen38 bins crates/gpu/src/host/prompt_stream.rs gate-gpu-clef-serve gate-gpu-glm5next-serve gate-gpu-lev-serve gate-gpu-nvtier gate-q38-serve-rules gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-qwen3moe-e2e gate-gpu-clef-hidden gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-serve the prompt call's stream state and its calls (host/mod.rs), reached through the bodies that hold it -crates/gpu/src/host/batch.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-linear gate-gpu-qwen3moe-e2e gate-app gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve BatchPort, the host tier's prompt batch (host/mod.rs), and the legs' ServeTimes -crates/gpu/src/head.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-head gate-gpu-mcol gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-chain-glue gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve Head and HeadNorm, which GpuModel holds (model.rs): the gates that open a model body +crates/gpu/src/host/batch.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-linear gate-gpu-qwen3moe-e2e gate-app gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower BatchPort, the host tier's prompt batch (host/mod.rs), and the legs' ServeTimes +crates/gpu/src/head.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-head gate-gpu-mcol gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-chain-glue gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower Head and HeadNorm, which GpuModel holds (model.rs): the gates that open a model body crates/gpu/src/rope_table.rs gate-gpu-e2e gate-gpu-qwen3moe-rope gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen35moe-attn gate-gpu-mimo2-flash gate-gpu-qwen3moe-e2e gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-q38-serve-rules gate-gpu-ds41-chain-glue gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve RopeTable (V4.1's params) and RopeRows (the qwen3moe bodies' scratch) -crates/gpu/src/media.rs gate-ds41-load gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-ds41-callstream gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-lib gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-prefill gate-gpu-ds41-residency-a gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-e2e gate-gpu-glm5next-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3moe-e2e gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-e2e gate-gpu-glm5next-residency gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-ds41-bind gate-ds41-place gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-qwen4exp-mtp gate-app gate-q38-serve-rules gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-nvtier gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve MediaSpan/ImageSpan/SeqRope and the MediaBody trait: the V4.1 and Qwen3.6/Clef media calls and the sessions that carry them +crates/gpu/src/media.rs gate-ds41-load gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-ds41-callstream gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-lib gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-prefill gate-gpu-ds41-residency-a gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-e2e gate-gpu-glm5next-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3moe-e2e gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-e2e gate-gpu-glm5next-residency gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-ds41-bind gate-ds41-place gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-qwen4exp-mtp gate-app gate-q38-serve-rules gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-nvtier gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve gate-gpu-gemma-embedding2-e2e MediaSpan/ImageSpan/SeqRope and the MediaBody trait: the V4.1 and Qwen3.6/Clef media calls and the sessions that carry them crates/gpu/src/iq_sel.rs gate-gpu-iq-sel gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-q38-serve-rules gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-mtp gate-gpu-glm5next-residency gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-serve IqSelKernels, the Qwen3.8 card's IQ selection (card38.rs), the common card leg's i-quant arms (card_leg.rs, which GLM's body and mimo2's reach), and its own gate crates/gpu/src/checkpoint/{saved,staging}.rs gate-ds41-load gate-gpu-clef-serve gate-gpu-ds41-callstream gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-lib gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-prefill gate-gpu-ds41-residency-a gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-glm5next-e2e gate-gpu-glm5next-mtp gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-lev-serve gate-gpu-nvtier gate-gpu-qwen3-serve gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-q38-serve-rules gate-ds41-bind gate-ds41-place gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-serve the host copy of a sequence state (SeqState, the pinned staging pool) under the Qwen3.8 and GLM bodies' seq_save and seq_resume, the V4.1 serve seat and the gates that open those bodies crates/gpu/src/checkpoint.rs gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-mtp gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-lev-serve gate-gpu-nvtier gate-gpu-qwen3-serve gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3moe-e2e gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-q38-serve-rules gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-serve the checkpoint ring (Checkpoints: the points a prompt call keeps, adopt, take, the save lane's point copy and its in-flight refusal) under the Qwen3.8 and GLM bodies and the gates that open them -crates/gpu/src/tensor.rs gate-app gate-ds41-bind gate-ds41-load gate-ds41-place gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-ds41-attn gate-gpu-ds41-callstream gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-comp gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-lib gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-moe gate-gpu-ds41-prefill gate-gpu-ds41-residency-a gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-woa gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-e2e gate-gpu-gemm gate-gpu-glm-mla gate-gpu-glm-router gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-gemm gate-gpu-glm5next-mtp gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-hc-gated gate-gpu-head gate-gpu-hybrid gate-gpu-iq-sel gate-gpu-kquant gate-gpu-lev-serve gate-gpu-mcol gate-gpu-moe gate-gpu-nvtier gate-gpu-p0b gate-gpu-p1 gate-gpu-p10 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p8 gate-gpu-p8b gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-q6k-ids gate-gpu-qwen3-serve gate-gpu-qwen35moe-attn gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35moe-moe gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-head gate-gpu-qwen3moe-router gate-gpu-qwen3vl-tower gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-swap gate-gpu-vision gate-q38-serve-rules gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve the device tensor and the 2-D copies (dtoh_2d, htod_2d) every gate that links the gpu crate through a body reads -crates/gpu-gates/src/lib.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-gpu-head gate-gpu-mcol gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-q38-serve-rules gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-dspark-read gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-vision gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve GateError and the family check (the oracle), which the bins' shared modules name -crates/gpu-gates/src/tier.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-gpu-head gate-gpu-mcol gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-q38-serve-rules gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-dspark-read gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-minicpm-text gate-gpu-qwen35moe-attn gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve Tier::from_env (lib.rs's family check) and tier::card, which shared/gate_card.rs runs in the card bins +crates/gpu/src/tensor.rs gate-app gate-ds41-bind gate-ds41-load gate-ds41-place gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-ds41-attn gate-gpu-ds41-callstream gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-comp gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-lib gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-moe gate-gpu-ds41-prefill gate-gpu-ds41-residency-a gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-woa gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-e2e gate-gpu-gemm gate-gpu-glm-mla gate-gpu-glm-router gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-gemm gate-gpu-glm5next-mtp gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-hc-gated gate-gpu-head gate-gpu-hybrid gate-gpu-iq-sel gate-gpu-kquant gate-gpu-lev-serve gate-gpu-mcol gate-gpu-moe gate-gpu-nvtier gate-gpu-p0b gate-gpu-p1 gate-gpu-p10 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p8 gate-gpu-p8b gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-q6k-ids gate-gpu-qwen3-serve gate-gpu-qwen35moe-attn gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35moe-moe gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-head gate-gpu-qwen3moe-router gate-gpu-qwen3vl-tower gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-swap gate-gpu-vision gate-q38-serve-rules gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower the device tensor and the 2-D copies (dtoh_2d, htod_2d) every gate that links the gpu crate through a body reads +crates/gpu-gates/src/lib.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-gpu-head gate-gpu-mcol gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-q38-serve-rules gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-dspark-read gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-vision gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower GateError and the family check (the oracle), which the bins' shared modules name +crates/gpu-gates/src/tier.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-gpu-head gate-gpu-mcol gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-q38-serve-rules gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-dspark-read gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-minicpm-text gate-gpu-qwen35moe-attn gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower Tier::from_env (lib.rs's family check) and tier::card, which shared/gate_card.rs runs in the card bins crates/gpu-gates/src/residency.rs gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3moe-e2e gate-q38-serve-rules gate-gpu-nvtier gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-chat weekly-gpu-mimo2-e2e weekly-gpu-mimo2-serve gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve the unset residency rules' plan inputs (PlanRoom) and the pair rule's call (at_plan): the qwen38, GLM, V4.1 and MiMo residency, serve and placement bins crates/gpu-gates/src/residency38.rs gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3moe-e2e gate-q38-serve-rules gate-gpu-nvtier gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-chat gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve the Qwen3.8 residency figures (checkpoint_bytes, CARD38, glm_seqs): the qwen38 and GLM residency, serve and placement bins crates/gpu-gates/src/q38_fixture.rs gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen38-twocard the Qwen3.8 fixture's Shape, which shared/q38_arch.rs reads crates/gpu-gates/src/q36_fixture.rs gate-gpu-qwen35moe-e2e gate-gpu-minicpm-text the Qwen3.6 body's per-layer launch counts from the header, which gate_qwen35moe_e2e reads, and which the MiniCPM text recipe runs through that bin -crates/gpu-gates/src/act_rule.rs gate-gpu-iq gate-gpu-ds41-hc gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-minicpm-text gate-gpu-ds41-index gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-chain-attn gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-gpu-ds41-step gate-gpu-dspark-experts gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-gemma-embedding2-e2e the activation rules: the MoE, chain, index, comp, hc, iq, gemm and dspark experts bins that name them, and the fixture bands' readers of site_rel (shared/glm5next_tier.rs, the qwen4exp e2e and MTP bins, ds41-step's head; the qwen3moe experts, down and e2e bins and the qwen35moe e2e bin through ik_q8_2's non-finite check) -crates/gpu-gates/src/flip.rs gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-ds41-step gate-gpu-glm5next-mtp gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-h-moe-e2e gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-qwen38-residency gate-gpu-gemma-embedding2-e2e Flip, which shared/e2e.rs, the qwen4exp MTP bin, ds41-step's head cap and the GLM MTP replay's margin cap read -crates/model/src/lib.rs gate-gpu-p4 gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve ModelError, which every model reader returns -crates/model/src/moe.rs gate-gpu-p4 gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve TierError in ModelError (lib.rs), and the host expert legs the bodies and tiers call -crates/model/src/ops.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-app gate-prompts gate-gpu-lib gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve DEFER_MAX_COLS (gpu's host/step.rs and host/tier.rs), Tensor2 and the matmul dispatch +crates/gpu-gates/src/act_rule.rs gate-gpu-iq gate-gpu-ds41-hc gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-minicpm-text gate-gpu-ds41-index gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-chain-attn gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-gpu-ds41-step gate-gpu-dspark-experts gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-gemma-embedding2-e2e gate-gpu-gemma4v-tower the activation rules: the MoE, chain, index, comp, hc, iq, gemm and dspark experts bins that name them, and the fixture bands' readers of site_rel (shared/glm5next_tier.rs, the qwen4exp e2e and MTP bins, ds41-step's head; the qwen3moe experts, down and e2e bins and the qwen35moe e2e bin through ik_q8_2's non-finite check) +crates/gpu-gates/src/flip.rs gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-ds41-step gate-gpu-glm5next-mtp gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-h-moe-e2e gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-qwen38-residency gate-gpu-gemma-embedding2-e2e gate-gpu-gemma4v-tower Flip, which shared/e2e.rs, the qwen4exp MTP bin, ds41-step's head cap and the GLM MTP replay's margin cap read +crates/model/src/lib.rs gate-gpu-p4 gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower ModelError, which every model reader returns +crates/model/src/moe.rs gate-gpu-p4 gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower TierError in ModelError (lib.rs), and the host expert legs the bodies and tiers call +crates/model/src/ops.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-app gate-prompts gate-gpu-lib gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower DEFER_MAX_COLS (gpu's host/step.rs and host/tier.rs), Tensor2 and the matmul dispatch crates/model/src/embed.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-ds41-lib gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen3-serve gate-ds41-load gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-e2e gate-q38-serve-rules gate-gpu-qwen38-serve gate-gpu-gemma-embedding2-serve RowLayout and EmbedTable: the V4.1 glue, the draft and the GLM body -crates/model/src/placement/workstation.rs gate-gpu-iq-sel gate-placement gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-fixture gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-app gate-gpu-ds41-chain-ffn gate-gpu-ds41-lib gate-gpu-e2e gate-gpu-hybrid gate-gpu-moe gate-gpu-p10 gate-gpu-p8 gate-gpu-p8b gate-gpu-swap gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve host_available, which GpuModel's placement (gpu's model.rs) and the qwen35moe plan read +crates/model/src/placement/workstation.rs gate-gpu-iq-sel gate-placement gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-fixture gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-app gate-gpu-ds41-chain-ffn gate-gpu-ds41-lib gate-gpu-e2e gate-gpu-hybrid gate-gpu-moe gate-gpu-p10 gate-gpu-p8 gate-gpu-p8b gate-gpu-swap gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower host_available, which GpuModel's placement (gpu's model.rs) and the qwen35moe plan read crates/model/src/arch/qwen35moe/fixture.rs gate-fixture gate-gpu-gemma-embedding2-e2e the Qwen3.8 fixture spec, which the fixture test reads crates/model/data/mtp-head-rows-qwen38-*.txt gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-mtp gate-q38-serve-rules gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve Qwen3.8's shipped MTP head list, which arch/qwen35moe/head_list.rs includes (include_str!): the gates whose targets run its reader (head_rows_of, SHIPPED) tools/fixture-variants.tsv gate-fixture gate-gpu-gemma-embedding2-e2e the fixture variants' table, which fixture/variant.rs reads by path at run time: the fixture test -crates/models/src/lib.rs gate-gpu-p4 gate-gpu-p6 gate-gpu-iq-sel gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-runtime gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-dspark-read gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gpu-qwen35moe-attn gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve its re-exports (Role, Need: every placement path names models::Role through them) and ModelSpec and Score, which the arch readers and models::shape read +crates/models/src/lib.rs gate-gpu-p4 gate-gpu-p6 gate-gpu-iq-sel gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-runtime gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-dspark-read gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gpu-qwen35moe-attn gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower its re-exports (Role, Need: every placement path names models::Role through them) and ModelSpec and Score, which the arch readers and models::shape read crates/models/src/shape.rs gate-gpu-p6 gate-gpu-iq-sel gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-e2e gate-fixture gate-gpu-head gate-gpu-lib gate-ds41-plan gate-gpu-ds41-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-ds41-bind gate-ds41-place gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-placement gate-nemotron-meta gate-nemotron-h-moe-meta gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve MoeShape and the router rows (RouterInst, router_row) the routers' kernels select by -crates/gguf/src/lib.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-qdot gate-engram gate-tokenizer gate-decision-clef gate-decision-lev gate-serve gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-vision gate-gpu-vision gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-refset gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-models gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gpu-mamba2 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-emb2-mel LoadError (in GpuError and placement's errors) and Gguf: every gate that reads a file or names a load error -crates/gguf/src/quant.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-qdot gate-engram gate-tokenizer gate-decision-clef gate-decision-lev gate-serve gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-refset gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-vision gate-gpu-vision gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-dspark-read gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-models gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-hf gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gpu-mamba2 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-emb2-mel GgmlType (gguf's lib.rs, placement) and the f16 conversion the card's fused kernels name; gate-dspark-read also runs gguf's own quant tests -crates/gguf/src/iq_tables.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-kquant gate-gpu-hc-gated gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-ds41-meta gate-qwen3moe-meta gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-gemm gate-gpu-glm5next-gemm gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-qdot gate-decision-clef gate-decision-lev gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve KVALUES_IQ4NL, which quant.rs's dequant and the card's IQ kernels read -crates/qdot/src/lib.rs gate-gpu-p4 gate-gpu-p6 gate-gpu-kquant gate-gpu-iq-sel gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen35moe-attn gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-qdot gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gpu-mamba2 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve QdotError in ModelError, and the fused dots coverage and quant ask for -crates/threads/src/lib.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-ds41-meta gate-qwen3moe-meta gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-glm5next-gemm gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-threads gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-ds41-plan gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gpu-p1 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve the pool: upload's memcpy_pool, ops' matmul and the host tier's legs -crates/engram/src/lib.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-lib gate-gpu-gates-lib gate-ds41-plan gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve EngramError (direct.rs) and Engram, which the V4.1 chain's glue holds +crates/gguf/src/lib.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-qdot gate-engram gate-tokenizer gate-decision-clef gate-decision-lev gate-serve gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-vision gate-gpu-vision gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-refset gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-models gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gpu-mamba2 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-emb2-mel gate-gpu-gemma4v-tower LoadError (in GpuError and placement's errors) and Gguf: every gate that reads a file or names a load error +crates/gguf/src/quant.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-qdot gate-engram gate-tokenizer gate-decision-clef gate-decision-lev gate-serve gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-refset gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-vision gate-gpu-vision gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-dspark-read gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-models gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-hf gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gpu-mamba2 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-emb2-mel gate-gpu-gemma4v-tower GgmlType (gguf's lib.rs, placement) and the f16 conversion the card's fused kernels name; gate-dspark-read also runs gguf's own quant tests +crates/gguf/src/iq_tables.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-kquant gate-gpu-hc-gated gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-ds41-meta gate-qwen3moe-meta gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-gemm gate-gpu-glm5next-gemm gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-qdot gate-decision-clef gate-decision-lev gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower gate-gpu-vision gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-gpu-glm5v-tower KVALUES_IQ4NL, which quant.rs's dequant and the card's IQ kernels read +crates/qdot/src/lib.rs gate-gpu-p4 gate-gpu-p6 gate-gpu-kquant gate-gpu-iq-sel gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen35moe-attn gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-qdot gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gpu-mamba2 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower QdotError in ModelError, and the fused dots coverage and quant ask for +crates/threads/src/lib.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-ds41-meta gate-qwen3moe-meta gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-glm5next-gemm gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-threads gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-ds41-plan gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gpu-p1 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower the pool: upload's memcpy_pool, ops' matmul and the host tier's legs +crates/engram/src/lib.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-lib gate-gpu-gates-lib gate-ds41-plan gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower EngramError (direct.rs) and Engram, which the V4.1 chain's glue holds crates/models/src/headpick.rs gate-deepseek4-meta gate-ds41-meta gate-ds41-plan gate-fixture gate-glm5next-meta gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-twocard gate-gpu-glm5next-serve gate-gpu-iq-sel gate-gpu-lev-serve gate-gpu-nvtier gate-gpu-qwen3-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35moe-media gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3moe-e2e gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-mimo2-meta gate-ops gate-q38-serve-rules gate-qwen3moe-meta gate-qwen4exp-meta gate-runtime gate-ds41-bind gate-ds41-place gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35-dspark gate-gpu-gemm gate-gpu-head gate-gpu-lib gate-gpu-ds41-lib gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-nvtier gate-gpu-ds41-long gate-gpu-ds41-prefill gate-gpu-ds41-media gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-nemotron-meta gate-nemotron-h-moe-meta gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve models::headpick, the row-list head's reader, digest and loaded set: every gate whose load reads a head list gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35-dspark gate-gpu-gemm gate-gpu-head gate-gpu-lib gate-gpu-ds41-lib gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-nvtier gate-gpu-ds41-long gate-gpu-ds41-prefill gate-gpu-ds41-media gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-twocard gate-gpu-glm5next-residency crates/engram/src/rows.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gates-lib gate-q38-serve-rules gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-serve SplitTable, which the Qwen3.8 body opens -crates/hf/src/fetch.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-fixture gate-ds41-bind gate-ds41-place gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-q38-serve-rules gate-gpu-qwen38-serve gate-gpu-gemma-embedding2-serve gate-gpu-gemma-embedding2-e2e gate-gpu-clef-hidden gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-qwen3moe-e2e gate-gpu-qwen38-twocard the fetch Event that model_file.rs reports; the fixture fetch's range reads +crates/hf/src/fetch.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-fixture gate-ds41-bind gate-ds41-place gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-q38-serve-rules gate-gpu-qwen38-serve gate-gpu-gemma-embedding2-serve gate-gpu-gemma-embedding2-e2e gate-gpu-clef-hidden gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-qwen3moe-e2e gate-gpu-qwen38-twocard gate-gpu-gemma4v-tower the fetch Event that model_file.rs reports; the fixture fetch's range reads crates/decision/src/lib.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-decision-lev gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve the decision crate's Error: the decision gates -crates/vision/src/lib.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-serve gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-vision gate-gpu-glm5next-serve gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-decision-clef gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-ds41-media gate-gpu-routed-legs gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve VisionError, which the media and preprocess paths return -crates/gpu/src/gelu.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-mamba2 gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-gpu-clef-hidden gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-app gate-gpu-head gate-gpu-mcol gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-nvtier gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-vision gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-glm5v-tower gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve the shared tanh GELU rule: the registered elementwise module and image towers that use it -crates/gpu-vision/src/{aligner,attn,attn_group,chain,encoder,gelu_tanh,gemm_bf16,glm5v,layer_norm,mean_groups,mlp,norm,pos_bilinear,qwen3vl,rope2d}.rs gate-gpu-clef-serve gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-glm5v-tower gate-gpu-qwen3-serve gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-gpu-vision gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-lev-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve the image towers' card kernels: the tower, serve and media-seat gates that run them -crates/gpu-vision/src/lib.rs gate-gpu-clef-serve gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-glm5v-tower gate-gpu-qwen3-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-gpu-vision gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-lev-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve the tower crate's root, which the Qwen3.5 media gates also link -crates/vision/src/arch/glm5v/{card,hparams,mod,names,tensors}.rs gate-ds41-bind gate-ds41-place gate-gpu-clef-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-glm5next-stagger gate-gpu-glm5v-tower gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-gpu-vision gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve the GLM-5V family's hparams, tensor names and card homes -crates/vision/src/arch/glm5v/{media,size}.rs gate-ds41-bind gate-ds41-place gate-gpu-clef-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-glm5next-stagger gate-gpu-glm5v-tower gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve the GLM-5V media span and size rule the serve seats plan with -crates/vision/src/arch/glm5v/rules.rs gate-gpu-clef-serve gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-glm5v-tower gate-gpu-qwen3-serve gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-gpu-vision gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-lev-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve the GLM-5V preprocess rules the towers read -crates/vision/src/{arch/deepseek41v/mod.rs,arch/deepseek41v/tensors.rs} gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-vision gate-gpu-glm5next-serve gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-ds41-bind gate-ds41-place gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-ds41-media gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve the V4.1 vision family's constants and its tensors' card homes -crates/vision/src/{image,preprocess}.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-decision-clef gate-serve gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-media gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-vision gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-gpu-glm5v-tower gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve the decoded image and its patch cut, which every image seat's prepare runs -crates/vision/src/resample.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-decision-clef gate-ds41-bind gate-ds41-place gate-gpu-ds41-media gate-gpu-qwen3-serve gate-gpu-vision gate-gpu-qwen3vl-tower gate-gpu-glm5v-tower gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-minicpmv-tower gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve the resize and pad an image seat's prepare runs -crates/vision/src/arch/deepseek41v/grid.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-ds41-bind gate-ds41-place gate-gpu-ds41-media gate-gpu-qwen3-serve gate-gpu-vision gate-gpu-qwen3vl-tower gate-gpu-glm5v-tower gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-minicpmv-tower gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve V4.1's grid plan, reached through the projector's arch dispatch -crates/vision/src/mrope/mod.rs gate-gpu-e2e gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-types gate-app gate-gpu-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-media gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-minicpmv-tower gate-gpu-minicpm-text gate-decision-clef gate-serve gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve the M-RoPE module's error and the public paths of its positions and rows +crates/vision/src/lib.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-serve gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-vision gate-gpu-glm5next-serve gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-decision-clef gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-ds41-media gate-gpu-routed-legs gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve gate-gpu-gemma-embedding2-e2e gate-gpu-gemma4v-tower VisionError, which the media and preprocess paths return +crates/gpu/src/gelu.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-mamba2 gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-gpu-clef-hidden gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-app gate-gpu-head gate-gpu-mcol gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-nvtier gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-vision gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-glm5v-tower gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower the shared tanh GELU rule: the registered elementwise module and image towers that use it +crates/gpu-vision/src/{aligner,attn,attn_group,chain,encoder,gelu_tanh,gemm_bf16,glm5v,layer_norm,mean_groups,mlp,norm,pos_bilinear,qwen3vl,rope2d}.rs gate-gpu-clef-serve gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-glm5v-tower gate-gpu-qwen3-serve gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-gpu-vision gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-lev-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve gate-gpu-gemma-embedding2-e2e gate-gpu-gemma4v-tower the image towers' card kernels: the tower, serve and media-seat gates that run them +crates/gpu-vision/src/lib.rs gate-gpu-clef-serve gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-glm5v-tower gate-gpu-qwen3-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-gpu-vision gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-lev-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve gate-gpu-gemma-embedding2-e2e gate-gpu-gemma4v-tower the tower crate's root, which the Qwen3.5 media gates also link +crates/vision/src/arch/glm5v/{card,hparams,mod,names,tensors}.rs gate-ds41-bind gate-ds41-place gate-gpu-clef-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-glm5next-stagger gate-gpu-glm5v-tower gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-gpu-vision gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve gate-gpu-gemma-embedding2-e2e gate-gpu-gemma4v-tower gate-q38-serve-rules gate-gpu-qwen38-serve the GLM-5V family's hparams, tensor names and card homes +crates/vision/src/arch/glm5v/{media,size}.rs gate-ds41-bind gate-ds41-place gate-gpu-clef-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-glm5next-stagger gate-gpu-glm5v-tower gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve gate-q38-serve-rules gate-gpu-qwen38-serve the GLM-5V media span and size rule the serve seats plan with +crates/vision/src/arch/glm5v/rules.rs gate-gpu-clef-serve gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-glm5v-tower gate-gpu-qwen3-serve gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-gpu-vision gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-lev-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve gate-gpu-gemma-embedding2-e2e gate-gpu-gemma4v-tower the GLM-5V preprocess rules the towers read +crates/vision/src/{arch/deepseek41v/mod.rs,arch/deepseek41v/tensors.rs} gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-vision gate-gpu-glm5next-serve gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-ds41-bind gate-ds41-place gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-ds41-media gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve gate-gpu-gemma-embedding2-e2e gate-gpu-gemma4v-tower gate-q38-serve-rules gate-gpu-qwen38-serve the V4.1 vision family's constants and its tensors' card homes +crates/vision/src/{image,preprocess}.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-decision-clef gate-serve gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-media gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-vision gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-gpu-glm5v-tower gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve gate-gpu-gemma-embedding2-e2e gate-gpu-gemma4v-tower the decoded image and its patch cut, which every image seat's prepare runs +crates/vision/src/resample.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-decision-clef gate-ds41-bind gate-ds41-place gate-gpu-ds41-media gate-gpu-qwen3-serve gate-gpu-vision gate-gpu-qwen3vl-tower gate-gpu-glm5v-tower gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-minicpmv-tower gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve gate-gpu-gemma-embedding2-e2e gate-gpu-gemma4v-tower gate-serve gate-q38-serve-rules gate-gpu-qwen38-serve the resize and pad an image seat's prepare runs +crates/vision/src/arch/deepseek41v/grid.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-ds41-bind gate-ds41-place gate-gpu-ds41-media gate-gpu-qwen3-serve gate-gpu-vision gate-gpu-qwen3vl-tower gate-gpu-glm5v-tower gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-minicpmv-tower gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve gate-gpu-gemma-embedding2-e2e gate-gpu-gemma4v-tower gate-q38-serve-rules gate-gpu-qwen38-serve V4.1's grid plan, reached through the projector's arch dispatch +crates/vision/src/mrope/mod.rs gate-gpu-e2e gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-types gate-app gate-gpu-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-media gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-minicpmv-tower gate-gpu-minicpm-text gate-decision-clef gate-serve gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve gate-gpu-gemma-embedding2-e2e the M-RoPE module's error and the public paths of its positions and rows crates/vision/src/mrope/{seq,rows}.rs gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-types gate-gpu-lib gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve the M-RoPE positions of a sequence's images and the table rows they rewrite crates/vision/src/arch/gemma4a.rs gate-emb2-mel gate-vision gemma4a host mel preprocessing and token counts @@ -216,7 +216,7 @@ crates/gpu/src/cand.rs weekly-gpu-ds41-cand the candidate mask's kernels, which crates/gpu-deepseek41/src/{indexer.rs,chain/attn.rs,chain/attn/batch.rs} weekly-gpu-ds41-cand the indexer's split launcher and the select path's candidate launches, per row of words in the step and per chunk in the prompt batch crates/gpu-deepseek41/src/{body.rs,body/ced.rs,body/prefill.rs} weekly-gpu-ds41-cand the decode and pair wiring, the triangle's candidate source, and the prompt batch's kept blocks per batch set crates/gpu-gates/src/bin/{gate_deepseek41_long.rs,gate_deepseek41_prefill.rs,shared/ds41_cand.rs} weekly-gpu-ds41-cand the weekly's two clauses and their candidate rule -crates/placement/src/** gate-app gate-deepseek4-meta gate-ds41-bind gate-ds41-host gate-ds41-kld gate-ds41-load gate-ds41-meta gate-ds41-oracle gate-ds41-plan gate-dspark-read gate-fixture gate-glm5next-meta gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-comp gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-index gate-gpu-ds41-lib gate-gpu-ds41-long gate-gpu-ds41-prefill gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-e2e gate-gpu-gates-lib gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-lib gate-gpu-mcol gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-qwen35moe-moe gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-head gate-gpu-qwen3moe-router gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-ops gate-placement gate-qwen35moe-meta gate-qwen3moe-meta gate-qwen4exp-meta gate-union gate-gpu-p4 gate-gpu-p6 gate-gpu-iq-sel gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-block gate-gpu-p10 gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-gpu-qwen3moe-rope gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-r8 gate-prompts gate-gpu-head gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-woa gate-gpu-ds41-media gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-p8b gate-mimo2-meta gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-nemotron-meta gate-gpu-qwen35moe-attn gate-gpu-p1 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-p2 gate-nemotron-h-moe-meta gate-gpu-nvfp4 gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-nemotron-h-moe-e2e gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve the placement planner, which bloomery-model re-exports as model::placement: every gate that plans a placement, reads a plan's figures or checks the r8 sidecar's refusal type +crates/placement/src/** gate-app gate-deepseek4-meta gate-ds41-bind gate-ds41-host gate-ds41-kld gate-ds41-load gate-ds41-meta gate-ds41-oracle gate-ds41-plan gate-dspark-read gate-fixture gate-glm5next-meta gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-comp gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-index gate-gpu-ds41-lib gate-gpu-ds41-long gate-gpu-ds41-prefill gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-e2e gate-gpu-gates-lib gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-lib gate-gpu-mcol gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-qwen35moe-moe gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-head gate-gpu-qwen3moe-router gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-ops gate-placement gate-qwen35moe-meta gate-qwen3moe-meta gate-qwen4exp-meta gate-union gate-gpu-p4 gate-gpu-p6 gate-gpu-iq-sel gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-block gate-gpu-p10 gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-gpu-qwen3moe-rope gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-r8 gate-prompts gate-gpu-head gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-woa gate-gpu-ds41-media gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-p8b gate-mimo2-meta gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-nemotron-meta gate-gpu-qwen35moe-attn gate-gpu-p1 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-p2 gate-nemotron-h-moe-meta gate-gpu-nvfp4 gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-nemotron-h-moe-e2e gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower the placement planner, which bloomery-model re-exports as model::placement: every gate that plans a placement, reads a plan's figures or checks the r8 sidecar's refusal type crates/model/src/arch/gemma_encoder.rs gate-ops gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gemma-embedding-meta gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-nemotron-h-moe-e2e gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve the shared Gemma encoder header checks, tensor shapes, roles and layer description except gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-gpu-clef-hidden gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-fixture gate-gpu-head gate-gpu-lib gate-gpu-ds41-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-nvtier gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41: they link the arch dispatch and never open a Gemma encoder file crates/model/src/arch/gemma_embedding/** gate-ops gate-gemma-embedding-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-nemotron-h-moe-e2e gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve EmbeddingGemma header reader and its synthetic and real-header contracts except gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-gpu-clef-hidden gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-fixture gate-gpu-head gate-gpu-lib gate-gpu-ds41-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-nvtier gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41: they link the arch dispatch and never open a Gemma encoder file crates/gpu-gates/src/beside.rs gate-gpu-clef-serve gate-gpu-glm5next-serve gate-gpu-lev-serve gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3-serve gate-gpu-qwen3moe-e2e gate-gpu-clef-hidden gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-chat gate-gpu-glm5next-twocard gate-gpu-nemotron-serve gate-q38-serve-rules gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve the reserves a plan holds beside its own tensors (the image encoder's tower in a placed plan and a whole-fit verdict): the qwen3 placement every Qwen3 bin includes, the V4.1 and GLM placements, and the servers that bind the seats that name a reserve diff --git a/tools/recipes.py b/tools/recipes.py index 51459af63..722e12ea0 100755 --- a/tools/recipes.py +++ b/tools/recipes.py @@ -10290,7 +10290,10 @@ def problems(*recipes: tuple[str, ...]) -> list[str]: return [f"raised {err}"] base = "gate_glm5next_e2e (bloomery-gpu-gates, release): " - wide, narrow = "{deepseek41,glm5next,gpu,vision}", "{deepseek41,glm5next,gpu}" + tree = Tree(ROOT) + pkg = tree.packages["bloomery-gpu-gates"] + closure = lambda feats: "{" + ",".join(sorted(tree.feature_closure(pkg, feats.split(","))[0])) + "}" + wide, narrow = closure("glm5next,vision"), closure("glm5next") got = problems(("a", call("glm5next")), ("b", call("glm5next,vision"))) expect(len(got) == 1 and got[0].startswith(f"{base}b {wide} vs a {narrow} "), f"bin features: glm5next vs glm5next,vision not refused by name: {got}") # the recipes' order and names do not move the line; a set shared by recipes lists them all, sorted diff --git a/tools/ref/emb2/emb2.sh b/tools/ref/emb2/emb2.sh index e68d3b252..4c06f5e54 100755 --- a/tools/ref/emb2/emb2.sh +++ b/tools/ref/emb2/emb2.sh @@ -11,6 +11,7 @@ # text.cpu ${SETPFX}_text.cpu: the same with no card in view # tower ${SETPFX}_tower: dump_emb2 tower over IMAGES on the card # tower.cpu ${SETPFX}_tower.cpu: the same with no card in view +# server.cpu ${SETPFX}_server.cpu: the same requests with both models on CPU # server ${SETPFX}_server: llama-server's /v1/embeddings vectors of every text # case that is UTF-8 and every image (server_vectors.py), the server on 127.0.0.1:${EMB2_PORT:-8097} for the # step's length @@ -24,7 +25,7 @@ # libllama.so it loads, and of the two cases files as they are now. Each set is written to .staging and moved over the # old set only when its writer exits 0, so a failed step leaves the old set whole: the old set waits as .old while the # new one takes its name, an exit or a caught signal in between moves it back, and a step that finds a .old left by a -# killed run restores it first (or discards it when the replacement is already there). The card steps (text, tower, server) +# killed run restores it first (or retains it with a timestamp, when the new set has its name already). The card steps (text, tower, server) # go through tools/gpu-gate.sh, as clefvis.sh's do: the card's gate lock and the bound EMB2_BOUND (default 1500 s, parsed as # tools/gate-bound.sh parses BLOOMERY_GATE_BOUND), for the call and for each step; the script links itself as # target/release/emb2_dump and runs the card steps under that name inside the runner. The twins open no card @@ -73,7 +74,7 @@ if [ "$(basename "${BASH_SOURCE[0]}")" != emb2_dump ] && [ ${#card_steps[@]} -gt a6000 | both) BLOOMERY_GATE_BOUND=$BOUND bash tools/gpu-gate.sh emb2_dump "${card_steps[@]}" ;; *) BLOOMERY_GATE_BOUND=$BOUND BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh emb2_dump "${card_steps[@]}" ;; esac - for s in text.cpu tower.cpu; do + for s in text.cpu tower.cpu server.cpu; do if selected "$s"; then bash "$REAL" "$s"; fi done exit 0 @@ -162,11 +163,19 @@ extra cases:3:$(md5sum < "crates/tokenizer/tests/$EMBED_CASES" | cut -d' ' -f1) EOF } -# .old left by a killed run: the old set when is missing, a leftover when the new set had its name already. +# Replaced sets and incomplete staging directories stay beside the published set under a unique retained name. +retain() { + if [ -e "$1" ]; then + local kept=$1.retained.$(date -u +%Y%m%dT%H%M%SZ).$$ + mv "$1" "$kept" + echo "emb2: retained $kept" + fi +} + recover() { # [ -e "$1.old" ] || return 0 if [ -e "$1" ]; then - rm -rf "$1.old" + retain "$1.old" else mv "$1.old" "$1" echo "emb2: restored $1 from $1.old" @@ -178,7 +187,7 @@ publish() { # : the old set stays whole until the new one has i if [ -e "$2" ]; then mv "$2" "$2.old"; fi mv "$1" "$2" publishing= - rm -rf "$2.old" + retain "$2.old" echo "emb2: $(basename "$2") -> $2" } @@ -191,7 +200,10 @@ for name in "${IMAGES[@]}"; do done server_step() { - local stage=$1 card_name=$2 port=${EMB2_PORT:-8097} up='' log=$DATA/${SETPFX}_server.log flags media_args + local stage=$1 card_name=$2 device=$3 port=${EMB2_PORT:-8097} up='' log flags media_args + log=$DATA/${SETPFX}_server + if [ "$device" = cpu ]; then log+=.cpu; fi + log+=.log if (: < "/dev/tcp/127.0.0.1/$port") 2> /dev/null; then die "something already listens on 127.0.0.1:$port; stop it or set EMB2_PORT" fi @@ -202,6 +214,12 @@ server_step() { flags+=(--mmproj "$MMPROJ") media_args=(--mmproj "$MMPROJ" --mmproj-sha256 "$MMPROJ_SHA256" "${image_args[@]}" --mix-image free-448x448) fi + local cpu=() + if [ "$device" = cpu ]; then + export CUDA_VISIBLE_DEVICES= + flags+=(-ngl 0 --no-mmproj-offload) + cpu=(--cpu) + fi "$SERVER_BIN" "${flags[@]}" > "$log" 2>&1 & server_pid=$! for _ in $(seq 120); do @@ -211,7 +229,7 @@ server_step() { done [ -n "$up" ] || die "llama-server did not come up (log: $log)" 1 timeout --kill-after=10 "$BOUND" python3 tools/ref/emb2/server_vectors.py --url "http://127.0.0.1:$port" --pid "$server_pid" \ - --model "$MODEL" --arch "$MODEL_NAME" --model-sha256 "$MODEL_SHA256" --card "$card_name" \ + "${cpu[@]}" --model "$MODEL" --arch "$MODEL_NAME" --model-sha256 "$MODEL_SHA256" --card "$card_name" \ --build-record "$BIN.build" --ids-dir "$IDS" "${case_args[@]}" --out "$stage" "${media_args[@]}" \ --server-flags "${flags[*]}" kill "$server_pid" @@ -230,7 +248,7 @@ for s in "${ALL[@]}"; do stage=$set_dir.staging recover "$set_dir" verify - rm -rf "$stage" + retain "$stage" SECONDS=0 case $s in text | text.cpu | tower | tower.cpu) @@ -249,10 +267,10 @@ for s in "${ALL[@]}"; do "${run[@]}" timeout --kill-after=10 "$BOUND" "$BIN" "${args[@]}" "${where[@]}" -t "$THREADS" --build-record "$BIN.build" \ --out "$stage" ;; - server) + server | server.cpu) ids_current - card_name=$(card) - server_step "$stage" "$card_name" + if [ "$s" = server.cpu ]; then card_name=cpu; device=cpu; else card_name=$(card); device=cuda; fi + server_step "$stage" "$card_name" "$device" ;; esac publish "$stage" "$set_dir" diff --git a/tools/ref/emb2/server_vectors.py b/tools/ref/emb2/server_vectors.py index 9604436b2..e26652b0f 100644 --- a/tools/ref/emb2/server_vectors.py +++ b/tools/ref/emb2/server_vectors.py @@ -28,19 +28,108 @@ `# mmproj sha256 `; then the server's flags and build, the build record's lines, and the `# complete` trailer, written last through a .partial name. Python's standard library only. + +Compare a running bloomery HTTP server with those saved vectors (no new oracle or threshold): + server_vectors.py compare --url --set [--ids-dir --case ...] + [--image = ...] [--mix-image ] +Prints cosine and maximum absolute difference for each selected raw and normalized vector; image inputs use the +collector's exact content-part construction. A successful command means the comparisons completed, not that a band passed. """ import argparse import base64 import hashlib +import importlib.util import json import math import os import struct +import sys import urllib.request from pathlib import Path MRL = (128, 256, 512) +MIX_TEXT = "A picture of coloured rings and ramps: " + + +def image_entries(images: list[str], mix: str | None): + """The same image-only and text-first inputs for collecting and comparing HTTP vectors.""" + found = False + for item in images: + name, png = item.split("=", 1) + data = Path(png).read_bytes() + url = "data:image/png;base64," + base64.b64encode(data).decode() + part = {"type": "image_url", "image_url": {"url": url}} + yield f"img_{name}", {"content": [part]}, png, data + if name == mix: + found = True + text = {"type": "text", "text": MIX_TEXT} + yield f"mix_{name}", {"content": [text, part]}, png, data + if mix and not found: + raise SystemExit(f"server_vectors: --mix-image {mix} names no --image") + + +def metrics(ours: list[float], reference: list[float]) -> tuple[float, float]: + """Cosine and maximum absolute difference, refusing empty, non-finite or unequal vectors.""" + if not ours or len(ours) != len(reference) or not all(math.isfinite(x) for x in ours + reference): + raise ValueError("empty, non-finite or unequal vector extents") + dot = sum(x * y for x, y in zip(ours, reference)) + norm = math.sqrt(sum(x * x for x in ours)) * math.sqrt(sum(y * y for y in reference)) + if norm == 0: + raise ValueError("cosine is undefined for a zero vector") + return dot / norm, max(abs(x - y) for x, y in zip(ours, reference)) + + +def compare_main(argv: list[str]) -> None: + """Compare a running HTTP server with saved llama-server raw and normalized vectors.""" + ap = argparse.ArgumentParser(description=compare_main.__doc__) + ap.add_argument("--url", required=True) + ap.add_argument("--set", required=True, type=Path) + ap.add_argument("--ids-dir", type=Path) + ap.add_argument("--case", action="append", default=[]) + ap.add_argument("--image", action="append", default=[]) + ap.add_argument("--mix-image") + a = ap.parse_args(argv) + if not a.case and not a.image: + ap.error("at least one --case or --image is required") + if a.case and a.ids_dir is None: + ap.error("--case requires --ids-dir") + spec = importlib.util.spec_from_file_location("manifest", Path(__file__).resolve().parents[2] / "bloomery/manifest.py") + manifest = importlib.util.module_from_spec(spec) + spec.loader.exec_module(manifest) + m = manifest.read(str(a.set)) + if m.complete is None or m.header.get("emb2") != "server" or m.header.get("arch") != "gemma-embedding2": + raise SystemExit(f"server_vectors: {a.set}: not a complete EmbeddingGemma 2 server set") + rows = {} + for row in m.rows("tensor"): + name = row["name"] + if name in rows: + raise SystemExit(f"server_vectors: {name}: duplicate tensor") + rows[name] = row + entries = [] + for name in a.case: + entries.append((name, (a.ids_dir / f"{name}.txt").read_text(encoding="utf-8"))) + entries.extend((name, entry) for name, entry, _, _ in image_entries(a.image, a.mix_image)) + for name, entry in entries: + for node, normalization in (("raw", -1), ("embd", 2)): + key = f"{name}/{node}" + if key not in rows: + raise SystemExit(f"server_vectors: {a.set}: no tensor {key}") + row = rows[key] + if row["type"] != "f32" or row.int("occurrence") != 0 or any(row.int(f"ne{i}") != 1 for i in (1, 2, 3)): + raise SystemExit(f"server_vectors: {key}: not one F32 vector") + path = Path(m.dir) / manifest.dump_file_name(key, 0, "output", "contiguous", "f32") + data = path.read_bytes() + if len(data) != row.int("bytes") or len(data) != row.int("ne0") * 4: + raise SystemExit(f"server_vectors: {key}: tensor byte count differs") + reference = list(struct.unpack(f"<{len(data) // 4}f", data)) + ours, count = embed(a.url, entry, normalization) + try: + cosine, maximum = metrics(ours, reference) + except ValueError as e: + raise SystemExit(f"server_vectors: {key}: {e}") from e + print(f"{key}: prompt_tokens {count} cosine {cosine:.9e} max_abs_diff {maximum:.9e}") + def post(url: str, path: str, body: dict) -> dict: @@ -119,6 +208,9 @@ def check_server(a: argparse.Namespace, build: str) -> dict: def main() -> None: + if sys.argv[1:2] == ["compare"]: + compare_main(sys.argv[2:]) + return ap = argparse.ArgumentParser() ap.add_argument("--url", required=True) ap.add_argument("--pid", required=True, type=int) @@ -128,6 +220,7 @@ def main() -> None: ap.add_argument("--mmproj-sha256") ap.add_argument("--arch", choices=("gemma-embedding", "gemma-embedding2"), required=True) ap.add_argument("--card", required=True) + ap.add_argument("--cpu", action="store_true") ap.add_argument("--build-record", required=True, type=Path) ap.add_argument("--ids-dir", required=True, type=Path) ap.add_argument("--case", action="append", required=True) @@ -137,6 +230,8 @@ def main() -> None: ap.add_argument("--server-flags", default="") a = ap.parse_args() + if a.cpu and os.environ.get("CUDA_VISIBLE_DEVICES") != "": + raise SystemExit("server_vectors: --cpu requires CUDA_VISIBLE_DEVICES empty") build = os.environ.get("BLOOMERY_REF_BUILD", "") if not build: raise SystemExit("server_vectors: BLOOMERY_REF_BUILD names no commit") @@ -157,7 +252,7 @@ def main() -> None: f"# build\t{build}", f"# arch\t{a.arch}", "# emb2\tserver", - f"# device\tcuda\tthreads\t-\tcard\t{a.card}", + f"# device\t{'cpu' if a.cpu else 'cuda'}\tthreads\t-\tcard\t{a.card}", f"# model_sha256\t{a.model_sha256}", f"# server_flags\t{a.server_flags}", f"# server_build\t{props['build_info']}", @@ -180,17 +275,10 @@ def main() -> None: if got != want or n != len(want): raise SystemExit(f"server_vectors: {name}: the server's ids are not the oracle's ({same}, prompt_tokens {n})") - images = dict(i.split("=", 1) for i in a.image) - for name, png in images.items(): - data = Path(png).read_bytes() - url = "data:image/png;base64," + base64.b64encode(data).decode() - part = {"type": "image_url", "image_url": {"url": url}} - n = pooled_rows(w, a.url, f"img_{name}", {"content": [part]}, False) - w.header.append(f"# image\timg_{name}\t{png}\t{hashlib.sha256(data).hexdigest()}\tprompt_tokens {n}") - if name == a.mix_image: - text = {"type": "text", "text": "A picture of coloured rings and ramps: "} - n = pooled_rows(w, a.url, f"mix_{name}", {"content": [text, part]}, False) - w.header.append(f"# image\tmix_{name}\t{png}\t{hashlib.sha256(data).hexdigest()}\tprompt_tokens {n}\ttext first") + for name, entry, png, data in image_entries(a.image, a.mix_image): + n = pooled_rows(w, a.url, name, entry, False) + suffix = "\ttext first" if name.startswith("mix_") else "" + w.header.append(f"# image\t{name}\t{png}\t{hashlib.sha256(data).hexdigest()}\tprompt_tokens {n}{suffix}") check_server(a, build) w.finish() print(f"server_vectors: {len(w.rows)} row(s) into {a.out}") diff --git a/tools/ref/models/gemma-embedding2.sh b/tools/ref/models/gemma-embedding2.sh index d3e6b29a7..6333f12fb 100644 --- a/tools/ref/models/gemma-embedding2.sh +++ b/tools/ref/models/gemma-embedding2.sh @@ -39,4 +39,4 @@ IMAGES=(small-200x200 min-96x64 free-448x448 free-640x480 pad-777x513 strip-1000 EMBED_VOCAB=gemma4 EMBED_CASES=cases-gemma4.txt EMBED_CONTEXT=8192 -EMBED_STEPS=(text tower server text.cpu tower.cpu) +EMBED_STEPS=(text tower server text.cpu tower.cpu server.cpu) diff --git a/tools/ref/ptx-shapes.tsv b/tools/ref/ptx-shapes.tsv index 4247024b5..ced6ab1db 100644 --- a/tools/ref/ptx-shapes.tsv +++ b/tools/ref/ptx-shapes.tsv @@ -620,20 +620,29 @@ gate_e2e tier_fault_copy 0 0 gate_e2e touch 0 0 gate_e2e weighted_sum 0 0 -# gate_qwen3vl_tower (--features vision), the `vis_` entries of bloomery-gpu-vision only: V4.1's six, the Qwen3-VL tower's five, the glm5v tower's three, and MiniCPM's two +# gate_qwen3vl_tower (--features vision), the shared vision entries, including Gemma4 residual and post-norm precision. +# PIN(2026-10-15): the seven Gemma4 entries have zero sm_86 ptxas spill; jit_local=0 is a derived cc8.6 ceiling, +# not a cc8.6 driver measurement. Both sm_86 and sm_121 PTX have zero JIT local bytes on GB10 cc12.1. +gate_qwen3vl_tower vis_add_f32 0 0 gate_qwen3vl_tower vis_attn 0 0 gate_qwen3vl_tower vis_attn_group 0 0 gate_qwen3vl_tower vis_attn_h72 0 0 +gate_qwen3vl_tower vis_clamp_f32 0 0 gate_qwen3vl_tower vis_gelu_erf 0 0 gate_qwen3vl_tower vis_gelu_tanh 0 0 gate_qwen3vl_tower vis_gemm_bf16 0 0 +gate_qwen3vl_tower vis_gemm_bf16_f32 0 0 gate_qwen3vl_tower vis_head_rms_norm 0 0 gate_qwen3vl_tower vis_layer_norm 0 0 gate_qwen3vl_tower vis_mean_groups 0 0 +gate_qwen3vl_tower vis_pool_order_f32 0 0 gate_qwen3vl_tower vis_pos_bilinear 0 0 +gate_qwen3vl_tower vis_position_rows 0 0 gate_qwen3vl_tower vis_rms_norm 0 0 +gate_qwen3vl_tower vis_rms_norm_f32 0 0 gate_qwen3vl_tower vis_rope2d 0 0 gate_qwen3vl_tower vis_rope2d_h72 0 0 +gate_qwen3vl_tower vis_row_op 0 0 gate_qwen3vl_tower vis_silu_mul 0 0 gate_qwen3vl_tower vis_silu_mul_clamp 0 0 gate_qwen3vl_tower vis_unfold 0 0 diff --git a/tools/ref/ptx-shapes/sm_121-cc12.1.tsv b/tools/ref/ptx-shapes/sm_121-cc12.1.tsv index a21a22263..d1213f4db 100644 --- a/tools/ref/ptx-shapes/sm_121-cc12.1.tsv +++ b/tools/ref/ptx-shapes/sm_121-cc12.1.tsv @@ -581,20 +581,28 @@ gate_e2e tier_fault_copy 0 0 gate_e2e touch 0 0 gate_e2e weighted_sum 0 0 -# gate_qwen3vl_tower (--features vision), the `vis_` entries of bloomery-gpu-vision only: V4.1's six, the Qwen3-VL tower's five, the glm5v tower's three, and MiniCPM's two +# gate_qwen3vl_tower (--features vision), the shared vision entries, including Gemma4 residual and post-norm precision. +# PIN(2026-10-15): the seven Gemma4 entries have zero ptxas spill and JIT local bytes on GB10 cc12.1. +gate_qwen3vl_tower vis_add_f32 0 0 gate_qwen3vl_tower vis_attn 0 0 gate_qwen3vl_tower vis_attn_group 0 0 gate_qwen3vl_tower vis_attn_h72 0 0 +gate_qwen3vl_tower vis_clamp_f32 0 0 gate_qwen3vl_tower vis_gelu_erf 0 0 gate_qwen3vl_tower vis_gelu_tanh 0 0 gate_qwen3vl_tower vis_gemm_bf16 0 0 +gate_qwen3vl_tower vis_gemm_bf16_f32 0 0 gate_qwen3vl_tower vis_head_rms_norm 0 0 gate_qwen3vl_tower vis_layer_norm 0 0 gate_qwen3vl_tower vis_mean_groups 0 0 +gate_qwen3vl_tower vis_pool_order_f32 0 0 gate_qwen3vl_tower vis_pos_bilinear 0 0 +gate_qwen3vl_tower vis_position_rows 0 0 gate_qwen3vl_tower vis_rms_norm 0 0 +gate_qwen3vl_tower vis_rms_norm_f32 0 0 gate_qwen3vl_tower vis_rope2d 0 0 gate_qwen3vl_tower vis_rope2d_h72 0 0 +gate_qwen3vl_tower vis_row_op 0 0 gate_qwen3vl_tower vis_silu_mul 0 0 gate_qwen3vl_tower vis_silu_mul_clamp 0 0 gate_qwen3vl_tower vis_unfold 0 0 diff --git a/tools/ref/ptx-shapes/sm_86-cc12.1.tsv b/tools/ref/ptx-shapes/sm_86-cc12.1.tsv index 2a60a7148..8e7118810 100644 --- a/tools/ref/ptx-shapes/sm_86-cc12.1.tsv +++ b/tools/ref/ptx-shapes/sm_86-cc12.1.tsv @@ -581,20 +581,28 @@ gate_e2e tier_fault_copy 0 0 gate_e2e touch 0 0 gate_e2e weighted_sum 0 0 -# gate_qwen3vl_tower (--features vision), the `vis_` entries of bloomery-gpu-vision only: V4.1's six, the Qwen3-VL tower's five, the glm5v tower's three, and MiniCPM's two +# gate_qwen3vl_tower (--features vision), the shared vision entries, including Gemma4 residual and post-norm precision. +# PIN(2026-10-15): the seven Gemma4 entries have zero ptxas spill and JIT local bytes on GB10 cc12.1. +gate_qwen3vl_tower vis_add_f32 0 0 gate_qwen3vl_tower vis_attn 0 0 gate_qwen3vl_tower vis_attn_group 0 0 gate_qwen3vl_tower vis_attn_h72 0 0 +gate_qwen3vl_tower vis_clamp_f32 0 0 gate_qwen3vl_tower vis_gelu_erf 0 0 gate_qwen3vl_tower vis_gelu_tanh 0 0 gate_qwen3vl_tower vis_gemm_bf16 0 0 +gate_qwen3vl_tower vis_gemm_bf16_f32 0 0 gate_qwen3vl_tower vis_head_rms_norm 0 0 gate_qwen3vl_tower vis_layer_norm 0 0 gate_qwen3vl_tower vis_mean_groups 0 0 +gate_qwen3vl_tower vis_pool_order_f32 0 0 gate_qwen3vl_tower vis_pos_bilinear 0 0 +gate_qwen3vl_tower vis_position_rows 0 0 gate_qwen3vl_tower vis_rms_norm 0 0 +gate_qwen3vl_tower vis_rms_norm_f32 0 0 gate_qwen3vl_tower vis_rope2d 0 0 gate_qwen3vl_tower vis_rope2d_h72 0 0 +gate_qwen3vl_tower vis_row_op 0 0 gate_qwen3vl_tower vis_silu_mul 0 0 gate_qwen3vl_tower vis_silu_mul_clamp 0 0 gate_qwen3vl_tower vis_unfold 0 0 diff --git a/tools/scan-args.sh b/tools/scan-args.sh index b11fb15b0..a11fa55c0 100755 --- a/tools/scan-args.sh +++ b/tools/scan-args.sh @@ -16,8 +16,8 @@ # more words than the scan script takes (ptx and lds: one entry substring; sass: the entry and its # decisions, which are a comma list of t/n or ADDR=t|n and are not refused for their commas). # A feature table it cannot read ends it with 2 too: an unread table would pass every word. -# The ptx scan alone is sm_86's (its blk/SM column is that arch's arithmetic): a BLOOMERY_ARCH that names another is refused -# by name, exit 64, before the build; the sass and lds scans follow the arch (tools/arch.sh). +# The ptx scan accepts only the architectures whose occupancy limits tools/ptx-scan.sh holds, before the build; +# the sass and lds scans follow the arch (tools/arch.sh). set -uo pipefail ROOT=$(cd "$(dirname "$0")/.." && pwd) @@ -37,9 +37,12 @@ check() { sass) max=2 flags=' --exact --raw --step ' form="just sass-scan $bin --features [-- --exact|--raw|--step HEAD] [entry substring [decisions|list]]" ;; *) echo "scan-args.sh: the scan is ptx, sass or lds, got '$scan'" >&2; return 64 ;; esac - if [ "$scan" = ptx ] && [ "${BLOOMERY_ARCH-sm_86}" != sm_86 ]; then - echo "ptx-scan: BLOOMERY_ARCH is '$BLOOMERY_ARCH': the scan's blk/SM column is sm_86's occupancy arithmetic and it holds no other arch's limits (unset BLOOMERY_ARCH for this recipe)" >&2 - return 64 + if [ "$scan" = ptx ]; then + case ${BLOOMERY_ARCH-sm_86} in + # PIN(2026-10-16): ptx-scan.sh tests cc12.x occupancy limits; the sm_86-only refusal rejected supported scans. + sm_86 | sm_120 | sm_120a | sm_120f | sm_121 | sm_121a | sm_121f) ;; + *) echo "ptx-scan: BLOOMERY_ARCH is '${BLOOMERY_ARCH-}': the scan holds occupancy limits for sm_86 and sm_120/sm_121 only" >&2; return 64 ;; + esac fi while [ $# -gt 0 ]; do w=$1 why='' @@ -123,7 +126,10 @@ self_test() { case_ sass-entry-comma 2 sass gemm,flash case_ scan-unknown 64 cubin gemm BLOOMERY_ARCH=sm_86 case_ ptx-arch-sm86 0 ptx - BLOOMERY_ARCH=sm_121 case_ ptx-arch-other 64 ptx + BLOOMERY_ARCH=sm_121 case_ ptx-arch-sm121 0 ptx + BLOOMERY_ARCH=sm_120a case_ ptx-arch-sm120a 0 ptx + BLOOMERY_ARCH=sm_121f case_ ptx-arch-sm121f 0 ptx + BLOOMERY_ARCH=sm_90 case_ ptx-arch-unknown 64 ptx BLOOMERY_ARCH=sm_121 case_ lds-arch-other 0 lds BLOOMERY_ARCH=sm_121 case_ sass-arch-other 0 sass # The real table: every feature crates/gpu-gates declares is refused as a word, and `gpu` is one. diff --git a/tools/unsafe-ratchet.txt b/tools/unsafe-ratchet.txt index faa1ba6ac..e3c382958 100644 --- a/tools/unsafe-ratchet.txt +++ b/tools/unsafe-ratchet.txt @@ -342,7 +342,11 @@ gpu-mimo2 0 0 0 # PIN(2026-10-13): gpu-vision 91 -> 97 blocks, new code, not old code relaxed: the MiniCPM-V merger's two new kernel entries read # and write their buffers through the same raw places as the others, each block with its SAFETY line — vis_attn_group 5 (its # loader, three word reads, one store), vis_mean_groups 1 (its loader). -gpu-vision 97 0 0 +# PIN(2026-10-15): gpu-vision 97 -> 116 blocks: Gemma4 row/position entries and F32 residual, clamp, pool, norm and GEMM +# keep Gemma4 post norms in their graph precision; every raw read and disjoint store carries its SAFETY invariant. +# PIN(2026-10-16): gpu-vision 116 -> 127 blocks: the same Gemma4 clamp, add, pool and F32 norm memory operations +# now have narrower unsafe scopes, each with its own SAFETY invariant. +gpu-vision 127 0 0 # PIN(2026-10-13): hf 0 -> 2 blocks, new code, not old code relaxed: the DSpark draft's 1.9 GB safetensors file is read # through a read-only mapping (Safetensors::open) instead of into memory, and a bf16 tensor's bytes are viewed as u16 # (Safetensors::bf16, a misaligned file refused), each block with its SAFETY line.