From a0994a74f089a2441d24c67a805b78a708fcef08 Mon Sep 17 00:00:00 2001 From: juliankang4 <200967947+juliankang4@users.noreply.github.com> Date: Thu, 15 Oct 2026 07:52:49 +0900 Subject: [PATCH 01/12] gpu: the tower crates use the common shape and tensor errors Vision and the text encoders share the named tensor and shape error contracts. --- crates/gpu/src/lib.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/crates/gpu/src/lib.rs b/crates/gpu/src/lib.rs index 6832eb58d..a7f451cac 100644 --- a/crates/gpu/src/lib.rs +++ b/crates/gpu/src/lib.rs @@ -239,7 +239,7 @@ pub enum GpuError { impl GpuError { /// A geometry contract broken by the caller of `what`. - pub(crate) fn shape(what: &'static str, detail: impl Into) -> GpuError { + pub fn shape(what: &'static str, detail: impl Into) -> GpuError { GpuError::Shape { what, detail: detail.into(), @@ -248,11 +248,7 @@ impl GpuError { /// A tensor `what` looked up that is absent or in the wrong format; /// `need` completes "is not …". - pub(crate) fn tensor( - what: &'static str, - name: impl Into, - need: &'static str, - ) -> GpuError { + pub fn tensor(what: &'static str, name: impl Into, need: &'static str) -> GpuError { GpuError::Tensor { what, name: name.into(), From c2536d535bc92dc326149e0c52032c57b7a57eb1 Mon Sep 17 00:00:00 2001 From: juliankang4 <200967947+juliankang4@users.noreply.github.com> Date: Thu, 15 Oct 2026 10:48:22 +0900 Subject: [PATCH 02/12] gpu-vision: share the BF16 GEMM accumulator walk Extract the existing tensor-core staging and accumulation into one macro. The BF16 entry keeps its epilogue and launch contract; no F32 entry or launcher is added here. --- crates/gpu-vision/src/gemm_bf16.rs | 153 +++++++++++++++-------------- 1 file changed, 81 insertions(+), 72 deletions(-) diff --git a/crates/gpu-vision/src/gemm_bf16.rs b/crates/gpu-vision/src/gemm_bf16.rs index 2790cec85..b867186b4 100644 --- a/crates/gpu-vision/src/gemm_bf16.rs +++ b/crates/gpu-vision/src/gemm_bf16.rs @@ -90,85 +90,48 @@ pub(crate) fn gelu_erf_dev(x: f32) -> f32 { mul_rn_f32(mul_rn_f32(x, 0.5), add_rn_f32(1.0, e)) } -// ---------------------------------------------------------------- kernels - -#[cuda_module] -mod gemm_kernels { - use super::*; - - /// `C = A·Bᵀ` with the module rule's bias and epilogue: `a` is `m` rows of `k` bf16, `b` is - /// `n` rows of `k` bf16, `c` (and `resid`, when read) `m` rows of `n` bf16, all row-major. - /// `use_bias` 1 adds `bias[j]` (f32) before the rounding; `gelu` 1 and `use_resid` 1 select - /// the epilogues (the launcher passes at most one). Block `b` computes tile row - /// `b / (n / TILE_N)`, tile column `b % (n / TILE_N)`. `n` a multiple of [`TILE_N`] and `k` - /// even and non-zero (host-checked). - #[allow( - clippy::too_many_arguments, - reason = "kernel entry: the device ABI takes the arguments flat (rust-quality R8)" - )] - #[kernel] - #[launch_bounds(128)] - #[launch_contract( - domain = 1, - block = (128, 1, 1), - requires = ( - a.len() >= m * k, - b.len() >= n * k, - bias.len() >= n * use_bias, - resid.len() >= m * n * use_resid, - c.len() >= m * n - ) - )] - pub fn vis_gemm_bf16( - a: &[u16], - b: &[u16], - bias: &[f32], - resid: &[u16], - m: u32, - n: u32, - k: u32, - use_bias: u32, - gelu: u32, - use_resid: u32, - mut c: DisjointSlice, - ) { - static mut AS: SharedArray = SharedArray::UNINIT; - static mut BS: SharedArray = SharedArray::UNINIT; +/// One tensor-core accumulator walk for the BF16 GEMM entry. +macro_rules! bf16_accumulate { + ($a:ident, $b:ident, $m:ident, $n:ident, $k:ident; + $acc:ident, $rows:ident, $cols:ident, $tm:ident, $tn:ident, $lane:ident, + $wm:ident, $wn:ident, $AS:ident, $BS:ident) => { + static mut $AS: SharedArray = SharedArray::UNINIT; + static mut $BS: SharedArray = SharedArray::UNINIT; let tid = cuda_device::thread::threadIdx_x() as usize; let blk = cuda_device::thread::blockIdx_x() as usize; - let rows = m as usize; - let cols = n as usize; - let kw = (k / 2) as usize; // u32 words per operand row - let tiles_n = cols / TILE_N; - let tm = blk / tiles_n; - let tn = blk - tm * tiles_n; - if tm * TILE_M >= rows { + let $rows = $m as usize; + let $cols = $n as usize; + let kw = ($k / 2) as usize; // u32 words per operand row + let tiles_n = $cols / TILE_N; + let $tm = blk / tiles_n; + let $tn = blk - $tm * tiles_n; + if $tm * TILE_M >= $rows { return; // block-uniform: no barrier and no warp collective is skipped } // SAFETY: block-shared, TILE_WORDS words each, written only by the staging loop between // the barriers that publish it. let (as_, bs_) = unsafe { ( - SharedArray::as_raw_mut_ptr(&raw mut AS), - SharedArray::as_raw_mut_ptr(&raw mut BS), + SharedArray::as_raw_mut_ptr(&raw mut $AS), + SharedArray::as_raw_mut_ptr(&raw mut $BS), ) }; - let aw = a.as_ptr().cast::(); - let bw = b.as_ptr().cast::(); + let aw = $a.as_ptr().cast::(); + let bw = $b.as_ptr().cast::(); - let lane = cuda_device::warp::lane_id() as usize; + let $lane = cuda_device::warp::lane_id() as usize; let wid = tid / 32; - let wm = wid / 2; // this warp's 32-row half of the tile - let wn = wid % 2; // and its 32-column half + let $wm = wid / 2; // this warp's 32-row half of the tile + let $wn = wid % 2; // and its 32-column half // `ldmatrix.x4` lane roles. A: row `lane % 16` of an m16 fragment at k half `lane / 16`. // B: row `(lane % 8) + 8 * (lane / 16)` of an n16 pair at k half `(lane / 8) % 2`. - let a_row = lane % 16; - let a_half = lane / 16; - let b_row = (lane % 8) + 8 * (lane / 16); - let b_half = (lane / 8) % 2; + let a_row = $lane % 16; + let a_half = $lane / 16; + let b_row = ($lane % 8) + 8 * ($lane / 16); + let b_half = ($lane / 8) % 2; - let mut acc = [[0.0f32; 4]; 8]; + let mut $acc = [[0.0f32; 4]; 8]; let k_steps = kw.div_ceil(TILE_K / 2); let mut ks = 0usize; @@ -185,14 +148,14 @@ mod gemm_kernels { let r = i / (TILE_K / 2); let w = i - r * (TILE_K / 2); let gw = w0 + w; - let ar = tm * TILE_M + r; - if ar < rows && gw < kw { + let ar = $tm * TILE_M + r; + if ar < $rows && gw < kw { // SAFETY: ar < m and gw < k / 2, so the word is inside row `ar` of `a`, // whose m·k bf16 the launch contract bounds; device buffers are 4-byte // aligned and k is even, so the word read is aligned. ra[s] = unsafe { *aw.add(ar * kw + gw) }; } - let br = tn * TILE_N + r; + let br = $tn * TILE_N + r; if gw < kw { // SAFETY: br < n (tiles cover n exactly) and gw < k / 2 bound the word // inside `b`'s n·k bf16 (launch contract), aligned as above. @@ -223,7 +186,7 @@ mod gemm_kernels { let mut mi = 0usize; while mi < 2 { cuda_device::thread::__unroll_config::<0>(); - let row = wm * 32 + mi * 16 + a_row; + let row = $wm * 32 + mi * 16 + a_row; // SAFETY: row < TILE_M and word kk·8 + a_half·4 + 4 <= TILE_K / 2 keep the // 16-byte row inside the A tile, published by the barrier above. let p = unsafe { as_.add(row * ROW_WORDS + kk * 8 + a_half * 4) }; @@ -241,7 +204,7 @@ mod gemm_kernels { let mut nj = 0usize; while nj < 2 { cuda_device::thread::__unroll_config::<0>(); - let row = wn * 32 + nj * 16 + b_row; + let row = $wn * 32 + nj * 16 + b_row; // SAFETY: row < TILE_N and the same word bound keep the row inside the B // tile, published by the barrier above. let p = unsafe { bs_.add(row * ROW_WORDS + kk * 8 + b_half * 4) }; @@ -260,13 +223,13 @@ mod gemm_kernels { // SAFETY: the whole warp issues these `mma.sync` (no lane-dependent // branch encloses them) with the fragments it loaded. unsafe { - acc[mi * 4 + nj * 2] = cuda_device::wmma::mma_m16n8k16_f32_bf16( - acc[mi * 4 + nj * 2], + $acc[mi * 4 + nj * 2] = cuda_device::wmma::mma_m16n8k16_f32_bf16( + $acc[mi * 4 + nj * 2], af[mi], [bf[0], bf[1]], ); - acc[mi * 4 + nj * 2 + 1] = cuda_device::wmma::mma_m16n8k16_f32_bf16( - acc[mi * 4 + nj * 2 + 1], + $acc[mi * 4 + nj * 2 + 1] = cuda_device::wmma::mma_m16n8k16_f32_bf16( + $acc[mi * 4 + nj * 2 + 1], af[mi], [bf[2], bf[3]], ); @@ -279,6 +242,52 @@ mod gemm_kernels { } ks += 1; } + }; +} + +// ---------------------------------------------------------------- kernels + +#[cuda_module] +mod gemm_kernels { + use super::*; + + /// `C = A·Bᵀ` with the module rule's bias and epilogue: `a` is `m` rows of `k` bf16, `b` is + /// `n` rows of `k` bf16, `c` (and `resid`, when read) `m` rows of `n` bf16, all row-major. + /// `use_bias` 1 adds `bias[j]` (f32) before the rounding; `gelu` 1 and `use_resid` 1 select + /// the epilogues (the launcher passes at most one). Block `b` computes tile row + /// `b / (n / TILE_N)`, tile column `b % (n / TILE_N)`. `n` a multiple of [`TILE_N`] and `k` + /// even and non-zero (host-checked). + #[allow( + clippy::too_many_arguments, + reason = "kernel entry: the device ABI takes the arguments flat (rust-quality R8)" + )] + #[kernel] + #[launch_bounds(128)] + #[launch_contract( + domain = 1, + block = (128, 1, 1), + requires = ( + a.len() >= m * k, + b.len() >= n * k, + bias.len() >= n * use_bias, + resid.len() >= m * n * use_resid, + c.len() >= m * n + ) + )] + pub fn vis_gemm_bf16( + a: &[u16], + b: &[u16], + bias: &[f32], + resid: &[u16], + m: u32, + n: u32, + k: u32, + use_bias: u32, + gelu: u32, + use_resid: u32, + mut c: DisjointSlice, + ) { + bf16_accumulate!(a, b, m, n, k; acc, rows, cols, tm, tn, lane, wm, wn, AS, BS); // Epilogue: accumulator register j of fragment (mi, ni) is row `group + 8·(j / 2)`, // column `2·(lane % 4) + j % 2` of that fragment. From 99fbcef397413e81dddc0b68de40a6569f89fbe9 Mon Sep 17 00:00:00 2001 From: juliankang4 <200967947+juliankang4@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:07:43 +0900 Subject: [PATCH 03/12] vision, serve: represent a missing chat image token A media rule may return no chat placeholder. Chat expansion, per-layer token checks and decision encoding refuse that case by name; the existing image models keep their token ids. --- crates/decision/src/encode.rs | 12 ++++++----- crates/gpu-gates/src/bin/gate_serve_start.rs | 5 ++++- .../src/bin/shared/glm5next_media.rs | 9 +++++++- .../src/bin/shared/serve_seats/glm.rs | 9 +++++++- crates/gpu-gates/src/vision_seat.rs | 21 +++++++++++-------- crates/serve/src/media.rs | 13 +++++++----- crates/serve/src/mock.rs | 4 ++-- crates/vision/src/arch/deepseek41v/media.rs | 10 +++++---- crates/vision/src/arch/glm5v/media.rs | 6 +++--- crates/vision/src/arch/minicpmv/media.rs | 14 ++++++++----- crates/vision/src/arch/mod.rs | 4 ++-- crates/vision/src/arch/qwen3vl/media.rs | 10 ++++----- crates/vision/src/media.rs | 3 ++- 13 files changed, 76 insertions(+), 44 deletions(-) diff --git a/crates/decision/src/encode.rs b/crates/decision/src/encode.rs index ffa55db6e..622ee5f58 100644 --- a/crates/decision/src/encode.rs +++ b/crates/decision/src/encode.rs @@ -107,7 +107,9 @@ pub(crate) fn encode_media_with( placeholders: (0..image_tokens.len()) .map(|i| media.image_placeholder(i)) .collect(), - token: media.image_token(), + token: media + .image_token() + .ok_or_else(|| Error::Images("the projector has no chat image token".into()))?, counts: image_tokens, }; encode_inner(tokens, req, Some(images), max_length) @@ -380,8 +382,8 @@ mod tests { PLACEHOLDER } - fn image_token(&self) -> u32 { - PAD + fn image_token(&self) -> Option { + Some(PAD) } fn prepare(&self, _: &vision::Rgb8) -> Result { @@ -484,8 +486,8 @@ mod tests { "<|image_pad|>" } - fn image_token(&self) -> u32 { - PAD + fn image_token(&self) -> Option { + Some(PAD) } fn prepare(&self, _: &vision::Rgb8) -> Result { diff --git a/crates/gpu-gates/src/bin/gate_serve_start.rs b/crates/gpu-gates/src/bin/gate_serve_start.rs index d11f1d00d..9c9101ba3 100644 --- a/crates/gpu-gates/src/bin/gate_serve_start.rs +++ b/crates/gpu-gates/src/bin/gate_serve_start.rs @@ -238,7 +238,10 @@ fn open_images(family: Family) -> Result { let seat = vision_seat::encoder_seat(Place::A, proj.card.total())?; let png = read_image()?; let span = proj.media.prepare(&vision::Rgb8::from_png(&png)?)?.span_len; - let token = proj.media.image_token(); + let token = proj + .media + .image_token() + .ok_or("the projector has no chat image token")?; Ok(Images { file, proj, diff --git a/crates/gpu-gates/src/bin/shared/glm5next_media.rs b/crates/gpu-gates/src/bin/shared/glm5next_media.rs index b9b7728e8..c0b5f17e4 100644 --- a/crates/gpu-gates/src/bin/shared/glm5next_media.rs +++ b/crates/gpu-gates/src/bin/shared/glm5next_media.rs @@ -328,7 +328,14 @@ pub mod seat { at.cap, at.card.total() ); - set_prefill_media(s.model_mut(), Some(at.media.image_token()))?; + set_prefill_media( + s.model_mut(), + Some( + at.media + .image_token() + .ok_or("the projector has no chat image token")?, + ), + )?; let ctx = s.model().gpu().context().clone(); let mut tower = Resident::open(&ctx, &at)?; let images: Vec = SIZES diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/glm.rs b/crates/gpu-gates/src/bin/shared/serve_seats/glm.rs index 01fbdaab5..850e39ed1 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/glm.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/glm.rs @@ -1317,7 +1317,14 @@ fn open_encoder( return Ok(None); }; let t = Instant::now(); - set_prefill_media(s.model_mut(), Some(v.media.image_token()))?; + set_prefill_media( + s.model_mut(), + Some( + v.media + .image_token() + .ok_or("the projector has no chat image token")?, + ), + )?; let resident = vision_seat::Resident::open(s.model().gpu().context(), &v)?; resident.announce(&v, vision_seat::ScratchHome::Own, t); Ok(Some(resident)) diff --git a/crates/gpu-gates/src/vision_seat.rs b/crates/gpu-gates/src/vision_seat.rs index 0ba68775e..ebfdd7813 100644 --- a/crates/gpu-gates/src/vision_seat.rs +++ b/crates/gpu-gates/src/vision_seat.rs @@ -420,20 +420,23 @@ impl Projector { )) ))); } - let image = self.media.image_token(); match text.ple { PleImage::Absent => {} PleImage::NoKey => { return Err(at(&"the text model carries per-layer embeddings and no \ ple.image_token_id, which must be the projector's image token")); } - PleImage::Id(id) if id != image => { - return Err(at(&format!( - "the text model's ple.image_token_id is {id}, the projector's image token \ - is {image}" - ))); + PleImage::Id(id) => { + let image = self.media.image_token().ok_or_else(|| { + at(&"the projector has no chat image token for ple.image_token_id") + })?; + if id != image { + return Err(at(&format!( + "the text model's ple.image_token_id is {id}, the projector's image token \ + is {image}" + ))); + } } - PleImage::Id(_) => {} } Ok(()) } @@ -1459,8 +1462,8 @@ mod tests { fn image_marker(&self) -> &str { "" } - fn image_token(&self) -> u32 { - self.0 + fn image_token(&self) -> Option { + Some(self.0) } fn prepare(&self, _: &vision::Rgb8) -> Result { Err(VisionError::Limits { diff --git a/crates/serve/src/media.rs b/crates/serve/src/media.rs index 48d491f91..a19dc1cf5 100644 --- a/crates/serve/src/media.rs +++ b/crates/serve/src/media.rs @@ -48,8 +48,8 @@ impl MediaModel for NumberedMedia { Some(vec![100, 200 + index as u32, 300]) } - fn image_token(&self) -> u32 { - 7 + fn image_token(&self) -> Option { + Some(7) } fn prepare(&self, _: &Rgb8) -> Result { @@ -243,6 +243,9 @@ impl From> for Prompt { /// are the request's fault. #[derive(Debug, thiserror::Error)] pub enum MediaError { + /// The projector supports embedding delimiters rather than a chat placeholder. + #[error("the projector has no chat image token; use its embedding input")] + NoChatImageToken, /// A content part that is not a JSON object. #[error("a content part must be an object")] PartNotObject, @@ -462,7 +465,7 @@ pub fn expand_prompt( urls: &[&str], model: &dyn MediaModel, ) -> Result { - let token = model.image_token(); + let token = model.image_token().ok_or(MediaError::NoChatImageToken)?; let placeholders = ids.iter().filter(|&&t| t == token).count(); if placeholders != urls.len() { return Err(MediaError::Expand(ExpandError::PlaceholderCount { @@ -686,8 +689,8 @@ mod tests { "" } - fn image_token(&self) -> u32 { - 7 + fn image_token(&self) -> Option { + Some(7) } fn part_join(&self) -> PartJoin { diff --git a/crates/serve/src/mock.rs b/crates/serve/src/mock.rs index 175bafa00..d7ce67cc3 100644 --- a/crates/serve/src/mock.rs +++ b/crates/serve/src/mock.rs @@ -1003,8 +1003,8 @@ impl MediaModel for MockMedia { IMAGE_SPECIAL } - fn image_token(&self) -> u32 { - IMAGE_ID + fn image_token(&self) -> Option { + Some(IMAGE_ID) } fn prepare(&self, image: &Rgb8) -> Result { diff --git a/crates/vision/src/arch/deepseek41v/media.rs b/crates/vision/src/arch/deepseek41v/media.rs index c86e33d43..d5def122a 100644 --- a/crates/vision/src/arch/deepseek41v/media.rs +++ b/crates/vision/src/arch/deepseek41v/media.rs @@ -61,8 +61,8 @@ impl MediaModel for Media { IMAGE_PLACEHOLDER } - fn image_token(&self) -> u32 { - IMAGE_TOKEN_ID + fn image_token(&self) -> Option { + Some(IMAGE_TOKEN_ID) } fn part_separator(&self) -> &str { @@ -111,7 +111,7 @@ mod tests { Cow::Borrowed(IMAGE_PLACEHOLDER) )); assert_eq!(m.image_marker(), IMAGE_PLACEHOLDER); - assert_eq!(m.image_token(), 129_264); + assert_eq!(m.image_token(), Some(129_264)); assert_eq!(m.part_separator(), "\n\n"); assert_eq!(m.part_join(), PartJoin::Uniform("\n\n")); } @@ -131,7 +131,9 @@ mod tests { assert_eq!((patches.n_vit_h, patches.n_vit_w), (39, 39)); assert_eq!(patches.bf16.len(), 39 * 39 * 3 * 14 * 14); assert_eq!( - image_span(&patches.plan, m.image_token()).ids.len(), + image_span(&patches.plan, m.image_token().expect("chat image token")) + .ids + .len(), p.span_len ); } diff --git a/crates/vision/src/arch/glm5v/media.rs b/crates/vision/src/arch/glm5v/media.rs index 475cc583a..cc102cbec 100644 --- a/crates/vision/src/arch/glm5v/media.rs +++ b/crates/vision/src/arch/glm5v/media.rs @@ -90,8 +90,8 @@ impl MediaModel for Media { IMAGE_PLACEHOLDER } - fn image_token(&self) -> u32 { - IMAGE_TOKEN_ID + fn image_token(&self) -> Option { + Some(IMAGE_TOKEN_ID) } fn part_join(&self) -> PartJoin { @@ -262,7 +262,7 @@ mod tests { Cow::Borrowed(IMAGE_PLACEHOLDER) )); assert_eq!(m.image_marker(), IMAGE_PLACEHOLDER); - assert_eq!(m.image_token(), IMAGE_TOKEN_ID); + assert_eq!(m.image_token(), Some(IMAGE_TOKEN_ID)); assert_eq!(m.part_join(), PartJoin::LlamaServer); assert_eq!(m.part_separator(), "\n"); } diff --git a/crates/vision/src/arch/minicpmv/media.rs b/crates/vision/src/arch/minicpmv/media.rs index 5e12868dc..ef89d1b12 100644 --- a/crates/vision/src/arch/minicpmv/media.rs +++ b/crates/vision/src/arch/minicpmv/media.rs @@ -61,8 +61,8 @@ impl MediaModel for Media { &self.marker } - fn image_token(&self) -> u32 { - self.pad + fn image_token(&self) -> Option { + Some(self.pad) } fn part_join(&self) -> PartJoin { @@ -108,7 +108,7 @@ mod tests { "12" ); assert_eq!(m.image_marker(), ""); - assert_eq!(m.image_token(), 248_077); + assert_eq!(m.image_token(), Some(248_077)); assert_eq!(m.part_join(), PartJoin::LlamaServer); // ``, the digits, ``, ``. let head = m.image_head(12).expect("a numbered model has a head"); @@ -135,8 +135,12 @@ mod tests { panic!("{w}x{h}: a MiniCPM-V image is a group of tiles"); }; assert_eq!((group.tiles.len(), group.rows()), (tiles, span), "{w}x{h}"); - let ids = - expand_spans(&[5, 248_077, 6], m.image_token(), &[p.span_len]).expect("expand"); + let ids = expand_spans( + &[5, 248_077, 6], + m.image_token().expect("chat image token"), + &[p.span_len], + ) + .expect("expand"); // One span, the range itself. `[1..1 + span]` is a one-element array of a // range, which is the lint's single-range init, not the positions inside it. assert_eq!(ids.spans.len(), 1, "{w}x{h}"); diff --git a/crates/vision/src/arch/mod.rs b/crates/vision/src/arch/mod.rs index bfdf092e1..5ed4db22b 100644 --- a/crates/vision/src/arch/mod.rs +++ b/crates/vision/src/arch/mod.rs @@ -197,7 +197,7 @@ mod tests { let t = table("qwen3vl_merger").with("clip.vision.projection_dim", Value::U32(out)); let (card, media) = open_table(&t, "clip").expect("open"); assert_eq!((card.weights, card.scratch), (weights, scratch), "{out}"); - assert_eq!(media.image_token(), 248_056); + assert_eq!(media.image_token(), Some(248_056)); assert_eq!(media.part_join(), PartJoin::LlamaServer); } } @@ -216,7 +216,7 @@ mod tests { (1_128_026_112, scratch), "{max}" ); - assert_eq!(media.image_token(), 154_854); + assert_eq!(media.image_token(), Some(154_854)); assert_eq!(media.part_join(), PartJoin::LlamaServer); } let image = crate::Rgb8::filled(1000, 600, [90, 120, 30]); diff --git a/crates/vision/src/arch/qwen3vl/media.rs b/crates/vision/src/arch/qwen3vl/media.rs index 198e432ae..68dc71c8d 100644 --- a/crates/vision/src/arch/qwen3vl/media.rs +++ b/crates/vision/src/arch/qwen3vl/media.rs @@ -7,11 +7,11 @@ use std::borrow::Cow; use super::TEMPORAL_FRAMES; -use super::size::SizeRule; use crate::GridPlan; use crate::media::{MediaModel, PartJoin, Prepared}; use crate::preprocess::{PatchLayout, Patches, patchify}; use crate::resample::pad_ceil; +use crate::size::SizeRule; use crate::{Rgb8, VisionError}; /// The placeholder Clef's prompt carries for one image. @@ -81,8 +81,8 @@ impl MediaModel for Media { IMAGE_PLACEHOLDER } - fn image_token(&self) -> u32 { - IMAGE_TOKEN_ID + fn image_token(&self) -> Option { + Some(IMAGE_TOKEN_ID) } fn part_join(&self) -> PartJoin { @@ -105,9 +105,9 @@ mod tests { use super::{IMAGE_PLACEHOLDER, Media, PAD_BLACK, VISION_END_ID, VISION_START_ID}; use crate::MropeSeq; use crate::Rgb8; - use crate::arch::qwen3vl::size::{SizeRule, TokenLimits}; use crate::media::{MediaModel, PartJoin}; use crate::preprocess::{f32_to_bf16, normalize}; + use crate::size::{SizeRule, TokenLimits}; fn media() -> Media { Media::new(SizeRule::new(16, 2, TokenLimits::DEFAULT).expect("rule")) @@ -126,7 +126,7 @@ mod tests { Cow::Borrowed(IMAGE_PLACEHOLDER) )); assert_eq!(m.image_marker(), IMAGE_PLACEHOLDER); - assert_eq!(m.image_token(), 248_056); + assert_eq!(m.image_token(), Some(248_056)); assert_eq!((VISION_START_ID, VISION_END_ID), (248_053, 248_054)); assert_eq!(PAD_BLACK, [0, 0, 0]); assert_eq!(m.part_separator(), "\n"); diff --git a/crates/vision/src/media.rs b/crates/vision/src/media.rs index 3a355daec..15dafd5a8 100644 --- a/crates/vision/src/media.rs +++ b/crates/vision/src/media.rs @@ -99,7 +99,8 @@ pub trait MediaModel { /// The id [`expand_spans`] finds in the tokenized prompt and expands to one copy /// per position of the image ([`Prepared::span_len`]): the one id V4.1's placeholder /// tokenizes to, and the `<|image_pad|>` between the two delimiter ids Clef's tokenizes to. - fn image_token(&self) -> u32; + /// None when the model has no chat placeholder token. + fn image_token(&self) -> Option; /// The separator between a message's content parts when they are flattened into one text: /// the separator of a [`PartJoin::Uniform`] join. From fe14def92723a79e4972245c7c2d883cf614eb6a Mon Sep 17 00:00:00 2001 From: juliankang4 <200967947+juliankang4@users.noreply.github.com> Date: Thu, 15 Oct 2026 07:48:47 +0900 Subject: [PATCH 04/12] vision: Gemma4 images use the mtmd canvas and raster positions Read the Gemma4 header, activation and pooling scale, plan its card bytes and prepare raster patches on the shared dynamic size rule. Embedding images use text delimiters and have no chat placeholder. --- crates/vision/src/arch/gemma4v/card.rs | 88 ++++++++++++ crates/vision/src/arch/gemma4v/hparams.rs | 138 +++++++++++++++++++ crates/vision/src/arch/gemma4v/media.rs | 92 +++++++++++++ crates/vision/src/arch/gemma4v/mod.rs | 61 ++++++++ crates/vision/src/arch/gemma4v/tensors.rs | 122 ++++++++++++++++ crates/vision/src/arch/header.rs | 11 +- crates/vision/src/arch/mod.rs | 32 +++-- crates/vision/src/arch/qwen3vl/card.rs | 2 +- crates/vision/src/arch/qwen3vl/hparams.rs | 2 +- crates/vision/src/arch/qwen3vl/mod.rs | 6 +- crates/vision/src/lib.rs | 3 +- crates/vision/src/media.rs | 8 +- crates/vision/src/{arch/qwen3vl => }/size.rs | 6 +- 13 files changed, 553 insertions(+), 18 deletions(-) create mode 100644 crates/vision/src/arch/gemma4v/card.rs create mode 100644 crates/vision/src/arch/gemma4v/hparams.rs create mode 100644 crates/vision/src/arch/gemma4v/media.rs create mode 100644 crates/vision/src/arch/gemma4v/mod.rs create mode 100644 crates/vision/src/arch/gemma4v/tensors.rs rename crates/vision/src/{arch/qwen3vl => }/size.rs (98%) diff --git a/crates/vision/src/arch/gemma4v/card.rs b/crates/vision/src/arch/gemma4v/card.rs new file mode 100644 index 000000000..715432ff5 --- /dev/null +++ b/crates/vision/src/arch/gemma4v/card.rs @@ -0,0 +1,88 @@ +//! The resident vision weights and the bounded activation buffers. + +use super::{Hparams, MERGE, tensors}; +use crate::{ + VisionError, + arch::{TokenLimits, card::CardBytes}, +}; +use gguf::Gguf; + +/// Element counts of the tower's buffers at the largest supported canvas. +#[derive(Clone, Copy)] +pub struct ScratchLens { + pub patches: usize, + pub raw: usize, + pub hidden: usize, + pub qkv: usize, + pub ff: usize, + pub rows: usize, + pub pooled: usize, +} + +impl ScratchLens { + /// Buffer extents for `tokens` pooled rows; refuse overflow before allocation. + pub fn new(hp: &Hparams, tokens: usize) -> Result { + if tokens == 0 || hp.patch == 0 || hp.dim == 0 || hp.ff == 0 || hp.out_dim == 0 { + return Err(VisionError::Limits { + detail: "gemma4v scratch extents must be positive".into(), + }); + } + let mul = |a: usize, b: usize| { + a.checked_mul(b) + .filter(|&v| v <= u32::MAX as usize) + .ok_or_else(|| VisionError::Limits { + detail: format!("gemma4v scratch {a} times {b} does not fit u32"), + }) + }; + let patches = mul(tokens, MERGE * MERGE)?; + let patch_values = mul(mul(hp.patch, hp.patch)?, 3)?; + let qkv_width = mul(hp.dim, 3)?; + Ok(Self { + patches, + raw: mul(patches, patch_values)?, + hidden: mul(patches, hp.dim)?, + qkv: mul(patches, qkv_width)?, + ff: mul(patches, hp.ff)?, + rows: mul(tokens, hp.out_dim)?, + pooled: mul(tokens, hp.dim)?, + }) + } + + /// Bytes of the buffers allocated by the tower. + #[must_use] + pub fn bytes(&self) -> u64 { + // F32 patches, patch output, residuals and post norms; BF16 linears, attention, + // pooling gather, gate/up/activation, output rows; f32 position angles. + 4 * (self.raw as u64 + 4 * self.hidden as u64) + + 2 * (4 * self.hidden as u64 + + self.qkv as u64 + + 5 * self.ff as u64 + + 3 * self.pooled as u64 + + self.rows as u64) + + 4 * 64 * self.patches as u64 + } +} + +/// The vision weights and scratch held on the side seat's card. +pub fn of(file: &Gguf, hp: &Hparams, limits: TokenLimits) -> Result { + hp.size_rule(limits)?; + let weights: u64 = tensors::check(file, hp)? + .iter() + .filter(|r| { + ![".input_min", ".input_max", ".output_min", ".output_max"] + .iter() + .any(|s| r.name.ends_with(s)) + }) + .map(|r| r.bytes()) + .sum(); + let weights = weights + + 4 * (hp.dim + 64) as u64 + + if file.find("v.std_bias").is_none() { + 8 * hp.dim as u64 + } else { + 0 + }; + // The shared BF16 GEMM holds one f32 bias and one bf16 residual stand-in. + let scratch = ScratchLens::new(hp, limits.max())?.bytes() + 6; + Ok(CardBytes { weights, scratch }) +} diff --git a/crates/vision/src/arch/gemma4v/hparams.rs b/crates/vision/src/arch/gemma4v/hparams.rs new file mode 100644 index 000000000..4bb42f5bf --- /dev/null +++ b/crates/vision/src/arch/gemma4v/hparams.rs @@ -0,0 +1,138 @@ +//! The vision half of the Gemma4 projector header. + +use super::{MERGE, PROJECTOR_TYPE}; +use crate::VisionError; +use crate::arch::{ + TokenLimits, + header::{ + KEY_HAS_VISION, KEY_MEAN, KEY_STD, check_architecture, check_eps, check_projector, + check_true, counts, need, refusal, + }, +}; +use crate::size::SizeRule; +use gguf::Gguf; + +/// Dimensions of the EmbeddingGemma 2 vision tower. +#[derive(Clone, Debug)] +pub struct Hparams { + pub n_layer: usize, + pub dim: usize, + pub ff: usize, + pub n_head: usize, + pub patch: usize, + pub out_dim: usize, + pub eps: f32, + pub pos_rows: usize, + /// The header chooses tanh GELU; without that key mainline uses Quick GELU. + pub gelu_tanh: bool, +} + +impl Hparams { + /// Read the vision header and refuse a geometry this tower does not run. + pub fn read(file: &Gguf) -> Result { + check_architecture(file)?; + check_projector(file, PROJECTOR_TYPE)?; + check_true(file, KEY_HAS_VISION, "the vision tower")?; + if let Some(v) = file.value("clip.vision.projector.scale_factor") + && v.as_u64() != Some(MERGE as u64) + { + return Err(refusal( + "clip.vision.projector.scale_factor", + format!("is {v:?}; gemma4v pools {MERGE} by {MERGE} patches"), + )); + } + let [n_layer, dim, ff, n_head, patch, out_dim] = counts( + file, + PROJECTOR_TYPE, + [ + ( + "clip.vision.block_count", + 16, + "the EmbeddingGemma 2 projector", + ), + ( + "clip.vision.embedding_length", + 768, + "the EmbeddingGemma 2 projector", + ), + ( + "clip.vision.feed_forward_length", + 3072, + "the EmbeddingGemma 2 projector", + ), + ( + "clip.vision.attention.head_count", + 12, + "the EmbeddingGemma 2 projector", + ), + ( + "clip.vision.patch_size", + 16, + "the EmbeddingGemma 2 projector", + ), + ( + "clip.vision.projection_dim", + 512, + "the EmbeddingGemma 2 projector", + ), + ], + )?; + for (key, want) in [(KEY_MEAN, 0.0f32), (KEY_STD, 1.0)] { + let v = need(file, key)?; + if !matches!(v, gguf::Value::Array(a) if a.len() == 3 && a.iter().all(|x| x.as_f32() == Some(want))) + { + return Err(refusal(key, format!("is {v:?}; gemma4v uses [{want}; 3]"))); + } + } + let eps = check_eps(file, 1e-6, "the Gemma4 vision RMS norm")?; + let gelu_tanh = activation(file)?; + Ok(Self { + n_layer, + dim, + ff, + n_head, + patch, + out_dim, + eps, + pos_rows: 10240, + gelu_tanh, + }) + } + + /// The mainline image budget when neither flag supplies a bound. + pub fn default_limits(&self) -> Result { + TokenLimits::new(70, 1120) + } + + /// The shared mtmd dynamic resize rule at `limits`. + pub fn size_rule(&self, limits: TokenLimits) -> Result { + SizeRule::new(self.patch, MERGE, limits) + } +} + +fn activation(m: &impl crate::arch::header::Meta) -> Result { + let flag = |key| match m.value(key) { + None => Ok(false), + Some(gguf::Value::Bool(v)) => Ok(*v), + Some(v) => Err(refusal(key, format!("is {v:?}, not a boolean"))), + }; + if flag("clip.use_silu")? { + return Err(refusal("clip.use_silu", "gemma4v does not run SiLU")); + } + flag("clip.use_gelu") +} + +#[cfg(test)] +mod tests { + use super::activation; + use crate::arch::header::testing::Table; + use gguf::Value; + #[test] + fn absent_activation_flags_select_mainline_quick_gelu() { + let table = Table(vec![]); + assert!(!activation(&table).unwrap()); + assert!(activation(&Table(vec![]).with("clip.use_gelu", Value::Bool(true))).unwrap()); + assert!(activation(&Table(vec![]).with("clip.use_silu", Value::Bool(true))).is_err()); + assert!(activation(&table.with("clip.use_gelu", Value::U32(1))).is_err()); + } +} diff --git a/crates/vision/src/arch/gemma4v/media.rs b/crates/vision/src/arch/gemma4v/media.rs new file mode 100644 index 000000000..3089dd27f --- /dev/null +++ b/crates/vision/src/arch/gemma4v/media.rs @@ -0,0 +1,92 @@ +//! Raster patches of the mtmd canvas. + +use super::{IMAGE_BEGIN, IMAGE_END}; +use crate::size::SizeRule; +use crate::{GridPlan, MediaModel, Patches, Prepared, Rgb8, VisionError}; +use std::borrow::Cow; + +/// Resize onto the mtmd canvas and cut raster patches after the graph's `2*x - 1`. +pub fn preprocess(image: &Rgb8, rule: &SizeRule) -> Result { + image.check()?; + let (best_w, best_h) = rule.aligned(image.width, image.height)?; + let (nx, ny) = rule.grid(best_w, best_h); + if nx.checked_mul(ny).is_none_or(|n| n > rule.max_tokens()) { + return Err(VisionError::Limits { + detail: "gemma4v aligned canvas exceeds the image token budget".into(), + }); + } + let plan = GridPlan { + n_llm_h: ny, + n_llm_w: nx, + best_h, + best_w, + }; + let resized = crate::resample::pad_ceil(image, best_w, best_h, [0; 3])?; + let patches = crate::preprocess::patchify( + &resized, + plan, + crate::preprocess::PatchLayout::row_major(rule.patch()), + )?; + if patches.n_vit_h > 10240 || patches.n_vit_w > 10240 { + return Err(VisionError::Limits { + detail: "gemma4v canvas exceeds the learned position table".into(), + }); + } + Ok(patches) +} + +/// Restore the original normalized bytes for the F32 convolution: their bf16 encodings are distinct. +pub fn conv_pixels(encoded: &[u16]) -> Result, VisionError> { + encoded + .iter() + .map(|&bits| { + let value = f32::from_bits(u32::from(bits) << 16); + let byte = ((value + 1.0) * 127.5).round(); + if !value.is_finite() + || !(0.0..=255.0).contains(&byte) + || crate::preprocess::normalize(byte as u8) != bits + { + return Err(VisionError::Format( + "gemma4v convolution input is not a normalized RGB byte".into(), + )); + } + Ok((byte / 255.0) * 2.0 - 1.0) + }) + .collect() +} + +/// The image preparation rule; the side seat inserts delimiters through its tokenizer. +#[derive(Clone, Copy, Debug)] +pub struct Media { + rule: SizeRule, +} + +impl Media { + /// An image preparer at the file's patch size and the request's token budget. + #[must_use] + pub fn new(rule: SizeRule) -> Self { + Self { rule } + } +} + +impl MediaModel for Media { + fn embedding_delimiters(&self) -> Option<(&str, &str)> { + Some((IMAGE_BEGIN, IMAGE_END)) + } + fn image_placeholder(&self, _: usize) -> Cow<'_, str> { + Cow::Owned(format!("{IMAGE_BEGIN}{IMAGE_END}")) + } + fn image_marker(&self) -> &str { + IMAGE_BEGIN + } + fn image_token(&self) -> Option { + None + } + fn prepare(&self, image: &Rgb8) -> Result { + let patches = preprocess(image, &self.rule)?; + Ok(Prepared::of_grid( + patches.plan.n_llm_h * patches.plan.n_llm_w, + patches, + )) + } +} diff --git a/crates/vision/src/arch/gemma4v/mod.rs b/crates/vision/src/arch/gemma4v/mod.rs new file mode 100644 index 000000000..646da8ae1 --- /dev/null +++ b/crates/vision/src/arch/gemma4v/mod.rs @@ -0,0 +1,61 @@ +//! Gemma4's vision input: dynamic aspect-preserving canvases, raster patches, +//! learned x/y positions and three-by-three pooled rows. + +pub mod card; +mod hparams; +mod media; +pub mod tensors; + +pub use hparams::Hparams; +pub use media::{Media, conv_pixels, preprocess}; + +/// The projector type of the vision half of a Gemma4 multimodal file. +pub const PROJECTOR_TYPE: &str = "gemma4v"; +/// Patches pooled along each axis. +pub const MERGE: usize = 3; +/// The tokenizer spelling before an image's rows. +pub const IMAGE_BEGIN: &str = "<|image>"; +/// The tokenizer spelling after an image's rows. +pub const IMAGE_END: &str = ""; + +#[cfg(test)] +mod tests { + use super::*; + use crate::size::SizeRule; + use crate::{Rgb8, arch::TokenLimits}; + + #[test] + fn gemma4v_canvas_and_patch_order() { + let rule = SizeRule::new(16, MERGE, TokenLimits::new(70, 1120).unwrap()).unwrap(); + for ((w, h), expected) in [ + ((224, 224), (432, 432)), + ((1000, 700), (1008, 720)), + ((2600, 1800), (1920, 1296)), + ((1, 1), (432, 432)), + ] { + assert_eq!(rule.aligned(w, h).unwrap(), expected); + } + let small = SizeRule::new(16, MERGE, TokenLimits::new(1, 4).unwrap()).unwrap(); + let p = preprocess(&Rgb8::filled(96, 48, [0, 128, 255]), &small).unwrap(); + assert_eq!((p.n_vit_w, p.n_vit_h, p.plan.span_grid()), (6, 3, (2, 1))); + assert_eq!(p.patch_len, 768); + assert_eq!(p.bf16[0], crate::preprocess::f32_to_bf16(-1.0)); + assert_eq!( + p.bf16[256], + crate::preprocess::f32_to_bf16(2.0 * (128.0f32 / 255.0) - 1.0) + ); + assert_eq!(p.bf16[512], crate::preprocess::f32_to_bf16(1.0)); + assert!(preprocess(&Rgb8::filled(0, 1, [0; 3]), &small).is_err()); + let encoded: Vec<_> = (0..=255).map(crate::preprocess::normalize).collect(); + let pixels = conv_pixels(&encoded).unwrap(); + for (byte, value) in pixels.into_iter().enumerate() { + assert_eq!( + value.to_bits(), + (2.0 * (byte as f32 / 255.0) - 1.0).to_bits() + ); + } + for invalid in [0, 0x7fc0, 0x7f80, 0xff80] { + assert!(conv_pixels(&[invalid]).is_err()); + } + } +} diff --git a/crates/vision/src/arch/gemma4v/tensors.rs b/crates/vision/src/arch/gemma4v/tensors.rs new file mode 100644 index 000000000..5aec37f58 --- /dev/null +++ b/crates/vision/src/arch/gemma4v/tensors.rs @@ -0,0 +1,122 @@ +//! The vision tensors of a multimodal Gemma4 projector; audio tensors stay on the host. + +use super::{Hparams, PROJECTOR_TYPE}; +use crate::{ + VisionError, + arch::table::{self, Expected}, +}; +use gguf::{GgmlType, Gguf}; + +/// Names and ggml shapes of every required vision tensor. +#[must_use] +pub fn expected(hp: &Hparams) -> Vec> { + let (d, f, out, p) = ( + hp.dim as u64, + hp.ff as u64, + hp.out_dim as u64, + hp.patch as u64, + ); + let mut rows = vec![ + Expected::row( + "v.patch_embd.weight".into(), + &[p, p, 3, d], + GgmlType::F32, + (), + ), + Expected::row( + "v.position_embd.weight".into(), + &[d, hp.pos_rows as u64, 2], + GgmlType::F32, + (), + ), + Expected::row( + "mm.input_projection.weight".into(), + &[d, out], + GgmlType::BF16, + (), + ), + ]; + for l in 0..hp.n_layer { + for stem in [ + "ln1", + "ln2", + "attn_post_norm", + "ffn_post_norm", + "attn_q_norm", + "attn_k_norm", + ] { + let n = if stem.ends_with("_q_norm") || stem.ends_with("_k_norm") { + 64 + } else { + d + }; + rows.push(Expected::row( + format!("v.blk.{l}.{stem}.weight"), + &[n], + GgmlType::F32, + (), + )); + } + for (stem, dims) in [ + ("attn_q", [d, d]), + ("attn_k", [d, d]), + ("attn_v", [d, d]), + ("attn_out", [d, d]), + ("ffn_gate", [d, f]), + ("ffn_up", [d, f]), + ("ffn_down", [f, d]), + ] { + rows.push(Expected::row( + format!("v.blk.{l}.{stem}.weight"), + &dims, + GgmlType::BF16, + (), + )); + } + } + rows +} + +/// Check the complete vision half, including optional standardization and clamp scalars. +pub fn check(file: &Gguf, hp: &Hparams) -> Result>, VisionError> { + let mut rows = expected(hp); + let has_bias = file.find("v.std_bias").is_some(); + let has_scale = file.find("v.std_scale").is_some(); + if has_bias != has_scale { + return Err(VisionError::Tensor { + name: "v.std_bias/v.std_scale".into(), + detail: "standardization needs both vectors".into(), + }); + } + if has_bias { + for name in ["v.std_bias", "v.std_scale"] { + rows.push(Expected::row( + name.into(), + &[hp.dim as u64], + GgmlType::F32, + (), + )); + } + } + let matrices: Vec = rows + .iter() + .filter(|r| r.dims.len() == 2) + .map(|r| r.name.clone()) + .collect(); + for name in matrices { + for suffix in ["input_min", "input_max", "output_min", "output_max"] { + let n = name.replace(".weight", &format!(".{suffix}")); + if file.find(&n).is_some() { + rows.push(Expected::row(n, &[1], GgmlType::F32, ())); + } + } + } + table::check( + PROJECTOR_TYPE, + &rows, + file.iter_tensors() + .filter(|t| !t.name.starts_with("a.") && !t.name.starts_with("mm.a.")) + .map(|t| (t.name.as_str(), t.dims.as_slice(), t.ty)), + )?; + Ok(rows) +} diff --git a/crates/vision/src/arch/header.rs b/crates/vision/src/arch/header.rs index d513f0a63..244614669 100644 --- a/crates/vision/src/arch/header.rs +++ b/crates/vision/src/arch/header.rs @@ -61,7 +61,7 @@ pub(crate) fn check_architecture(gguf: &Gguf) -> Result<(), VisionError> { /// The file names `projector` as its projector type. pub(crate) fn check_projector(m: &impl Meta, projector: &str) -> Result<(), VisionError> { - match need(m, KEY_PROJECTOR)?.as_str() { + match projector_value(m)?.as_str() { Some(p) if p == projector => Ok(()), Some(other) => Err(refusal( KEY_PROJECTOR, @@ -71,6 +71,15 @@ pub(crate) fn check_projector(m: &impl Meta, projector: &str) -> Result<(), Visi } } +pub(crate) fn projector_value(m: &impl Meta) -> Result<&Value, VisionError> { + match m.value(KEY_PROJECTOR) { + Some(v) => Ok(v), + None => m + .value("clip.vision.projector_type") + .ok_or_else(|| refusal(KEY_PROJECTOR, "is absent")), + } +} + /// A boolean key that must be true; `why` says what needs it. pub(crate) fn check_true(m: &impl Meta, key: &str, why: &str) -> Result<(), VisionError> { match need(m, key)?.as_bool() { diff --git a/crates/vision/src/arch/mod.rs b/crates/vision/src/arch/mod.rs index 5ed4db22b..f5d5667bd 100644 --- a/crates/vision/src/arch/mod.rs +++ b/crates/vision/src/arch/mod.rs @@ -14,6 +14,7 @@ pub mod card; pub mod deepseek41v; pub mod gemma4a; +pub mod gemma4v; pub mod glm5v; pub(crate) mod header; pub mod minicpmv; @@ -27,12 +28,12 @@ use std::sync::Arc; use gguf::Gguf; use self::card::CardBytes; -use self::header::{KEY_PROJECTOR, Meta, check_architecture, need, refusal}; +use self::header::{KEY_PROJECTOR, Meta, check_architecture, projector_value, refusal}; use crate::{MediaModel, VisionError}; /// The bounds on how many tokens an image takes: [`open`]'s limits, which every projector that /// plans from them reads in this one type. -pub use self::qwen3vl::TokenLimits; +pub use crate::size::TokenLimits; /// What an encoder file opens to: the tower's bytes on its card, and the model's image input. pub type Opened = (CardBytes, Arc); @@ -54,6 +55,11 @@ pub type Opened = (CardBytes, Arc); pub fn open(file: &Gguf, limits: TokenLimits) -> Result { check_architecture(file)?; match projector(file)? { + Projector::Gemma4v => { + let hp = gemma4v::Hparams::read(file)?; + let media = gemma4v::Media::new(hp.size_rule(limits)?); + Ok((gemma4v::card::of(file, &hp, limits)?, Arc::new(media))) + } Projector::Deepseek41v => { let hp = deepseek41v::Hparams::read(file)?; Ok(( @@ -92,6 +98,7 @@ pub fn open(file: &Gguf, limits: TokenLimits) -> Result { /// The projector types this crate runs. #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum Projector { + Gemma4v, Deepseek41v, Glm5v, Qwen3vl, @@ -100,8 +107,9 @@ enum Projector { /// The file's `clip.projector_type`, refused by name when it is a tower this crate does not run. fn projector(m: &impl Meta) -> Result { - let v = need(m, KEY_PROJECTOR)?; + let v = projector_value(m)?; match v.as_str() { + Some(gemma4v::PROJECTOR_TYPE) => Ok(Projector::Gemma4v), Some(deepseek41v::PROJECTOR_TYPE) => Ok(Projector::Deepseek41v), Some(glm5v::PROJECTOR_TYPE) => Ok(Projector::Glm5v), Some(qwen3vl::PROJECTOR_TYPE) => Ok(Projector::Qwen3vl), @@ -109,8 +117,9 @@ fn projector(m: &impl Meta) -> Result { Some(other) => Err(refusal( KEY_PROJECTOR, format!( - "is \"{other}\"; this crate reads \"{}\", \"{}\", \"{}\" and \"{}\"", + "is \"{other}\"; this crate reads \"{}\", \"{}\", \"{}\", \"{}\" and \"{}\"", deepseek41v::PROJECTOR_TYPE, + gemma4v::PROJECTOR_TYPE, glm5v::PROJECTOR_TYPE, minicpmv::PROJECTOR_TYPE, qwen3vl::PROJECTOR_TYPE @@ -120,6 +129,13 @@ fn projector(m: &impl Meta) -> Result { } } +/// The vision projector type: the common key, or the vision key of a multimodal file. +pub fn projector_type(file: &Gguf) -> Result<&str, VisionError> { + projector_value(file)? + .as_str() + .ok_or_else(|| refusal(KEY_PROJECTOR, "is not a string")) +} + #[cfg(test)] mod tests { use gguf::{Gguf, Value}; @@ -135,8 +151,8 @@ mod tests { clef().with("clip.projector_type", Value::String(projector.into())) } - /// The projector type picks the module; a stranger is refused by its name, which lists the - /// four this crate reads. + /// The projector type picks the module; a stranger is refused by its name, which lists + /// the types this crate reads. #[test] fn the_projector_type_picks_the_tower() { assert_eq!( @@ -155,7 +171,7 @@ mod tests { assert_eq!( projector(&table("qwen2vl_merger")).unwrap_err().to_string(), "metadata clip.projector_type: is \"qwen2vl_merger\"; this crate reads \ - \"deepseek41v\", \"glm5v\", \"minicpmv4_7\" and \"qwen3vl_merger\"" + \"deepseek41v\", \"gemma4v\", \"glm5v\", \"minicpmv4_7\" and \"qwen3vl_merger\"" ); assert_eq!( projector(&clef().with("clip.projector_type", Value::U32(7))) @@ -257,7 +273,7 @@ mod tests { assert_eq!( refused(&table("qwen2vl_merger"), "clip"), "metadata clip.projector_type: is \"qwen2vl_merger\"; this crate reads \ - \"deepseek41v\", \"glm5v\", \"minicpmv4_7\" and \"qwen3vl_merger\"" + \"deepseek41v\", \"gemma4v\", \"glm5v\", \"minicpmv4_7\" and \"qwen3vl_merger\"" ); assert!(refused(&table("qwen3vl_merger"), "llama").starts_with("metadata architecture:")); let huge = TokenLimits::new(16, 3_000_000).expect("limits"); diff --git a/crates/vision/src/arch/qwen3vl/card.rs b/crates/vision/src/arch/qwen3vl/card.rs index 441e1da26..69aa7f2b8 100644 --- a/crates/vision/src/arch/qwen3vl/card.rs +++ b/crates/vision/src/arch/qwen3vl/card.rs @@ -12,11 +12,11 @@ use std::collections::HashSet; use super::hparams::TILE_N; -use super::size::TokenLimits; use super::tensors::{Export, expected}; use super::{Hparams, TEMPORAL_FRAMES, names}; pub use crate::arch::card::CardBytes; use crate::arch::card::activation_bytes; +use crate::size::TokenLimits; /// The tower of a file of `hp` at `limits` image tokens (module doc). #[must_use] diff --git a/crates/vision/src/arch/qwen3vl/hparams.rs b/crates/vision/src/arch/qwen3vl/hparams.rs index 7e0561f08..f3a156326 100644 --- a/crates/vision/src/arch/qwen3vl/hparams.rs +++ b/crates/vision/src/arch/qwen3vl/hparams.rs @@ -9,12 +9,12 @@ use gguf::{Gguf, Value}; use super::PROJECTOR_TYPE; -use super::size::{SizeRule, TokenLimits}; use crate::VisionError; use crate::arch::header::{ KEY_HAS_VISION, Meta, check_architecture, check_eps, check_mean_std, check_projector, check_true, count_multiple, counts, need, refusal, }; +use crate::size::{SizeRule, TokenLimits}; const KEY_GELU: &str = "clip.use_gelu"; const KEY_DEEPSTACK: &str = "clip.vision.is_deepstack_layers"; diff --git a/crates/vision/src/arch/qwen3vl/mod.rs b/crates/vision/src/arch/qwen3vl/mod.rs index e58c66be5..1c2cb82e5 100644 --- a/crates/vision/src/arch/qwen3vl/mod.rs +++ b/crates/vision/src/arch/qwen3vl/mod.rs @@ -3,7 +3,7 @@ //! MLP, a learned 48×48 position table, a Conv3d patch embedding stored as two 16×16 kernels — //! and the merger (a 2×2 merge into a 4608-wide MLP). //! -//! The host side of an image is llama.cpp's mtmd rule for this projector: [`size`] plans the +//! The host side of an image is llama.cpp's mtmd rule for this projector: [`crate::size`] plans the //! pixel size, [`media`] pads the image onto a black canvas, normalizes it and cuts it in merge //! order with each patch repeated over the two temporal frames. @@ -11,12 +11,12 @@ pub mod card; pub mod hparams; pub mod media; pub mod names; -pub mod size; pub mod tensors; +pub use crate::size; +pub use crate::size::{SizeRule, TokenLimits}; pub use hparams::Hparams; pub use media::{Media, preprocess}; -pub use size::{SizeRule, TokenLimits}; /// `projector_type` of the files this module reads. pub const PROJECTOR_TYPE: &str = "qwen3vl_merger"; diff --git a/crates/vision/src/lib.rs b/crates/vision/src/lib.rs index 15c6ac850..1f33133a4 100644 --- a/crates/vision/src/lib.rs +++ b/crates/vision/src/lib.rs @@ -18,7 +18,7 @@ //! GGUF), each read once and refused by name when this crate does not run it, and what the //! projector's own image rule needs: V4.1's resize plan and span layout (`deepseek41v::grid`, //! `deepseek41v::span`), its reference `load_image` and its bytes on its card; Clef's size rule -//! and pad colour (`qwen3vl::size`, `qwen3vl::media`). +//! and pad colour (`size`, `qwen3vl::media`). //! //! The reference of V4.1 is deepseek-ai/DeepSeek-V4.1-Flash `inference/image_processor.py` and //! `inference/vision.py`, and Pillow's `libImaging/Resample.c` and `ImageOps.py` of the version @@ -31,6 +31,7 @@ pub mod media; pub mod mrope; pub mod preprocess; pub mod resample; +pub mod size; pub use image::{FileKind, Rgb8}; pub use media::{EncoderInput, MediaModel, Prepared}; diff --git a/crates/vision/src/media.rs b/crates/vision/src/media.rs index 15dafd5a8..4d5deda38 100644 --- a/crates/vision/src/media.rs +++ b/crates/vision/src/media.rs @@ -78,6 +78,10 @@ impl EncoderInput { /// A model's image input, as data plus one prepare. pub trait MediaModel { + /// Ordinary text around image rows in an embedding input; no placeholder token is needed. + fn embedding_delimiters(&self) -> Option<(&str, &str)> { + None + } /// The placeholder text the chat template carries for the image at `index`: its position /// among all the images of the prompt, from 0, in the order of the images. A model that /// numbers its images writes the number into the text; one that does not ignores it. @@ -96,10 +100,10 @@ pub trait MediaModel { None } - /// The id [`expand_spans`] finds in the tokenized prompt and expands to one copy + /// The optional id [`expand_spans`] finds in a tokenized chat prompt and expands to one copy /// per position of the image ([`Prepared::span_len`]): the one id V4.1's placeholder /// tokenizes to, and the `<|image_pad|>` between the two delimiter ids Clef's tokenizes to. - /// None when the model has no chat placeholder token. + /// `None` when images use embedding delimiters instead of a chat placeholder. fn image_token(&self) -> Option; /// The separator between a message's content parts when they are flattened into one text: diff --git a/crates/vision/src/arch/qwen3vl/size.rs b/crates/vision/src/size.rs similarity index 98% rename from crates/vision/src/arch/qwen3vl/size.rs rename to crates/vision/src/size.rs index 8533b56a5..9adc9f690 100644 --- a/crates/vision/src/arch/qwen3vl/size.rs +++ b/crates/vision/src/size.rs @@ -1,5 +1,5 @@ //! The pixel size an image is brought to before it is cut into patches: llama.cpp's mtmd rule -//! for this projector (`img_tool::calc_size_preserved_ratio`, the dyn-size preprocessor), ported +//! shared by dynamic-size projectors (`img_tool::calc_size_preserved_ratio`), ported //! operation for operation. //! //! Each side is rounded to a multiple of `align = patch * merge`; a size over the pixel limit is @@ -108,6 +108,10 @@ impl SizeRule { self.merge } + pub(crate) fn max_tokens(&self) -> usize { + self.limits.max() + } + /// Pixels a token covers on a side: `patch * merge`. #[must_use] pub fn align(&self) -> usize { From 159bf666e2209429e68396e4ae74544d2a9f8857 Mon Sep 17 00:00:00 2001 From: juliankang4 <200967947+juliankang4@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:06:52 +0900 Subject: [PATCH 05/12] gpu-vision: Gemma4 composes the shared tower kernels Keep the convolution, learned positions, residual stream and post norms in F32, with BF16 transformer linears and raster pooling. Add the F32 GEMM store and explicit F32 tap handlers; row operations expose only the modes their launcher emits. --- crates/gpu-vision/src/arch/gemma4v.rs | 863 ++++++++++++++++++++++++++ crates/gpu-vision/src/chain.rs | 31 +- crates/gpu-vision/src/gemm_bf16.rs | 100 ++- crates/gpu-vision/src/lib.rs | 12 +- crates/gpu-vision/src/norm.rs | 167 +++++ crates/gpu-vision/src/pointwise.rs | 482 ++++++++++++++ crates/gpu-vision/src/rope2d.rs | 19 + tools/unsafe-ratchet.txt | 6 +- 8 files changed, 1673 insertions(+), 7 deletions(-) create mode 100644 crates/gpu-vision/src/arch/gemma4v.rs create mode 100644 crates/gpu-vision/src/pointwise.rs diff --git a/crates/gpu-vision/src/arch/gemma4v.rs b/crates/gpu-vision/src/arch/gemma4v.rs new file mode 100644 index 000000000..11ebb7940 --- /dev/null +++ b/crates/gpu-vision/src/arch/gemma4v.rs @@ -0,0 +1,863 @@ +//! Gemma4 vision: F32 patch convolution and learned positions, BF16 ViT +//! linears, sandwich RMS norms, gainless V norm, scale-one non-causal +//! attention, the header's GELU GLU, three-by-three pooling and the input projection. +//! Audio tensors of the shared projector are never uploaded. +//! +//! Q/K rows and gains use [0..16, 32..48, 16..32, 48..64] in each head. +//! This expresses the two NEOX half-head turns through the shared whole-head +//! RoPE entry; V and the attention output keep their original channel order. +//! Residuals stay F32; the shared GEMMs and attention consume BF16 rows. + +use crate::{ + attn::{AttnArgs, AttnKernels, QkvLayout}, + chain::{Encoded, NoTaps, TapSink, read_bf16, read_f32, tap, tap_f32}, + gemm_bf16::{Epilogue, FloatGemmArgs, GemmArgs, GemmKernels}, + mean_groups::MeanGroupsKernels, + norm::{FloatNormArgs, HeadNormArgs, NormKernels}, + pointwise::{self, Op}, + rope2d::{RopeArgs, RopeKernels, RopeTable}, +}; +use bloomery_gpu::gemm::Gemm32Kernels; +use bloomery_gpu::{DeviceTensor, GpuError, WindowMut}; +use cuda_core::{CudaContext, CudaStream, DeviceBuffer}; +use gguf::Gguf; +use std::collections::BTreeMap; +use std::sync::Arc; +use vision::arch::gemma4v::{ + Hparams, MERGE, + card::{self, ScratchLens}, +}; +use vision::{Patches, arch::TokenLimits}; + +const WHAT: &str = "gemma4v tower"; + +struct Linear { + w: DeviceBuffer, + input: (f32, f32), + output: (f32, f32), +} +struct Weights { + patch: DeviceTensor, + pos: DeviceBuffer, + linear: BTreeMap, + norm: BTreeMap>, + std_bias: DeviceBuffer, + std_scale: DeviceBuffer, + ones: DeviceBuffer, + head_ones: DeviceBuffer, +} + +/// A resident Gemma4 vision tower and its bounded buffers, one image at a time. +pub struct Encoder { + hp: Hparams, + limits: TokenLimits, + standardize: bool, + w: Weights, + gemm: GemmKernels, + patch_gemm: Gemm32Kernels, + norm: NormKernels, + ops: pointwise::Kernels, + rope: RopeKernels, + attn: AttnKernels, + mean: MeanGroupsKernels, + s: Scratch, + weight_bytes: u64, + scratch_bytes: u64, + launches: usize, + stream: Arc, +} + +struct Scratch { + raw: DeviceBuffer, + patch: DeviceBuffer, + x: DeviceBuffer, + residual: DeviceBuffer, + h: DeviceBuffer, + h32: DeviceBuffer, + work: DeviceBuffer, + att: DeviceBuffer, + gathered: DeviceBuffer, + qkv: DeviceBuffer, + gate: DeviceBuffer, + up: DeviceBuffer, + act: DeviceBuffer, + tmp1: DeviceBuffer, + tmp2: DeviceBuffer, + pooled: DeviceBuffer, + scaled: DeviceBuffer, + pooled_norm: DeviceBuffer, + rows: DeviceBuffer, + cs: DeviceBuffer, +} + +impl Scratch { + fn new(stream: &CudaStream, l: ScratchLens) -> Result { + let b = |n| DeviceBuffer::::zeroed(stream, n); + Ok(Self { + raw: DeviceBuffer::zeroed(stream, l.raw)?, + patch: DeviceBuffer::zeroed(stream, l.hidden)?, + x: DeviceBuffer::zeroed(stream, l.hidden)?, + residual: DeviceBuffer::zeroed(stream, l.hidden)?, + h: b(l.hidden)?, + h32: DeviceBuffer::zeroed(stream, l.hidden)?, + work: b(l.hidden)?, + att: b(l.hidden)?, + gathered: b(l.hidden)?, + qkv: b(l.qkv)?, + gate: b(l.ff)?, + up: b(l.ff)?, + act: b(l.ff)?, + tmp1: b(l.ff)?, + tmp2: b(l.ff)?, + pooled: b(l.pooled)?, + scaled: b(l.pooled)?, + pooled_norm: b(l.pooled)?, + rows: b(l.rows)?, + cs: DeviceBuffer::zeroed(stream, 64 * l.patches)?, + }) + } + fn bytes(&self) -> u64 { + 4 * (self.raw.len() + + self.patch.len() + + self.cs.len() + + self.x.len() + + self.residual.len() + + self.h32.len()) as u64 + + 2 * [ + &self.h, + &self.work, + &self.att, + &self.gathered, + &self.qkv, + &self.gate, + &self.up, + &self.act, + &self.tmp1, + &self.tmp2, + &self.pooled, + &self.scaled, + &self.pooled_norm, + &self.rows, + ] + .iter() + .map(|b| b.len() as u64) + .sum::() + } +} + +fn permute(v: &[T], k: usize) -> Vec { + (0..v.len() / k) + .flat_map(|r| { + let h = r / 64 * 64; + let j = r % 64; + let src = h + [0, 32, 16, 48][j / 16] + j % 16; + v[src * k..(src + 1) * k].iter().copied() + }) + .collect() +} + +fn clamp(file: &Gguf, name: &str, side: &str) -> Result<(f32, f32), GpuError> { + let read = |suffix: &str, default| { + let key = name.replace(".weight", &format!(".{side}_{suffix}")); + if file.find(&key).is_none() { + return Ok(default); + } + let v = read_f32(file, &key, WHAT)?; + match v.as_slice() { + [x] if x.is_finite() => Ok(*x), + _ => Err(GpuError::shape( + WHAT, + format!("{key}: not one finite scalar"), + )), + } + }; + let bounds = (read("min", -f32::MAX)?, read("max", f32::MAX)?); + if bounds.0 > bounds.1 { + return Err(GpuError::shape( + WHAT, + format!("{name}: {side} clamp is reversed"), + )); + } + Ok(bounds) +} + +impl Encoder { + /// Load the BF16 vision half of `file` and buffers for `limits.max()` pooled tokens. + pub fn load( + ctx: &Arc, + stream: &Arc, + file: &Gguf, + limits: TokenLimits, + ) -> Result { + let hp = Hparams::read(file).map_err(|e| GpuError::plan(WHAT, e.to_string()))?; + let bytes = card::of(file, &hp, limits).map_err(|e| GpuError::plan(WHAT, e.to_string()))?; + let table = vision::arch::gemma4v::tensors::check(file, &hp) + .map_err(|e| GpuError::plan(WHAT, e.to_string()))?; + let mut linear = BTreeMap::new(); + let mut norm = BTreeMap::new(); + for t in &table { + if t.ty == gguf::GgmlType::BF16 { + let mut v = read_bf16(file, &t.name, WHAT)?; + if v.iter().any(|&bits| !crate::bf16_f32(bits).is_finite()) { + return Err(GpuError::shape( + WHAT, + format!("{} contains a non-finite weight", t.name), + )); + } + if t.name.ends_with(".attn_q.weight") || t.name.ends_with(".attn_k.weight") { + v = permute(&v, hp.dim); + } + linear.insert( + t.name.clone(), + Linear { + w: DeviceBuffer::from_host(stream, &v)?, + input: clamp(file, &t.name, "input")?, + output: clamp(file, &t.name, "output")?, + }, + ); + } else if t.name.starts_with("v.blk.") && t.name.ends_with(".weight") { + let mut v = read_f32(file, &t.name, WHAT)?; + if t.name.ends_with(".attn_q_norm.weight") + || t.name.ends_with(".attn_k_norm.weight") + { + v = permute(&v, 1); + } + if v.iter().any(|x| !x.is_finite()) { + return Err(GpuError::shape( + WHAT, + format!("{} contains a non-finite gain", t.name), + )); + } + norm.insert(t.name.clone(), DeviceBuffer::from_host(stream, &v)?); + } + } + let vector = |name, default| -> Result<_, GpuError> { + let v = if file.find(name).is_some() { + read_f32(file, name, WHAT)? + } else { + vec![default; hp.dim] + }; + if v.iter().any(|x| !x.is_finite()) { + return Err(GpuError::shape( + WHAT, + format!("{name} contains a non-finite value"), + )); + } + Ok(DeviceBuffer::from_host(stream, &v)?) + }; + let patch = read_f32(file, "v.patch_embd.weight", WHAT)?; + let pos = read_f32(file, "v.position_embd.weight", WHAT)?; + if patch.iter().chain(&pos).any(|x| !x.is_finite()) { + return Err(GpuError::shape(WHAT, "non-finite patch or position weight")); + } + let w = Weights { + patch: DeviceTensor::upload(stream, &patch, hp.dim, 3 * hp.patch * hp.patch)?, + pos: DeviceBuffer::from_host(stream, &pos)?, + linear, + norm, + std_bias: vector("v.std_bias", 0.0)?, + std_scale: vector("v.std_scale", 1.0)?, + ones: DeviceBuffer::from_host(stream, &vec![1.0; hp.dim])?, + head_ones: DeviceBuffer::from_host(stream, &[1.0f32; 64])?, + }; + let weight_bytes = 4 + * (w.patch.buf().len() + + w.pos.len() + + w.std_bias.len() + + w.std_scale.len() + + w.ones.len() + + w.head_ones.len() + + w.norm.values().map(DeviceBuffer::len).sum::()) as u64 + + 2 * w.linear.values().map(|l| l.w.len() as u64).sum::(); + let lens = + ScratchLens::new(&hp, limits.max()).map_err(|e| GpuError::plan(WHAT, e.to_string()))?; + let s = Scratch::new(stream, lens)?; + let gemm = GemmKernels::load(ctx, stream)?; + let scratch_bytes = s.bytes() + gemm.scratch_bytes(); + if (weight_bytes, scratch_bytes) != (bytes.weights, bytes.scratch) { + return Err(GpuError::shape( + WHAT, + format!( + "uploaded weights/scratch {weight_bytes}/{scratch_bytes}, reserved {}/{}", + bytes.weights, bytes.scratch + ), + )); + } + let standardize = file.find("v.std_bias").is_some(); + let clamps = w + .linear + .iter() + .map(|(name, w)| { + let post = name.ends_with(".attn_out.weight") || name.ends_with(".ffn_down.weight"); + usize::from(w.input != (-f32::MAX, f32::MAX)) + + if w.output == (-f32::MAX, f32::MAX) { + 0 + } else if post { + 1 + } else { + 2 + } + }) + .sum::(); + let launches = 7 + 22 * hp.n_layer + usize::from(standardize) + clamps; + Ok(Self { + hp, + limits, + standardize, + w, + s, + weight_bytes, + scratch_bytes, + launches, + stream: Arc::clone(stream), + gemm, + patch_gemm: Gemm32Kernels::load(ctx)?, + norm: NormKernels::load(ctx)?, + ops: pointwise::Kernels::load(ctx)?, + rope: RopeKernels::load(ctx)?, + attn: AttnKernels::load(ctx)?, + mean: MeanGroupsKernels::load(ctx)?, + }) + } + + /// Encode one raster patch grid with no intermediate readback. + pub fn encode(&mut self, patches: &Patches) -> Result, GpuError> { + self.encode_tapped(patches, &mut NoTaps) + } + + /// Encode one grid, handing requested graph-node rows to `taps`. + pub fn encode_tapped( + &mut self, + p: &Patches, + taps: &mut dyn TapSink, + ) -> Result, GpuError> { + let n = p + .n_vit_h + .checked_mul(p.n_vit_w) + .ok_or_else(|| GpuError::shape(WHAT, "patch count overflows"))?; + let raw = n + .checked_mul(p.patch_len) + .ok_or_else(|| GpuError::shape(WHAT, "patch values overflow"))?; + let hp = &self.hp; + let d = hp.dim; + if n == 0 + || n > 9 * self.limits.max() + || !p.n_vit_w.is_multiple_of(MERGE) + || !p.n_vit_h.is_multiple_of(MERGE) + || p.n_vit_w > hp.pos_rows + || p.n_vit_h > hp.pos_rows + || p.patch_len != 3 * hp.patch * hp.patch + || raw != p.bf16.len() + || p.plan.span_grid() != (p.n_vit_w / MERGE, p.n_vit_h / MERGE) + { + return Err(GpuError::shape( + WHAT, + "patch geometry, token budget or learned positions do not match", + )); + } + let stream = &self.stream; + let s = &mut self.s; + let input = vision::arch::gemma4v::conv_pixels(&p.bf16) + .map_err(|e| GpuError::shape(WHAT, e.to_string()))?; + WindowMut::of_mut(&mut s.raw, 0, raw)?.copy_from_host(stream, &input)?; + self.patch_gemm + .enqueue_f32_tile(stream, &self.w.patch, &s.raw, n, &mut s.patch)?; + self.ops.positions( + stream, + pointwise::PositionArgs { + x: &s.patch, + table: &self.w.pos, + rows: n, + width: d, + grid_w: p.n_vit_w, + pos_rows: hp.pos_rows, + out: &mut s.x, + }, + )?; + tap_f32(stream, taps, "pos_embd", &s.x, n, d)?; + let table = RopeTable::neox_halves(p.n_vit_h, p.n_vit_w, 100.0); + WindowMut::of_mut(&mut s.cs, 0, table.cs.len())?.copy_from_host(stream, &table.cs)?; + let c = Chain { + hp, + norm: &self.norm, + rope: &self.rope, + attn: &self.attn, + mean: &self.mean, + standardize: self.standardize, + gemm: &self.gemm, + ops: &self.ops, + w: &self.w, + }; + for l in 0..hp.n_layer { + c.attention(stream, s, n, l)?; + c.ffn(stream, s, n, l)?; + tap_f32(stream, taps, &format!("layer_out-{l}"), &s.x, n, d)?; + } + c.finish(stream, s, n, p.n_vit_w, taps)?; + let rows = n / 9; + Ok(Encoded { + rows: DeviceTensor::window_of(&s.rows, 0, rows, hp.out_dim)?, + launches: self.launches, + }) + } +} + +// Patch input and output buffers are free after embedding and hold the post-norm work. +struct PostScratch<'a> { + input: &'a mut DeviceBuffer, + raw: &'a mut DeviceBuffer, + clamped: &'a mut DeviceBuffer, + norm16: &'a mut DeviceBuffer, + norm32: &'a mut DeviceBuffer, +} + +struct Chain<'a> { + hp: &'a Hparams, + norm: &'a NormKernels, + rope: &'a RopeKernels, + attn: &'a AttnKernels, + mean: &'a MeanGroupsKernels, + standardize: bool, + gemm: &'a GemmKernels, + ops: &'a pointwise::Kernels, + w: &'a Weights, +} +impl Chain<'_> { + fn attention( + &self, + stream: &CudaStream, + s: &mut Scratch, + n: usize, + l: usize, + ) -> Result<(), GpuError> { + let hp = self.hp; + let d = hp.dim; + let name = |stem| format!("v.blk.{l}.{stem}.weight"); + self.norm.enqueue_f32( + stream, + FloatNormArgs { + x: &s.x, + gain: self.gain(&name("ln1"))?, + eps: hp.eps, + rows: n, + to_f32: false, + y16: &mut s.h, + y32: &mut s.h32, + }, + )?; + for (stem, col) in [("attn_q", 0), ("attn_k", d), ("attn_v", 2 * d)] { + self.linear( + stream, + &name(stem), + &s.h, + (n, d, d), + &mut s.work, + (&mut s.tmp1, &mut s.tmp2), + )?; + self.op( + stream, + Op::Copy, + &s.work, + &s.work, + (n, d, 3 * d, col), + &mut s.qkv, + )?; + } + for (stem, col) in [("attn_q_norm", 0), ("attn_k_norm", d)] { + self.norm.enqueue_heads( + stream, + HeadNormArgs { + x: &mut s.qkv, + gain: self.gain(&name(stem))?, + eps: hp.eps, + rows: n, + row_width: 3 * d, + col0: col, + n_heads: hp.n_head, + }, + )?; + } + self.norm.enqueue_heads( + stream, + HeadNormArgs { + x: &mut s.qkv, + gain: &self.w.head_ones, + eps: hp.eps, + rows: n, + row_width: 3 * d, + col0: 2 * d, + n_heads: hp.n_head, + }, + )?; + self.rope.enqueue( + stream, + RopeArgs { + cs: &s.cs, + n, + row_width: 3 * d, + q0: 0, + k0: d, + n_heads: hp.n_head, + x: &mut s.qkv, + }, + )?; + self.attn.enqueue( + stream, + AttnArgs { + qkv: &s.qkv, + layout: QkvLayout { + row_width: 3 * d, + q0: 0, + k0: d, + v0: 2 * d, + n_heads: hp.n_head, + }, + n, + scale: 1.0, + out_width: d, + out: &mut s.att, + }, + )?; + self.post( + stream, + (&name("attn_out"), &s.att, d), + &name("attn_post_norm"), + n, + PostScratch { + input: &mut s.tmp1, + raw: &mut s.raw, + clamped: &mut s.patch, + norm16: &mut s.h, + norm32: &mut s.h32, + }, + )?; + self.ops + .add_f32(stream, &s.x, &s.h32, n * d, &mut s.residual)?; + Ok(()) + } + fn ffn( + &self, + stream: &CudaStream, + s: &mut Scratch, + n: usize, + l: usize, + ) -> Result<(), GpuError> { + let hp = self.hp; + let (d, f) = (hp.dim, hp.ff); + let name = |stem| format!("v.blk.{l}.{stem}.weight"); + self.norm.enqueue_f32( + stream, + FloatNormArgs { + x: &s.residual, + gain: self.gain(&name("ln2"))?, + eps: hp.eps, + rows: n, + to_f32: false, + y16: &mut s.h, + y32: &mut s.h32, + }, + )?; + self.linear( + stream, + &name("ffn_gate"), + &s.h, + (n, f, d), + &mut s.gate, + (&mut s.tmp1, &mut s.tmp2), + )?; + self.linear( + stream, + &name("ffn_up"), + &s.h, + (n, f, d), + &mut s.up, + (&mut s.tmp1, &mut s.tmp2), + )?; + let op = if hp.gelu_tanh { + Op::GeGlu + } else { + Op::QuickGeGlu + }; + self.op(stream, op, &s.gate, &s.up, (n, f, f, 0), &mut s.act)?; + self.post( + stream, + (&name("ffn_down"), &s.act, f), + &name("ffn_post_norm"), + n, + PostScratch { + input: &mut s.tmp1, + raw: &mut s.raw, + clamped: &mut s.patch, + norm16: &mut s.h, + norm32: &mut s.h32, + }, + )?; + self.ops + .add_f32(stream, &s.residual, &s.h32, n * d, &mut s.x)?; + Ok(()) + } + fn finish( + &self, + stream: &CudaStream, + s: &mut Scratch, + n: usize, + grid_w: usize, + taps: &mut dyn TapSink, + ) -> Result<(), GpuError> { + let hp = self.hp; + let d = hp.dim; + let rows = n / 9; + self.ops.pool_f32( + stream, + pointwise::FloatPoolArgs { + x: &s.x, + rows: n, + width: d, + grid_w, + out: &mut s.gathered, + }, + )?; + self.mean + .enqueue(stream, n, 9, d, &s.gathered, &mut s.pooled)?; + + self.ops.enqueue( + stream, + pointwise::Args { + op: Op::Scale, + x: &s.pooled, + b: &s.pooled, + gain: &self.w.ones, + bias: &self.w.std_bias, + rows, + width: d, + out_width: d, + out_col: 0, + scale: (d as f32).sqrt(), + clamp: (-f32::MAX, f32::MAX), + out: &mut s.scaled, + }, + )?; + tap(stream, taps, "pooled", &s.scaled, rows, d)?; + if self.standardize { + self.ops.enqueue( + stream, + pointwise::Args { + op: Op::Affine, + x: &s.scaled, + b: &s.scaled, + gain: &self.w.std_scale, + bias: &self.w.std_bias, + rows, + width: d, + out_width: d, + out_col: 0, + scale: 1.0, + clamp: (-f32::MAX, f32::MAX), + out: &mut s.pooled, + }, + )?; + tap(stream, taps, "std_scaled", &s.pooled, rows, d)?; + } + let pooled = if self.standardize { + &s.pooled + } else { + &s.scaled + }; + self.norm.enqueue( + stream, + pooled, + &self.w.ones, + hp.eps, + rows, + &mut s.pooled_norm, + )?; + self.linear( + stream, + "mm.input_projection.weight", + &s.pooled_norm, + (rows, hp.out_dim, d), + &mut s.rows, + (&mut s.tmp1, &mut s.tmp2), + )?; + tap(stream, taps, "projected", &s.rows, rows, hp.out_dim)?; + Ok(()) + } + + fn gain(&self, name: &str) -> Result<&DeviceBuffer, GpuError> { + self.w + .norm + .get(name) + .ok_or_else(|| GpuError::tensor(WHAT, name, "a norm gain")) + } + fn op( + &self, + stream: &CudaStream, + op: Op, + x: &DeviceBuffer, + b: &DeviceBuffer, + shape: (usize, usize, usize, usize), + out: &mut DeviceBuffer, + ) -> Result<(), GpuError> { + let (rows, width, out_width, out_col) = shape; + self.ops.enqueue( + stream, + pointwise::Args { + op, + x, + b, + gain: &self.w.ones, + bias: &self.w.std_bias, + rows, + width, + out_width, + out_col, + scale: 1.0, + clamp: (-f32::MAX, f32::MAX), + out, + }, + ) + } + fn post( + &self, + stream: &CudaStream, + (name, x, k): (&str, &DeviceBuffer, usize), + gain: &str, + n: usize, + s: PostScratch<'_>, + ) -> Result<(), GpuError> { + let d = self.hp.dim; + let w = self + .w + .linear + .get(name) + .ok_or_else(|| GpuError::tensor(WHAT, name, "a BF16 linear weight"))?; + let input = if w.input != (-f32::MAX, f32::MAX) { + self.ops.enqueue( + stream, + pointwise::Args { + op: Op::Clamp, + x, + b: x, + gain: &self.w.ones, + bias: &self.w.std_bias, + rows: n, + width: k, + out_width: k, + out_col: 0, + scale: 1.0, + clamp: w.input, + out: s.input, + }, + )?; + &*s.input + } else { + x + }; + self.gemm.enqueue_f32( + stream, + FloatGemmArgs { + a: input, + b: &w.w, + m: n, + n: d, + k, + c: s.raw, + }, + )?; + let input = if w.output != (-f32::MAX, f32::MAX) { + self.ops + .clamp_f32(stream, s.raw, w.output, n * d, s.clamped)?; + &*s.clamped + } else { + &*s.raw + }; + self.norm.enqueue_f32( + stream, + FloatNormArgs { + x: input, + gain: self.gain(gain)?, + eps: self.hp.eps, + rows: n, + to_f32: true, + y16: s.norm16, + y32: s.norm32, + }, + ) + } + + fn linear( + &self, + stream: &CudaStream, + name: &str, + x: &DeviceBuffer, + shape: (usize, usize, usize), + y: &mut DeviceBuffer, + temp: (&mut DeviceBuffer, &mut DeviceBuffer), + ) -> Result<(), GpuError> { + let (n, out, k) = shape; + let w = self + .w + .linear + .get(name) + .ok_or_else(|| GpuError::tensor(WHAT, name, "a BF16 linear weight"))?; + let clamp_into = |x: &DeviceBuffer, width, bounds, out: &mut DeviceBuffer| { + self.ops.enqueue( + stream, + pointwise::Args { + op: Op::Clamp, + x, + b: x, + gain: &self.w.ones, + bias: &self.w.std_bias, + rows: n, + width, + out_width: width, + out_col: 0, + scale: 1.0, + clamp: bounds, + out, + }, + ) + }; + let src = if w.input != (-f32::MAX, f32::MAX) { + clamp_into(x, k, w.input, temp.0)?; + &*temp.0 + } else { + x + }; + self.gemm.enqueue( + stream, + GemmArgs { + a: src, + b: &w.w, + bias: None, + epilogue: Epilogue::None, + resid: None, + m: n, + n: out, + k, + c: y, + }, + )?; + if w.output != (-f32::MAX, f32::MAX) { + clamp_into(y, out, w.output, temp.1)?; + self.op(stream, Op::Copy, temp.1, temp.1, (n, out, out, 0), y)?; + } + Ok(()) + } +} + +impl crate::Tower for Encoder { + fn encode(&mut self, p: &Patches) -> Result, GpuError> { + Encoder::encode(self, p) + } + fn out_dim(&self) -> usize { + self.hp.out_dim + } + fn weight_bytes(&self) -> u64 { + self.weight_bytes + } + fn scratch_bytes(&self) -> u64 { + self.scratch_bytes + } +} diff --git a/crates/gpu-vision/src/chain.rs b/crates/gpu-vision/src/chain.rs index 7b2a0ac69..fd3e9b39f 100644 --- a/crates/gpu-vision/src/chain.rs +++ b/crates/gpu-vision/src/chain.rs @@ -11,11 +11,19 @@ use gguf::{GgmlType, Gguf}; /// Where an encode's intermediate tensors go when a caller asks for them (a gate). `wants` is /// asked before each named point; a `true` synchronizes the stream and hands the tensor over -/// as bf16 bits, `cols` values per row. The names are each tower's own (listed in its module +/// through `take` for BF16 or `take_f32` for F32, `cols` values per row. Every sink handles both +/// storage formats. The names are each tower's own (listed in its module /// doc), the oracle's file stems or graph node names a gate maps them to. pub trait TapSink { fn wants(&self, name: &str) -> bool; fn take(&mut self, name: &str, cols: usize, bits: Vec); + fn take_f32(&mut self, name: &str, cols: usize, values: Vec) { + self.take( + name, + cols, + values.into_iter().map(crate::f32_bf16).collect(), + ); + } } /// The sink of a plain encode. @@ -26,6 +34,7 @@ impl TapSink for NoTaps { false } fn take(&mut self, _: &str, _: usize, _: Vec) {} + fn take_f32(&mut self, _: &str, _: usize, _: Vec) {} } /// An encode's result: the rows the text model reads in place of an image's tokens (`out_dim` bf16 @@ -53,6 +62,26 @@ pub(crate) fn tap( Ok(()) } +/// Hand an F32 residual stream to a sink without changing its storage precision. +pub(crate) fn tap_f32( + stream: &CudaStream, + taps: &mut dyn TapSink, + name: &str, + buf: &DeviceBuffer, + rows: usize, + cols: usize, +) -> Result<(), GpuError> { + if taps.wants(name) { + stream.synchronize()?; + taps.take_f32( + name, + cols, + Window::::of(buf, 0, rows * cols)?.to_host_vec(stream)?, + ); + } + Ok(()) +} + /// The first `len` values of `buf`, after the stream's work so far. pub(crate) fn to_host( stream: &CudaStream, diff --git a/crates/gpu-vision/src/gemm_bf16.rs b/crates/gpu-vision/src/gemm_bf16.rs index b867186b4..ae28c333e 100644 --- a/crates/gpu-vision/src/gemm_bf16.rs +++ b/crates/gpu-vision/src/gemm_bf16.rs @@ -2,6 +2,7 @@ //! accumulation on `mma.sync.m16n8k16`, bf16 output, with the epilogues the encoder's linear //! layers need. `B` is a linear layer's weight as the file stores it (one row per output), so no //! operand is transposed on the host. +//! The unbiased F32 store shares the same accumulator walk and keeps the result for a following F32 norm. //! //! Tiling: one block of [`THREADS`] threads computes a [`TILE_M`] × [`TILE_N`] tile of `C`; its four //! warps each own a 32 × 32 quarter (2 m16 × 4 n8 fragments, 32 f32 accumulators per lane). The K @@ -90,7 +91,7 @@ pub(crate) fn gelu_erf_dev(x: f32) -> f32 { mul_rn_f32(mul_rn_f32(x, 0.5), add_rn_f32(1.0, e)) } -/// One tensor-core accumulator walk for the BF16 GEMM entry. +/// One tensor-core accumulator walk for both output storage formats. macro_rules! bf16_accumulate { ($a:ident, $b:ident, $m:ident, $n:ident, $k:ident; $acc:ident, $rows:ident, $cols:ident, $tm:ident, $tn:ident, $lane:ident, @@ -330,6 +331,44 @@ mod gemm_kernels { f += 1; } } + /// Unbiased BF16 operands with the same tensor-core accumulation, stored in F32. + #[kernel] + #[launch_bounds(128)] + #[launch_contract(domain = 1, block = (128, 1, 1), + requires = (a.len() >= m * k, b.len() >= n * k, c.len() >= m * n))] + pub fn vis_gemm_bf16_f32( + a: &[u16], + b: &[u16], + m: u32, + n: u32, + k: u32, + mut c: DisjointSlice, + ) { + bf16_accumulate!(a, b, m, n, k; acc, rows, cols, tm, tn, lane, wm, wn, AS, BS); + + let group = lane / 4; + let t4 = lane % 4; + let mut f = 0usize; + while f < 8 { + cuda_device::thread::__unroll_config::<0>(); + let mi = f / 4; + let ni = f % 4; + let mut j = 0usize; + while j < 4 { + cuda_device::thread::__unroll_config::<0>(); + let row = tm * TILE_M + wm * 32 + mi * 16 + group + 8 * (j / 2); + let col = tn * TILE_N + wn * 32 + ni * 8 + 2 * t4 + j % 2; + if row < rows { + // SAFETY: the tile covers n columns exactly and each live cell has one owner lane. + unsafe { + *c.get_unchecked_mut(row * cols + col) = acc[f][j]; + } + } + j += 1; + } + f += 1; + } + } } // -------------------------------------------------------------- launchers @@ -348,6 +387,16 @@ pub struct GemmArgs<'a> { pub c: &'a mut DeviceBuffer, } +/// An unbiased BF16 GEMM whose accumulator is stored as F32. +pub(crate) struct FloatGemmArgs<'a> { + pub a: &'a DeviceBuffer, + pub b: &'a DeviceBuffer, + pub m: usize, + pub n: usize, + pub k: usize, + pub c: &'a mut DeviceBuffer, +} + /// The loaded GEMM module. Owns no stream: each enqueue takes the caller's stream. pub struct GemmKernels { module: gemm_kernels::LoadedModule, @@ -358,6 +407,10 @@ pub struct GemmKernels { } impl GemmKernels { + /// Bytes of the unused-operand stand-ins this module holds on the card. + pub(crate) fn scratch_bytes(&self) -> u64 { + 4 * self.dummy_f32.len() as u64 + 2 * self.dummy_u16.len() as u64 + } /// Load this file's device bundle into `ctx`. Load-time only. pub fn load(ctx: &Arc, stream: &CudaStream) -> Result { // SAFETY: this crate owns the embedded device bundle produced for the module above; the @@ -370,6 +423,51 @@ impl GemmKernels { }) } + /// Enqueue an unbiased BF16 GEMM into F32 storage, asynchronously and without allocation. + pub(crate) fn enqueue_f32( + &self, + stream: &CudaStream, + a: FloatGemmArgs<'_>, + ) -> Result<(), GpuError> { + let what = "GemmKernels::enqueue_f32"; + if a.m == 0 || a.n == 0 || !a.n.is_multiple_of(TILE_N) || a.k == 0 || !a.k.is_multiple_of(2) + { + return Err(GpuError::shape( + what, + "need positive m, n a TILE_N multiple and positive even k", + )); + } + let extent = |x: usize, y| { + x.checked_mul(y) + .ok_or_else(|| GpuError::shape(what, "extent overflows")) + }; + let (mk, nk, mn) = (extent(a.m, a.k)?, extent(a.n, a.k)?, extent(a.m, a.n)?); + for count in [mk, nk, mn] { + launch_u32(what, "element count", count)?; + } + if a.a.len() < mk || a.b.len() < nk || a.c.len() < mn { + return Err(GpuError::shape( + what, + "operands do not hold the matrix extents", + )); + } + let grid = launch_u32(what, "grid", a.m.div_ceil(TILE_M) * (a.n / TILE_N))?; + let prep = + self.module + .prepare_vis_gemm_bf16_f32(LaunchConfig1D::new(grid, THREADS_U32, 0))?; + self.module.vis_gemm_bf16_f32( + stream, + &prep, + a.a, + a.b, + launch_u32(what, "m", a.m)?, + launch_u32(what, "n", a.n)?, + launch_u32(what, "k", a.k)?, + a.c, + )?; + Ok(()) + } + /// Enqueue one GEMM ([`GemmArgs`]). `n` a multiple of [`TILE_N`], `k` even and non-zero, `m` /// non-zero; a `Residual` epilogue needs `resid`, and a `resid` without it is refused. /// Asynchronous, allocation-free. diff --git a/crates/gpu-vision/src/lib.rs b/crates/gpu-vision/src/lib.rs index 47d0969ed..584443b6b 100644 --- a/crates/gpu-vision/src/lib.rs +++ b/crates/gpu-vision/src/lib.rs @@ -57,12 +57,15 @@ pub mod chain; pub mod encoder; pub mod gelu_tanh; pub mod gemm_bf16; +#[path = "arch/gemma4v.rs"] +pub mod gemma4v; pub mod glm5v; pub mod layer_norm; pub mod mean_groups; pub mod minicpmv; pub mod mlp; pub mod norm; +pub(crate) mod pointwise; pub mod pos_bilinear; pub mod qwen3vl; pub mod rope2d; @@ -96,7 +99,7 @@ use vision::arch::{ /// One image tower on the card, whichever projector type its file declares. A call takes /// `&mut self`: one image at a time owns the tower's activations, which are allocated at load, and /// the rows it returns borrow them until the next call. -pub trait Tower { +pub trait Tower: Send { /// Encode one image's patches into the rows of its tokens, without waiting for the card; /// read them on the stream the tower was opened with. fn encode(&mut self, patches: &Patches) -> Result, GpuError>; @@ -140,10 +143,11 @@ pub fn open( file: &Gguf, limits: TokenLimits, ) -> Result, GpuError> { - let projector = file - .value("clip.projector_type") - .and_then(gguf::Value::as_str); + let projector = vision::arch::projector_type(file).ok(); match projector { + Some(vision::arch::gemma4v::PROJECTOR_TYPE) => { + Ok(Box::new(gemma4v::Encoder::load(ctx, stream, file, limits)?)) + } Some(deepseek41v::PROJECTOR_TYPE) => { Ok(Box::new(encoder::Encoder::load(ctx, stream, file)?)) } diff --git a/crates/gpu-vision/src/norm.rs b/crates/gpu-vision/src/norm.rs index e09dbbeab..36714a0f4 100644 --- a/crates/gpu-vision/src/norm.rs +++ b/crates/gpu-vision/src/norm.rs @@ -188,6 +188,118 @@ mod norm_kernels { } } + /// RMSNorm of an F32 residual stream, stored as F32 or narrowed for a BF16 GEMM. + #[allow( + clippy::too_many_arguments, + reason = "kernel entry uses the flat device ABI (rust-quality R8)" + )] + #[kernel] + #[launch_bounds(256)] + #[launch_contract( + domain = 1, + block = (256, 1, 1), + requires = (x.len() >= rows * dim, gain.len() >= dim, y16.len() >= rows * dim, y32.len() >= rows * dim) + )] + pub fn vis_rms_norm_f32( + x: &[f32], + gain: &[f32], + eps: f32, + dim: u32, + rows: u32, + to_f32: u32, + mut y16: DisjointSlice, + mut y32: DisjointSlice, + ) { + static mut WSUM: SharedArray = SharedArray::UNINIT; + + let r = thread::blockIdx_x() as usize; + if r >= rows as usize { + return; + } + let tid = thread::threadIdx_x() as usize; + let d = dim as usize; + let words = d / 2; + let base = r * d; + let mut acc = 0.0f32; + let mut w = tid; + while w < words { + let at = base + 2 * w; + // SAFETY: r < rows and 2w < dim keep at inside the input row. + let a = unsafe { *x.get_unchecked(at) }; + // SAFETY: r < rows and 2w + 1 < dim keep at + 1 inside the input row. + let b = unsafe { *x.get_unchecked(at + 1) }; + acc = fma_rn_f32(a, a, acc); + acc = fma_rn_f32(b, b, acc); + w += THREADS; + } + acc = add_rn_f32(acc, warp::shuffle_xor_f32(acc, 16)); + acc = add_rn_f32(acc, warp::shuffle_xor_f32(acc, 8)); + acc = add_rn_f32(acc, warp::shuffle_xor_f32(acc, 4)); + acc = add_rn_f32(acc, warp::shuffle_xor_f32(acc, 2)); + acc = add_rn_f32(acc, warp::shuffle_xor_f32(acc, 1)); + // SAFETY: block-shared, WARPS == blockDim.x / 32, written before the barrier that + // publishes it. + let ws = unsafe { SharedArray::as_raw_mut_ptr(&raw mut WSUM) }; + if warp::lane_id() == 0 { + // SAFETY: tid / 32 < WARPS bounds the slot; lane 0 is its one writer. + unsafe { + *ws.add(tid / 32) = acc; + } + } + thread::sync_threads(); + // SAFETY: the eight slots, published by the barrier above. + let sums = unsafe { + [ + *ws, + *ws.add(1), + *ws.add(2), + *ws.add(3), + *ws.add(4), + *ws.add(5), + *ws.add(6), + *ws.add(7), + ] + }; + let mean = div_rn_f32(warp_tree(sums), d as f32); + let scale = div_rn_f32(1.0, sqrt_rn_f32(add_rn_f32(mean, eps))); + let mut w = tid; + while w < words { + let at = base + 2 * w; + // SAFETY: r < rows and 2*w < d bound this input lane. + let a = unsafe { *x.get_unchecked(at) }; + // SAFETY: r < rows and 2*w + 1 < d bound this input lane. + let b = unsafe { *x.get_unchecked(at + 1) }; + // SAFETY: 2*w < d <= gain.len(). + let g0 = unsafe { *gain.get_unchecked(2 * w) }; + // SAFETY: 2*w + 1 < d <= gain.len(). + let g1 = unsafe { *gain.get_unchecked(2 * w + 1) }; + let y0 = mul_rn_f32(g0, mul_rn_f32(a, scale)); + let y1 = mul_rn_f32(g1, mul_rn_f32(b, scale)); + if to_f32 != 0 { + // SAFETY: at < rows*d <= y32.len(); this thread owns this lane. + unsafe { + *y32.get_unchecked_mut(at) = y0; + } + // SAFETY: at + 1 < rows*d <= y32.len(); this thread owns this lane. + unsafe { + *y32.get_unchecked_mut(at + 1) = y1; + } + } else { + let y0 = f32_to_bf16_rne(y0); + // SAFETY: at < rows*d <= y16.len(); this thread owns this lane. + unsafe { + *y16.get_unchecked_mut(at) = y0; + } + let y1 = f32_to_bf16_rne(y1); + // SAFETY: at + 1 < rows*d <= y16.len(); this thread owns this lane. + unsafe { + *y16.get_unchecked_mut(at + 1) = y1; + } + } + w += THREADS; + } + } + /// `x = RMSNorm(x)` in place over the `n_heads` heads of [`HEAD`] bf16 values at columns /// `col0 + h·64` of each of `rows` rows of `row_width`, every head with the gain of 64: the /// module rule per head. One warp per (row, head), eight to a block; lane `t` owns the pair at @@ -265,6 +377,17 @@ pub struct NormKernels { module: norm_kernels::LoadedModule, } +/// Both output buffers hold the row extent; only the selected storage format is written. +pub(crate) struct FloatNormArgs<'a> { + pub x: &'a DeviceBuffer, + pub gain: &'a DeviceBuffer, + pub eps: f32, + pub rows: usize, + pub to_f32: bool, + pub y16: &'a mut DeviceBuffer, + pub y32: &'a mut DeviceBuffer, +} + impl NormKernels { /// Load this file's device bundle into `ctx`. Load-time only. pub fn load(ctx: &Arc) -> Result { @@ -318,6 +441,50 @@ impl NormKernels { )?; Ok(()) } + pub(crate) fn enqueue_f32( + &self, + stream: &CudaStream, + a: FloatNormArgs<'_>, + ) -> Result<(), GpuError> { + let what = "NormKernels::enqueue_f32"; + let dim = a.gain.len(); + let n = a + .rows + .checked_mul(dim) + .ok_or_else(|| GpuError::shape(what, "element count overflows"))?; + launch_u32(what, "element count", n)?; + if dim == 0 + || !dim.is_multiple_of(2) + || a.rows == 0 + || a.x.len() < n + || a.y16.len() < n + || a.y32.len() < n + { + return Err(GpuError::shape( + what, + "the input and both output buffers must hold every even-width row", + )); + } + let prep = self.module.prepare_vis_rms_norm_f32(LaunchConfig1D::new( + launch_u32(what, "rows", a.rows)?, + THREADS_U32, + 0, + ))?; + self.module.vis_rms_norm_f32( + stream, + &prep, + a.x, + a.gain, + a.eps, + launch_u32(what, "dim", dim)?, + launch_u32(what, "rows", a.rows)?, + u32::from(a.to_f32), + a.y16, + a.y32, + )?; + Ok(()) + } + /// Enqueue the per-head norm ([`HeadNormArgs`]): in place, one launch per segment. The gain is /// [`HEAD`] values; the heads must lie inside the row. Asynchronous. pub fn enqueue_heads( diff --git a/crates/gpu-vision/src/pointwise.rs b/crates/gpu-vision/src/pointwise.rs new file mode 100644 index 000000000..205212059 --- /dev/null +++ b/crates/gpu-vision/src/pointwise.rs @@ -0,0 +1,482 @@ +//! Small row operations between a tower's shared GEMMs and norms. + +use bloomery_gpu::{GpuError, launch_u32}; +use cuda_core::{CudaContext, CudaStream, DeviceBuffer, LaunchConfig1D}; +use cuda_device::convert::{bf16_to_f32, f32_to_bf16_rne}; +use cuda_device::float::{add_rn_f32, div_rn_f32, mul_rn_f32}; +use cuda_device::{DisjointSlice, kernel, launch_bounds, launch_contract, thread}; +use cuda_host::cuda_module; +use std::sync::Arc; + +/// A row operation, each with a separate f32 rounding at its arithmetic boundaries. +#[derive(Clone, Copy)] +pub(crate) enum Op { + GeGlu, + QuickGeGlu, + Affine, + Clamp, + Copy, + Scale, +} + +#[cuda_module] +mod kernels { + use super::*; + + /// Apply one row operation, writing each destination lane once. + #[allow( + clippy::too_many_arguments, + reason = "kernel entry uses the flat device ABI (rust-quality R8)" + )] + #[kernel] + #[launch_bounds(256)] + #[launch_contract(domain = 1, block = (256, 1, 1), requires = (x.len() >= n, out.len() >= n))] + pub(crate) fn vis_row_op( + x: &[u16], + b: &[u16], + gain: &[f32], + bias: &[f32], + n: u32, + width: u32, + stride: u32, + col0: u32, + op: u32, + scale: f32, + lo: f32, + hi: f32, + mut out: DisjointSlice, + ) { + let i = thread::index_1d().get(); + if i >= n as usize { + return; + } + let w = width as usize; + let row = i / w; + let d = i % w; + // SAFETY: i < n <= x.len(). + let a = bf16_to_f32(unsafe { *x.get_unchecked(i) }); + let y = match op { + 1 | 7 => { + // SAFETY: both gated operations require b.len() >= n; i < n. + let other = bf16_to_f32(unsafe { *b.get_unchecked(i) }); + if op == 1 { + mul_rn_f32(bloomery_gpu::gelu::gelu_tanh_dev(a), other) + } else { + let e = mul_rn_f32(-1.702, a).exp(); + let gelu = mul_rn_f32(a, div_rn_f32(1.0, add_rn_f32(1.0, e))); + mul_rn_f32(gelu, other) + } + } + 2 => { + // SAFETY: Affine checks gain at width; d < width. + let gain = unsafe { *gain.get_unchecked(d) }; + // SAFETY: Affine checks bias at width; d < width. + let bias = unsafe { *bias.get_unchecked(d) }; + mul_rn_f32(add_rn_f32(mul_rn_f32(a, scale), -bias), gain) + } + 3 => { + if a < lo { + lo + } else if a > hi { + hi + } else { + a + } + } + 6 => mul_rn_f32(a, scale), + _ => a, + }; + let dst = row * stride as usize + col0 as usize + d; + // SAFETY: the host proves the row extent and col0 + width <= stride; + // each source lane has one distinct destination lane. + unsafe { + *out.get_unchecked_mut(dst) = f32_to_bf16_rne(y); + } + } + + /// Clamp F32 values without narrowing the result. + #[kernel] + #[launch_bounds(256)] + #[launch_contract(domain = 1, block = (256, 1, 1), requires = (x.len() >= n, out.len() >= n))] + pub(crate) fn vis_clamp_f32(x: &[f32], n: u32, lo: f32, hi: f32, mut out: DisjointSlice) { + let i = thread::index_1d().get(); + if i >= n as usize { + return; + } + // SAFETY: i < n <= x.len(). + let value = unsafe { *x.get_unchecked(i) }; + let value = if value < lo { + lo + } else if value > hi { + hi + } else { + value + }; + // SAFETY: i < n <= out.len(); thread i owns this output. + unsafe { + *out.get_unchecked_mut(i) = value; + } + } + + /// Add two F32 residual streams without an intervening narrowing. + #[kernel] + #[launch_bounds(256)] + #[launch_contract(domain = 1, block = (256, 1, 1), requires = (a.len() >= n, b.len() >= n, out.len() >= n))] + pub(crate) fn vis_add_f32(a: &[f32], b: &[f32], n: u32, mut out: DisjointSlice) { + let i = thread::index_1d().get(); + if i >= n as usize { + return; + } + // SAFETY: i < n <= a.len(). + let av = unsafe { *a.get_unchecked(i) }; + // SAFETY: i < n <= b.len(). + let bv = unsafe { *b.get_unchecked(i) }; + let value = add_rn_f32(av, bv); + // SAFETY: i < n <= out.len(); thread i owns this output. + unsafe { + *out.get_unchecked_mut(i) = value; + } + } + + /// Gather three-by-three raster cells and narrow each value for the shared group mean. + #[kernel] + #[launch_bounds(256)] + #[launch_contract(domain = 1, block = (256, 1, 1), requires = (x.len() >= n, out.len() >= n))] + pub(crate) fn vis_pool_order_f32( + x: &[f32], + n: u32, + width: u32, + grid_w: u32, + mut out: DisjointSlice, + ) { + let i = thread::index_1d().get(); + if i >= n as usize { + return; + } + let w = width as usize; + let row = i / w; + let group = row / 9; + let cell = row % 9; + let cols = grid_w as usize / 3; + let src = ((group / cols * 3 + cell / 3) * grid_w as usize + group % cols * 3 + cell % 3) + * w + + i % w; + // SAFETY: complete three-by-three groups permute n lanes, so src < n <= x.len(). + let value = unsafe { *x.get_unchecked(src) }; + let value = f32_to_bf16_rne(value); + // SAFETY: i < n <= out.len(); thread i owns this output. + unsafe { + *out.get_unchecked_mut(i) = value; + } + } + + /// Add the learned x and y rows to the F32 residual stream. + #[kernel] + #[launch_bounds(256)] + #[launch_contract(domain = 1, block = (256, 1, 1), requires = (x.len() >= n, out.len() >= n))] + pub(crate) fn vis_position_rows( + x: &[f32], + table: &[f32], + n: u32, + width: u32, + grid_w: u32, + pos_rows: u32, + mut out: DisjointSlice, + ) { + let i = thread::index_1d().get(); + if i >= n as usize { + return; + } + let w = width as usize; + let row = i / w; + let d = i % w; + let px = row % grid_w as usize; + let py = row / grid_w as usize; + // SAFETY: i < n <= x.len(). + let x = unsafe { *x.get_unchecked(i) }; + // SAFETY: px < pos_rows and d < width bound the first table plane. + let px = unsafe { *table.get_unchecked(px * w + d) }; + // SAFETY: py < pos_rows and d < width bound the second table plane. + let py = unsafe { *table.get_unchecked((pos_rows as usize + py) * w + d) }; + let a = add_rn_f32(x, px); + let y = add_rn_f32(a, py); + if let Some(dst) = out.get_mut(thread::index_1d()) { + *dst = y; + } + } +} + +/// Operands and extents of one row operation; `out_col` places a segment inside wider rows. +pub(crate) struct Args<'a> { + pub op: Op, + pub x: &'a DeviceBuffer, + pub b: &'a DeviceBuffer, + pub gain: &'a DeviceBuffer, + pub bias: &'a DeviceBuffer, + pub rows: usize, + pub width: usize, + pub out_width: usize, + pub out_col: usize, + pub scale: f32, + pub clamp: (f32, f32), + pub out: &'a mut DeviceBuffer, +} + +/// The learned position addition's input and output. +pub(crate) struct PositionArgs<'a> { + pub x: &'a DeviceBuffer, + pub table: &'a DeviceBuffer, + pub rows: usize, + pub width: usize, + pub grid_w: usize, + pub pos_rows: usize, + pub out: &'a mut DeviceBuffer, +} + +pub(crate) struct FloatPoolArgs<'a> { + pub x: &'a DeviceBuffer, + pub rows: usize, + pub width: usize, + pub grid_w: usize, + pub out: &'a mut DeviceBuffer, +} + +/// Loaded row-operation kernels. +pub(crate) struct Kernels { + module: kernels::LoadedModule, +} + +impl Kernels { + pub(crate) fn clamp_f32( + &self, + stream: &CudaStream, + x: &DeviceBuffer, + bounds: (f32, f32), + n: usize, + out: &mut DeviceBuffer, + ) -> Result<(), GpuError> { + let what = "vision F32 clamp"; + if n == 0 + || x.len() < n + || out.len() < n + || !bounds.0.is_finite() + || !bounds.1.is_finite() + || bounds.0 > bounds.1 + { + return Err(GpuError::shape(what, "invalid clamp bounds or buffers")); + } + let prep = self.module.prepare_vis_clamp_f32(LaunchConfig1D::new( + launch_u32(what, "grid", n.div_ceil(256))?, + 256, + 0, + ))?; + self.module.vis_clamp_f32( + stream, + &prep, + x, + launch_u32(what, "n", n)?, + bounds.0, + bounds.1, + out, + )?; + Ok(()) + } + + pub(crate) fn add_f32( + &self, + stream: &CudaStream, + a: &DeviceBuffer, + b: &DeviceBuffer, + n: usize, + out: &mut DeviceBuffer, + ) -> Result<(), GpuError> { + let what = "vision F32 residual add"; + if n == 0 || a.len() < n || b.len() < n || out.len() < n { + return Err(GpuError::shape( + what, + "residual operands do not hold the output extent", + )); + } + let prep = self.module.prepare_vis_add_f32(LaunchConfig1D::new( + launch_u32(what, "grid", n.div_ceil(256))?, + 256, + 0, + ))?; + self.module + .vis_add_f32(stream, &prep, a, b, launch_u32(what, "n", n)?, out)?; + Ok(()) + } + + pub(crate) fn pool_f32( + &self, + stream: &CudaStream, + a: FloatPoolArgs<'_>, + ) -> Result<(), GpuError> { + let FloatPoolArgs { + x, + rows, + width, + grid_w, + out, + } = a; + let what = "vision F32 pooling gather"; + let group_rows = grid_w + .checked_mul(3) + .ok_or_else(|| GpuError::shape(what, "pool grid overflows"))?; + let n = rows + .checked_mul(width) + .ok_or_else(|| GpuError::shape(what, "element count overflows"))?; + if n == 0 + || grid_w == 0 + || !grid_w.is_multiple_of(3) + || !rows.is_multiple_of(group_rows) + || x.len() < n + || out.len() < n + { + return Err(GpuError::shape( + what, + "pooling buffers must hold complete three-by-three groups", + )); + } + let u = |name, value| launch_u32(what, name, value); + let prep = self.module.prepare_vis_pool_order_f32(LaunchConfig1D::new( + u("grid", n.div_ceil(256))?, + 256, + 0, + ))?; + self.module.vis_pool_order_f32( + stream, + &prep, + x, + u("n", n)?, + u("width", width)?, + u("grid width", grid_w)?, + out, + )?; + Ok(()) + } + + /// Load this module's device bundle into the tower's context. + pub(crate) fn load(ctx: &Arc) -> Result { + // SAFETY: this crate owns the module's embedded device bundle. + let module = unsafe { bloomery_gpu::shared_module!(kernels, ctx)? }; + Ok(Self { module }) + } + + /// Apply one checked row operation asynchronously. + pub(crate) fn enqueue(&self, stream: &CudaStream, a: Args<'_>) -> Result<(), GpuError> { + let what = "vision row operation"; + let n = a + .rows + .checked_mul(a.width) + .ok_or_else(|| GpuError::shape(what, "element count overflows"))?; + let need = a + .rows + .checked_mul(a.out_width) + .ok_or_else(|| GpuError::shape(what, "output count overflows"))?; + if n == 0 + || a.out_col + .checked_add(a.width) + .is_none_or(|v| v > a.out_width) + || a.x.len() < n + || a.out.len() < need + || !a.scale.is_finite() + || !a.clamp.0.is_finite() + || !a.clamp.1.is_finite() + || a.clamp.0 > a.clamp.1 + { + return Err(GpuError::shape( + what, + "invalid row extent, buffers, scale or clamp", + )); + } + let op = match a.op { + Op::GeGlu | Op::QuickGeGlu if a.b.len() < n => { + return Err(GpuError::shape(what, "second operand is short")); + } + Op::Affine if a.gain.len() < a.width || a.bias.len() < a.width => { + return Err(GpuError::shape(what, "affine vectors are short")); + } + Op::GeGlu => 1, + Op::QuickGeGlu => 7, + Op::Affine => 2, + Op::Clamp => 3, + Op::Copy => 5, + Op::Scale => 6, + }; + let u = |name, v| launch_u32(what, name, v); + let prep = self.module.prepare_vis_row_op(LaunchConfig1D::new( + u("grid", n.div_ceil(256))?, + 256, + 0, + ))?; + self.module.vis_row_op( + stream, + &prep, + a.x, + a.b, + a.gain, + a.bias, + u("n", n)?, + u("width", a.width)?, + u("stride", a.out_width)?, + u("column", a.out_col)?, + op, + a.scale, + a.clamp.0, + a.clamp.1, + a.out, + )?; + Ok(()) + } + + /// Add learned x/y position rows to the patch embeddings asynchronously. + pub(crate) fn positions( + &self, + stream: &CudaStream, + a: PositionArgs<'_>, + ) -> Result<(), GpuError> { + let what = "vision learned position rows"; + let n = a + .rows + .checked_mul(a.width) + .ok_or_else(|| GpuError::shape(what, "element count overflows"))?; + let table = a + .pos_rows + .checked_mul(a.width) + .and_then(|n| n.checked_mul(2)) + .ok_or_else(|| GpuError::shape(what, "position table count overflows"))?; + if n == 0 + || a.grid_w == 0 + || !a.rows.is_multiple_of(a.grid_w) + || a.grid_w > a.pos_rows + || a.rows / a.grid_w > a.pos_rows + || a.x.len() < n + || a.out.len() < n + || a.table.len() < table + { + return Err(GpuError::shape( + what, + "patch or position extent does not fit its buffer", + )); + } + let u = |name, v| launch_u32(what, name, v); + let prep = self.module.prepare_vis_position_rows(LaunchConfig1D::new( + u("grid", n.div_ceil(256))?, + 256, + 0, + ))?; + self.module.vis_position_rows( + stream, + &prep, + a.x, + a.table, + u("n", n)?, + u("width", a.width)?, + u("grid width", a.grid_w)?, + u("position rows", a.pos_rows)?, + a.out, + )?; + Ok(()) + } +} diff --git a/crates/gpu-vision/src/rope2d.rs b/crates/gpu-vision/src/rope2d.rs index 9424724d0..465ab5533 100644 --- a/crates/gpu-vision/src/rope2d.rs +++ b/crates/gpu-vision/src/rope2d.rs @@ -55,6 +55,25 @@ pub struct RopeTable { } impl RopeTable { + /// Gemma4's two NEOX half-head turns, applied through this module's + /// whole-head pairing after the q/k rows and gains are permuted at load. + #[must_use] + pub(crate) fn neox_halves(n_h: usize, n_w: usize, theta: f32) -> RopeTable { + let step = f64::from(theta).powf(f64::from(-2.0f32 / PAIRS as f32)) as f32; + let mut cs = Vec::with_capacity(n_h * n_w * 2 * PAIRS); + for (y, x) in raster(n_h, n_w) { + for pos in [x, y] { + let mut angle = pos as f32; + for _ in 0..PER_AXIS { + cs.push(f64::from(angle).cos() as f32); + cs.push(f64::from(angle).sin() as f32); + angle *= step; + } + } + } + RopeTable { n_h, n_w, cs } + } + /// The table of an `n_h × n_w` patch grid at RoPE base `theta`, as the reference builds it /// in f32: `f_i = 1.0 / theta^(i / 16)` — the exponent `arange(0, 32, 2) / 32` is exact in /// f32, the power correctly rounded (the device's `powf` the reference calls is within an diff --git a/tools/unsafe-ratchet.txt b/tools/unsafe-ratchet.txt index faa1ba6ac..e3c382958 100644 --- a/tools/unsafe-ratchet.txt +++ b/tools/unsafe-ratchet.txt @@ -342,7 +342,11 @@ gpu-mimo2 0 0 0 # PIN(2026-10-13): gpu-vision 91 -> 97 blocks, new code, not old code relaxed: the MiniCPM-V merger's two new kernel entries read # and write their buffers through the same raw places as the others, each block with its SAFETY line — vis_attn_group 5 (its # loader, three word reads, one store), vis_mean_groups 1 (its loader). -gpu-vision 97 0 0 +# PIN(2026-10-15): gpu-vision 97 -> 116 blocks: Gemma4 row/position entries and F32 residual, clamp, pool, norm and GEMM +# keep Gemma4 post norms in their graph precision; every raw read and disjoint store carries its SAFETY invariant. +# PIN(2026-10-16): gpu-vision 116 -> 127 blocks: the same Gemma4 clamp, add, pool and F32 norm memory operations +# now have narrower unsafe scopes, each with its own SAFETY invariant. +gpu-vision 127 0 0 # PIN(2026-10-13): hf 0 -> 2 blocks, new code, not old code relaxed: the DSpark draft's 1.9 GB safetensors file is read # through a read-only mapping (Safetensors::open) instead of into memory, and a bf16 tensor's bytes are viewed as u16 # (Safetensors::bf16, a misaligned file refused), each block with its SAFETY line. From 7479add2363c5dfa44a77511ebd35372ba0306aa Mon Sep 17 00:00:00 2001 From: juliankang4 <200967947+juliankang4@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:07:43 +0900 Subject: [PATCH 06/12] gpu-gates, serve: image rows join the embedding side seat pass Budget the image tower before chat placement and run its rows with ordered text parts in one embedding pass. The HTTP route and gates share serve::embeddings::assemble. Build the encoder explicitly for serving instead of enabling it for every DeepSeek gate, and refuse duplicate side flags by name. --- crates/gpu-gates/Cargo.toml | 4 +- crates/gpu-gates/src/arch/embedding.rs | 2 + crates/gpu-gates/src/bin/bloomery_serve.rs | 4 +- crates/gpu-gates/src/side_seat.rs | 252 +++++++++++++++++- .../src/side_seat/gemma_embedding2.rs | 60 ++++- crates/gpu-gates/src/vision_seat.rs | 28 +- crates/gpu-gemma-embedding2/src/lib.rs | 5 + crates/gpu-gemma-embedding2/src/pass.rs | 77 +++++- crates/serve/src/api.rs | 2 +- crates/serve/src/arch/gemma4v.rs | 59 ++++ crates/serve/src/embed.rs | 14 + crates/serve/src/embeddings.rs | 163 ++++++++++- crates/serve/src/lib.rs | 1 + justfile | 6 +- 14 files changed, 637 insertions(+), 40 deletions(-) create mode 100644 crates/serve/src/arch/gemma4v.rs diff --git a/crates/gpu-gates/Cargo.toml b/crates/gpu-gates/Cargo.toml index dbb138857..734c749f9 100644 --- a/crates/gpu-gates/Cargo.toml +++ b/crates/gpu-gates/Cargo.toml @@ -122,7 +122,7 @@ vision = ["gpu", "dep:bloomery-gpu-vision", "dep:bloomery-vision"] clef = ["gpu", "dep:bloomery-decision", "dep:bloomery-serve", "dep:serde_json"] glm5next = ["deepseek41", "dep:bloomery-gpu-glm5next", "bloomery-app/glm5next"] mimo2 = ["gpu", "dep:bloomery-gpu-mimo2", "bloomery-app/mimo2"] -gemma-embedding2 = ["gpu", "dep:bloomery-gpu-gemma-embedding2"] +gemma-embedding2 = ["gpu", "dep:bloomery-gpu-gemma-embedding2", "dep:bloomery-serve"] # A bin whose `main` sits behind a feature requires that feature: cargo refuses to build it without the # feature before it links anything, instead of linking the feature-off stub `main` over the working @@ -541,7 +541,7 @@ required-features = ["gpu"] [[bin]] name = "gate_gemma_embedding2_e2e" -required-features = ["gemma-embedding2"] +required-features = ["gemma-embedding2", "vision", "deepseek41"] [[bin]] name = "gate_gemma_embedding2_serve" diff --git a/crates/gpu-gates/src/arch/embedding.rs b/crates/gpu-gates/src/arch/embedding.rs index b3fb0d920..79571e7fc 100644 --- a/crates/gpu-gates/src/arch/embedding.rs +++ b/crates/gpu-gates/src/arch/embedding.rs @@ -15,6 +15,8 @@ pub const ROWS: &[Row] = &[ ctx: model::arch::gemma_embedding2::hparams::CARD_CTX, bytes, open, + projector_file: Some("mmproj-embeddinggemma-2-BF16.gguf"), + projector_types: &[vision::arch::gemma4v::PROJECTOR_TYPE], }, ]; diff --git a/crates/gpu-gates/src/bin/bloomery_serve.rs b/crates/gpu-gates/src/bin/bloomery_serve.rs index 30b54ccc9..484242571 100644 --- a/crates/gpu-gates/src/bin/bloomery_serve.rs +++ b/crates/gpu-gates/src/bin/bloomery_serve.rs @@ -140,7 +140,9 @@ mod drive { const USAGE: &str = "usage: bloomery-serve [--model ds41|qwen38|glm|qwen3|mimo2|nemotron|decide] \ [-m PATH | --model-file PATH | --hf [:]] [--head \ [--head-config ]] [--embd-model FILE | --embd-hf \ - [:]] [:]] [--embd-ctx N] [--embd-mmproj FILE | \ + --embd-no-mmproj] [--embd-image-min-tokens N] [--embd-image-max-tokens N] \ + | --version"; diff --git a/crates/gpu-gates/src/side_seat.rs b/crates/gpu-gates/src/side_seat.rs index c09d021fb..396e2f7cc 100644 --- a/crates/gpu-gates/src/side_seat.rs +++ b/crates/gpu-gates/src/side_seat.rs @@ -32,6 +32,8 @@ pub use architectures::ROWS; #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct Flags { model: source::Flags, + image: vision_seat::Flags, + ctx: Option, } impl Flags { @@ -39,15 +41,47 @@ impl Flags { /// file is read here. A flag with no value or given twice, and `--embd-model` beside /// `--embd-hf`, are refused by name. pub fn take(args: &[String]) -> Result<(Flags, Vec), GateError> { + let mut f = Flags::default(); let mut selected = Vec::new(); let mut rest = Vec::with_capacity(args.len()); + let mut image_seen = std::collections::HashSet::new(); let mut it = args.iter(); - while let Some(arg) = it.next() { - let flag = match arg.as_str() { + while let Some(a) = it.next() { + let flag = a.as_str(); + if let Some(image_flag) = match flag { + "--embd-mmproj" => Some("--mmproj"), + "--embd-no-mmproj" => Some("--no-mmproj"), + "--embd-image-min-tokens" => Some("--image-min-tokens"), + "--embd-image-max-tokens" => Some("--image-max-tokens"), + _ => None, + } { + if !image_seen.insert(flag) { + return Err(format!("{flag} is given twice").into()); + } + f.image + .take(image_flag, &mut it.by_ref().map(String::as_str)) + .map_err(|e| format!("{flag}: {e}"))?; + continue; + } + if flag == "--embd-ctx" { + let v = it.next().ok_or("--embd-ctx needs a value")?; + let ctx = v + .parse::() + .map_err(|e| format!("--embd-ctx {v:?}: {e}"))?; + if f.ctx.is_some() { + return Err("--embd-ctx is given twice".into()); + } + if ctx == 0 { + return Err("--embd-ctx needs one positive count".into()); + } + f.ctx = Some(ctx); + continue; + } + let flag = match flag { "--embd-model" => "--embd-model", "--embd-hf" => "--hf", _ => { - rest.push(arg.clone()); + rest.push(a.clone()); continue; } }; @@ -56,7 +90,11 @@ impl Flags { } let (model, _) = source::take(&selected, &["--embd-model"]) .map_err(|e| format!("embedding flags (--embd-model/--embd-hf): {e}"))?; - Ok((Flags { model }, rest)) + f.model = model; + if !f.names() && (!image_seen.is_empty() || f.ctx.is_some()) { + return Err("embedding image/context flags need --embd-model or --embd-hf".into()); + } + Ok((f, rest)) } /// Whether the flags name an embedding model. @@ -87,14 +125,68 @@ impl Flags { .arch_get_u64("context_length") .and_then(|n| usize::try_from(n).ok()) .ok_or_else(|| format!("{}: no {arch}.context_length", path.display()))?; - let ctx = row.ctx.min(train); - let bytes = (row.bytes)(&split, ctx) + let limit = row.ctx.min(train); + let ctx = self.ctx.unwrap_or(limit); + if ctx > limit { + return Err( + format!("--embd-ctx {ctx}: the side model supports at most {limit}").into(), + ); + } + let mut bytes = (row.bytes)(&split, ctx) .map_err(|e| format!("--embd-model {}: {e}", path.display()))?; + let projector_path = if let Some(p) = self.image.mmproj()? { + Some(p.to_path_buf()) + } else if let (Some(hf), Some(mmproj)) = (&self.model.hf, row.projector_file) { + if cfg!(feature = "vision") && !self.image.disabled() { + let repo = hf::RepoRef::parse(hf)?; + Some( + model_file::fetch_exact(&repo.repo, &[mmproj])? + .into_iter() + .next() + .ok_or("the projector fetch returned no file")?, + ) + } else { + None + } + } else { + None + }; + let projector = match projector_path.as_deref() { + Some(_) if !cfg!(feature = "vision") => { + return Err("--embd-mmproj needs the vision build feature".into()); + } + Some(p) => { + let p = vision_seat::Projector::open(p, &self.image, row.projector_types)?; + let n_embd = split + .arch_get_u64("embedding_length") + .and_then(|n| usize::try_from(n).ok()) + .ok_or("the embedding width is absent")?; + p.check_text( + &vision_seat::TextFacts { + n_embd, + imrope: None, + ple: vision_seat::PleImage::Absent, + }, + &|_| None, + )?; + bytes = bytes + .checked_add(p.card.total()) + .ok_or("embedding model and tower byte count overflows")?; + Some(p) + } + None if self.image.sets_limits() => { + return Err( + "--embd-image-min-tokens/--embd-image-max-tokens need --embd-mmproj".into(), + ); + } + None => None, + }; Ok(Some(Side { row, path, ctx, bytes, + projector, })) } } @@ -111,6 +203,10 @@ pub struct Row { pub bytes: fn(&Split, usize) -> Result, /// The model of `side`'s file opened on `seat`'s card, on the thread that runs its passes. pub open: Open, + /// The BF16 projector this row reads from an `--embd-hf` repository. + pub projector_file: Option<&'static str>, + /// The projector types this embedding architecture accepts. + pub projector_types: &'static [&'static str], } /// A row's open: the model of `side`'s file on `seat`'s card. @@ -148,6 +244,8 @@ pub struct Side { pub ctx: usize, /// The bytes it takes on its card ([`Row::bytes`]). pub bytes: u64, + /// The image tower sized before the chat plan; its bytes share this side seat's reserve. + pub projector: Option, } impl Side { @@ -177,7 +275,11 @@ impl Side { } /// One pass: the input's ids and where its answer goes. -type Job = (Vec, mpsc::Sender, EmbedError>>); +type Job = ( + Vec, + Vec, + mpsc::Sender, EmbedError>>, +); /// The model the server's embeddings endpoints call, held on a thread of its own: the facts read /// once at its open, and the channel each pass goes through. Dropping it closes the channel and @@ -189,6 +291,7 @@ pub struct SideEngine { name: String, ctx: usize, dims: Vec, + media: Option, } impl SideEngine { @@ -214,12 +317,24 @@ impl SideEngine { return; } }; - let facts = (model.tokenizer(), model.name(), model.ctx(), model.dims()); + let facts = ( + model.tokenizer(), + model.name(), + model.ctx(), + model.dims(), + model.media(), + ); if opened_tx.send(Ok(facts)).is_err() { return; } - for (ids, answer) in rx { - match catch_unwind(|| model.embed(&ids)) { + for (ids, images, answer) in rx { + match catch_unwind(|| { + if images.is_empty() { + model.embed(&ids) + } else { + model.embed_media(&ids, &images) + } + }) { Ok(v) => { let _ = answer.send(v); } @@ -233,7 +348,7 @@ impl SideEngine { } }) .map_err(|e| format!("starting the embedding model's thread: {e}"))?; - let (tokenizer, name, ctx, dims) = match opened.recv() { + let (tokenizer, name, ctx, dims, media) = match opened.recv() { Ok(Ok(facts)) => facts, refused => { // The thread ended with its open; nothing is left on its card. @@ -251,6 +366,7 @@ impl SideEngine { name, ctx, dims, + media, }) } } @@ -271,6 +387,9 @@ impl Drop for SideEngine { } impl Embed for SideEngine { + fn media(&self) -> Option { + self.media.clone() + } fn tokenizer(&self) -> Arc { Arc::clone(&self.tokenizer) } @@ -288,10 +407,19 @@ impl Embed for SideEngine { } fn embed(&mut self, ids: &[u32]) -> Result, EmbedError> { + self.embed_media(ids, &[]) + } + + fn embed_media( + &mut self, + ids: &[u32], + images: &[serve::media::MediaFeed], + ) -> Result, EmbedError> { let ended = || EmbedError("the embedding model's thread ended".to_owned()); let jobs = self.jobs.as_ref().ok_or_else(ended)?; let (answer, answered) = mpsc::channel(); - jobs.send((ids.to_vec(), answer)).map_err(|_| ended())?; + jobs.send((ids.to_vec(), images.to_vec(), answer)) + .map_err(|_| ended())?; answered.recv().map_err(|_| ended())? } } @@ -336,6 +464,49 @@ mod tests { let (none, _) = Flags::take(&args(&["--port", "1"])).expect("none"); assert_eq!(none, Flags::default()); assert!(!none.names()); + let (image, rest) = Flags::take(&args(&[ + "--embd-model", + "/e.gguf", + "--embd-ctx", + "8192", + "--embd-mmproj", + "/p.gguf", + "--embd-image-min-tokens", + "70", + "--embd-image-max-tokens", + "1120", + "--port", + "8080", + ])) + .expect("image flags"); + assert_eq!(rest, args(&["--port", "8080"])); + assert_eq!(image.ctx, Some(8192)); + assert_eq!( + image.image.mmproj().expect("projector"), + Some(std::path::Path::new("/p.gguf")) + ); + assert!(image.image.sets_limits()); + let default = crate::vision_seat::TokenLimits::new(1, 1).expect("limits"); + let limits = image.image.limits_over(default).expect("image limits"); + assert_eq!((limits.min(), limits.max()), (70, 1120)); + for flag in ["--embd-image-min-tokens", "--embd-image-max-tokens"] { + let (zero, _) = + Flags::take(&args(&["--embd-model", "/a", flag, "0"])).expect("count parsed"); + let e = zero + .image + .limits_over(default) + .expect_err("zero limit refused") + .to_string(); + assert!( + e.starts_with("--image-min-tokens/--image-max-tokens: image size limits: "), + "{flag}: {e}" + ); + assert!(e.contains("need 1 <= min <= max"), "{flag}: {e}"); + let (one, _) = + Flags::take(&args(&["--embd-model", "/a", flag, "1"])).expect("count parsed"); + let limits = one.image.limits_over(default).expect("one token limit"); + assert_eq!((limits.min(), limits.max()), (1, 1)); + } for (bad, why) in [ (&["--embd-model"][..], "--embd-model needs a value"), (&["--embd-hf"], "--hf needs a value"), @@ -343,6 +514,63 @@ mod tests { (&["--embd-hf", "x/y", "--embd-hf", "a/b"], "named twice"), (&["--embd-model", "/a", "--embd-hf", "x/y"], "named twice"), (&["--embd-hf", "x/y", "--embd-model", "/a"], "named twice"), + (&["--embd-mmproj", "/p"], "need --embd-model or --embd-hf"), + (&["--embd-no-mmproj"], "need --embd-model or --embd-hf"), + ( + &["--embd-image-min-tokens", "70"], + "need --embd-model or --embd-hf", + ), + ( + &["--embd-image-max-tokens", "1120"], + "need --embd-model or --embd-hf", + ), + (&["--embd-ctx", "8192"], "need --embd-model or --embd-hf"), + ( + &["--embd-model", "/a", "--embd-ctx", "1", "--embd-ctx", "2"], + "--embd-ctx is given twice", + ), + ( + &["--embd-model", "/a", "--embd-ctx", "0"], + "needs one positive count", + ), + ( + &[ + "--embd-model", + "/a", + "--embd-mmproj", + "/p", + "--embd-mmproj", + "/q", + ], + "--embd-mmproj is given twice", + ), + ( + &["--embd-model", "/a", "--embd-no-mmproj", "--embd-no-mmproj"], + "--embd-no-mmproj is given twice", + ), + ( + &[ + "--embd-model", + "/a", + "--embd-image-min-tokens", + "70", + "--embd-image-min-tokens", + "80", + ], + "--embd-image-min-tokens is given twice", + ), + ( + &[ + "--embd-model", + "/a", + "--embd-image-max-tokens", + "1120", + "--embd-image-max-tokens", + "1200", + ], + "--embd-image-max-tokens is given twice", + ), + (&["--embd-model", "/a", "--embd-mmproj"], "--embd-mmproj"), ] { let e = Flags::take(&args(bad)).expect_err("refused").to_string(); assert!(e.contains(why), "{bad:?}: {e}"); diff --git a/crates/gpu-gates/src/side_seat/gemma_embedding2.rs b/crates/gpu-gates/src/side_seat/gemma_embedding2.rs index 8a2f7f551..3d5a1c3ef 100644 --- a/crates/gpu-gates/src/side_seat/gemma_embedding2.rs +++ b/crates/gpu-gates/src/side_seat/gemma_embedding2.rs @@ -11,6 +11,8 @@ use serve::embed::{Embed, EmbedError}; use super::{Side, architectures}; use crate::bind::Vocab; use crate::vision_seat::EncoderSeat; +#[cfg(feature = "vision")] +use crate::vision_seat::{Image, Opened, Resident}; pub(super) fn bytes(split: &Split, ctx: usize) -> Result { card_bytes(split, ctx).map_err(|e| e.to_string()) @@ -20,7 +22,9 @@ pub(super) fn open(side: &Side, seat: &EncoderSeat) -> Result, St let load = || -> Result, crate::GateError> { let split = Split::open(&side.path)?; let dims = architectures::gemma_embedding2_dims(&split)?; - let actual = card_bytes(&split, side.ctx)?; + let actual = card_bytes(&split, side.ctx)? + .checked_add(side.projector.as_ref().map_or(0, |p| p.card.total())) + .ok_or("embedding model and tower byte count overflows")?; if actual != side.bytes || seat.reserve.is_some_and(|b| b != actual) { return Err(format!( "embedding model: load needs {actual} B, sized {} B, reserved {:?}", @@ -41,6 +45,22 @@ pub(super) fn open(side: &Side, seat: &EncoderSeat) -> Result, St .device(); let gpu = Gpu::open_card(&seat.name, Some(device))?; let encoder = Encoder::open(split, side.ctx, gpu)?; + #[cfg(feature = "vision")] + let vision = side + .projector + .as_ref() + .map(|p| { + let at = Opened { + path: p.path.clone(), + seat: seat.clone(), + media: p.media.clone(), + cap: p.cap, + card: p.card, + limits: p.limits, + }; + Resident::open(encoder.context(), &at) + }) + .transpose()?; let name = side .path .file_name() @@ -49,6 +69,8 @@ pub(super) fn open(side: &Side, seat: &EncoderSeat) -> Result, St .into_owned(); Ok(Box::new(Model { encoder, + #[cfg(feature = "vision")] + vision, tok, name, dims, @@ -59,12 +81,25 @@ pub(super) fn open(side: &Side, seat: &EncoderSeat) -> Result, St struct Model { encoder: Encoder, + #[cfg(feature = "vision")] + vision: Option, tok: Arc, name: String, dims: Vec, } impl Embed for Model { + fn media(&self) -> Option { + #[cfg(feature = "vision")] + { + self.vision.as_ref().map(Resident::media) + } + #[cfg(not(feature = "vision"))] + { + None + } + } + fn tokenizer(&self) -> Arc { self.tok.clone() } @@ -87,4 +122,27 @@ impl Embed for Model { .map(|pass| pass.pooled) .map_err(|e| EmbedError(e.to_string())) } + + #[cfg(feature = "vision")] + fn embed_media( + &mut self, + ids: &[u32], + images: &[serve::media::MediaFeed], + ) -> Result, EmbedError> { + let tower = self + .vision + .as_mut() + .ok_or_else(|| EmbedError("image input needs --embd-mmproj".into()))?; + let encoded = tower + .encode_with( + self.encoder.context(), + images.iter().map(|i| Image::of(i.at, &i.prepared)), + crate::vision_seat::merged_cells, + ) + .map_err(|e| EmbedError(e.to_string()))?; + self.encoder + .pass_media(ids, &encoded.rows().media_spans(), false) + .map(|p| p.pooled) + .map_err(|e| EmbedError(e.to_string())) + } } diff --git a/crates/gpu-gates/src/vision_seat.rs b/crates/gpu-gates/src/vision_seat.rs index ebfdd7813..07ec6cf25 100644 --- a/crates/gpu-gates/src/vision_seat.rs +++ b/crates/gpu-gates/src/vision_seat.rs @@ -56,8 +56,8 @@ use gguf::Gguf; use model::placement::workstation::{CardSpec, DeviceInfo}; use vision::arch::card::CardBytes; use vision::arch::minicpmv::{self, Group}; -use vision::arch::qwen3vl::TokenLimits; -use vision::arch::{deepseek41v, glm5v, qwen3vl}; +pub(crate) use vision::arch::qwen3vl::TokenLimits; +use vision::arch::{deepseek41v, gemma4v, glm5v, qwen3vl}; use vision::mrope::Canvas; use vision::{EncoderInput, MediaModel, Patches, Prepared, VisionError}; @@ -175,6 +175,12 @@ impl Flags { self.min_tokens.is_some() || self.max_tokens.is_some() } + /// Whether the caller explicitly disables a repository's projector. + #[must_use] + pub fn disabled(&self) -> bool { + self.no_mmproj + } + /// The image-token limits: the flags' values over `default`, the limits the projector starts /// from when no flag gives one ([`Rule`]), refused by name when `min` is 0 or above `max`. pub fn limits_over(&self, default: TokenLimits) -> Result { @@ -186,7 +192,7 @@ impl Flags { } } -/// A flag's count: a positive integer, or refused by name. +/// A flag's count: an unsigned integer, or refused by name. fn count(flag: &str, v: &str) -> Result { v.parse::() .map_err(|e| format!("{flag} {v:?}: {e}").into()) @@ -270,6 +276,15 @@ pub fn rules() -> &'static [Rule] { default_limits: |_| TokenLimits::new(1, MINICPM_TILE_TOKENS), cap: |_, limits| Ok(limits.max()), }, + Rule { + projector: gemma4v::PROJECTOR_TYPE, + ids: &[], + text_rope: None, + takes_limits: true, + out_dim: |f| Ok(gemma4v::Hparams::read(f)?.out_dim), + default_limits: |f| gemma4v::Hparams::read(f)?.default_limits(), + cap: |_, limits| Ok(limits.max()), + }, ]; RULES } @@ -323,10 +338,7 @@ impl Projector { pub fn open(path: &Path, flags: &Flags, takes: &[&str]) -> Result { let at = |e: &dyn Display| format!("--mmproj {}: {e}", path.display()); let file = Gguf::open(path).map_err(|e| at(&e))?; - let kind = file - .value("clip.projector_type") - .and_then(gguf::Value::as_str) - .ok_or_else(|| at(&"clip.projector_type is absent or not a string"))?; + let kind = vision::arch::projector_type(&file).map_err(|e| at(&e))?; if !takes.contains(&kind) { return Err(at(&format!( "projector type {kind:?}; this seat's text model takes {takes:?}" @@ -1856,7 +1868,7 @@ mod tests { assert_eq!( open(&bare, &none, &["deepseek41v"]), format!( - "--mmproj {}: clip.projector_type is absent or not a string", + "--mmproj {}: metadata clip.projector_type: is absent", bare.0.display() ) ); diff --git a/crates/gpu-gemma-embedding2/src/lib.rs b/crates/gpu-gemma-embedding2/src/lib.rs index 0241dc85f..fbf0deba2 100644 --- a/crates/gpu-gemma-embedding2/src/lib.rs +++ b/crates/gpu-gemma-embedding2/src/lib.rs @@ -512,6 +512,11 @@ impl Encoder { pub fn ctx(&self) -> usize { self.ctx } + + /// The context the encoder's pass owns; a tower call binds it back after its encode. + pub fn context(&self) -> &std::sync::Arc { + self.gpu.context() + } } /// Rows `rows` of `name` in `split`, `k` values each, decoded to f32 and diff --git a/crates/gpu-gemma-embedding2/src/pass.rs b/crates/gpu-gemma-embedding2/src/pass.rs index 9be7b1d05..8959fb9a2 100644 --- a/crates/gpu-gemma-embedding2/src/pass.rs +++ b/crates/gpu-gemma-embedding2/src/pass.rs @@ -398,6 +398,17 @@ impl Encoder { /// the error, and so is an output value that is not finite, by its /// position. pub fn pass(&mut self, ids: &[u32], taps: bool) -> Result { + self.pass_media(ids, &[], taps) + } + + /// One bidirectional pass with image rows already scaled by the tower, + /// replacing their spans after the text rows receive `sqrt(hidden)`. + pub fn pass_media( + &mut self, + ids: &[u32], + spans: &[bloomery_gpu::media::MediaSpan<'_>], + taps: bool, + ) -> Result { let n = ids.len(); if n == 0 || n > self.ctx { return Err(GpuError::Shape { @@ -405,7 +416,18 @@ impl Encoder { detail: format!("{n} positions; a pass takes 1 to {}", self.ctx), }); } - let inp_scaled = self.token_rows(ids)?; + bloomery_gpu::media::check_spans(PASS, spans, n, self.hp.n_embd)?; + let mut inp_scaled = self.token_rows(ids)?; + for span in spans { + let dst = &mut inp_scaled[span.at.start * self.hp.n_embd..span.at.end * self.hp.n_embd]; + bloomery_gpu::media::widen_bf16(span.rows, dst); + if dst.iter().any(|v| !v.is_finite()) { + return Err(GpuError::shape( + PASS, + "image row contains a non-finite value", + )); + } + } let Encoder { gpu, hp, @@ -522,6 +544,59 @@ fn mean(rows: &[f32], width: usize) -> Vec { mod tests { use super::*; + #[test] + #[ignore = "needs a CUDA card"] + fn hw_media_spans_are_checked_before_the_pass() { + use bloomery_gpu::media::MediaSpan; + + crate::tests::with_model(64, 2048, |split, hp| -> Result<(), GpuError> { + let mut enc = Encoder::open(split, hp.n_ctx, Gpu::new()?)?; + let rows = vec![0; hp.n_embd]; + for spans in [ + vec![MediaSpan { + at: 0..0, + rows: &[], + }], + vec![MediaSpan { + at: 1..2, + rows: &rows, + }], + vec![MediaSpan { + at: 0..1, + rows: &[], + }], + vec![ + MediaSpan { + at: 0..1, + rows: &rows, + }, + MediaSpan { + at: 0..1, + rows: &rows, + }, + ], + ] { + assert!(matches!( + enc.pass_media(&[0], &spans, false), + Err(GpuError::Shape { .. }) + )); + } + assert!( + enc.pass_media( + &[0], + &[MediaSpan { + at: 0..1, + rows: &rows + }], + false + ) + .is_ok() + ); + Ok(()) + }) + .unwrap(); + } + #[test] #[ignore = "needs a CUDA card"] fn hw_per_layer_projection_scale_near_epsilon() { diff --git a/crates/serve/src/api.rs b/crates/serve/src/api.rs index 3db8eca11..9ea005d57 100644 --- a/crates/serve/src/api.rs +++ b/crates/serve/src/api.rs @@ -93,7 +93,7 @@ mod anthropic; /// The embeddings endpoints over the side model, a child of this module so /// they read a prompt as the completion path does. #[path = "embeddings.rs"] -mod embeddings; +pub mod embeddings; /// Per-token log-probabilities, which the generation loop collects and the /// completion and chat answers render. #[path = "logprobs.rs"] diff --git a/crates/serve/src/arch/gemma4v.rs b/crates/serve/src/arch/gemma4v.rs new file mode 100644 index 000000000..0d938ab12 --- /dev/null +++ b/crates/serve/src/arch/gemma4v.rs @@ -0,0 +1,59 @@ +//! The embedding input contract with the Gemma4 image model. + +use super::*; +use crate::mock::MockEmbed; + +#[test] +fn image_parts_preserve_delimiters_and_place_spans_in_one_input() { + let mut side = Side::new(Box::new(MockEmbed::new(128))); + let rule = + vision::size::SizeRule::new(16, 3, vision::arch::TokenLimits::new(1, 4).unwrap()).unwrap(); + side.media = Some(Arc::new(vision::arch::gemma4v::Media::new(rule))); + let image = json!({"type":"image_url","image_url":{"url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAIAAAABCAIAAAB7QOjdAAAADUlEQVR4nGM4kWIERAAKxQK9KV6Y1QAAAABJRU5ErkJggg=="}}); + let body = json!({"content":[{"type":"text","text":"a<|image>"},image.clone(),{"type":"text","text":"b"},image]}); + let parsed = inputs(&side, "input", &json!([body])).unwrap_or_else(|e| panic!("{}", e.message)); + assert_eq!(parsed.len(), 1); + let got = &parsed[0]; + assert_eq!(got.images.len(), 2); + let mut want = vec![side.tok.bos()]; + want.extend(side.tok.encode("a<|image>")); + for (i, after) in [(0, "b"), (1, "")] { + want.extend(side.tok.encode("<|image>")); + assert_eq!(got.images[i].at, want.len()); + assert_eq!(got.images[i].prepared.span_len, 1); + want.extend([side.tok.bos(); 1]); + want.extend(side.tok.encode("")); + want.extend(side.tok.encode(after)); + } + if side.tok.add_eos() { + want.push(side.tok.eos()); + } + assert_eq!(got.ids, want); + let model = side.media.as_ref().expect("image preparer"); + assert_eq!(model.image_token(), None); + assert!(matches!( + media::expand_prompt(&[side.tok.bos()], &[], &**model), + Err(media::MediaError::NoChatImageToken) + )); + side.ctx = got.ids.len(); + assert!(inputs(&side, "input", &json!([body.clone()])).is_ok()); + side.ctx = got.images[1].at; + let mut over = body.clone(); + let parts = over["content"].as_array_mut().unwrap(); + parts.extend(std::iter::repeat_n(parts[1].clone(), 64)); + parts.push(json!({"type":"image_url","image_url":"https://example.com/not-prepared.png"})); + let error = inputs(&side, "input", &json!([over])) + .err() + .expect("context refusal"); + assert_eq!(error.message, "image input exceeds the embedding context"); + side.media = None; + assert!(inputs(&side, "input", &json!([body])).is_err()); + assert!( + inputs( + &side, + "input", + &json!({"content":[{"type":"image_url","image_url":"https://example.com/x.png"}]}) + ) + .is_err() + ); +} diff --git a/crates/serve/src/embed.rs b/crates/serve/src/embed.rs index d21dbb742..f8f4a4e62 100644 --- a/crates/serve/src/embed.rs +++ b/crates/serve/src/embed.rs @@ -28,9 +28,23 @@ pub trait Embed: Send { /// The widths a request's `dimensions` may cut the pooled vector to, ascending; the last is /// the pooled vector's own width. fn dims(&self) -> Vec; + /// The side model's image preparer, present when a projector is loaded. + fn media(&self) -> Option { + None + } /// One pass over one input: at least one id and at most [`Embed::ctx`], each below the /// vocabulary's size. The pooled vector, the full width (the last of [`Embed::dims`]). fn embed(&mut self, ids: &[u32]) -> Result, EmbedError>; + /// One pass with prepared images replacing the positions they occupy. + fn embed_media( + &mut self, + _ids: &[u32], + _images: &[crate::media::MediaFeed], + ) -> Result, EmbedError> { + Err(EmbedError( + "the embedding model has no image encoder (--embd-mmproj)".into(), + )) + } } /// A normalization llama.cpp's `common_embd_normalize` names (`embd_normalize`): `-1` none, `0` diff --git a/crates/serve/src/embeddings.rs b/crates/serve/src/embeddings.rs index 64ed12fcd..0c43ada2b 100644 --- a/crates/serve/src/embeddings.rs +++ b/crates/serve/src/embeddings.rs @@ -4,7 +4,7 @@ //! `6868a52c`), with these differences: a field of another type than it takes is a 400 naming it //! (optional fields treat `null` as unset); `/v1/embeddings` refuses a `model` of another type //! than a string or a name other than the side model's, while the native routes ignore it; an object -//! input (`{"prompt_string": …}`, or a `content` array of parts) is a 400, an input past the model's +//! input with `prompt_string` is a 400, an input past the model's //! context is a 400 with no batch limit before it, an `embd_normalize` below `-1` or one whose //! norm f32 cannot scale by is a 400 ([`crate::embed::normalize`]), and `dimensions` cuts the //! pooled vector to one of the model's widths before it is normalized. @@ -13,6 +13,8 @@ //! array of token ids, an array of strings and token ids, or an array of these with no id at its //! top level, one vector each. Text is tokenized with the side model's vocabulary, the first text //! element of each input with its special tokens, as llama-server's `tokenize_mixed` puts them. +//! An object with a `content` array prepares image parts through the side model's media rule. +//! Delimiter text and every image span share one pass, with BOS and EOS around the whole input. //! //! The side model runs one request at a time. A failure in it ends the server as an engine's does: //! the request gets a 500, `/health` and every later request a 503, and the server exits after its @@ -21,6 +23,7 @@ use std::io; use std::net::TcpStream; +use std::ops::Range; use std::panic::{self, AssertUnwindSafe}; use std::sync::{Arc, Mutex}; @@ -46,6 +49,7 @@ pub(super) struct Side { pub(super) name: String, pub(super) ctx: usize, dims: Vec, + media: Option, } impl Side { @@ -58,6 +62,7 @@ impl Side { "an embedding model names its width, the last of its dims: {dims:?}" ); Side { + media: embed.media(), tok: embed.tokenizer(), name: embed.name(), ctx: embed.ctx(), @@ -131,7 +136,7 @@ fn answer(state: &State, req: &Request, oai: bool) -> Result { .collect(), )); } - let n_tokens: usize = inputs.iter().map(Vec::len).sum(); + let n_tokens: usize = inputs.iter().map(|i| i.ids.len()).sum(); let data: Vec = vectors .iter() .enumerate() @@ -202,22 +207,141 @@ fn cut(side: &Side, b: &Map) -> Result { }) } -/// The request's inputs as ids: an array with no id at its top level is one input an element, +/// The request's inputs as ids and prepared images: an array with no id at its top level is one input an element, /// anything else one input, each read as a `/completion` prompt with the special tokens on. No /// input, an input of no id and one of more ids than the model's context are refused by name. -fn inputs(side: &Side, key: &str, v: &Value) -> Result>, ApiError> { +struct Input { + ids: Vec, + images: Vec, +} + +/// One ordered embedding input part, after an image has been prepared and sized. +pub enum InputPart<'a> { + Text(&'a str), + /// Tokenized delimiters around `positions` image rows. + Image { + begin: &'a str, + end: &'a str, + positions: usize, + }, +} + +/// Token ids and the image spans that replace their filler rows, in part order. +pub struct Assembled { + pub ids: Vec, + pub spans: Vec>, +} + +/// Tokenize each part separately, with BOS and EOS around the whole input. Image spans are +/// bounded by `ctx` before allocation; the caller checks the total length, including text and EOS. +pub fn assemble<'a>( + parts: impl Iterator, String>>, + tok: &dyn Tokenizer, + ctx: usize, +) -> Result { + let mut parts = parts.peekable(); + if parts.peek().is_none() { + return Err("Input content cannot be empty".into()); + } + let mut ids = Vec::new(); + let mut spans = Vec::new(); + if tok.add_bos() { + ids.push(tok.bos()); + } + for part in parts { + match part? { + InputPart::Text(text) => ids.extend(tok.encode(text)), + InputPart::Image { + begin, + end, + positions, + } => { + if positions == 0 { + return Err("an embedding image has an empty span".into()); + } + ids.extend(tok.encode(begin)); + let at = ids.len(); + let last = at + .checked_add(positions) + .filter(|&n| n <= ctx) + .ok_or("image input exceeds the embedding context")?; + ids.resize(last, tok.bos()); + spans.push(at..last); + ids.extend(tok.encode(end)); + } + } + } + if tok.add_eos() { + ids.push(tok.eos()); + } + if ids.is_empty() { + return Err("Input content cannot be empty".into()); + } + Ok(Assembled { ids, spans }) +} + +fn input(side: &Side, key: &str, v: &Value) -> Result { + if let Some(parts) = v + .as_object() + .and_then(|o| o.get("content")) + .and_then(Value::as_array) + { + let mut images = Vec::new(); + let ordered = parts.iter().map(|part| -> Result, String> { + match media::parse_part(part).map_err(|e| e.to_string())? { + media::Part::Text(text) => Ok(InputPart::Text(text)), + media::Part::Image(url) => { + let model = side + .media + .as_ref() + .ok_or("image input needs --embd-mmproj")?; + let (begin, end) = model + .embedding_delimiters() + .ok_or("the side projector has no embedding delimiters")?; + let file = media::load(url).map_err(|e| e.to_string())?; + let prepared = + Arc::new(media::prepare(&file, &**model).map_err(|e| e.to_string())?); + let positions = prepared.span_len; + images.push(media::MediaFeed { + at: 0, + key: file.key, + prepared, + }); + Ok(InputPart::Image { + begin, + end, + positions, + }) + } + } + }); + let Assembled { ids, spans } = assemble(ordered, &*side.tok, side.ctx).map_err(invalid)?; + for (image, span) in images.iter_mut().zip(spans) { + image.at = span.start; + } + Ok(Input { ids, images }) + } else { + Ok(Input { + ids: prompt_ids(&*side.tok, key, v, true)?, + images: Vec::new(), + }) + } +} + +fn inputs(side: &Side, key: &str, v: &Value) -> Result, ApiError> { let is_id = |x: &Value| x.is_u64() || x.is_i64(); let inputs = match v { Value::Array(a) if !a.iter().any(is_id) => a .iter() - .map(|x| prompt_ids(&*side.tok, key, x, true)) + .map(|x| input(side, key, x)) .collect::, _>>()?, - v => vec![prompt_ids(&*side.tok, key, v, true)?], + v => vec![input(side, key, v)?], }; if inputs.is_empty() { return Err(invalid("\"input\" must not be empty")); } - for ids in &inputs { + for input in &inputs { + let ids = &input.ids; if ids.is_empty() { return Err(invalid("Input content cannot be empty")); } @@ -240,7 +364,7 @@ fn inputs(side: &Side, key: &str, v: &Value) -> Result>, ApiError> /// Every input's pooled vector, through the side model under its lock, each pass after /// [`open`] holds. A model failure, a panic in it, its lock poisoned, and a vector that is not its /// width or not finite are fatal ([`fail`]). -fn run(state: &State, side: &Side, inputs: &[Vec]) -> Result>, ApiError> { +fn run(state: &State, side: &Side, inputs: &[Input]) -> Result>, ApiError> { let Ok(mut model) = side.embed.lock() else { return Err(fail( state, @@ -250,9 +374,15 @@ fn run(state: &State, side: &Side, inputs: &[Vec]) -> Result>, }; let width = side.width(); let mut pooled = Vec::with_capacity(inputs.len()); - for ids in inputs { + for input in inputs { open(state)?; - let v = match panic::catch_unwind(AssertUnwindSafe(|| model.embed(ids))) { + let v = match panic::catch_unwind(AssertUnwindSafe(|| { + if input.images.is_empty() { + model.embed(&input.ids) + } else { + model.embed_media(&input.ids, &input.images) + } + })) { Ok(Ok(v)) => v, Ok(Err(e)) => return Err(fail(state, side, e.0)), Err(p) => { @@ -301,6 +431,10 @@ fn fail(state: &State, side: &Side, error: String) -> ApiError { } } +#[cfg(test)] +#[path = "arch/gemma4v.rs"] +mod image_tests; + #[cfg(test)] mod tests { use std::net::SocketAddr; @@ -584,7 +718,14 @@ mod tests { assert!(open(&waiting_state).is_ok()); entered.send(()).expect("entered"); let side = waiting_state.side.as_ref().expect("a side model"); - run(&waiting_state, side, &[vec![id("a")]]) + run( + &waiting_state, + side, + &[Input { + ids: vec![id("a")], + images: Vec::new(), + }], + ) }); ready .recv_timeout(Duration::from_secs(5)) diff --git a/crates/serve/src/lib.rs b/crates/serve/src/lib.rs index aced69594..e9228cf5c 100644 --- a/crates/serve/src/lib.rs +++ b/crates/serve/src/lib.rs @@ -20,6 +20,7 @@ //! reads at a terminal. mod api; +pub use api::embeddings; pub mod decide; pub mod dsml; pub mod embed; diff --git a/justfile b/justfile index 97079c8eb..b71c4e5fb 100644 --- a/justfile +++ b/justfile @@ -1261,11 +1261,11 @@ gate-gpu-gemma-embedding2-serve: # text case of mainline's card set (just dump-ref-emb2) through one pass, its taps within the CPU twin's band, # a rerun bit for bit, and the refusals by name. The # key bounds first hold every key of the symmetric mask, independently of the tap bands. -gate-gpu-gemma-embedding2-e2e: +gate-gpu-gemma-embedding2-e2e *ARGS: ./tools/box.sh 'bash tools/gate.sh --oxide --release -p bloomery-gpu-gemma-embedding2 --lib -- --include-ignored' ./tools/box.sh 'bash tools/gate.sh --release -p bloomery-model --lib -- arch::gemma_embedding2::spec::tests::key_bounds_are_the_symmetric_mask --exact' - ./tools/box.sh 'bash tools/gate.sh --oxide --release -p bloomery-gpu-gates --features gemma-embedding2 --bin gate_gemma_embedding2_e2e' - ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features gemma-embedding2 --release --bin gate_gemma_embedding2_e2e && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_gemma_embedding2_e2e' + ./tools/box.sh 'bash tools/gate.sh --oxide --release -p bloomery-gpu-gates --features gemma-embedding2,vision,deepseek41 --bin gate_gemma_embedding2_e2e' + ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features gemma-embedding2,vision,deepseek41 --release --bin gate_gemma_embedding2_e2e && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_gemma_embedding2_e2e {{ARGS}}' # GLM-5.3-Flash MLA 층의 카드 부분 — 잠재 RMS + f16 추가, 인덱서 키 LayerNorm + [키; 게이트] 추가(`latent`), # 창 0·싱크 −∞·스케일 1/16으로 다시 쓰는 V4.1 어텐션 — 을 모델 없이 합성 입력으로 호스트 규칙에 맞춘다. From f8e6c31db92442786a97efe2dfe558e919003d19 Mon Sep 17 00:00:00 2001 From: juliankang4 <200967947+juliankang4@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:40:52 +0900 Subject: [PATCH 07/12] refset: dump the EmbeddingGemma 2 server CPU twin Register the tower and server CPU families, collect image-only and text-first vectors, and keep CPU server logs apart from card logs. The collector also compares a running HTTP server with saved llama-server vectors, printing cosine and maximum absolute difference. --- .../refset/src/arch/gemma_embedding2/mod.rs | 86 ++++++++++---- .../refset/src/arch/gemma_embedding2/tests.rs | 11 +- crates/refset/src/emb2.rs | 7 +- crates/refset/src/emb2/tests.rs | 2 +- tools/ref/emb2/emb2.sh | 40 +++++-- tools/ref/emb2/server_vectors.py | 112 ++++++++++++++++-- tools/ref/models/gemma-embedding2.sh | 2 +- 7 files changed, 207 insertions(+), 53 deletions(-) diff --git a/crates/refset/src/arch/gemma_embedding2/mod.rs b/crates/refset/src/arch/gemma_embedding2/mod.rs index 5f5413212..8b699e08f 100644 --- a/crates/refset/src/arch/gemma_embedding2/mod.rs +++ b/crates/refset/src/arch/gemma_embedding2/mod.rs @@ -14,7 +14,7 @@ //! its `# model` is the projector file and its `# arch` the projector's //! (`clip`). [`TOWER_CPU`] is its CPU twin. //! - [`SERVER`]: llama-server's `/v1/embeddings` vectors of every text case and -//! image, normalized and raw (`tools/ref/emb2/server_vectors.py`). +//! image, normalized and raw (`tools/ref/emb2/server_vectors.py`). [`SERVER_CPU`] is its CPU twin. //! - [`AUDIO_MEL`]: the pinned host preprocessor's mel from synthetic decoded //! mono PCM (`tools/ref/emb2/mel.sh`), with no model or device opened. //! @@ -61,8 +61,23 @@ pub const TEXT_CPU_SET: &str = "ref_gemma_embedding2_text.cpu"; pub const TOWER_SET: &str = "ref_gemma_embedding2_tower"; pub const TOWER_CPU_SET: &str = "ref_gemma_embedding2_tower.cpu"; +/// The image inputs the mainline tower dump supplies, in the profile's order. +pub const TOWER_IMAGES: &[&str] = &[ + "small-200x200", + "min-96x64", + "free-448x448", + "free-640x480", + "pad-777x513", + "strip-1000x1", + "free-1024x768", +]; + +/// The gate that reads the tower sets. +pub const TOWER_GATE: &str = "gate-gpu-gemma4v-tower"; + /// llama-server's vectors. pub const SERVER_SET: &str = "ref_gemma_embedding2_server"; +pub const SERVER_CPU_SET: &str = "ref_gemma_embedding2_server.cpu"; /// A row's band in the encoder and serving gates: this many times the CPU /// twin's distance from the card set (`shared/glm5next_media.rs`'s `TWIN`, @@ -97,6 +112,7 @@ fn mmproj() -> Result { /// The gate that reads the text sets. const E2E: &[&str] = &["gate-gpu-gemma-embedding2-e2e"]; +const IMAGE: &[&str] = &[TOWER_GATE, "gate-gpu-gemma-embedding2-e2e"]; /// The text cases' token rows, pooled vectors and layer taps on the card. pub static TEXT: Family = Family { @@ -128,28 +144,34 @@ pub static TEXT_CPU: Family = Family { ) }; /// The image cases' tower taps and rows on the card. -pub static TOWER: Family = family( - "tower-gemma4v", - &[TOWER_SET], - Kind::Tower, - &PROFILE, - crate::clefvis::TOWER_ARCH, - mmproj, - RECIPE, -); +pub static TOWER: Family = Family { + consumers: IMAGE, + ..family( + "tower-gemma4v", + &[TOWER_SET], + Kind::Tower, + &PROFILE, + crate::clefvis::TOWER_ARCH, + mmproj, + RECIPE, + ) +}; /// The same, with no card in view. -pub static TOWER_CPU: Family = family( - "tower-gemma4v-cpu", - &[TOWER_CPU_SET], - Kind::TowerCpu, - &PROFILE, - crate::clefvis::TOWER_ARCH, - mmproj, - RECIPE, -); +pub static TOWER_CPU: Family = Family { + consumers: IMAGE, + ..family( + "tower-gemma4v-cpu", + &[TOWER_CPU_SET], + Kind::TowerCpu, + &PROFILE, + crate::clefvis::TOWER_ARCH, + mmproj, + RECIPE, + ) +}; /// llama-server's `/v1/embeddings` vectors. pub static SERVER: Family = Family { - consumers: &["gate-gpu-gemma-embedding2-serve"], + consumers: &["gate-gpu-gemma-embedding2-e2e", "gate-gpu-gemma-embedding2-serve"], ..family( "server-gemma-embedding2", &[SERVER_SET], @@ -183,8 +205,30 @@ pub static AUDIO_MEL: Family = Family { consumers: &["gate-emb2-mel"], }; +/// The same requests with the text model and projector on the CPU. +pub static SERVER_CPU: Family = Family { + consumers: E2E, + ..family( + "server-gemma-embedding2-cpu", + &[SERVER_CPU_SET], + Kind::ServerCpu, + &PROFILE, + ARCH, + model, + RECIPE, + ) +}; + /// The architecture's families, in the order `refset-check` lists them. -pub static FAMILIES: &[&Family] = &[&TEXT, &TEXT_CPU, &TOWER, &TOWER_CPU, &SERVER, &AUDIO_MEL]; +pub static FAMILIES: &[&Family] = &[ + &TEXT, + &TEXT_CPU, + &TOWER, + &TOWER_CPU, + &SERVER, + &SERVER_CPU, + &AUDIO_MEL, +]; #[cfg(test)] mod tests; diff --git a/crates/refset/src/arch/gemma_embedding2/tests.rs b/crates/refset/src/arch/gemma_embedding2/tests.rs index 899c44a44..1a7df91db 100644 --- a/crates/refset/src/arch/gemma_embedding2/tests.rs +++ b/crates/refset/src/arch/gemma_embedding2/tests.rs @@ -1,6 +1,6 @@ use super::{ - ARCH, MMPROJ_BF16, MMPROJ_BF16_SHA256, MODEL, MODEL_SHA256, PROFILE, SERVER, TEXT, TEXT_CPU, - TOWER, TOWER_CPU, + ARCH, MMPROJ_BF16, MMPROJ_BF16_SHA256, MODEL, MODEL_SHA256, PROFILE, SERVER, SERVER_CPU, + TEXT, TEXT_CPU, TOWER, TOWER_CPU, }; use crate::RefError; use crate::emb2::tests::{check, header, remove, set_dir}; @@ -40,12 +40,13 @@ type FileCase<'a> = (Kind, &'a Family, Vec<(&'a str, &'a str)>, &'a str); /// the trailer is written, and the text the error names. type RefusedCase<'a> = (Kind, &'a Family, (&'a str, &'a str), bool, &'a str); -const ALL: [(Kind, &Family); 5] = [ +const ALL: [(Kind, &Family); 6] = [ (Kind::Text, &TEXT), (Kind::TextCpu, &TEXT_CPU), (Kind::Tower, &TOWER), (Kind::TowerCpu, &TOWER_CPU), (Kind::Server, &SERVER), + (Kind::ServerCpu, &SERVER_CPU), ]; /// Each family takes a set of its own kind and device, of the pinned files, @@ -75,12 +76,14 @@ fn each_family_takes_a_set_of_its_kind() -> Result<(), RefError> { #[test] fn a_set_of_another_kind_or_device_is_foreign() -> Result<(), RefError> { let dir = set_dir("kind")?; - let cases: [(Kind, &Family, &str); 5] = [ + let cases: [(Kind, &Family, &str); 7] = [ (Kind::TextCpu, &TEXT, "device \"cpu\""), (Kind::Text, &TEXT_CPU, "device \"cuda\""), (Kind::TowerCpu, &TOWER, "device \"cpu\""), (Kind::Tower, &TOWER_CPU, "device \"cuda\""), (Kind::Server, &TEXT, "emb2 \"server\""), + (Kind::ServerCpu, &SERVER, "device \"cpu\""), + (Kind::Server, &SERVER_CPU, "device \"cuda\""), ]; for (kind, family, want) in cases { match check(&dir, family, &header(&PROFILE, ARCH, kind, &[]), true) { diff --git a/crates/refset/src/emb2.rs b/crates/refset/src/emb2.rs index 1dda4d98e..cfbd12c5c 100644 --- a/crates/refset/src/emb2.rs +++ b/crates/refset/src/emb2.rs @@ -18,6 +18,7 @@ pub enum Kind { Tower, TowerCpu, Server, + ServerCpu, } impl Kind { @@ -27,7 +28,7 @@ impl Kind { match self { Kind::Text | Kind::TextCpu => "text", Kind::Tower | Kind::TowerCpu => "tower", - Kind::Server => "server", + Kind::Server | Kind::ServerCpu => "server", } } @@ -35,7 +36,7 @@ impl Kind { #[must_use] pub const fn device(self) -> &'static str { match self { - Kind::TextCpu | Kind::TowerCpu => "cpu", + Kind::TextCpu | Kind::TowerCpu | Kind::ServerCpu => "cpu", Kind::Text | Kind::Tower | Kind::Server => "cuda", } } @@ -114,7 +115,7 @@ impl Emb2Set { )); } }, - Kind::Text | Kind::TextCpu | Kind::Server => { + Kind::Text | Kind::TextCpu | Kind::Server | Kind::ServerCpu => { let model = one(&man, "model")?; let [path] = model[..] else { return Err(RefError::malformed( diff --git a/crates/refset/src/emb2/tests.rs b/crates/refset/src/emb2/tests.rs index 689f658cc..6c7fc757a 100644 --- a/crates/refset/src/emb2/tests.rs +++ b/crates/refset/src/emb2/tests.rs @@ -27,7 +27,7 @@ pub(crate) fn header( let device = format!( "{}\tthreads\t{}\tcard\t{}", kind.device(), - if kind == Kind::Server { "-" } else { "16" }, + if matches!(kind, Kind::Server | Kind::ServerCpu) { "-" } else { "16" }, if kind.device() == "cpu" { "cpu" } else { diff --git a/tools/ref/emb2/emb2.sh b/tools/ref/emb2/emb2.sh index e68d3b252..4c06f5e54 100755 --- a/tools/ref/emb2/emb2.sh +++ b/tools/ref/emb2/emb2.sh @@ -11,6 +11,7 @@ # text.cpu ${SETPFX}_text.cpu: the same with no card in view # tower ${SETPFX}_tower: dump_emb2 tower over IMAGES on the card # tower.cpu ${SETPFX}_tower.cpu: the same with no card in view +# server.cpu ${SETPFX}_server.cpu: the same requests with both models on CPU # server ${SETPFX}_server: llama-server's /v1/embeddings vectors of every text # case that is UTF-8 and every image (server_vectors.py), the server on 127.0.0.1:${EMB2_PORT:-8097} for the # step's length @@ -24,7 +25,7 @@ # libllama.so it loads, and of the two cases files as they are now. Each set is written to .staging and moved over the # old set only when its writer exits 0, so a failed step leaves the old set whole: the old set waits as .old while the # new one takes its name, an exit or a caught signal in between moves it back, and a step that finds a .old left by a -# killed run restores it first (or discards it when the replacement is already there). The card steps (text, tower, server) +# killed run restores it first (or retains it with a timestamp, when the new set has its name already). The card steps (text, tower, server) # go through tools/gpu-gate.sh, as clefvis.sh's do: the card's gate lock and the bound EMB2_BOUND (default 1500 s, parsed as # tools/gate-bound.sh parses BLOOMERY_GATE_BOUND), for the call and for each step; the script links itself as # target/release/emb2_dump and runs the card steps under that name inside the runner. The twins open no card @@ -73,7 +74,7 @@ if [ "$(basename "${BASH_SOURCE[0]}")" != emb2_dump ] && [ ${#card_steps[@]} -gt a6000 | both) BLOOMERY_GATE_BOUND=$BOUND bash tools/gpu-gate.sh emb2_dump "${card_steps[@]}" ;; *) BLOOMERY_GATE_BOUND=$BOUND BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh emb2_dump "${card_steps[@]}" ;; esac - for s in text.cpu tower.cpu; do + for s in text.cpu tower.cpu server.cpu; do if selected "$s"; then bash "$REAL" "$s"; fi done exit 0 @@ -162,11 +163,19 @@ extra cases:3:$(md5sum < "crates/tokenizer/tests/$EMBED_CASES" | cut -d' ' -f1) EOF } -# .old left by a killed run: the old set when is missing, a leftover when the new set had its name already. +# Replaced sets and incomplete staging directories stay beside the published set under a unique retained name. +retain() { + if [ -e "$1" ]; then + local kept=$1.retained.$(date -u +%Y%m%dT%H%M%SZ).$$ + mv "$1" "$kept" + echo "emb2: retained $kept" + fi +} + recover() { # [ -e "$1.old" ] || return 0 if [ -e "$1" ]; then - rm -rf "$1.old" + retain "$1.old" else mv "$1.old" "$1" echo "emb2: restored $1 from $1.old" @@ -178,7 +187,7 @@ publish() { # : the old set stays whole until the new one has i if [ -e "$2" ]; then mv "$2" "$2.old"; fi mv "$1" "$2" publishing= - rm -rf "$2.old" + retain "$2.old" echo "emb2: $(basename "$2") -> $2" } @@ -191,7 +200,10 @@ for name in "${IMAGES[@]}"; do done server_step() { - local stage=$1 card_name=$2 port=${EMB2_PORT:-8097} up='' log=$DATA/${SETPFX}_server.log flags media_args + local stage=$1 card_name=$2 device=$3 port=${EMB2_PORT:-8097} up='' log flags media_args + log=$DATA/${SETPFX}_server + if [ "$device" = cpu ]; then log+=.cpu; fi + log+=.log if (: < "/dev/tcp/127.0.0.1/$port") 2> /dev/null; then die "something already listens on 127.0.0.1:$port; stop it or set EMB2_PORT" fi @@ -202,6 +214,12 @@ server_step() { flags+=(--mmproj "$MMPROJ") media_args=(--mmproj "$MMPROJ" --mmproj-sha256 "$MMPROJ_SHA256" "${image_args[@]}" --mix-image free-448x448) fi + local cpu=() + if [ "$device" = cpu ]; then + export CUDA_VISIBLE_DEVICES= + flags+=(-ngl 0 --no-mmproj-offload) + cpu=(--cpu) + fi "$SERVER_BIN" "${flags[@]}" > "$log" 2>&1 & server_pid=$! for _ in $(seq 120); do @@ -211,7 +229,7 @@ server_step() { done [ -n "$up" ] || die "llama-server did not come up (log: $log)" 1 timeout --kill-after=10 "$BOUND" python3 tools/ref/emb2/server_vectors.py --url "http://127.0.0.1:$port" --pid "$server_pid" \ - --model "$MODEL" --arch "$MODEL_NAME" --model-sha256 "$MODEL_SHA256" --card "$card_name" \ + "${cpu[@]}" --model "$MODEL" --arch "$MODEL_NAME" --model-sha256 "$MODEL_SHA256" --card "$card_name" \ --build-record "$BIN.build" --ids-dir "$IDS" "${case_args[@]}" --out "$stage" "${media_args[@]}" \ --server-flags "${flags[*]}" kill "$server_pid" @@ -230,7 +248,7 @@ for s in "${ALL[@]}"; do stage=$set_dir.staging recover "$set_dir" verify - rm -rf "$stage" + retain "$stage" SECONDS=0 case $s in text | text.cpu | tower | tower.cpu) @@ -249,10 +267,10 @@ for s in "${ALL[@]}"; do "${run[@]}" timeout --kill-after=10 "$BOUND" "$BIN" "${args[@]}" "${where[@]}" -t "$THREADS" --build-record "$BIN.build" \ --out "$stage" ;; - server) + server | server.cpu) ids_current - card_name=$(card) - server_step "$stage" "$card_name" + if [ "$s" = server.cpu ]; then card_name=cpu; device=cpu; else card_name=$(card); device=cuda; fi + server_step "$stage" "$card_name" "$device" ;; esac publish "$stage" "$set_dir" diff --git a/tools/ref/emb2/server_vectors.py b/tools/ref/emb2/server_vectors.py index 9604436b2..e26652b0f 100644 --- a/tools/ref/emb2/server_vectors.py +++ b/tools/ref/emb2/server_vectors.py @@ -28,19 +28,108 @@ `# mmproj sha256 `; then the server's flags and build, the build record's lines, and the `# complete` trailer, written last through a .partial name. Python's standard library only. + +Compare a running bloomery HTTP server with those saved vectors (no new oracle or threshold): + server_vectors.py compare --url --set [--ids-dir --case ...] + [--image = ...] [--mix-image ] +Prints cosine and maximum absolute difference for each selected raw and normalized vector; image inputs use the +collector's exact content-part construction. A successful command means the comparisons completed, not that a band passed. """ import argparse import base64 import hashlib +import importlib.util import json import math import os import struct +import sys import urllib.request from pathlib import Path MRL = (128, 256, 512) +MIX_TEXT = "A picture of coloured rings and ramps: " + + +def image_entries(images: list[str], mix: str | None): + """The same image-only and text-first inputs for collecting and comparing HTTP vectors.""" + found = False + for item in images: + name, png = item.split("=", 1) + data = Path(png).read_bytes() + url = "data:image/png;base64," + base64.b64encode(data).decode() + part = {"type": "image_url", "image_url": {"url": url}} + yield f"img_{name}", {"content": [part]}, png, data + if name == mix: + found = True + text = {"type": "text", "text": MIX_TEXT} + yield f"mix_{name}", {"content": [text, part]}, png, data + if mix and not found: + raise SystemExit(f"server_vectors: --mix-image {mix} names no --image") + + +def metrics(ours: list[float], reference: list[float]) -> tuple[float, float]: + """Cosine and maximum absolute difference, refusing empty, non-finite or unequal vectors.""" + if not ours or len(ours) != len(reference) or not all(math.isfinite(x) for x in ours + reference): + raise ValueError("empty, non-finite or unequal vector extents") + dot = sum(x * y for x, y in zip(ours, reference)) + norm = math.sqrt(sum(x * x for x in ours)) * math.sqrt(sum(y * y for y in reference)) + if norm == 0: + raise ValueError("cosine is undefined for a zero vector") + return dot / norm, max(abs(x - y) for x, y in zip(ours, reference)) + + +def compare_main(argv: list[str]) -> None: + """Compare a running HTTP server with saved llama-server raw and normalized vectors.""" + ap = argparse.ArgumentParser(description=compare_main.__doc__) + ap.add_argument("--url", required=True) + ap.add_argument("--set", required=True, type=Path) + ap.add_argument("--ids-dir", type=Path) + ap.add_argument("--case", action="append", default=[]) + ap.add_argument("--image", action="append", default=[]) + ap.add_argument("--mix-image") + a = ap.parse_args(argv) + if not a.case and not a.image: + ap.error("at least one --case or --image is required") + if a.case and a.ids_dir is None: + ap.error("--case requires --ids-dir") + spec = importlib.util.spec_from_file_location("manifest", Path(__file__).resolve().parents[2] / "bloomery/manifest.py") + manifest = importlib.util.module_from_spec(spec) + spec.loader.exec_module(manifest) + m = manifest.read(str(a.set)) + if m.complete is None or m.header.get("emb2") != "server" or m.header.get("arch") != "gemma-embedding2": + raise SystemExit(f"server_vectors: {a.set}: not a complete EmbeddingGemma 2 server set") + rows = {} + for row in m.rows("tensor"): + name = row["name"] + if name in rows: + raise SystemExit(f"server_vectors: {name}: duplicate tensor") + rows[name] = row + entries = [] + for name in a.case: + entries.append((name, (a.ids_dir / f"{name}.txt").read_text(encoding="utf-8"))) + entries.extend((name, entry) for name, entry, _, _ in image_entries(a.image, a.mix_image)) + for name, entry in entries: + for node, normalization in (("raw", -1), ("embd", 2)): + key = f"{name}/{node}" + if key not in rows: + raise SystemExit(f"server_vectors: {a.set}: no tensor {key}") + row = rows[key] + if row["type"] != "f32" or row.int("occurrence") != 0 or any(row.int(f"ne{i}") != 1 for i in (1, 2, 3)): + raise SystemExit(f"server_vectors: {key}: not one F32 vector") + path = Path(m.dir) / manifest.dump_file_name(key, 0, "output", "contiguous", "f32") + data = path.read_bytes() + if len(data) != row.int("bytes") or len(data) != row.int("ne0") * 4: + raise SystemExit(f"server_vectors: {key}: tensor byte count differs") + reference = list(struct.unpack(f"<{len(data) // 4}f", data)) + ours, count = embed(a.url, entry, normalization) + try: + cosine, maximum = metrics(ours, reference) + except ValueError as e: + raise SystemExit(f"server_vectors: {key}: {e}") from e + print(f"{key}: prompt_tokens {count} cosine {cosine:.9e} max_abs_diff {maximum:.9e}") + def post(url: str, path: str, body: dict) -> dict: @@ -119,6 +208,9 @@ def check_server(a: argparse.Namespace, build: str) -> dict: def main() -> None: + if sys.argv[1:2] == ["compare"]: + compare_main(sys.argv[2:]) + return ap = argparse.ArgumentParser() ap.add_argument("--url", required=True) ap.add_argument("--pid", required=True, type=int) @@ -128,6 +220,7 @@ def main() -> None: ap.add_argument("--mmproj-sha256") ap.add_argument("--arch", choices=("gemma-embedding", "gemma-embedding2"), required=True) ap.add_argument("--card", required=True) + ap.add_argument("--cpu", action="store_true") ap.add_argument("--build-record", required=True, type=Path) ap.add_argument("--ids-dir", required=True, type=Path) ap.add_argument("--case", action="append", required=True) @@ -137,6 +230,8 @@ def main() -> None: ap.add_argument("--server-flags", default="") a = ap.parse_args() + if a.cpu and os.environ.get("CUDA_VISIBLE_DEVICES") != "": + raise SystemExit("server_vectors: --cpu requires CUDA_VISIBLE_DEVICES empty") build = os.environ.get("BLOOMERY_REF_BUILD", "") if not build: raise SystemExit("server_vectors: BLOOMERY_REF_BUILD names no commit") @@ -157,7 +252,7 @@ def main() -> None: f"# build\t{build}", f"# arch\t{a.arch}", "# emb2\tserver", - f"# device\tcuda\tthreads\t-\tcard\t{a.card}", + f"# device\t{'cpu' if a.cpu else 'cuda'}\tthreads\t-\tcard\t{a.card}", f"# model_sha256\t{a.model_sha256}", f"# server_flags\t{a.server_flags}", f"# server_build\t{props['build_info']}", @@ -180,17 +275,10 @@ def main() -> None: if got != want or n != len(want): raise SystemExit(f"server_vectors: {name}: the server's ids are not the oracle's ({same}, prompt_tokens {n})") - images = dict(i.split("=", 1) for i in a.image) - for name, png in images.items(): - data = Path(png).read_bytes() - url = "data:image/png;base64," + base64.b64encode(data).decode() - part = {"type": "image_url", "image_url": {"url": url}} - n = pooled_rows(w, a.url, f"img_{name}", {"content": [part]}, False) - w.header.append(f"# image\timg_{name}\t{png}\t{hashlib.sha256(data).hexdigest()}\tprompt_tokens {n}") - if name == a.mix_image: - text = {"type": "text", "text": "A picture of coloured rings and ramps: "} - n = pooled_rows(w, a.url, f"mix_{name}", {"content": [text, part]}, False) - w.header.append(f"# image\tmix_{name}\t{png}\t{hashlib.sha256(data).hexdigest()}\tprompt_tokens {n}\ttext first") + for name, entry, png, data in image_entries(a.image, a.mix_image): + n = pooled_rows(w, a.url, name, entry, False) + suffix = "\ttext first" if name.startswith("mix_") else "" + w.header.append(f"# image\t{name}\t{png}\t{hashlib.sha256(data).hexdigest()}\tprompt_tokens {n}{suffix}") check_server(a, build) w.finish() print(f"server_vectors: {len(w.rows)} row(s) into {a.out}") diff --git a/tools/ref/models/gemma-embedding2.sh b/tools/ref/models/gemma-embedding2.sh index d3e6b29a7..6333f12fb 100644 --- a/tools/ref/models/gemma-embedding2.sh +++ b/tools/ref/models/gemma-embedding2.sh @@ -39,4 +39,4 @@ IMAGES=(small-200x200 min-96x64 free-448x448 free-640x480 pad-777x513 strip-1000 EMBED_VOCAB=gemma4 EMBED_CASES=cases-gemma4.txt EMBED_CONTEXT=8192 -EMBED_STEPS=(text tower server text.cpu tower.cpu) +EMBED_STEPS=(text tower server text.cpu tower.cpu server.cpu) From 957a374342b3c9ba32fccae39528c7f700c3051b Mon Sep 17 00:00:00 2001 From: juliankang4 <200967947+juliankang4@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:42:20 +0900 Subject: [PATCH 08/12] gpu-gates: compare Gemma4 tower and image passes with their CPU twins Check F32 convolution pixels, raster pooling, every layer and projected rows. Image-only and mix_free-448x448 inputs use the serving assembler and compare raw and normalized vectors with both server sets. Reuse the family opener and the shared CPU-twin multiplier. PIN(2026-10-15): Remove the duplicate embd comparison because projected reads the same output buffer; the projected comparison keeps its band. --- crates/gpu-gates/Cargo.toml | 4 + .../gpu-gates/src/bin/gate_gemma4v_tower.rs | 152 ++++++++++++++++++ .../src/bin/gate_gemma_embedding2_e2e.rs | 116 +++++++++++-- crates/gpu-gates/src/bin/shared/emb2_image.rs | 87 ++++++++++ justfile | 7 +- tools/gate-paths.tsv | 102 ++++++------ tools/recipes.py | 5 +- 7 files changed, 409 insertions(+), 64 deletions(-) create mode 100644 crates/gpu-gates/src/bin/gate_gemma4v_tower.rs create mode 100644 crates/gpu-gates/src/bin/shared/emb2_image.rs diff --git a/crates/gpu-gates/Cargo.toml b/crates/gpu-gates/Cargo.toml index 734c749f9..767860a4f 100644 --- a/crates/gpu-gates/Cargo.toml +++ b/crates/gpu-gates/Cargo.toml @@ -547,6 +547,10 @@ required-features = ["gemma-embedding2", "vision", "deepseek41"] name = "gate_gemma_embedding2_serve" required-features = ["deepseek41", "gemma-embedding2"] +[[bin]] +name = "gate_gemma4v_tower" +required-features = ["vision"] + [[bin]] name = "gate_qwen3moe_router" required-features = ["gpu"] diff --git a/crates/gpu-gates/src/bin/gate_gemma4v_tower.rs b/crates/gpu-gates/src/bin/gate_gemma4v_tower.rs new file mode 100644 index 000000000..3598fd86b --- /dev/null +++ b/crates/gpu-gates/src/bin/gate_gemma4v_tower.rs @@ -0,0 +1,152 @@ +//! The Gemma4 tower from pinned image bytes through every layer to projected rows. +//! Each numerical band is 2.5 times that row's CPU/card oracle distance. + +#[path = "shared/emb2_image.rs"] +mod image; + +#[allow( + dead_code, + reason = "the shared runner serves several gates; this one reads its distance helper" +)] +#[path = "shared/e2e.rs"] +mod e2e; + +use bloomery_gpu::Gpu; +use bloomery_gpu_gates::{GateError, checks_failed, exit_with, ref_model_path, verdict}; +use bloomery_gpu_vision::{bf16_f32, chain::TapSink, gemma4v::Encoder}; +use gguf::Gguf; +use refset::arch::gemma_embedding2::{TOWER, TOWER_CPU, TOWER_IMAGES}; +use std::collections::BTreeMap; +use vision::arch::TokenLimits; +use vision::arch::gemma4v::{Hparams, MERGE, conv_pixels, preprocess}; + +#[derive(Default)] +struct Taps(BTreeMap>); +impl TapSink for Taps { + fn wants(&self, _: &str) -> bool { + true + } + fn take(&mut self, name: &str, _: usize, bits: Vec) { + self.0 + .insert(name.to_string(), bits.into_iter().map(bf16_f32).collect()); + } + fn take_f32(&mut self, name: &str, _: usize, values: Vec) { + self.0.insert(name.to_string(), values); + } +} + +fn pooling(p: &vision::Patches, taps: &Taps, hp: &Hparams) -> Result { + use bloomery_gpu_vision::{f32_bf16, mean_groups::mean_groups_ref}; + let last = taps + .0 + .get(&format!("layer_out-{}", hp.n_layer - 1)) + .ok_or("no last layer for pooling")?; + let pooled = taps.0.get("pooled").ok_or("no pooled tap")?; + let mut gathered = Vec::with_capacity(last.len()); + for gy in 0..p.n_vit_h / MERGE { + for gx in 0..p.n_vit_w / MERGE { + for dy in 0..MERGE { + for dx in 0..MERGE { + let row = ((gy * MERGE + dy) * p.n_vit_w + gx * MERGE + dx) * hp.dim; + gathered.extend(last[row..row + hp.dim].iter().map(|&v| f32_bf16(v))); + } + } + } + } + let expected: Vec<_> = mean_groups_ref(&gathered, hp.dim, MERGE * MERGE) + .into_iter() + .map(|bits| bf16_f32(f32_bf16(bf16_f32(bits) * (hp.dim as f32).sqrt()))) + .collect(); + Ok(pooled + .iter() + .map(|v| v.to_bits()) + .eq(expected.iter().map(|v| v.to_bits()))) +} + +fn run() -> Result<(), GateError> { + bloomery_levers::at_main(&[])?; + let card = image::open(&TOWER)?; + let cpu = image::open(&TOWER_CPU)?; + let model = ref_model_path()?; + let file = Gguf::open(image::projector(&model)?)?; + let hp = Hparams::read(&file)?; + let limits = TokenLimits::new(70, 1120)?; + let rule = hp.size_rule(limits)?; + let gpu = Gpu::new()?; + let stream = gpu.stream_handle(); + let mut enc = Encoder::load(gpu.context(), &stream, &file, limits)?; + let mut ok = true; + for name in TOWER_IMAGES { + let (rgb, _) = image::image(&card, &cpu, name)?; + let p = preprocess(&rgb, &rule)?; + let n = p.n_vit_h * p.n_vit_w; + let tokens = n / (MERGE * MERGE); + let raw_shape = [p.plan.best_w as u64, p.plan.best_h as u64, 3, 1]; + let raw = image::row(&card, &format!("{name}/inp_raw_scaled"), raw_shape)?; + let raw_cpu = image::row(&cpu, &format!("{name}/inp_raw_scaled"), raw_shape)?; + let mut patches = Vec::with_capacity(raw.len()); + let mut pixels = Vec::with_capacity(raw.len()); + for py in 0..p.n_vit_h { + for px in 0..p.n_vit_w { + for c in 0..3 { + for y in 0..hp.patch { + for x in 0..hp.patch { + let at = (c * p.plan.best_h + py * hp.patch + y) * p.plan.best_w + + px * hp.patch + + x; + patches.push(bloomery_gpu_vision::f32_bf16(raw[at])); + pixels.push(raw[at]); + } + } + } + } + } + let exact = p.bf16 == patches + && e2e::same_bits(&raw, &raw_cpu) + && e2e::same_bits(&conv_pixels(&p.bf16)?, &pixels); + println!( + "(p) {name}: canvas and normalized patch bits {}", + verdict(exact) + ); + ok &= exact; + let mut taps = Taps::default(); + enc.encode_tapped(&p, &mut taps)?; + let pool_ok = pooling(&p, &taps, &hp)?; + println!( + "(o) {name}: 3x3 raster pooling and scale bits {}", + verdict(pool_ok) + ); + ok &= pool_ok; + for (node, ours) in &taps.0 { + let cols = if node == "projected" { + hp.out_dim + } else { + hp.dim + }; + let count = if node == "pos_embd" || node.starts_with("layer_out-") { + n + } else { + tokens + }; + let shape = [cols as u64, count as u64, 1, 1]; + let key = format!("{name}/{node}"); + let reference = image::row(&card, &key, shape)?; + let twin = image::row(&cpu, &key, shape)?; + ok &= image::compare(&format!("(t) {key}"), ours, &reference, &twin); + } + for node in ["pos_embd", "pooled", "projected"] + .into_iter() + .map(String::from) + .chain((0..hp.n_layer).map(|l| format!("layer_out-{l}"))) + { + if !taps.0.contains_key(&node) { + return Err(format!("{name}: no tap {node}").into()); + } + } + } + if ok { Ok(()) } else { Err(checks_failed()) } +} + +fn main() -> std::process::ExitCode { + exit_with("gate_gemma4v_tower", run()) +} diff --git a/crates/gpu-gates/src/bin/gate_gemma_embedding2_e2e.rs b/crates/gpu-gates/src/bin/gate_gemma_embedding2_e2e.rs index f2cbf939b..cf1146b0e 100644 --- a/crates/gpu-gates/src/bin/gate_gemma_embedding2_e2e.rs +++ b/crates/gpu-gates/src/bin/gate_gemma_embedding2_e2e.rs @@ -2,7 +2,7 @@ //! end to end on the Q8_0 file the sets are dumped from ([`MODEL`], held to //! its sha256), against llama.cpp mainline's text set dumped on the card //! ([`TEXT`]) and its CPU twin ([`TEXT_CPU`]), both read through -//! [`Emb2Set::open`]: each case the set names, its ids read from the +//! [`refset::emb2::Emb2Set::open`]: each case the set names, its ids read from the //! tokenizer oracle's gemma4 set ([`IDS_SET`]) and held to the set's count //! and digest, through one pass at the load's context with its taps read //! back. @@ -22,7 +22,9 @@ //! case holds the global layers. //! 2. (d) The longest case run again with no taps is bit for bit the first //! run's rows. -//! 3. (r) Refused by name, the load untouched after each: a pass of no ids, +//! 3. (i) Image-only and `mix_free-448x448` inputs assembled by the serving helper, +//! pooled and normalized against the server card set and its CPU twin. +//! 4. (r) Refused by name, the load untouched after each: a pass of no ids, //! of one id past the load's context and of an id past the vocabulary, //! and a load at one position past [`CARD_CTX`]. @@ -39,22 +41,26 @@ fn main() -> std::process::ExitCode { #[path = "shared/e2e.rs"] mod e2e; +#[path = "shared/emb2_image.rs"] +mod image; + #[cfg(feature = "gemma-embedding2")] mod gate { use std::path::Path; use std::time::Instant; use crate::e2e::{clause, elapsed, refused_by, rel, same_bits, tap}; + use crate::image::open; use bloomery_gpu::Gpu; - use bloomery_gpu_gates::{GateError, RefManifest, checks_failed, data_dir, sha256, verdict}; + use bloomery_gpu_gates::{ + GateError, RefManifest, checks_failed, data_dir, ref_model_path, sha256, verdict, + }; use bloomery_gpu_gemma_embedding2::{Encoder, Pass, card_bytes}; use gguf::Split; use model::arch::gemma_embedding2::hparams::{CARD_CTX, Hparams}; use refset::arch::gemma_embedding2::{ IDS_SET, MODEL, MODEL_SHA256, REL_FLOOR as FLOOR, TEXT, TEXT_CPU, TWIN, }; - use refset::emb2::Emb2Set; - use refset::family::Family; use refset::ik::RefHeader; use refset::tokenizer::read_ids; @@ -270,9 +276,95 @@ mod gate { Ok(ok && same) } - /// `family`'s set, read through its reader. - fn open(family: &Family) -> Result { - Ok(Emb2Set::open(&family.path(family.sets[0]), family)?.man) + fn images(enc: &mut Encoder, model: &Path) -> Result { + use crate::image; + use bloomery_gpu::media::MediaSpan; + use bloomery_gpu_gates::bind::Vocab; + use bloomery_gpu_vision::gemma4v::Encoder as Tower; + use refset::arch::gemma_embedding2::{SERVER, SERVER_CPU, TOWER, TOWER_CPU, TOWER_IMAGES}; + use serve::embeddings::{InputPart, assemble}; + use vision::arch::{TokenLimits, gemma4v}; + + let card = image::open(&TOWER)?; + let cpu = image::open(&TOWER_CPU)?; + let server = image::open(&SERVER)?; + let server_cpu = image::open(&SERVER_CPU)?; + let file = gguf::Gguf::open(image::projector(model)?)?; + let hp = gemma4v::Hparams::read(&file)?; + let limits = TokenLimits::new(70, 1120)?; + let rule = hp.size_rule(limits)?; + let stream = enc.context().new_stream_with_priority(0)?; + let mut tower = Tower::load(enc.context(), &stream, &file, limits)?; + let tok = Vocab::new(tokenizer::Tokenizer::from_gguf(model)?)?; + let mut ok = true; + for name in TOWER_IMAGES { + let (rgb, digest) = image::image(&card, &cpu, name)?; + let p = gemma4v::preprocess(&rgb, &rule)?; + let tokens = p.plan.n_llm_h * p.plan.n_llm_w; + let rows = tower.encode(&p)?.rows.buf().to_host_vec(&stream)?; + for mixed in [false, true] { + if mixed && *name != "free-448x448" { + continue; + } + let mut parts = Vec::new(); + if mixed { + parts.push(InputPart::Text("A picture of coloured rings and ramps: ")); + } + parts.push(InputPart::Image { + begin: gemma4v::IMAGE_BEGIN, + end: gemma4v::IMAGE_END, + positions: tokens, + }); + let assembled = assemble(parts.into_iter().map(Ok), &tok, CARD_CTX)?; + let ids = &assembled.ids; + let span = assembled + .spans + .first() + .ok_or("image assembly returned no span")?; + let key = format!("{}_{name}", if mixed { "mix" } else { "img" }); + // The server pair must have consumed these same bytes and position count. + if image::image(&server, &server_cpu, &key)?.1 != digest { + return Err(format!("{key}: server/tower image sha256 mismatch").into()); + } + for man in [&server, &server_cpu] { + let prefix = format!("# image\t{key}\t"); + let expected = format!("prompt_tokens {}", ids.len()); + if !man + .header + .other + .iter() + .any(|l| l.starts_with(&prefix) && l.split('\t').any(|f| f == expected)) + { + return Err(format!( + "{key}: server position count differs from {}", + ids.len() + ) + .into()); + } + } + let pass = enc.pass_media( + ids, + &[MediaSpan { + at: span.clone(), + rows: &rows, + }], + false, + )?; + let norm = serve::embed::normalize(&pass.pooled, serve::embed::Norm::default())?; + for (node, ours) in [("raw", &pass.pooled), ("embd", &norm)] { + let key = format!("{key}/{node}"); + let expected = image::row(&server, &key, [768, 1, 1, 1])?; + let twin = image::row(&server_cpu, &key, [768, 1, 1, 1])?; + ok &= image::compare( + &format!("(i) {key} pass_media {} positions", ids.len()), + ours, + &expected, + &twin, + ); + } + } + } + Ok(ok) } pub fn run() -> Result<(), GateError> { @@ -281,14 +373,15 @@ mod gate { if !args.is_empty() { return Err(format!("no arguments; got {args:?}").into()); } - let digest = sha256(Path::new(MODEL))?; + let model = ref_model_path()?; + let digest = sha256(&model)?; if digest != MODEL_SHA256 { return Err(format!( "{MODEL}: sha256 {digest}, the sets are dumped from {MODEL_SHA256}" ) .into()); } - let split = Split::open(MODEL).map_err(|e| format!("open {MODEL}: {e}"))?; + let split = Split::open(&model).map_err(|e| format!("open {}: {e}", model.display()))?; let hp = Hparams::read(&split).map_err(|e| format!("{MODEL}: {e}"))?; let read = model::arch::spec(&split).map_err(|e| format!("{MODEL}: {e}"))?; let encoder = read @@ -350,7 +443,8 @@ mod gate { refusals(&mut enc, c, &first.rows, hp.n_vocab), ); } - let past = Split::open(MODEL).map_err(|e| format!("open {MODEL}: {e}"))?; + ok &= clause("(i)", "image inputs", images(&mut enc, &model)); + let past = Split::open(&model).map_err(|e| format!("open {}: {e}", model.display()))?; let refused = refused_by( &Encoder::open(past, ctx + 1, Gpu::new()?), "Encoder::open", diff --git a/crates/gpu-gates/src/bin/shared/emb2_image.rs b/crates/gpu-gates/src/bin/shared/emb2_image.rs new file mode 100644 index 000000000..4153ab28b --- /dev/null +++ b/crates/gpu-gates/src/bin/shared/emb2_image.rs @@ -0,0 +1,87 @@ +//! The EmbeddingGemma image gates' pinned inputs and CPU-twin comparisons. + +use crate::e2e::rel; +use bloomery_gpu_gates::{GateError, RefManifest, sha256, verdict}; +use refset::arch::gemma_embedding2::{MMPROJ_BF16, MMPROJ_BF16_SHA256, TWIN}; +use refset::emb2::Emb2Set; +use refset::family::Family; +use refset::ik::ref_tensor_of_in; +use std::path::{Path, PathBuf}; +use vision::Rgb8; + +pub fn open(family: &Family) -> Result { + Ok(Emb2Set::open(&family.path(family.sets[0]), family)?.man) +} + +pub fn row(man: &RefManifest, name: &str, shape: [u64; 4]) -> Result, GateError> { + let r = man.tensor(name, 0)?; + if r.ty != "f32" || r.ne != shape { + return Err(format!("{name}: {} {:?}, expected f32 {shape:?}", r.ty, r.ne).into()); + } + let data = ref_tensor_of_in(&man.dir, r)?; + if data.iter().any(|v| !v.is_finite()) { + return Err(format!("{name}: non-finite oracle value").into()); + } + Ok(data) +} + +pub fn compare(name: &str, ours: &[f32], card: &[f32], cpu: &[f32]) -> bool { + let distance = rel(ours, card); + let twin = rel(cpu, card); + let band = TWIN * twin; + let ok = !card.is_empty() && distance.is_finite() && twin.is_finite() && distance <= band; + println!( + "{name}: rel {distance:.9e}, cpu-twin {twin:.9e}, band {band:.9e} ({TWIN} x twin) {}", + verdict(ok) + ); + ok +} + +pub fn projector(model: &Path) -> Result { + let path = model.with_file_name( + Path::new(MMPROJ_BF16) + .file_name() + .ok_or("no projector filename")?, + ); + let digest = sha256(&path)?; + if digest != MMPROJ_BF16_SHA256 { + return Err(format!( + "{}: projector sha256 {digest}, expected {MMPROJ_BF16_SHA256}", + path.display() + ) + .into()); + } + Ok(path) +} + +/// Both sets must name the same image bytes; the file is checked before decoding. +pub fn image( + card: &RefManifest, + cpu: &RefManifest, + name: &str, +) -> Result<(Rgb8, String), GateError> { + let header = |man: &RefManifest| -> Result<(PathBuf, String), GateError> { + let prefix = format!("# image\t{name}\t"); + let mut lines = man + .header + .other + .iter() + .filter_map(|l| l.strip_prefix(&prefix)); + let line = lines + .next() + .ok_or_else(|| format!("{name}: missing image identity"))?; + if lines.next().is_some() { + return Err(format!("{name}: duplicate image identity").into()); + } + let f: Vec<_> = line.split('\t').collect(); + if f.len() < 2 { + return Err(format!("{name}: malformed image identity").into()); + } + Ok((PathBuf::from(f[0]), f[1].to_string())) + }; + let (path, digest) = header(card)?; + if header(cpu)?.1 != digest || sha256(&path)? != digest { + return Err(format!("{name}: CPU/card image sha256 mismatch").into()); + } + Ok((Rgb8::from_png(&std::fs::read(path)?)?, digest)) +} diff --git a/justfile b/justfile index b71c4e5fb..93d9f6902 100644 --- a/justfile +++ b/justfile @@ -785,7 +785,7 @@ build-emb1-ref: (build-embedding-ref "gemma-embedding") build-embedding-ref MODEL: BLOOMERY_MODEL={{MODEL}} ./tools/box.sh 'bash tools/ref/clefvis/build-clefvis.sh' -# EmbeddingGemma 2, the oracle sets (tools/ref/emb2/emb2.sh; refset families in crates/refset/src/arch/gemma_embedding2) into $BLOOMERY_DATA: every text case's token rows, pooled vector and layer taps (ref_gemma_embedding2_text), the image cases' tower taps and rows (ref_gemma_embedding2_tower), llama-server's /v1/embeddings vectors of both (ref_gemma_embedding2_server), and the CPU twins of the first two (.cpu). The text cases and their ids are the tokenizer oracle's gemma4 set: `just dump-ref-tokenizer gemma4` first. STEPS names some of text, tower, server, text.cpu and tower.cpu (none: all five). The card steps go through the card's gate lock (EMB2_BOUND is the call's and each step's bound); no lease. +# EmbeddingGemma 2, the oracle sets (tools/ref/emb2/emb2.sh; refset families in crates/refset/src/arch/gemma_embedding2) into $BLOOMERY_DATA: every text case's token rows, pooled vector and layer taps (ref_gemma_embedding2_text), the image cases' tower taps and rows (ref_gemma_embedding2_tower), llama-server's /v1/embeddings vectors of both (ref_gemma_embedding2_server), and the CPU twins of all three (.cpu). The text cases and their ids are the tokenizer oracle's gemma4 set: `just dump-ref-tokenizer gemma4` first. STEPS names some of text, tower, server, text.cpu, tower.cpu and server.cpu (none: all six). The card steps go through the card's gate lock (EMB2_BOUND is the call's and each step's bound); no lease. dump-ref-emb2 *STEPS: (dump-ref-embedding "gemma-embedding2" STEPS) # Gemma4a host mel from synthetic decoded PCM, using the existing pinned mainline tree. @@ -1267,6 +1267,11 @@ gate-gpu-gemma-embedding2-e2e *ARGS: ./tools/box.sh 'bash tools/gate.sh --oxide --release -p bloomery-gpu-gates --features gemma-embedding2,vision,deepseek41 --bin gate_gemma_embedding2_e2e' ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features gemma-embedding2,vision,deepseek41 --release --bin gate_gemma_embedding2_e2e && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_gemma_embedding2_e2e {{ARGS}}' +# Gemma4 vision's mainline card and CPU sets, including every block's rows. +# Every layer and projected row is held to its CPU/card twin distance. +gate-gpu-gemma4v-tower: + BLOOMERY_MODEL=gemma-embedding2 ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features vision --release --bin gate_gemma4v_tower && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_gemma4v_tower' + # GLM-5.3-Flash MLA 층의 카드 부분 — 잠재 RMS + f16 추가, 인덱서 키 LayerNorm + [키; 게이트] 추가(`latent`), # 창 0·싱크 −∞·스케일 1/16으로 다시 쓰는 V4.1 어텐션 — 을 모델 없이 합성 입력으로 호스트 규칙에 맞춘다. # 추가는 캐시 비트, 어텐션은 오차 모형에서 끌어낸 밴드, 싱크 −∞는 분할 부분합의 무싱크 접기와 비트로 본다. diff --git a/tools/gate-paths.tsv b/tools/gate-paths.tsv index 4a7cf368e..e91a5c7ba 100644 --- a/tools/gate-paths.tsv +++ b/tools/gate-paths.tsv @@ -41,7 +41,7 @@ crates/gpu-deepseek41/src/{draft/**,experts_mxfp4.rs,markov.rs,hc_f32.rs} gate-g # crates/gpu-gates' lib, below its bins. crates/gpu-gates/src/record.rs gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-swap gate-gpu-qwen35moe-e2e gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3moe-e2e gate-q38-serve-rules gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media-seat gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text weekly-gpu-ds41-flowcounts gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve gate-gpu-qwen38-nodes the record kinds: the bins that print records (generate_ds41, bloomery-chat, bloomery-serve-ds41, the prefill, residency, callstream and swap gates) and generate.rs's users crates/gpu-gates/src/generate.rs gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-serve the V4.1 generate loop shared by generate_ds41, the chat and serve bins and the tier and two-card gates -crates/gpu-gates/src/{bind.rs,serve_client.rs} gate-ds41-bind gate-gpu-clef-serve gate-gpu-lev-serve gate-q38-serve-rules gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve the V4.1 server's engine binding and its HTTP client: gpu-gates' bind tests; bloomery-serve-ds41 and its gate are weekly-gpu-ds41-serve (a trigger below) +crates/gpu-gates/src/{bind.rs,serve_client.rs} gate-ds41-bind gate-gpu-clef-serve gate-gpu-lev-serve gate-q38-serve-rules gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve gate-gpu-gemma-embedding2-e2e the V4.1 server's engine binding and its HTTP client: gpu-gates' bind tests; bloomery-serve-ds41 and its gate are weekly-gpu-ds41-serve (a trigger below) crates/gpu-gates/src/vision_seat.rs gate-ds41-bind gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve weekly-gpu-ds41-serve gate-gpu-glm5next-media-seat gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-ds41-media gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-routed-legs gate-gpu-ds41-chat gate-gpu-glm5next-twocard weekly-gpu-glm5next-media-serve gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-q38-serve-rules gate-gpu-qwen38-serve gate-gpu-gemma-embedding2-serve the vision seat every serve seat shares: gate-ds41-bind's unit tests (the vision_seat:: run), and the servers that bind it (the V4.1 seat's images) crates/gpu-gates/src/image_plan.rs gate-gpu-clef-serve gate-gpu-glm5next-serve gate-gpu-lev-serve gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3-serve gate-gpu-qwen3moe-e2e gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve the plan's side of an image input (a sequence's rotation table a position): the qwen3 placement every Qwen3 bin includes, and the servers that bind its seat crates/gpu-gates/src/engine.rs gate-gpu-e2e AnyEngine, the V2-Lite and Qwen3 engine behind gate_e2e @@ -59,17 +59,17 @@ crates/app/src/{lib.rs,arch/mod.rs} gate-gpu-ds41-callstream gate-gpu-ds41-resid crates/app/src/arch/deepseek41/** gate-gpu-clef-serve gate-gpu-ds41-callstream gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-media gate-gpu-ds41-residency-a gate-gpu-ds41-twocard gate-gpu-glm5next-serve gate-gpu-qwen3-serve gate-gpu-lev-serve gate-gpu-ds41-step gate-gpu-ds41-prefill gate-gpu-ds41-nvtier gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve the V4.1 session impls and its draft: generate_ds41, the chat and serve bins, and the gates that drive them crates/app/src/arch/glm5next/** gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen3-serve gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve the GLM session impls: the GLM e2e gate crates/app/src/{mtp.rs,mtp/**,arch/qwen3moe/**} gate-app gate-q38-serve-rules gate-gpu-qwen3moe-e2e gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen4exp-mtp gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-glm5next-twocard gate-gpu-qwen3-serve gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-serve the shared MTP window and the qwen3moe session impls: the app tests, the qwen3moe e2e gate, the Qwen3.8 serve seat and MTP gates, and the Qwen3.6 draft gate (its Session taps) -crates/runtime/src/** gate-ops gate-placement gate-union gate-fixture gate-app gate-deepseek4-meta gate-ds41-bind gate-ds41-host gate-ds41-kld gate-ds41-load gate-ds41-meta gate-ds41-oracle gate-ds41-plan gate-dspark-read gate-glm5next-meta gate-qwen35moe-meta gate-qwen3moe-meta gate-qwen4exp-meta gate-ptx-spill gate-gpu-e2e gate-gpu-gemm gate-gpu-lib gate-gpu-gates-lib gate-gpu-head gate-gpu-hc-gated gate-gpu-kquant gate-gpu-mcol gate-gpu-moe gate-gpu-p10 gate-gpu-swap gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-gemm gate-gpu-glm5next-mtp gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-qwen3-serve gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-head gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-attn gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35moe-moe gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-ds41-callstream gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-comp gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-index gate-gpu-ds41-lib gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-moe gate-gpu-ds41-prefill gate-gpu-ds41-residency-a gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-woa gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-q38-serve-rules gate-gpu-nvtier gate-mimo2-meta gate-gpu-hybrid gate-gpu-p8 gate-gpu-linear gate-ds41-place gate-gpu-p8b gate-gpu-iq-sel gate-gpu-xstream gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-hf gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gpu-p1 gate-gpu-mamba2 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gemma-embedding-meta gate-gpu-qwen38-nodes gate-gpu-nemotron-h-moe-e2e gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve the generation loop and its stores: every gate that names a runtime path in its own code (runtime::combine in gate_kquant and the chain-ffn/prefill bins, hc_gated, qsa, seqstate, swaprule, stores), loads weights through bloomery_gpu::weights (whose every loader entry funnels into upload_rows → runtime::words::stream_words), or opens a model or session (arch place.rs → runtime::stores; clef-hidden opens the Qwen3.5 body, whose body35.rs reads runtime::seqstate). The gates left out load no model and name no runtime path: the synthetic kernel gates upload through DeviceTensor::upload/from_host and read reference values host-side, the CPU oracle gates read the model host-side, and gate-runtime is picked as its own lib's tests +crates/runtime/src/** gate-ops gate-placement gate-union gate-fixture gate-app gate-deepseek4-meta gate-ds41-bind gate-ds41-host gate-ds41-kld gate-ds41-load gate-ds41-meta gate-ds41-oracle gate-ds41-plan gate-dspark-read gate-glm5next-meta gate-qwen35moe-meta gate-qwen3moe-meta gate-qwen4exp-meta gate-ptx-spill gate-gpu-e2e gate-gpu-gemm gate-gpu-lib gate-gpu-gates-lib gate-gpu-head gate-gpu-hc-gated gate-gpu-kquant gate-gpu-mcol gate-gpu-moe gate-gpu-p10 gate-gpu-swap gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-gemm gate-gpu-glm5next-mtp gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-qwen3-serve gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-head gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-attn gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35moe-moe gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-ds41-callstream gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-comp gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-index gate-gpu-ds41-lib gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-moe gate-gpu-ds41-prefill gate-gpu-ds41-residency-a gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-woa gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-q38-serve-rules gate-gpu-nvtier gate-mimo2-meta gate-gpu-hybrid gate-gpu-p8 gate-gpu-linear gate-ds41-place gate-gpu-p8b gate-gpu-iq-sel gate-gpu-xstream gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-hf gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gpu-p1 gate-gpu-mamba2 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gemma-embedding-meta gate-gpu-qwen38-nodes gate-gpu-nemotron-h-moe-e2e gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower the generation loop and its stores: every gate that names a runtime path in its own code (runtime::combine in gate_kquant and the chain-ffn/prefill bins, hc_gated, qsa, seqstate, swaprule, stores), loads weights through bloomery_gpu::weights (whose every loader entry funnels into upload_rows → runtime::words::stream_words), or opens a model or session (arch place.rs → runtime::stores; clef-hidden opens the Qwen3.5 body, whose body35.rs reads runtime::seqstate). The gates left out load no model and name no runtime path: the synthetic kernel gates upload through DeviceTensor::upload/from_host and read reference values host-side, the CPU oracle gates read the model host-side, and gate-runtime is picked as its own lib's tests crates/hf/src/{safetensors.rs,dspark.rs} gate-gpu-qwen35-dspark gate-gpu-clef-serve gate-gpu-lev-serve gate-decision-clef gate-decision-lev gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve hf::safetensors is the Clef head reader (decision re-exports its errors through decision::Error, so the serve seats that open the decide row reach it) and the DSpark draft file reader; hf::dspark is read only by the Qwen3.6 draft gate -crates/serve/{src/**,build.rs} gate-ds41-bind gate-gpu-clef-serve gate-gpu-lev-serve gate-serve gate-q38-serve-rules gate-ds41-place gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve the llama-server-compatible API: gpu-gates' bind tests (bloomery-serve-ds41's gate is weekly-gpu-ds41-serve, a trigger below); gate-serve is its own +crates/serve/{src/**,build.rs} gate-ds41-bind gate-gpu-clef-serve gate-gpu-lev-serve gate-serve gate-q38-serve-rules gate-ds41-place gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve gate-gpu-gemma-embedding2-e2e the llama-server-compatible API: gpu-gates' bind tests (bloomery-serve-ds41's gate is weekly-gpu-ds41-serve, a trigger below); gate-serve is its own crates/jinja/src/** gate-jinja gate-serve gate-ds41-bind gate-decision-clef gate-decision-lev gate-gpu-clef-serve gate-gpu-lev-serve gate-q38-serve-rules gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve the template engine: its own tests, serve's chat templates (the serve row's readers) and the decision models' templates (Clef's and lev's request renders) crates/tokenizer/src/** gate-app gate-decision-clef gate-deepseek4-meta gate-ds41-bind gate-ds41-host gate-ds41-kld gate-ds41-load gate-ds41-meta gate-ds41-oracle gate-ds41-plan gate-dspark-read gate-fixture gate-glm5next-meta gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-comp gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-index gate-gpu-ds41-lib gate-gpu-ds41-long gate-gpu-ds41-prefill gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-e2e gate-gpu-gates-lib gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-lib gate-gpu-mcol gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35moe-moe gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-head gate-gpu-qwen3moe-router gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-ops gate-placement gate-qwen35moe-meta gate-qwen3moe-meta gate-qwen4exp-meta gate-union gate-gpu-clef-serve gate-gpu-lev-serve gate-decision-lev gate-q38-serve-rules gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-iq-sel gate-gpu-clef-hidden gate-gpu-qwen4exp-iqleg gate-gpu-glm5next-gemm gate-gpu-ds41-media gate-gpu-nvtier gate-gpu-qwen38-residency gate-mimo2-meta gate-gpu-glm5next-residency gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-ds41-place gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-qwen35-dspark gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve the encoder and decoder, and runs_pre_tokenizer, which coverage::check asks at every placement plan (model::arch::*::place): every gate that plans a placement or tokenizes text # crates/refset: the reference-set readers. Every oracle read goes through refset::arch's dispatch (node_dumps, # families) and a family's check; which family a gate asks for is its run-time arch, so a family's row keeps every # dispatch user. -crates/refset/src/{lib,family,mtpref,dsref,greedy,kld,vision,ik,columns}.rs gate-attn gate-derived gate-ds41-bind gate-ds41-host gate-ds41-kld gate-ds41-load gate-ds41-meta gate-ds41-oracle gate-ds41-plan gate-ffn gate-forward gate-gpu-block gate-gpu-clef-hidden gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-gates-lib gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-mtp gate-gpu-glm5next-residency gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-head gate-gpu-linear gate-gpu-mcol gate-gpu-moe gate-gpu-p0b gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p8 gate-gpu-p8b gate-gpu-qwen35moe-attn gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-qwen35moe-moe gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-vision gate-head gate-kv gate-moe gate-mt gate-ops gate-profile gate-qwen3moe-meta gate-union gate-vision gate-1-1 gate-tokenizer gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-ds41-place gate-q38-serve-rules gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-iq gate-gpu-iq-sel gate-gpu-e2e gate-gpu-hybrid gate-gpu-gemm gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-dspark-read gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-clef-serve gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-minicpmv-tower gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-emb2-mel the readers and the family check every set goes through: every gate that reads a reference set +crates/refset/src/{lib,family,mtpref,dsref,greedy,kld,vision,ik,columns}.rs gate-attn gate-derived gate-ds41-bind gate-ds41-host gate-ds41-kld gate-ds41-load gate-ds41-meta gate-ds41-oracle gate-ds41-plan gate-ffn gate-forward gate-gpu-block gate-gpu-clef-hidden gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-gates-lib gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-mtp gate-gpu-glm5next-residency gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-head gate-gpu-linear gate-gpu-mcol gate-gpu-moe gate-gpu-p0b gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p8 gate-gpu-p8b gate-gpu-qwen35moe-attn gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-qwen35moe-moe gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-vision gate-head gate-kv gate-moe gate-mt gate-ops gate-profile gate-qwen3moe-meta gate-union gate-vision gate-1-1 gate-tokenizer gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-ds41-place gate-q38-serve-rules gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-iq gate-gpu-iq-sel gate-gpu-e2e gate-gpu-hybrid gate-gpu-gemm gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-dspark-read gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-clef-serve gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-minicpmv-tower gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-emb2-mel gate-gpu-gemma4v-tower the readers and the family check every set goes through: every gate that reads a reference set crates/refset/src/arch/mod.rs gate-ds41-bind gate-ds41-kld gate-ds41-load gate-ds41-oracle gate-ds41-plan gate-gpu-block gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-media gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-gates-lib gate-gpu-head gate-gpu-linear gate-gpu-moe gate-gpu-p0b gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p8 gate-gpu-p8b gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-1-1 gate-tokenizer gate-ds41-place gate-gpu-qwen4exp-mtp gate-q38-serve-rules gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-media-seat gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve the per-arch dispatch (node_dumps, families, named): every oracle user except gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-glm5next-serve: bloomery-serve opens one seat a run (--model, or the file's architecture); these run its decide and glm seats, and only the qwen38 seat's module (its DraftFrom) reaches this file | except gate-gpu-serve-start-glm: bloomery-serve opens one seat a run (--model, or the file's architecture); this recipe runs its glm seat, and only another seat's module reaches this file | except gate-gpu-serve-start-ds41: bloomery-serve opens one seat a run (--model, or the file's architecture); this recipe runs its ds41 seat, and only another seat's module reaches this file crates/refset/src/arch/deepseek41/** gate-ds41-bind gate-ds41-host gate-ds41-kld gate-ds41-load gate-ds41-meta gate-ds41-oracle gate-ds41-plan gate-gpu-block gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-long gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-gates-lib gate-gpu-glm5next-serve gate-gpu-head gate-gpu-linear gate-gpu-mcol gate-gpu-moe gate-gpu-p0b gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p8 gate-gpu-p8b gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-union gate-gpu-qwen3-serve gate-gpu-clef-serve gate-gpu-lev-serve gate-ds41-place gate-tokenizer gate-1-1 gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve the deepseek41 family's sets: the dispatch users and the gates that name the family crates/refset/src/arch/deepseek41v/** gate-ds41-bind gate-ds41-kld gate-ds41-load gate-ds41-oracle gate-ds41-plan gate-gpu-block gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-media gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-gates-lib gate-gpu-head gate-gpu-linear gate-gpu-moe gate-gpu-p0b gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p8 gate-gpu-p8b gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-vision gate-vision gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 the deepseek41v family's sets: the dispatch users and the gates that name the family except gate-ds41-place: they read no deepseek41v set: only refset::arch's table (BY_ARCH), which picks a set's family by the file it names at run time, reaches it @@ -77,11 +77,11 @@ crates/refset/src/arch/glm5next/** gate-ds41-bind gate-ds41-kld gate-ds41-load g crates/refset/src/arch/mimo2/** gate-ds41-bind gate-ds41-kld gate-ds41-load gate-ds41-oracle gate-ds41-plan gate-gpu-block gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-gates-lib gate-gpu-head gate-gpu-linear gate-gpu-mimo2-flash gate-gpu-moe gate-gpu-p0b gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p8 gate-gpu-p8b gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router weekly-gpu-mimo2-e2e the mimo2 family's sets: the dispatch users and the gates that name the family except gate-ds41-place: they read no mimo2 set: only refset::arch's table (BY_ARCH), which picks a set's family by the file it names at run time, reaches it crates/refset/src/arch/qwen35moe/** gate-ds41-bind gate-ds41-kld gate-ds41-load gate-ds41-oracle gate-ds41-plan gate-gpu-block gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-gates-lib gate-gpu-head gate-gpu-linear gate-gpu-moe gate-gpu-p0b gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p8 gate-gpu-p8b gate-gpu-qwen35moe-attn gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-qwen35moe-moe gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-qwen3-serve gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp the qwen35moe family's sets: the dispatch users and the gates that name the family except gate-ds41-place: they read no qwen35moe set: only refset::arch's table (BY_ARCH), which picks a set's family by the file it names at run time, reaches it crates/refset/src/arch/qwen4exp/** gate-ds41-bind gate-ds41-kld gate-ds41-load gate-ds41-oracle gate-ds41-plan gate-gpu-block gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-gates-lib gate-gpu-glm5next-serve gate-gpu-head gate-gpu-linear gate-gpu-moe gate-gpu-p0b gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p8 gate-gpu-p8b gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3-serve gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-nvtier gate-q38-serve-rules gate-tokenizer gate-ds41-place gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve the qwen4exp family's sets: the dispatch users and the gates that name the family -crates/refset/src/{clefvis.rs,clefvis/**} gate-refset gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-ds41-hc gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-derived gate-mt gate-profile gate-ds41-meta gate-qwen3moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-tokenizer gate-1-1 gate-gpu-head gate-gpu-mcol gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-vision gate-gpu-vision gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-minicpmv-tower gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-h-moe-e2e gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-emb2-mel the Clef image-input sets' reader: no gate reads a clefvis set yet, gate-refset runs its tests and walks every set in place +crates/refset/src/{clefvis.rs,clefvis/**} gate-refset gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-ds41-hc gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-derived gate-mt gate-profile gate-ds41-meta gate-qwen3moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-tokenizer gate-1-1 gate-gpu-head gate-gpu-mcol gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-vision gate-gpu-vision gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-minicpmv-tower gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-h-moe-e2e gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-emb2-mel gate-gpu-gemma4v-tower the Clef image-input sets' reader: no gate reads a clefvis set yet, gate-refset runs its tests and walks every set in place crates/refset/src/arch/minicpmv4_7/** gate-ds41-bind gate-ds41-kld gate-ds41-load gate-ds41-oracle gate-ds41-plan gate-gpu-block gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-gates-lib gate-gpu-linear gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-ds41-place gate-gpu-minicpmv-tower gate-refset the MiniCPM-V 4.7 image sets' reader (the tower per tile, the hidden states and four position lanes of an image prompt, the chat ids): the dispatch users, the tower gate that reads the tower sets, and gate-refset, which runs the reader's tests and walks every set in place crates/refset/src/arch/qwen35/** gate-refset gate-gpu-clef-hidden gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media the qwen35 families' sets: the hidden-state gate that names the families, and the table walk except gate-gpu-ds41-hc gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen3moe-down gate-gpu-linear gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-ds41-bind gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-ds41-load: they read no qwen35 set: only refset::arch's table (BY_ARCH), which picks a set's family by the file it names at run time, reaches it -crates/refset/src/emb2.rs gate-refset gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-h-moe-e2e gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-emb2-mel the EmbeddingGemma 2 oracle sets: gate-refset checks these families except gate-1-1 gate-attn gate-derived gate-ds41-bind gate-ds41-host gate-ds41-kld gate-ds41-load gate-ds41-meta gate-ds41-oracle gate-ds41-place gate-ds41-plan gate-ffn gate-forward gate-gpu-block gate-gpu-clef-hidden gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-gates-lib gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-gemm gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-residency gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-glm5v-tower gate-gpu-head gate-gpu-linear gate-gpu-mcol gate-gpu-mimo2-flash gate-gpu-minicpm-text gate-gpu-minicpmv-tower gate-gpu-moe gate-gpu-p0b gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p8 gate-gpu-p8b gate-gpu-qwen3-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-attn gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-media gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-verify gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen3vl-tower gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-serve-start-ds41 gate-gpu-serve-start-glm gate-gpu-vision gate-head gate-kv gate-moe gate-mt gate-ops gate-profile gate-qwen3moe-meta gate-tokenizer gate-union gate-vision: they read no EmbeddingGemma 2 set: only the runtime family dispatch reaches this reader and these families -crates/refset/src/arch/gemma_embedding2/** gate-refset gate-gpu-gemma-embedding2-e2e gate-gemma-embedding2-meta gate-gpu-gemma-embedding2-serve gate-emb2-mel the EmbeddingGemma 2 oracle sets: gate-refset checks these families except gate-ds41-bind gate-ds41-kld gate-ds41-load gate-ds41-oracle gate-ds41-place gate-ds41-plan gate-gpu-block gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-gates-lib gate-gpu-linear gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router: they read no EmbeddingGemma 2 set: only the runtime family dispatch reaches this reader and these families +crates/refset/src/emb2.rs gate-refset gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-h-moe-e2e gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-emb2-mel gate-gpu-gemma4v-tower the EmbeddingGemma 2 oracle sets: gate-refset checks these families except gate-1-1 gate-attn gate-derived gate-ds41-bind gate-ds41-host gate-ds41-kld gate-ds41-load gate-ds41-meta gate-ds41-oracle gate-ds41-place gate-ds41-plan gate-ffn gate-forward gate-gpu-block gate-gpu-clef-hidden gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-gates-lib gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-gemm gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-residency gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-glm5v-tower gate-gpu-head gate-gpu-linear gate-gpu-mcol gate-gpu-mimo2-flash gate-gpu-minicpm-text gate-gpu-minicpmv-tower gate-gpu-moe gate-gpu-p0b gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p8 gate-gpu-p8b gate-gpu-qwen3-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-attn gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-media gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-verify gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen3vl-tower gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-serve-start-ds41 gate-gpu-serve-start-glm gate-gpu-vision gate-head gate-kv gate-moe gate-mt gate-ops gate-profile gate-qwen3moe-meta gate-tokenizer gate-union gate-vision: they read no EmbeddingGemma 2 set: only the runtime family dispatch reaches this reader and these families +crates/refset/src/arch/gemma_embedding2/** gate-refset gate-gpu-gemma-embedding2-e2e gate-gemma-embedding2-meta gate-gpu-gemma-embedding2-serve gate-emb2-mel gate-gpu-gemma4v-tower the EmbeddingGemma 2 oracle sets: gate-refset checks these families except gate-ds41-bind gate-ds41-kld gate-ds41-load gate-ds41-oracle gate-ds41-place gate-ds41-plan gate-gpu-block gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-moe gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-woa gate-gpu-gates-lib gate-gpu-linear gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router: they read no EmbeddingGemma 2 set: only the runtime family dispatch reaches this reader and these families crates/refset/src/{md5,tokenizer,dequant}.rs gate-tokenizer gate-1-1 gate-gpu-ds41-hc gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-attn gate-gpu-qwen35moe-e2e gate-gpu-minicpm-text gate-gpu-nemotron-h-moe-e2e gate-gpu-gemma-embedding2-e2e the tokenizer and dequant set readers and the md5 they check by: the two gates that open those sets (gate-refset runs the crate's own tests) except gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen3moe-down gate-gpu-linear gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-ds41-bind gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-ds41-load: they read no dequant and tokenizer set: only refset::arch's table (BY_ARCH), which picks a set's family by the file it names at run time, reaches it | except gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency: they open their own families' sets through refset::family::Family::check_set, which reaches DequantSet::open only in its DequantManifest arm, an identity no set of theirs carries: they read no dequant set crates/refset/src/arch/{tokenizer,dequant}.rs gate-tokenizer gate-1-1 gate-gpu-nemotron-e2e the tokenizer and dequant families' rows and pins: the two gates that open those sets (gate-refset runs the crate's own tests) except gate-gpu-ds41-hc gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen3moe-down gate-gpu-linear gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-ds41-bind gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-ds41-load: they read no dequant and tokenizer set: only refset::arch's table (BY_ARCH), which picks a set's family by the file it names at run time, reaches it crates/refset/src/census.rs gate-gpu-routed-legs the census stub reader (the manifest and the digest check): the routed-legs gate plans every set through it @@ -98,7 +98,7 @@ tools/census/stubs.tsv gate-gpu-routed-legs the census manifest (refset::census: # files a gate opens. One row: every gate that links the modules, less the gates whose bins only read tensors # through gguf::Gguf or build synthetic blocks and call no arch module (their own code, and the bloomery-gpu # kernel modules and bloomery-gpu-gates helpers they call, name no model::arch:: path). the fixture generator, arch/coverage.rs and arch/qwen35moe/mtp.rs are carved out to their own rows below: their readers are bounded (a leaf the family's fixture module names; the place/engine/plan modules that call them). -crates/model/src/arch/{mod.rs,dspark.rs,classify.rs,stems.rs,deepseek2/**,qwen3moe/**,glm5next/**} gate-gpu-p4 gate-gpu-p6 gate-gpu-kquant gate-gpu-iq-sel gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-ptx-spill gate-dspark-read gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-q38-serve-rules gate-gpu-block gate-gpu-nvtier gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p5 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-cand gate-gpu-iq gate-gpu-q6k-ids gate-gpu-xstream gate-gpu-mimo2-flash gate-gpu-ple gate-gpu-vision gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve every gate that links an arch module, less the gguf-only kernel gates (above): a family's code runs wherever any family's file is opened +crates/model/src/arch/{mod.rs,dspark.rs,classify.rs,stems.rs,deepseek2/**,qwen3moe/**,glm5next/**} gate-gpu-p4 gate-gpu-p6 gate-gpu-kquant gate-gpu-iq-sel gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-ptx-spill gate-dspark-read gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-q38-serve-rules gate-gpu-block gate-gpu-nvtier gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p5 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-cand gate-gpu-iq gate-gpu-q6k-ids gate-gpu-xstream gate-gpu-mimo2-flash gate-gpu-ple gate-gpu-vision gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower every gate that links an arch module, less the gguf-only kernel gates (above): a family's code runs wherever any family's file is opened crates/model/src/arch/mimo2/** gate-ops gate-mimo2-meta gate-gpu-mimo2-flash gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-ds41-bind gate-ds41-place gate-gpu-routed-legs gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-minicpm-text weekly-gpu-mimo2-e2e weekly-gpu-mimo2-serve gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve MiMo-V2.6-Flash's reader and plan: its code runs only where a mimo2 file opens (its meta test, its flash gate) in bloomery-model's own unit tests (gate-ops runs --lib) and in bloomery-serve's mimo2 seat, which every serve gate's bin links except gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-clef-hidden gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen3moe-e2e gate-fixture gate-gpu-head gate-gpu-lib gate-gpu-ds41-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency: they open no mimo2 file: only model::arch's dispatch by the file's general.architecture (arch/mod.rs) reaches it crates/model/src/arch/nemotron_h/** gate-ops gate-nemotron-meta gate-fixture gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-iq-sel gate-placement gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-ds41-bind gate-ds41-place gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve Nemotron 3 Nano 4B's header reader and fixture spec: its code runs where a nemotron_h file is described (its meta test), where its fixture is planned and verified (the fixture generator's gate), in bloomery-model's own unit tests, where the card body reads the layer kinds and widths at load (the end-to-end gate), and behind the arch dispatch and, through its plan (place.rs), in bloomery-serve's nemotron seat, which gate-gpu-nemotron-serve starts and every serve gate's bin links except gate-gpu-e2e gate-gpu-hybrid gate-gpu-p6 gate-gpu-ds41-moe gate-gpu-moe gate-gpu-p8 gate-gpu-block gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen35moe-moe gate-gpu-qwen3moe-router gate-qwen35moe-meta gate-gpu-qwen35-dspark gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-minicpm-text gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-types gate-gpu-head gate-gpu-routed-legs gate-gpu-ds41-lib gate-gpu-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-nvtier gate-gpu-ds41-residency-a gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-long gate-gpu-ds41-chat gate-gpu-ds41-media gate-gpu-ds41-prefill gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-ds41-load gate-gpu-qwen38-serve gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-p8b gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-residency gate-gpu-glm5next-twocard: they link the arch dispatch and never open a nemotron_h file crates/model/src/arch/gemma_embedding2/** gate-ops gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-iq-sel gate-placement gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-ds41-bind gate-ds41-place gate-gpu-gemma-embedding2-serve EmbeddingGemma 2's header reader: no program runs it, so its code runs where a gemma-embedding2 file is described (its meta test), in bloomery-model's own unit tests, and behind the arch dispatch except gate-gpu-e2e gate-gpu-hybrid gate-gpu-p6 gate-gpu-ds41-moe gate-gpu-moe gate-gpu-p8 gate-gpu-block gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen35moe-moe gate-gpu-qwen3moe-router gate-qwen35moe-meta gate-gpu-qwen35-dspark gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-minicpm-text gate-gpu-qwen35-media gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen35moe-media gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-types gate-fixture gate-gpu-head gate-gpu-routed-legs gate-gpu-ds41-lib gate-gpu-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-nvtier gate-gpu-ds41-residency-a gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-long gate-gpu-ds41-chat gate-gpu-ds41-media gate-gpu-ds41-prefill gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-ds41-load gate-gpu-qwen3-serve gate-gpu-qwen38-serve gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-p8b gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-residency gate-gpu-glm5next-twocard gate-gpu-glm5next-serve gate-gpu-serve-start-ds41 gate-gpu-serve-start-glm: they link the arch dispatch and never open a gemma-embedding2 file @@ -107,9 +107,9 @@ crates/model/src/arch/deepseek41/{hparams.rs,host.rs,kv.rs,mod.rs,names.rs,place crates/model/src/arch/qwen35moe/{head_list.rs,hparams.rs,host.rs,mod.rs,names.rs,place.rs,roles.rs,spec.rs} gate-gpu-p4 gate-gpu-p6 gate-gpu-kquant gate-gpu-iq-sel gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-ptx-spill gate-dspark-read gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-gpu-nvtier gate-q38-serve-rules gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-qwen38-nodes gate-gpu-nemotron-h-moe-e2e gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve every gate that links an arch module, less the gguf-only kernel gates (above): a family's code runs wherever any family's file is opened except gate-gpu-block: they open no qwen35moe file: only model::arch's dispatch by the file's general.architecture (arch/mod.rs) reaches it crates/model/src/{fixture/**,arch/deepseek41/fixture.rs} gate-ops gate-ds41-meta gate-fixture gate-deepseek4-meta gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-ds41-load gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-qwen35moe-e2e gate-gpu-minicpm-text gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e the fixture generator and V4.1's FixtureSpec: no file but bin fixture, the fixture test and arch/deepseek41/fixture.rs names crate::fixture (grep), so the gates are the fixture gate and the --lib runs that execute arch::deepseek41::fixture's own unit tests (gate-ops unfiltered, gate-ds41-meta and gate-deepseek4-meta filtered arch::deepseek41) crates/model/src/arch/{coverage.rs,qwen35moe/mtp.rs} gate-gpu-iq-sel gate-gpu-e2e gate-ops gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-glm5next-gemm gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-app gate-gpu-lib gate-gpu-gates-lib gate-ds41-plan gate-gpu-ds41-lib gate-ds41-bind gate-ds41-place gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-ptx-spill gate-deepseek4-meta gate-glm5next-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-gpu-glm-sel gate-fixture gate-q38-serve-rules gate-gpu-nvtier gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-p6 gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-gpu-head gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-qwen38-nodes gate-gpu-nemotron-h-moe-e2e gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve coverage::check's and qwen35moe's mtp readers: deepseek41/glm5next/qwen35moe place.rs (unimplemented, super::mtp), gpu-gates' engine.rs, gpu's qwen3moe plan/plan38/body38 (turns_as_neox, .unimplemented()) and the meta tests that call coverage::check — every gate that opens or plans those families' files; the two modules' lists coincide -crates/model/src/arch/names.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-q38-serve-rules gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-lib gate-gpu-glm5next-nvtier gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-qwen3moe-e2e gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-iq-sel gate-gpu-ds41-moe gate-gpu-moe gate-gpu-p8 gate-gpu-block gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-router gate-placement gate-qwen35moe-meta gate-gpu-clef-hidden gate-gpu-qwen35-dspark gate-gpu-qwen35moe-moe gate-gpu-gemm gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-experts gate-ds41-host gate-fixture gate-union gate-gpu-ds41-comp gate-gpu-head gate-gpu-mcol gate-ds41-bind gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-place gate-gpu-ds41-chain-glue gate-gpu-ds41-engram gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-twocard gate-gpu-ds41-draft gate-gpu-ds41-long gate-gpu-ds41-nvtier gate-gpu-ds41-dspark-loop gate-gpu-ds41-media gate-gpu-ds41-prefill gate-ds41-load gate-gpu-ds41-chat gate-gpu-p8b gate-dspark-read gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-qwen35moe-mtp gate-app gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve the routed stacks' common names (routed_stacks): the GLM, Qwen3.6 and Qwen3.8 stack tables (swap.rs, qwen3moe/swap.rs for swap35.rs and swap38.rs) and the GLM card check (glm5next_card.rs) call it — every gate that opens those bodies or runs that check -crates/model/src/arch/mod.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-ptx-spill gate-dspark-read gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-vision gate-deepseek4-meta gate-glm5next-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-q38-serve-rules gate-gpu-xstream gate-gpu-mimo2-flash gate-gpu-nvtier gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve the dispatch and Arch: every gate that links it, the gguf-only kernel gates too — bloomery-gpu's fault sites (fault.rs) and bloomery-gpu-gates' family check (lib.rs, the oracle) read Arch in them -crates/gpu/src/weights.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq-sel gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-qwen35moe-attn gate-gpu-glm5next-gemm gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-app gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-ds41-place gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-ptx-spill gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-q38-serve-rules gate-gpu-nvtier gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve the card weight loaders: the gates whose own targets name weights, or reach it through an open path (gpu's arch bodies -> model.rs -> host/swap_source.rs, gpu-deepseek41's body/chain/dense/experts/draft, gpu-glm5next's body/ffn; gate-gpu-xstream names GpuModel), less the synthetic kernel gates whose bins build blocks and name no weights path +crates/model/src/arch/names.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-q38-serve-rules gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-lib gate-gpu-glm5next-nvtier gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-qwen3moe-e2e gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-iq-sel gate-gpu-ds41-moe gate-gpu-moe gate-gpu-p8 gate-gpu-block gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-router gate-placement gate-qwen35moe-meta gate-gpu-clef-hidden gate-gpu-qwen35-dspark gate-gpu-qwen35moe-moe gate-gpu-gemm gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-experts gate-ds41-host gate-fixture gate-union gate-gpu-ds41-comp gate-gpu-head gate-gpu-mcol gate-ds41-bind gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-place gate-gpu-ds41-chain-glue gate-gpu-ds41-engram gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-twocard gate-gpu-ds41-draft gate-gpu-ds41-long gate-gpu-ds41-nvtier gate-gpu-ds41-dspark-loop gate-gpu-ds41-media gate-gpu-ds41-prefill gate-ds41-load gate-gpu-ds41-chat gate-gpu-p8b gate-dspark-read gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-qwen35moe-mtp gate-app gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower the routed stacks' common names (routed_stacks): the GLM, Qwen3.6 and Qwen3.8 stack tables (swap.rs, qwen3moe/swap.rs for swap35.rs and swap38.rs) and the GLM card check (glm5next_card.rs) call it — every gate that opens those bodies or runs that check +crates/model/src/arch/mod.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-ptx-spill gate-dspark-read gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-vision gate-deepseek4-meta gate-glm5next-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-q38-serve-rules gate-gpu-xstream gate-gpu-mimo2-flash gate-gpu-nvtier gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower the dispatch and Arch: every gate that links it, the gguf-only kernel gates too — bloomery-gpu's fault sites (fault.rs) and bloomery-gpu-gates' family check (lib.rs, the oracle) read Arch in them +crates/gpu/src/weights.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq-sel gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-qwen35moe-attn gate-gpu-glm5next-gemm gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-app gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-ds41-place gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-ptx-spill gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-q38-serve-rules gate-gpu-nvtier gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower the card weight loaders: the gates whose own targets name weights, or reach it through an open path (gpu's arch bodies -> model.rs -> host/swap_source.rs, gpu-deepseek41's body/chain/dense/experts/draft, gpu-glm5next's body/ffn; gate-gpu-xstream names GpuModel), less the synthetic kernel gates whose bins build blocks and name no weights path # crates/gpu's per-arch bodies: GpuModel is generic; a body's impls run where B is that body. crates/gpu/src/arch/qwen3moe/** gate-app gate-ds41-bind gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-e2e gate-gpu-gemm gate-gpu-glm5next-serve gate-gpu-lev-serve gate-gpu-nvtier gate-gpu-qwen3-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-qwen35moe-media gate-gpu-qwen35moe-moe gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-head gate-gpu-qwen3moe-router gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-q38-serve-rules gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-serve the Qwen3, Qwen3.5 and Qwen4 bodies: the bins that name them (the qwen gates, gate_gemm) and AnyEngine (gate_e2e; bind's tests build it under deepseek41) @@ -119,64 +119,64 @@ crates/gpu/src/arch/deepseek2/** gate-ds41-bind gate-gpu-e2e gate-gpu-hybrid gat # Derived rows: each row's recipes are `python3 tools/recipes.py reach --row ` at the time of writing, the # gates whose targets can run code of the files through another lib (tools/recipes.py, Reach); files grouped only # where the derivation gives each the same gates. `just check-recipes` holds each to the derivation. -crates/gpu/src/{host/mod.rs,host/swap.rs,lib.rs} gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-app gate-gpu-head gate-gpu-mcol gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-vision gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve the Gpu and GpuError, whose Poison and Unresident variants hold host::Poison and host::swap::Unresident: every gate whose code names a bloomery_gpu error or the Gpu +crates/gpu/src/{host/mod.rs,host/swap.rs,lib.rs} gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-app gate-gpu-head gate-gpu-mcol gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-vision gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower the Gpu and GpuError, whose Poison and Unresident variants hold host::Poison and host::swap::Unresident: every gate whose code names a bloomery_gpu error or the Gpu crates/gpu/src/lane.rs gate-gpu-clef-serve gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-glm5next-serve gate-gpu-glm5next-twocard gate-gpu-lev-serve gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3-serve gate-q38-serve-rules gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve the lane probe a load prints beside its load line (probe_card, record::lane): the seats and generators that call it -crates/gpu/src/{host/lane.rs,watchdog.rs,flash_gqa.rs,elem.rs} gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-app gate-gpu-head gate-gpu-mcol gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve the Gpu's own parts: watchdog's wait_done and the swap lane's Shared, flash's seg_span and ElemKernels, which lib.rs's Gpu and host/mod.rs hold -crates/gpu/src/model.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-head gate-gpu-mcol gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-p2 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve GpuModel, the ChainBody and Rollback traits and StepMode: the gates that open a model body or name its step mode -crates/gpu/src/host/swap_source.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve ResidencyGlue (host/served.rs) and DEADLINE (the qwen3moe bodies' swap38): the bodies that serve a residency -crates/gpu/src/model/slots.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-ds41-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve GpuModel's slot impls (slots, selected) and SlotRows: the gates that open a body with slots -crates/gpu/src/{host/nvtier.rs,host/nvtier/reader.rs,host/step.rs} gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve the host tier's step (hybrid::Chain in model.rs) and the NvTier arena (host/mod.rs): the gates that open a body with a host tier +crates/gpu/src/{host/lane.rs,watchdog.rs,flash_gqa.rs,elem.rs} gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-app gate-gpu-head gate-gpu-mcol gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower gate-gpu-vision gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-gpu-glm5v-tower the Gpu's own parts: watchdog's wait_done and the swap lane's Shared, flash's seg_span and ElemKernels, which lib.rs's Gpu and host/mod.rs hold +crates/gpu/src/model.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-head gate-gpu-mcol gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-p2 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower GpuModel, the ChainBody and Rollback traits and StepMode: the gates that open a model body or name its step mode +crates/gpu/src/host/swap_source.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower ResidencyGlue (host/served.rs) and DEADLINE (the qwen3moe bodies' swap38): the bodies that serve a residency +crates/gpu/src/model/slots.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-ds41-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower GpuModel's slot impls (slots, selected) and SlotRows: the gates that open a body with slots +crates/gpu/src/{host/nvtier.rs,host/nvtier/reader.rs,host/step.rs} gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower the host tier's step (hybrid::Chain in model.rs) and the NvTier arena (host/mod.rs): the gates that open a body with a host tier crates/gpu/src/host/run.rs gate-gpu-e2e gate-gpu-ds41-chain-ffn gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-gpu-ds41-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-serve HostRun, the host expert leg: the V4.1, Qwen3.8 and GLM bodies that hold one -crates/gpu/src/host/xstream.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve XStream, the host tier's cross-stream lane (host/mod.rs), and XMode in the qwen38 bins +crates/gpu/src/host/xstream.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower XStream, the host tier's cross-stream lane (host/mod.rs), and XMode in the qwen38 bins crates/gpu/src/host/prompt_stream.rs gate-gpu-clef-serve gate-gpu-glm5next-serve gate-gpu-lev-serve gate-gpu-nvtier gate-q38-serve-rules gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-qwen3moe-e2e gate-gpu-clef-hidden gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-serve the prompt call's stream state and its calls (host/mod.rs), reached through the bodies that hold it -crates/gpu/src/host/batch.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-linear gate-gpu-qwen3moe-e2e gate-app gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve BatchPort, the host tier's prompt batch (host/mod.rs), and the legs' ServeTimes -crates/gpu/src/head.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-head gate-gpu-mcol gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-chain-glue gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve Head and HeadNorm, which GpuModel holds (model.rs): the gates that open a model body +crates/gpu/src/host/batch.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-linear gate-gpu-qwen3moe-e2e gate-app gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower BatchPort, the host tier's prompt batch (host/mod.rs), and the legs' ServeTimes +crates/gpu/src/head.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-head gate-gpu-mcol gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-chain-glue gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower Head and HeadNorm, which GpuModel holds (model.rs): the gates that open a model body crates/gpu/src/rope_table.rs gate-gpu-e2e gate-gpu-qwen3moe-rope gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen35moe-attn gate-gpu-mimo2-flash gate-gpu-qwen3moe-e2e gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-q38-serve-rules gate-gpu-ds41-chain-glue gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve RopeTable (V4.1's params) and RopeRows (the qwen3moe bodies' scratch) -crates/gpu/src/media.rs gate-ds41-load gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-ds41-callstream gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-lib gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-prefill gate-gpu-ds41-residency-a gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-e2e gate-gpu-glm5next-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3moe-e2e gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-e2e gate-gpu-glm5next-residency gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-ds41-bind gate-ds41-place gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-qwen4exp-mtp gate-app gate-q38-serve-rules gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-nvtier gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve MediaSpan/ImageSpan/SeqRope and the MediaBody trait: the V4.1 and Qwen3.6/Clef media calls and the sessions that carry them +crates/gpu/src/media.rs gate-ds41-load gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-ds41-callstream gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-lib gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-prefill gate-gpu-ds41-residency-a gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-e2e gate-gpu-glm5next-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3moe-e2e gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-e2e gate-gpu-glm5next-residency gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-ds41-bind gate-ds41-place gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-qwen4exp-mtp gate-app gate-q38-serve-rules gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-nvtier gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve gate-gpu-gemma-embedding2-e2e MediaSpan/ImageSpan/SeqRope and the MediaBody trait: the V4.1 and Qwen3.6/Clef media calls and the sessions that carry them crates/gpu/src/iq_sel.rs gate-gpu-iq-sel gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-q38-serve-rules gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-mtp gate-gpu-glm5next-residency gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-serve IqSelKernels, the Qwen3.8 card's IQ selection (card38.rs), the common card leg's i-quant arms (card_leg.rs, which GLM's body and mimo2's reach), and its own gate crates/gpu/src/checkpoint/{saved,staging}.rs gate-ds41-load gate-gpu-clef-serve gate-gpu-ds41-callstream gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-lib gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-prefill gate-gpu-ds41-residency-a gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-glm5next-e2e gate-gpu-glm5next-mtp gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-lev-serve gate-gpu-nvtier gate-gpu-qwen3-serve gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-q38-serve-rules gate-ds41-bind gate-ds41-place gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-serve the host copy of a sequence state (SeqState, the pinned staging pool) under the Qwen3.8 and GLM bodies' seq_save and seq_resume, the V4.1 serve seat and the gates that open those bodies crates/gpu/src/checkpoint.rs gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-mtp gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-lev-serve gate-gpu-nvtier gate-gpu-qwen3-serve gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3moe-e2e gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-q38-serve-rules gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-serve the checkpoint ring (Checkpoints: the points a prompt call keeps, adopt, take, the save lane's point copy and its in-flight refusal) under the Qwen3.8 and GLM bodies and the gates that open them -crates/gpu/src/tensor.rs gate-app gate-ds41-bind gate-ds41-load gate-ds41-place gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-ds41-attn gate-gpu-ds41-callstream gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-comp gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-lib gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-moe gate-gpu-ds41-prefill gate-gpu-ds41-residency-a gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-woa gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-e2e gate-gpu-gemm gate-gpu-glm-mla gate-gpu-glm-router gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-gemm gate-gpu-glm5next-mtp gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-hc-gated gate-gpu-head gate-gpu-hybrid gate-gpu-iq-sel gate-gpu-kquant gate-gpu-lev-serve gate-gpu-mcol gate-gpu-moe gate-gpu-nvtier gate-gpu-p0b gate-gpu-p1 gate-gpu-p10 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p8 gate-gpu-p8b gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-q6k-ids gate-gpu-qwen3-serve gate-gpu-qwen35moe-attn gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35moe-moe gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-head gate-gpu-qwen3moe-router gate-gpu-qwen3vl-tower gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-swap gate-gpu-vision gate-q38-serve-rules gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve the device tensor and the 2-D copies (dtoh_2d, htod_2d) every gate that links the gpu crate through a body reads -crates/gpu-gates/src/lib.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-gpu-head gate-gpu-mcol gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-q38-serve-rules gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-dspark-read gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-vision gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve GateError and the family check (the oracle), which the bins' shared modules name -crates/gpu-gates/src/tier.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-gpu-head gate-gpu-mcol gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-q38-serve-rules gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-dspark-read gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-minicpm-text gate-gpu-qwen35moe-attn gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve Tier::from_env (lib.rs's family check) and tier::card, which shared/gate_card.rs runs in the card bins +crates/gpu/src/tensor.rs gate-app gate-ds41-bind gate-ds41-load gate-ds41-place gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-ds41-attn gate-gpu-ds41-callstream gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-comp gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-hc gate-gpu-ds41-index gate-gpu-ds41-lib gate-gpu-ds41-long gate-gpu-ds41-media gate-gpu-ds41-moe gate-gpu-ds41-prefill gate-gpu-ds41-residency-a gate-gpu-ds41-rope gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-woa gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-e2e gate-gpu-gemm gate-gpu-glm-mla gate-gpu-glm-router gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-gemm gate-gpu-glm5next-mtp gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-glm5next-stagger gate-gpu-glm5next-twocard gate-gpu-hc-gated gate-gpu-head gate-gpu-hybrid gate-gpu-iq-sel gate-gpu-kquant gate-gpu-lev-serve gate-gpu-mcol gate-gpu-moe gate-gpu-nvtier gate-gpu-p0b gate-gpu-p1 gate-gpu-p10 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p8 gate-gpu-p8b gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-q6k-ids gate-gpu-qwen3-serve gate-gpu-qwen35moe-attn gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35moe-moe gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3moe-down gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-head gate-gpu-qwen3moe-router gate-gpu-qwen3vl-tower gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-swap gate-gpu-vision gate-q38-serve-rules gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower the device tensor and the 2-D copies (dtoh_2d, htod_2d) every gate that links the gpu crate through a body reads +crates/gpu-gates/src/lib.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-gpu-head gate-gpu-mcol gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-q38-serve-rules gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-dspark-read gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-vision gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower GateError and the family check (the oracle), which the bins' shared modules name +crates/gpu-gates/src/tier.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-gpu-head gate-gpu-mcol gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-q38-serve-rules gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-dspark-read gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-minicpm-text gate-gpu-qwen35moe-attn gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower Tier::from_env (lib.rs's family check) and tier::card, which shared/gate_card.rs runs in the card bins crates/gpu-gates/src/residency.rs gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3moe-e2e gate-q38-serve-rules gate-gpu-nvtier gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-chat weekly-gpu-mimo2-e2e weekly-gpu-mimo2-serve gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve the unset residency rules' plan inputs (PlanRoom) and the pair rule's call (at_plan): the qwen38, GLM, V4.1 and MiMo residency, serve and placement bins crates/gpu-gates/src/residency38.rs gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3moe-e2e gate-q38-serve-rules gate-gpu-nvtier gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-chat gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve the Qwen3.8 residency figures (checkpoint_bytes, CARD38, glm_seqs): the qwen38 and GLM residency, serve and placement bins crates/gpu-gates/src/q38_fixture.rs gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen38-twocard the Qwen3.8 fixture's Shape, which shared/q38_arch.rs reads crates/gpu-gates/src/q36_fixture.rs gate-gpu-qwen35moe-e2e gate-gpu-minicpm-text the Qwen3.6 body's per-layer launch counts from the header, which gate_qwen35moe_e2e reads, and which the MiniCPM text recipe runs through that bin -crates/gpu-gates/src/act_rule.rs gate-gpu-iq gate-gpu-ds41-hc gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-minicpm-text gate-gpu-ds41-index gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-chain-attn gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-gpu-ds41-step gate-gpu-dspark-experts gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-gemma-embedding2-e2e the activation rules: the MoE, chain, index, comp, hc, iq, gemm and dspark experts bins that name them, and the fixture bands' readers of site_rel (shared/glm5next_tier.rs, the qwen4exp e2e and MTP bins, ds41-step's head; the qwen3moe experts, down and e2e bins and the qwen35moe e2e bin through ik_q8_2's non-finite check) -crates/gpu-gates/src/flip.rs gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-ds41-step gate-gpu-glm5next-mtp gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-h-moe-e2e gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-qwen38-residency gate-gpu-gemma-embedding2-e2e Flip, which shared/e2e.rs, the qwen4exp MTP bin, ds41-step's head cap and the GLM MTP replay's margin cap read -crates/model/src/lib.rs gate-gpu-p4 gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve ModelError, which every model reader returns -crates/model/src/moe.rs gate-gpu-p4 gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve TierError in ModelError (lib.rs), and the host expert legs the bodies and tiers call -crates/model/src/ops.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-app gate-prompts gate-gpu-lib gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve DEFER_MAX_COLS (gpu's host/step.rs and host/tier.rs), Tensor2 and the matmul dispatch +crates/gpu-gates/src/act_rule.rs gate-gpu-iq gate-gpu-ds41-hc gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-minicpm-text gate-gpu-ds41-index gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-chain-attn gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-gpu-ds41-step gate-gpu-dspark-experts gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-gemma-embedding2-e2e gate-gpu-gemma4v-tower the activation rules: the MoE, chain, index, comp, hc, iq, gemm and dspark experts bins that name them, and the fixture bands' readers of site_rel (shared/glm5next_tier.rs, the qwen4exp e2e and MTP bins, ds41-step's head; the qwen3moe experts, down and e2e bins and the qwen35moe e2e bin through ik_q8_2's non-finite check) +crates/gpu-gates/src/flip.rs gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-ds41-step gate-gpu-glm5next-mtp gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-h-moe-e2e gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-qwen38-residency gate-gpu-gemma-embedding2-e2e gate-gpu-gemma4v-tower Flip, which shared/e2e.rs, the qwen4exp MTP bin, ds41-step's head cap and the GLM MTP replay's margin cap read +crates/model/src/lib.rs gate-gpu-p4 gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower ModelError, which every model reader returns +crates/model/src/moe.rs gate-gpu-p4 gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower TierError in ModelError (lib.rs), and the host expert legs the bodies and tiers call +crates/model/src/ops.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-app gate-prompts gate-gpu-lib gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower DEFER_MAX_COLS (gpu's host/step.rs and host/tier.rs), Tensor2 and the matmul dispatch crates/model/src/embed.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-ds41-lib gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen3-serve gate-ds41-load gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-e2e gate-q38-serve-rules gate-gpu-qwen38-serve gate-gpu-gemma-embedding2-serve RowLayout and EmbedTable: the V4.1 glue, the draft and the GLM body -crates/model/src/placement/workstation.rs gate-gpu-iq-sel gate-placement gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-fixture gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-app gate-gpu-ds41-chain-ffn gate-gpu-ds41-lib gate-gpu-e2e gate-gpu-hybrid gate-gpu-moe gate-gpu-p10 gate-gpu-p8 gate-gpu-p8b gate-gpu-swap gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve host_available, which GpuModel's placement (gpu's model.rs) and the qwen35moe plan read +crates/model/src/placement/workstation.rs gate-gpu-iq-sel gate-placement gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-fixture gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-app gate-gpu-ds41-chain-ffn gate-gpu-ds41-lib gate-gpu-e2e gate-gpu-hybrid gate-gpu-moe gate-gpu-p10 gate-gpu-p8 gate-gpu-p8b gate-gpu-swap gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower host_available, which GpuModel's placement (gpu's model.rs) and the qwen35moe plan read crates/model/src/arch/qwen35moe/fixture.rs gate-fixture gate-gpu-gemma-embedding2-e2e the Qwen3.8 fixture spec, which the fixture test reads crates/model/data/mtp-head-rows-qwen38-*.txt gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-mtp gate-q38-serve-rules gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve Qwen3.8's shipped MTP head list, which arch/qwen35moe/head_list.rs includes (include_str!): the gates whose targets run its reader (head_rows_of, SHIPPED) tools/fixture-variants.tsv gate-fixture gate-gpu-gemma-embedding2-e2e the fixture variants' table, which fixture/variant.rs reads by path at run time: the fixture test -crates/models/src/lib.rs gate-gpu-p4 gate-gpu-p6 gate-gpu-iq-sel gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-runtime gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-dspark-read gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gpu-qwen35moe-attn gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve its re-exports (Role, Need: every placement path names models::Role through them) and ModelSpec and Score, which the arch readers and models::shape read +crates/models/src/lib.rs gate-gpu-p4 gate-gpu-p6 gate-gpu-iq-sel gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-runtime gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-dspark-read gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gpu-qwen35moe-attn gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower its re-exports (Role, Need: every placement path names models::Role through them) and ModelSpec and Score, which the arch readers and models::shape read crates/models/src/shape.rs gate-gpu-p6 gate-gpu-iq-sel gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-e2e gate-fixture gate-gpu-head gate-gpu-lib gate-ds41-plan gate-gpu-ds41-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-ds41-bind gate-ds41-place gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-placement gate-nemotron-meta gate-nemotron-h-moe-meta gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve MoeShape and the router rows (RouterInst, router_row) the routers' kernels select by -crates/gguf/src/lib.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-qdot gate-engram gate-tokenizer gate-decision-clef gate-decision-lev gate-serve gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-vision gate-gpu-vision gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-refset gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-models gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gpu-mamba2 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-emb2-mel LoadError (in GpuError and placement's errors) and Gguf: every gate that reads a file or names a load error -crates/gguf/src/quant.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-qdot gate-engram gate-tokenizer gate-decision-clef gate-decision-lev gate-serve gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-refset gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-vision gate-gpu-vision gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-dspark-read gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-models gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-hf gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gpu-mamba2 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-emb2-mel GgmlType (gguf's lib.rs, placement) and the f16 conversion the card's fused kernels name; gate-dspark-read also runs gguf's own quant tests -crates/gguf/src/iq_tables.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-kquant gate-gpu-hc-gated gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-ds41-meta gate-qwen3moe-meta gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-gemm gate-gpu-glm5next-gemm gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-qdot gate-decision-clef gate-decision-lev gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve KVALUES_IQ4NL, which quant.rs's dequant and the card's IQ kernels read -crates/qdot/src/lib.rs gate-gpu-p4 gate-gpu-p6 gate-gpu-kquant gate-gpu-iq-sel gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen35moe-attn gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-qdot gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gpu-mamba2 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve QdotError in ModelError, and the fused dots coverage and quant ask for -crates/threads/src/lib.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-ds41-meta gate-qwen3moe-meta gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-glm5next-gemm gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-threads gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-ds41-plan gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gpu-p1 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve the pool: upload's memcpy_pool, ops' matmul and the host tier's legs -crates/engram/src/lib.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-lib gate-gpu-gates-lib gate-ds41-plan gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve EngramError (direct.rs) and Engram, which the V4.1 chain's glue holds +crates/gguf/src/lib.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-qdot gate-engram gate-tokenizer gate-decision-clef gate-decision-lev gate-serve gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-vision gate-gpu-vision gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-refset gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-models gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gpu-mamba2 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-emb2-mel gate-gpu-gemma4v-tower LoadError (in GpuError and placement's errors) and Gguf: every gate that reads a file or names a load error +crates/gguf/src/quant.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-qdot gate-engram gate-tokenizer gate-decision-clef gate-decision-lev gate-serve gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-refset gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-vision gate-gpu-vision gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-dspark-read gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-models gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-hf gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gpu-mamba2 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-emb2-mel gate-gpu-gemma4v-tower GgmlType (gguf's lib.rs, placement) and the f16 conversion the card's fused kernels name; gate-dspark-read also runs gguf's own quant tests +crates/gguf/src/iq_tables.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-kquant gate-gpu-hc-gated gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-ds41-meta gate-qwen3moe-meta gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-gemm gate-gpu-glm5next-gemm gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-qdot gate-decision-clef gate-decision-lev gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower gate-gpu-vision gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-gpu-glm5v-tower KVALUES_IQ4NL, which quant.rs's dequant and the card's IQ kernels read +crates/qdot/src/lib.rs gate-gpu-p4 gate-gpu-p6 gate-gpu-kquant gate-gpu-iq-sel gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen35moe-attn gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-qdot gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gpu-mamba2 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower QdotError in ModelError, and the fused dots coverage and quant ask for +crates/threads/src/lib.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-ds41-meta gate-qwen3moe-meta gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-glm5next-gemm gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-threads gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-ds41-plan gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gpu-p1 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower the pool: upload's memcpy_pool, ops' matmul and the host tier's legs +crates/engram/src/lib.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-lib gate-gpu-gates-lib gate-ds41-plan gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower EngramError (direct.rs) and Engram, which the V4.1 chain's glue holds crates/models/src/headpick.rs gate-deepseek4-meta gate-ds41-meta gate-ds41-plan gate-fixture gate-glm5next-meta gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-twocard gate-gpu-glm5next-serve gate-gpu-iq-sel gate-gpu-lev-serve gate-gpu-nvtier gate-gpu-qwen3-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35moe-media gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3moe-e2e gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-mimo2-meta gate-ops gate-q38-serve-rules gate-qwen3moe-meta gate-qwen4exp-meta gate-runtime gate-ds41-bind gate-ds41-place gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35-dspark gate-gpu-gemm gate-gpu-head gate-gpu-lib gate-gpu-ds41-lib gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-nvtier gate-gpu-ds41-long gate-gpu-ds41-prefill gate-gpu-ds41-media gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-nemotron-meta gate-nemotron-h-moe-meta gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve models::headpick, the row-list head's reader, digest and loaded set: every gate whose load reads a head list gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35-dspark gate-gpu-gemm gate-gpu-head gate-gpu-lib gate-gpu-ds41-lib gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-nvtier gate-gpu-ds41-long gate-gpu-ds41-prefill gate-gpu-ds41-media gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-twocard gate-gpu-glm5next-residency crates/engram/src/rows.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gates-lib gate-q38-serve-rules gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-serve SplitTable, which the Qwen3.8 body opens -crates/hf/src/fetch.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-fixture gate-ds41-bind gate-ds41-place gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-q38-serve-rules gate-gpu-qwen38-serve gate-gpu-gemma-embedding2-serve gate-gpu-gemma-embedding2-e2e gate-gpu-clef-hidden gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-qwen3moe-e2e gate-gpu-qwen38-twocard the fetch Event that model_file.rs reports; the fixture fetch's range reads +crates/hf/src/fetch.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-fixture gate-ds41-bind gate-ds41-place gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-q38-serve-rules gate-gpu-qwen38-serve gate-gpu-gemma-embedding2-serve gate-gpu-gemma-embedding2-e2e gate-gpu-clef-hidden gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-qwen3moe-e2e gate-gpu-qwen38-twocard gate-gpu-gemma4v-tower the fetch Event that model_file.rs reports; the fixture fetch's range reads crates/decision/src/lib.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-decision-lev gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve the decision crate's Error: the decision gates -crates/vision/src/lib.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-serve gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-vision gate-gpu-glm5next-serve gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-decision-clef gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-ds41-media gate-gpu-routed-legs gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve VisionError, which the media and preprocess paths return -crates/gpu/src/gelu.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-mamba2 gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-gpu-clef-hidden gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-app gate-gpu-head gate-gpu-mcol gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-nvtier gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-vision gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-glm5v-tower gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve the shared tanh GELU rule: the registered elementwise module and image towers that use it -crates/gpu-vision/src/{aligner,attn,attn_group,chain,encoder,gelu_tanh,gemm_bf16,glm5v,layer_norm,mean_groups,mlp,norm,pos_bilinear,qwen3vl,rope2d}.rs gate-gpu-clef-serve gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-glm5v-tower gate-gpu-qwen3-serve gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-gpu-vision gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-lev-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve the image towers' card kernels: the tower, serve and media-seat gates that run them -crates/gpu-vision/src/lib.rs gate-gpu-clef-serve gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-glm5v-tower gate-gpu-qwen3-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-gpu-vision gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-lev-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve the tower crate's root, which the Qwen3.5 media gates also link -crates/vision/src/arch/glm5v/{card,hparams,mod,names,tensors}.rs gate-ds41-bind gate-ds41-place gate-gpu-clef-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-glm5next-stagger gate-gpu-glm5v-tower gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-gpu-vision gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve the GLM-5V family's hparams, tensor names and card homes -crates/vision/src/arch/glm5v/{media,size}.rs gate-ds41-bind gate-ds41-place gate-gpu-clef-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-glm5next-stagger gate-gpu-glm5v-tower gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve the GLM-5V media span and size rule the serve seats plan with -crates/vision/src/arch/glm5v/rules.rs gate-gpu-clef-serve gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-glm5v-tower gate-gpu-qwen3-serve gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-gpu-vision gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-lev-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve the GLM-5V preprocess rules the towers read -crates/vision/src/{arch/deepseek41v/mod.rs,arch/deepseek41v/tensors.rs} gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-vision gate-gpu-glm5next-serve gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-ds41-bind gate-ds41-place gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-ds41-media gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve the V4.1 vision family's constants and its tensors' card homes -crates/vision/src/{image,preprocess}.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-decision-clef gate-serve gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-media gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-vision gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-gpu-glm5v-tower gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve the decoded image and its patch cut, which every image seat's prepare runs -crates/vision/src/resample.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-decision-clef gate-ds41-bind gate-ds41-place gate-gpu-ds41-media gate-gpu-qwen3-serve gate-gpu-vision gate-gpu-qwen3vl-tower gate-gpu-glm5v-tower gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-minicpmv-tower gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve the resize and pad an image seat's prepare runs -crates/vision/src/arch/deepseek41v/grid.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-ds41-bind gate-ds41-place gate-gpu-ds41-media gate-gpu-qwen3-serve gate-gpu-vision gate-gpu-qwen3vl-tower gate-gpu-glm5v-tower gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-minicpmv-tower gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve V4.1's grid plan, reached through the projector's arch dispatch -crates/vision/src/mrope/mod.rs gate-gpu-e2e gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-types gate-app gate-gpu-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-media gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-minicpmv-tower gate-gpu-minicpm-text gate-decision-clef gate-serve gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve the M-RoPE module's error and the public paths of its positions and rows +crates/vision/src/lib.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-serve gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-vision gate-gpu-glm5next-serve gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-decision-clef gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-ds41-media gate-gpu-routed-legs gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve gate-gpu-gemma-embedding2-e2e gate-gpu-gemma4v-tower VisionError, which the media and preprocess paths return +crates/gpu/src/gelu.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-mamba2 gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-gpu-clef-hidden gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-app gate-gpu-head gate-gpu-mcol gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-nvtier gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-vision gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-glm5v-tower gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower the shared tanh GELU rule: the registered elementwise module and image towers that use it +crates/gpu-vision/src/{aligner,attn,attn_group,chain,encoder,gelu_tanh,gemm_bf16,glm5v,layer_norm,mean_groups,mlp,norm,pos_bilinear,qwen3vl,rope2d}.rs gate-gpu-clef-serve gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-glm5v-tower gate-gpu-qwen3-serve gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-gpu-vision gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-lev-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve gate-gpu-gemma-embedding2-e2e gate-gpu-gemma4v-tower the image towers' card kernels: the tower, serve and media-seat gates that run them +crates/gpu-vision/src/lib.rs gate-gpu-clef-serve gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-glm5v-tower gate-gpu-qwen3-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-gpu-vision gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-lev-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve gate-gpu-gemma-embedding2-e2e gate-gpu-gemma4v-tower the tower crate's root, which the Qwen3.5 media gates also link +crates/vision/src/arch/glm5v/{card,hparams,mod,names,tensors}.rs gate-ds41-bind gate-ds41-place gate-gpu-clef-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-glm5next-stagger gate-gpu-glm5v-tower gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-gpu-vision gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve gate-gpu-gemma-embedding2-e2e gate-gpu-gemma4v-tower gate-q38-serve-rules gate-gpu-qwen38-serve the GLM-5V family's hparams, tensor names and card homes +crates/vision/src/arch/glm5v/{media,size}.rs gate-ds41-bind gate-ds41-place gate-gpu-clef-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-glm5next-stagger gate-gpu-glm5v-tower gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve gate-q38-serve-rules gate-gpu-qwen38-serve the GLM-5V media span and size rule the serve seats plan with +crates/vision/src/arch/glm5v/rules.rs gate-gpu-clef-serve gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-glm5v-tower gate-gpu-qwen3-serve gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-gpu-vision gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-lev-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve gate-gpu-gemma-embedding2-e2e gate-gpu-gemma4v-tower the GLM-5V preprocess rules the towers read +crates/vision/src/{arch/deepseek41v/mod.rs,arch/deepseek41v/tensors.rs} gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-vision gate-gpu-glm5next-serve gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-ds41-bind gate-ds41-place gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-ds41-media gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve gate-gpu-gemma-embedding2-e2e gate-gpu-gemma4v-tower gate-q38-serve-rules gate-gpu-qwen38-serve the V4.1 vision family's constants and its tensors' card homes +crates/vision/src/{image,preprocess}.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-decision-clef gate-serve gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-media gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-vision gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-gpu-glm5v-tower gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve gate-gpu-gemma-embedding2-e2e gate-gpu-gemma4v-tower the decoded image and its patch cut, which every image seat's prepare runs +crates/vision/src/resample.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-decision-clef gate-ds41-bind gate-ds41-place gate-gpu-ds41-media gate-gpu-qwen3-serve gate-gpu-vision gate-gpu-qwen3vl-tower gate-gpu-glm5v-tower gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-minicpmv-tower gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve gate-gpu-gemma-embedding2-e2e gate-gpu-gemma4v-tower gate-serve gate-q38-serve-rules gate-gpu-qwen38-serve the resize and pad an image seat's prepare runs +crates/vision/src/arch/deepseek41v/grid.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-ds41-bind gate-ds41-place gate-gpu-ds41-media gate-gpu-qwen3-serve gate-gpu-vision gate-gpu-qwen3vl-tower gate-gpu-glm5v-tower gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-minicpmv-tower gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve gate-gpu-gemma-embedding2-e2e gate-gpu-gemma4v-tower gate-q38-serve-rules gate-gpu-qwen38-serve V4.1's grid plan, reached through the projector's arch dispatch +crates/vision/src/mrope/mod.rs gate-gpu-e2e gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-types gate-app gate-gpu-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-media gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-minicpmv-tower gate-gpu-minicpm-text gate-decision-clef gate-serve gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve gate-gpu-gemma-embedding2-e2e the M-RoPE module's error and the public paths of its positions and rows crates/vision/src/mrope/{seq,rows}.rs gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-types gate-gpu-lib gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve the M-RoPE positions of a sequence's images and the table rows they rewrite crates/vision/src/arch/gemma4a.rs gate-emb2-mel gate-vision gemma4a host mel preprocessing and token counts @@ -216,7 +216,7 @@ crates/gpu/src/cand.rs weekly-gpu-ds41-cand the candidate mask's kernels, which crates/gpu-deepseek41/src/{indexer.rs,chain/attn.rs,chain/attn/batch.rs} weekly-gpu-ds41-cand the indexer's split launcher and the select path's candidate launches, per row of words in the step and per chunk in the prompt batch crates/gpu-deepseek41/src/{body.rs,body/ced.rs,body/prefill.rs} weekly-gpu-ds41-cand the decode and pair wiring, the triangle's candidate source, and the prompt batch's kept blocks per batch set crates/gpu-gates/src/bin/{gate_deepseek41_long.rs,gate_deepseek41_prefill.rs,shared/ds41_cand.rs} weekly-gpu-ds41-cand the weekly's two clauses and their candidate rule -crates/placement/src/** gate-app gate-deepseek4-meta gate-ds41-bind gate-ds41-host gate-ds41-kld gate-ds41-load gate-ds41-meta gate-ds41-oracle gate-ds41-plan gate-dspark-read gate-fixture gate-glm5next-meta gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-comp gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-index gate-gpu-ds41-lib gate-gpu-ds41-long gate-gpu-ds41-prefill gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-e2e gate-gpu-gates-lib gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-lib gate-gpu-mcol gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-qwen35moe-moe gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-head gate-gpu-qwen3moe-router gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-ops gate-placement gate-qwen35moe-meta gate-qwen3moe-meta gate-qwen4exp-meta gate-union gate-gpu-p4 gate-gpu-p6 gate-gpu-iq-sel gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-block gate-gpu-p10 gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-gpu-qwen3moe-rope gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-r8 gate-prompts gate-gpu-head gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-woa gate-gpu-ds41-media gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-p8b gate-mimo2-meta gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-nemotron-meta gate-gpu-qwen35moe-attn gate-gpu-p1 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-p2 gate-nemotron-h-moe-meta gate-gpu-nvfp4 gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-nemotron-h-moe-e2e gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve the placement planner, which bloomery-model re-exports as model::placement: every gate that plans a placement, reads a plan's figures or checks the r8 sidecar's refusal type +crates/placement/src/** gate-app gate-deepseek4-meta gate-ds41-bind gate-ds41-host gate-ds41-kld gate-ds41-load gate-ds41-meta gate-ds41-oracle gate-ds41-plan gate-dspark-read gate-fixture gate-glm5next-meta gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-comp gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-index gate-gpu-ds41-lib gate-gpu-ds41-long gate-gpu-ds41-prefill gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-e2e gate-gpu-gates-lib gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-lib gate-gpu-mcol gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-qwen35moe-moe gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-head gate-gpu-qwen3moe-router gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-ops gate-placement gate-qwen35moe-meta gate-qwen3moe-meta gate-qwen4exp-meta gate-union gate-gpu-p4 gate-gpu-p6 gate-gpu-iq-sel gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-block gate-gpu-p10 gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-gpu-qwen3moe-rope gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-r8 gate-prompts gate-gpu-head gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-woa gate-gpu-ds41-media gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-p8b gate-mimo2-meta gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-nemotron-meta gate-gpu-qwen35moe-attn gate-gpu-p1 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-p2 gate-nemotron-h-moe-meta gate-gpu-nvfp4 gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-nemotron-h-moe-e2e gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower the placement planner, which bloomery-model re-exports as model::placement: every gate that plans a placement, reads a plan's figures or checks the r8 sidecar's refusal type crates/model/src/arch/gemma_encoder.rs gate-ops gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gemma-embedding-meta gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-nemotron-h-moe-e2e gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve the shared Gemma encoder header checks, tensor shapes, roles and layer description except gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-gpu-clef-hidden gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-fixture gate-gpu-head gate-gpu-lib gate-gpu-ds41-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-nvtier gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41: they link the arch dispatch and never open a Gemma encoder file crates/model/src/arch/gemma_embedding/** gate-ops gate-gemma-embedding-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-nemotron-h-moe-e2e gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve EmbeddingGemma header reader and its synthetic and real-header contracts except gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-gpu-clef-hidden gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-fixture gate-gpu-head gate-gpu-lib gate-gpu-ds41-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-nvtier gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41: they link the arch dispatch and never open a Gemma encoder file crates/gpu-gates/src/beside.rs gate-gpu-clef-serve gate-gpu-glm5next-serve gate-gpu-lev-serve gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3-serve gate-gpu-qwen3moe-e2e gate-gpu-clef-hidden gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-chat gate-gpu-glm5next-twocard gate-gpu-nemotron-serve gate-q38-serve-rules gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve the reserves a plan holds beside its own tensors (the image encoder's tower in a placed plan and a whole-fit verdict): the qwen3 placement every Qwen3 bin includes, the V4.1 and GLM placements, and the servers that bind the seats that name a reserve diff --git a/tools/recipes.py b/tools/recipes.py index 51459af63..722e12ea0 100755 --- a/tools/recipes.py +++ b/tools/recipes.py @@ -10290,7 +10290,10 @@ def problems(*recipes: tuple[str, ...]) -> list[str]: return [f"raised {err}"] base = "gate_glm5next_e2e (bloomery-gpu-gates, release): " - wide, narrow = "{deepseek41,glm5next,gpu,vision}", "{deepseek41,glm5next,gpu}" + tree = Tree(ROOT) + pkg = tree.packages["bloomery-gpu-gates"] + closure = lambda feats: "{" + ",".join(sorted(tree.feature_closure(pkg, feats.split(","))[0])) + "}" + wide, narrow = closure("glm5next,vision"), closure("glm5next") got = problems(("a", call("glm5next")), ("b", call("glm5next,vision"))) expect(len(got) == 1 and got[0].startswith(f"{base}b {wide} vs a {narrow} "), f"bin features: glm5next vs glm5next,vision not refused by name: {got}") # the recipes' order and names do not move the line; a set shared by recipes lists them all, sorted From 36f89e9f26d4b8f08820e6ce034e622b733d1fda Mon Sep 17 00:00:00 2001 From: juliankang4 <200967947+juliankang4@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:16:27 +0900 Subject: [PATCH 09/12] docs: serving EmbeddingGemma 2 images beside a chat model Document the image side seat, input parts, reserve and reference gates, with a command for comparing a running HTTP server to saved llama-server vectors. --- docs/serving.md | 75 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 73 insertions(+), 2 deletions(-) diff --git a/docs/serving.md b/docs/serving.md index 874cd3f8b..a082f9e46 100644 --- a/docs/serving.md +++ b/docs/serving.md @@ -37,8 +37,8 @@ Your OpenAI client works unchanged. bloomery speaks llama-server's HTTP API and other than the embedding model's is a 400 `invalid_request_error` and runs no pass; `null` is unset. `/embedding` and `/embeddings` ignore `model`. An input past the model's context is a 400 `exceed_context_size_error` naming both lengths. llama-server checks its physical batch first and answers an input - past it with a 500; this server has no batch limit there. An object input (`prompt_string`, or a `content` array - of parts) is a 400. + past it with a 500; this server has no batch limit there. An object with a `content` array takes text and image + parts as described below. An object with `prompt_string` is a 400. - **Tool calls on every generative model**: DeepSeek-V4.1-Flash (DSML), GLM-5.3-Flash (its XML), Qwen3-30B (Hermes JSON), Qwen3.6-35B and Qwen3.8-Flash-Next (the `` markup) parse into `tool_calls` and `tool_use` blocks, so agent clients work against each of them. @@ -256,3 +256,74 @@ binary's `--help`. - A pinned nightly with a pinned cuda-oxide revision from our fork ([`BUILD.md`](BUILD.md)). In progress: serving on N cards for every model; Clef from the IQ and Q2_K files; DeepSeek-V4-Flash-0731. + +## EmbeddingGemma 2 images + +A side seat serves the `gemma-embedding2` encoder beside a generative model. Build `bloomery-serve` with +`--features glm5next,clef,mimo2,gemma-embedding2,vision` through `cargo oxide`, and add these flags to the chat +model's command: + +```text +--embd-model embeddinggemma-2-Q8_0.gguf +--embd-mmproj mmproj-embeddinggemma-2-BF16.gguf +--embd-ctx 8192 +``` + +`--embd-hf [:]` selects the text file and the row's BF16 projector from that repository. +`--embd-no-mmproj` disables that automatic projector. A local `--embd-model` loads images only with an explicit +`--embd-mmproj`. The projector's vision matrices must be BF16; a Q8_0 projector is refused by tensor name. +The F32 patch convolution and learned position tables retain their file precision. With no activation keys, +the tower follows mainline's Quick GELU; `clip.use_gelu=true` selects its tanh GELU and SiLU is refused. Audio tensors of a combined +vision/audio file stay on the host and are not counted in the card reserve. + +The encoder and tower share the embedding side seat's card and thread, with streams of their own. Before the chat placement +is made, their weight and activation bytes are counted together. On a card outside the chat plan this costs the +plan no bytes; otherwise the plan holds them in its named `embedding model` reserve, reducing the chat expert +budget. The side context bounds the complete input, including image rows and delimiter text. + +Send one object per input inside OpenAI's `input` array: + +```json +{ + "input": [ + { + "content": [ + {"type": "text", "text": "A description of this picture:"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,..."}} + ] + } + ], + "dimensions": 256 +} +``` + +An image URL is a base64 `data:` URL for PNG or JPEG, as on the chat path; HTTP URLs are refused. Multiple image +parts remain in input order. Each image is resized with mtmd's aspect-preserving dynamic canvas, bicubic filter +and centered black padding. The default budget is 70 to 1120 pooled image tokens. Set +`--embd-image-min-tokens` and `--embd-image-max-tokens` to change it independently of the chat model's image flags. +A canvas exceeding the reserved token budget or learned position table is a request error before a tower call. + +`<|image>` and `` are tokenized around each image's rows with the server's vocabulary rule, +included in the single bidirectional pass and included in mean pooling, as in the pinned mainline server. +Only text embeddings receive the text encoder's `sqrt(hidden)` scale: image rows enter already projected. +The entire input runs once; image chunks are not decoded separately. A request without a loaded image tower, +with an unsupported image part or beyond the side context is refused with a 400. + +The reference families in `crates/refset/src/arch/gemma_embedding2` own the projector path, its digest and +llama.cpp build pin. `just gate-gpu-gemma4v-tower` reads the mainline card and CPU tower sets and every required +layer tensor, comparing the device to the card with each band derived from the CPU twin. +`just gate-gpu-gemma-embedding2-e2e` also runs image-only and `mix_free-448x448` text-first inputs through +`pass_media`, using `serve::embeddings::assemble` for the same part ordering and token spans as the HTTP route. +It compares raw and normalized vectors with the server and its CPU twin (`just dump-ref-emb2 server.cpu`). + +To compare a running HTTP server directly with saved llama-server vectors: + +```sh +python3 tools/ref/emb2/server_vectors.py compare --url http://127.0.0.1:8080 \ + --set "$BLOOMERY_DATA/ref_gemma_embedding2_server" \ + --image free-448x448=/path/to/free-448x448.png --mix-image free-448x448 +``` + +Use the exact image bytes named by the reference set. Add `--ids-dir "$BLOOMERY_DATA/ref-tokenizer-gemma4"` +and `--case NAME` for text cases. Each selected `raw` and `embd` row prints cosine and maximum absolute +difference. Exit 0 means the comparison completed; it applies no numerical acceptance band. From b58244800b19310a9b53d0c5993c12f260fa207c Mon Sep 17 00:00:00 2001 From: juliankang4 <200967947+juliankang4@users.noreply.github.com> Date: Thu, 15 Oct 2026 10:05:44 +0900 Subject: [PATCH 10/12] tools: admit the PTX scan architectures with occupancy limits Allow the architecture spellings whose occupancy limits ptx-scan already defines. This changes no server collection path. --- tools/scan-args.sh | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/tools/scan-args.sh b/tools/scan-args.sh index b11fb15b0..a11fa55c0 100755 --- a/tools/scan-args.sh +++ b/tools/scan-args.sh @@ -16,8 +16,8 @@ # more words than the scan script takes (ptx and lds: one entry substring; sass: the entry and its # decisions, which are a comma list of t/n or ADDR=t|n and are not refused for their commas). # A feature table it cannot read ends it with 2 too: an unread table would pass every word. -# The ptx scan alone is sm_86's (its blk/SM column is that arch's arithmetic): a BLOOMERY_ARCH that names another is refused -# by name, exit 64, before the build; the sass and lds scans follow the arch (tools/arch.sh). +# The ptx scan accepts only the architectures whose occupancy limits tools/ptx-scan.sh holds, before the build; +# the sass and lds scans follow the arch (tools/arch.sh). set -uo pipefail ROOT=$(cd "$(dirname "$0")/.." && pwd) @@ -37,9 +37,12 @@ check() { sass) max=2 flags=' --exact --raw --step ' form="just sass-scan $bin --features [-- --exact|--raw|--step HEAD] [entry substring [decisions|list]]" ;; *) echo "scan-args.sh: the scan is ptx, sass or lds, got '$scan'" >&2; return 64 ;; esac - if [ "$scan" = ptx ] && [ "${BLOOMERY_ARCH-sm_86}" != sm_86 ]; then - echo "ptx-scan: BLOOMERY_ARCH is '$BLOOMERY_ARCH': the scan's blk/SM column is sm_86's occupancy arithmetic and it holds no other arch's limits (unset BLOOMERY_ARCH for this recipe)" >&2 - return 64 + if [ "$scan" = ptx ]; then + case ${BLOOMERY_ARCH-sm_86} in + # PIN(2026-10-16): ptx-scan.sh tests cc12.x occupancy limits; the sm_86-only refusal rejected supported scans. + sm_86 | sm_120 | sm_120a | sm_120f | sm_121 | sm_121a | sm_121f) ;; + *) echo "ptx-scan: BLOOMERY_ARCH is '${BLOOMERY_ARCH-}': the scan holds occupancy limits for sm_86 and sm_120/sm_121 only" >&2; return 64 ;; + esac fi while [ $# -gt 0 ]; do w=$1 why='' @@ -123,7 +126,10 @@ self_test() { case_ sass-entry-comma 2 sass gemm,flash case_ scan-unknown 64 cubin gemm BLOOMERY_ARCH=sm_86 case_ ptx-arch-sm86 0 ptx - BLOOMERY_ARCH=sm_121 case_ ptx-arch-other 64 ptx + BLOOMERY_ARCH=sm_121 case_ ptx-arch-sm121 0 ptx + BLOOMERY_ARCH=sm_120a case_ ptx-arch-sm120a 0 ptx + BLOOMERY_ARCH=sm_121f case_ ptx-arch-sm121f 0 ptx + BLOOMERY_ARCH=sm_90 case_ ptx-arch-unknown 64 ptx BLOOMERY_ARCH=sm_121 case_ lds-arch-other 0 lds BLOOMERY_ARCH=sm_121 case_ sass-arch-other 0 sass # The real table: every feature crates/gpu-gates declares is refused as a word, and `gpu` is one. From 9935117203223ca96a5bcd86c833ec85f4193d1e Mon Sep 17 00:00:00 2001 From: juliankang4 <200967947+juliankang4@users.noreply.github.com> Date: Thu, 15 Oct 2026 10:53:53 +0900 Subject: [PATCH 11/12] gpu-gates: pin the Gemma4 vision entries on both PTX targets Carry the measured Gemma4 entry rows for the default table and both architecture-specific PTX tables. --- tools/ref/ptx-shapes.tsv | 11 ++++++++++- tools/ref/ptx-shapes/sm_121-cc12.1.tsv | 10 +++++++++- tools/ref/ptx-shapes/sm_86-cc12.1.tsv | 10 +++++++++- 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/tools/ref/ptx-shapes.tsv b/tools/ref/ptx-shapes.tsv index 4247024b5..ced6ab1db 100644 --- a/tools/ref/ptx-shapes.tsv +++ b/tools/ref/ptx-shapes.tsv @@ -620,20 +620,29 @@ gate_e2e tier_fault_copy 0 0 gate_e2e touch 0 0 gate_e2e weighted_sum 0 0 -# gate_qwen3vl_tower (--features vision), the `vis_` entries of bloomery-gpu-vision only: V4.1's six, the Qwen3-VL tower's five, the glm5v tower's three, and MiniCPM's two +# gate_qwen3vl_tower (--features vision), the shared vision entries, including Gemma4 residual and post-norm precision. +# PIN(2026-10-15): the seven Gemma4 entries have zero sm_86 ptxas spill; jit_local=0 is a derived cc8.6 ceiling, +# not a cc8.6 driver measurement. Both sm_86 and sm_121 PTX have zero JIT local bytes on GB10 cc12.1. +gate_qwen3vl_tower vis_add_f32 0 0 gate_qwen3vl_tower vis_attn 0 0 gate_qwen3vl_tower vis_attn_group 0 0 gate_qwen3vl_tower vis_attn_h72 0 0 +gate_qwen3vl_tower vis_clamp_f32 0 0 gate_qwen3vl_tower vis_gelu_erf 0 0 gate_qwen3vl_tower vis_gelu_tanh 0 0 gate_qwen3vl_tower vis_gemm_bf16 0 0 +gate_qwen3vl_tower vis_gemm_bf16_f32 0 0 gate_qwen3vl_tower vis_head_rms_norm 0 0 gate_qwen3vl_tower vis_layer_norm 0 0 gate_qwen3vl_tower vis_mean_groups 0 0 +gate_qwen3vl_tower vis_pool_order_f32 0 0 gate_qwen3vl_tower vis_pos_bilinear 0 0 +gate_qwen3vl_tower vis_position_rows 0 0 gate_qwen3vl_tower vis_rms_norm 0 0 +gate_qwen3vl_tower vis_rms_norm_f32 0 0 gate_qwen3vl_tower vis_rope2d 0 0 gate_qwen3vl_tower vis_rope2d_h72 0 0 +gate_qwen3vl_tower vis_row_op 0 0 gate_qwen3vl_tower vis_silu_mul 0 0 gate_qwen3vl_tower vis_silu_mul_clamp 0 0 gate_qwen3vl_tower vis_unfold 0 0 diff --git a/tools/ref/ptx-shapes/sm_121-cc12.1.tsv b/tools/ref/ptx-shapes/sm_121-cc12.1.tsv index a21a22263..d1213f4db 100644 --- a/tools/ref/ptx-shapes/sm_121-cc12.1.tsv +++ b/tools/ref/ptx-shapes/sm_121-cc12.1.tsv @@ -581,20 +581,28 @@ gate_e2e tier_fault_copy 0 0 gate_e2e touch 0 0 gate_e2e weighted_sum 0 0 -# gate_qwen3vl_tower (--features vision), the `vis_` entries of bloomery-gpu-vision only: V4.1's six, the Qwen3-VL tower's five, the glm5v tower's three, and MiniCPM's two +# gate_qwen3vl_tower (--features vision), the shared vision entries, including Gemma4 residual and post-norm precision. +# PIN(2026-10-15): the seven Gemma4 entries have zero ptxas spill and JIT local bytes on GB10 cc12.1. +gate_qwen3vl_tower vis_add_f32 0 0 gate_qwen3vl_tower vis_attn 0 0 gate_qwen3vl_tower vis_attn_group 0 0 gate_qwen3vl_tower vis_attn_h72 0 0 +gate_qwen3vl_tower vis_clamp_f32 0 0 gate_qwen3vl_tower vis_gelu_erf 0 0 gate_qwen3vl_tower vis_gelu_tanh 0 0 gate_qwen3vl_tower vis_gemm_bf16 0 0 +gate_qwen3vl_tower vis_gemm_bf16_f32 0 0 gate_qwen3vl_tower vis_head_rms_norm 0 0 gate_qwen3vl_tower vis_layer_norm 0 0 gate_qwen3vl_tower vis_mean_groups 0 0 +gate_qwen3vl_tower vis_pool_order_f32 0 0 gate_qwen3vl_tower vis_pos_bilinear 0 0 +gate_qwen3vl_tower vis_position_rows 0 0 gate_qwen3vl_tower vis_rms_norm 0 0 +gate_qwen3vl_tower vis_rms_norm_f32 0 0 gate_qwen3vl_tower vis_rope2d 0 0 gate_qwen3vl_tower vis_rope2d_h72 0 0 +gate_qwen3vl_tower vis_row_op 0 0 gate_qwen3vl_tower vis_silu_mul 0 0 gate_qwen3vl_tower vis_silu_mul_clamp 0 0 gate_qwen3vl_tower vis_unfold 0 0 diff --git a/tools/ref/ptx-shapes/sm_86-cc12.1.tsv b/tools/ref/ptx-shapes/sm_86-cc12.1.tsv index 2a60a7148..8e7118810 100644 --- a/tools/ref/ptx-shapes/sm_86-cc12.1.tsv +++ b/tools/ref/ptx-shapes/sm_86-cc12.1.tsv @@ -581,20 +581,28 @@ gate_e2e tier_fault_copy 0 0 gate_e2e touch 0 0 gate_e2e weighted_sum 0 0 -# gate_qwen3vl_tower (--features vision), the `vis_` entries of bloomery-gpu-vision only: V4.1's six, the Qwen3-VL tower's five, the glm5v tower's three, and MiniCPM's two +# gate_qwen3vl_tower (--features vision), the shared vision entries, including Gemma4 residual and post-norm precision. +# PIN(2026-10-15): the seven Gemma4 entries have zero ptxas spill and JIT local bytes on GB10 cc12.1. +gate_qwen3vl_tower vis_add_f32 0 0 gate_qwen3vl_tower vis_attn 0 0 gate_qwen3vl_tower vis_attn_group 0 0 gate_qwen3vl_tower vis_attn_h72 0 0 +gate_qwen3vl_tower vis_clamp_f32 0 0 gate_qwen3vl_tower vis_gelu_erf 0 0 gate_qwen3vl_tower vis_gelu_tanh 0 0 gate_qwen3vl_tower vis_gemm_bf16 0 0 +gate_qwen3vl_tower vis_gemm_bf16_f32 0 0 gate_qwen3vl_tower vis_head_rms_norm 0 0 gate_qwen3vl_tower vis_layer_norm 0 0 gate_qwen3vl_tower vis_mean_groups 0 0 +gate_qwen3vl_tower vis_pool_order_f32 0 0 gate_qwen3vl_tower vis_pos_bilinear 0 0 +gate_qwen3vl_tower vis_position_rows 0 0 gate_qwen3vl_tower vis_rms_norm 0 0 +gate_qwen3vl_tower vis_rms_norm_f32 0 0 gate_qwen3vl_tower vis_rope2d 0 0 gate_qwen3vl_tower vis_rope2d_h72 0 0 +gate_qwen3vl_tower vis_row_op 0 0 gate_qwen3vl_tower vis_silu_mul 0 0 gate_qwen3vl_tower vis_silu_mul_clamp 0 0 gate_qwen3vl_tower vis_unfold 0 0 From c83b59009da378e4749b38688bfd873b0987cb11 Mon Sep 17 00:00:00 2001 From: juliankang4 <200967947+juliankang4@users.noreply.github.com> Date: Fri, 16 Oct 2026 12:31:20 +0900 Subject: [PATCH 12/12] gpu-gates: the image gates follow the rebased embedding seat Regenerate the gate path table against the embedding seat's current encoder and keep the no-argument e2e recipe in step with its binary. Format the merged reference families and shared header fixture. --- crates/refset/src/arch/gemma_embedding2/mod.rs | 5 ++++- crates/refset/src/arch/gemma_embedding2/tests.rs | 4 ++-- crates/refset/src/emb2/tests.rs | 6 +++++- justfile | 4 ++-- 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/crates/refset/src/arch/gemma_embedding2/mod.rs b/crates/refset/src/arch/gemma_embedding2/mod.rs index 8b699e08f..d25343b92 100644 --- a/crates/refset/src/arch/gemma_embedding2/mod.rs +++ b/crates/refset/src/arch/gemma_embedding2/mod.rs @@ -171,7 +171,10 @@ pub static TOWER_CPU: Family = Family { }; /// llama-server's `/v1/embeddings` vectors. pub static SERVER: Family = Family { - consumers: &["gate-gpu-gemma-embedding2-e2e", "gate-gpu-gemma-embedding2-serve"], + consumers: &[ + "gate-gpu-gemma-embedding2-e2e", + "gate-gpu-gemma-embedding2-serve", + ], ..family( "server-gemma-embedding2", &[SERVER_SET], diff --git a/crates/refset/src/arch/gemma_embedding2/tests.rs b/crates/refset/src/arch/gemma_embedding2/tests.rs index 1a7df91db..e8ce7239e 100644 --- a/crates/refset/src/arch/gemma_embedding2/tests.rs +++ b/crates/refset/src/arch/gemma_embedding2/tests.rs @@ -1,6 +1,6 @@ use super::{ - ARCH, MMPROJ_BF16, MMPROJ_BF16_SHA256, MODEL, MODEL_SHA256, PROFILE, SERVER, SERVER_CPU, - TEXT, TEXT_CPU, TOWER, TOWER_CPU, + ARCH, MMPROJ_BF16, MMPROJ_BF16_SHA256, MODEL, MODEL_SHA256, PROFILE, SERVER, SERVER_CPU, TEXT, + TEXT_CPU, TOWER, TOWER_CPU, }; use crate::RefError; use crate::emb2::tests::{check, header, remove, set_dir}; diff --git a/crates/refset/src/emb2/tests.rs b/crates/refset/src/emb2/tests.rs index 6c7fc757a..e1f55c3a1 100644 --- a/crates/refset/src/emb2/tests.rs +++ b/crates/refset/src/emb2/tests.rs @@ -27,7 +27,11 @@ pub(crate) fn header( let device = format!( "{}\tthreads\t{}\tcard\t{}", kind.device(), - if matches!(kind, Kind::Server | Kind::ServerCpu) { "-" } else { "16" }, + if matches!(kind, Kind::Server | Kind::ServerCpu) { + "-" + } else { + "16" + }, if kind.device() == "cpu" { "cpu" } else { diff --git a/justfile b/justfile index 93d9f6902..419b21e5e 100644 --- a/justfile +++ b/justfile @@ -1261,11 +1261,11 @@ gate-gpu-gemma-embedding2-serve: # text case of mainline's card set (just dump-ref-emb2) through one pass, its taps within the CPU twin's band, # a rerun bit for bit, and the refusals by name. The # key bounds first hold every key of the symmetric mask, independently of the tap bands. -gate-gpu-gemma-embedding2-e2e *ARGS: +gate-gpu-gemma-embedding2-e2e: ./tools/box.sh 'bash tools/gate.sh --oxide --release -p bloomery-gpu-gemma-embedding2 --lib -- --include-ignored' ./tools/box.sh 'bash tools/gate.sh --release -p bloomery-model --lib -- arch::gemma_embedding2::spec::tests::key_bounds_are_the_symmetric_mask --exact' ./tools/box.sh 'bash tools/gate.sh --oxide --release -p bloomery-gpu-gates --features gemma-embedding2,vision,deepseek41 --bin gate_gemma_embedding2_e2e' - ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features gemma-embedding2,vision,deepseek41 --release --bin gate_gemma_embedding2_e2e && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_gemma_embedding2_e2e {{ARGS}}' + BLOOMERY_MODEL=gemma-embedding2 ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features gemma-embedding2,vision,deepseek41 --release --bin gate_gemma_embedding2_e2e && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_gemma_embedding2_e2e' # Gemma4 vision's mainline card and CPU sets, including every block's rows. # Every layer and projected row is held to its CPU/card twin distance.