diff --git a/crates/gpu-vision/src/gelu_tanh.rs b/crates/gpu-vision/src/gelu_tanh.rs index d519b4a01..50dfbcdbc 100644 --- a/crates/gpu-vision/src/gelu_tanh.rs +++ b/crates/gpu-vision/src/gelu_tanh.rs @@ -6,9 +6,7 @@ //! Not [`crate::gemm_bf16::Epilogue::Gelu`]: that epilogue is the erf GELU after a GEMM, in the //! GEMM's registers; the two kinds differ by up to 4.7e-4 in the range the network's rows take. //! -//! Numeric rule, per value: `x` widened exactly, the expression of llama.cpp's CUDA `op_gelu` in -//! f32 in its op order — the polynomial `1 + (0.044715·x)·x` contracted to one fused multiply-add -//! as nvcc contracts it, every other op rounded on its own — with libdevice's `tanhf`, and the +//! Numeric rule, per value: `x` widened exactly, [`bloomery_gpu::gelu`]'s f32 rule, and the //! result rounded once to bf16. [`gelu_tanh_ref`] is the rule on the host with the host's `tanh`, //! which differs from the device's by ulps; the gate pins the count of values that round apart. //! The erf rule is `gemm_bf16`'s `gelu_erf_dev`, the one device function of the crate that calls @@ -17,19 +15,15 @@ //! rounded on its own, once to bf16; [`gelu_erf_ref`] is it on the host with an f64 `erf`. use crate::gemm_bf16::gelu_erf_dev; +use bloomery_gpu::gelu::{gelu_tanh_dev, gelu_tanh_f32}; use bloomery_gpu::{GpuError, launch_u32}; use cuda_core::{CudaContext, CudaStream, DeviceBuffer, LaunchConfig1D}; use cuda_device::convert::{bf16_to_f32, f32_to_bf16_rne}; -use cuda_device::float::{add_rn_f32, fma_rn_f32, mul_rn_f32}; use cuda_device::{DisjointSlice, kernel, launch_bounds, launch_contract, thread}; use cuda_host::cuda_module; use std::sync::Arc; const BLOCK: u32 = 256; -/// `√(2/π)` as the f32 llama.cpp's `SQRT_2_OVER_PI` rounds to. -const SQRT_2_OVER_PI: f32 = 0.797_884_6; -/// llama.cpp's `GELU_COEF_A`. -const COEF_A: f32 = 0.044_715; /// Which GELU a launch applies. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -40,14 +34,6 @@ pub enum GeluKind { Erf, } -/// The rule on one f32 on the host. -#[must_use] -pub fn gelu_tanh_f32(x: f32) -> f32 { - let poly = (COEF_A * x).mul_add(x, 1.0); - let t = (SQRT_2_OVER_PI * x * poly).tanh(); - (0.5 * x) * (1.0 + t) -} - /// The host rule of one value, bf16 bits in and out. #[must_use] pub fn gelu_tanh_ref(x: u16) -> u16 { @@ -75,10 +61,7 @@ mod gelu_tanh_kernels { } // SAFETY: i < n <= x.len() (contract); one thread per value. let p = unsafe { x.get_unchecked_mut(i) }; - let v = bf16_to_f32(*p); - let poly = fma_rn_f32(mul_rn_f32(COEF_A, v), v, 1.0); - let t = mul_rn_f32(mul_rn_f32(SQRT_2_OVER_PI, v), poly).tanh(); - *p = f32_to_bf16_rne(mul_rn_f32(mul_rn_f32(0.5, v), add_rn_f32(1.0, t))); + *p = f32_to_bf16_rne(gelu_tanh_dev(bf16_to_f32(*p))); } /// `x[i] = gelu_erf(x[i])` for `i < n`, the module rule, in place. One thread per value. diff --git a/crates/gpu-vision/src/minicpmv/host.rs b/crates/gpu-vision/src/minicpmv/host.rs index 388f05ab8..79e0c4478 100644 --- a/crates/gpu-vision/src/minicpmv/host.rs +++ b/crates/gpu-vision/src/minicpmv/host.rs @@ -3,8 +3,8 @@ use super::GROUP; use crate::attn::{HEAD_DIM_72, QkvLayout, attn_72_ref}; -use crate::gelu_tanh::gelu_tanh_f32; use crate::gemm_bf16::gelu_erf; +use bloomery_gpu::gelu::gelu_tanh_f32; /// One linear layer on the host: `y = x · Wᵀ + b`, `w` row-major `[out, inp]`. #[derive(Clone, Debug)] diff --git a/crates/gpu/src/elem.rs b/crates/gpu/src/elem.rs index b5e7fcddb..54a8d80dd 100644 --- a/crates/gpu/src/elem.rs +++ b/crates/gpu/src/elem.rs @@ -1,6 +1,7 @@ //! P4: every kernel of a decode step that is not a matmul, attention or the //! router — the embedding row dequant, rms_norm, rope (the YaRN cos/sin cache -//! is computed on the host and uploaded; the kernel applies it), swiglu, the +//! is computed on the host and uploaded; the kernel applies it), swiglu and +//! the tanh GELU-gated unit (`geglu_tanh`, [`crate::gelu`]'s rule), the //! residual add, the routed-expert weighted sum and argmax. One //! `#[cuda_module]` in its own file (docs/gpu-design.md decision 6); the //! arithmetic bodies are cores above the module so a later fused block kernel @@ -21,12 +22,14 @@ use crate::GpuError; use crate::cores::{funnel16, half_to_f32, q3k_aux_scales, q3k_sub_scale, q4k_scale_min}; use crate::fault::{FaultSink, FaultSite, LAYER_NONE}; use crate::flash::{f32_to_f16_bits, f32x2_to_f16x2_bits}; +use crate::gelu::gelu_tanh_dev; use crate::hybrid::HOST; use crate::launch_u32; use crate::tensor::DeviceTensor; use crate::view::{Apply, Elem, Elem2, RopePair}; use cuda_core::{CudaContext, CudaStream, DeviceBuffer, LaunchConfig1D}; use cuda_device::atomic::{AtomicOrdering, DeviceAtomicU32}; +use cuda_device::float::mul_rn_f32; use cuda_device::{ DisjointSlice, SharedArray, kernel, launch_bounds, launch_contract, thread, warp, }; @@ -1101,6 +1104,29 @@ mod elem_kernels { e.map(silu_mul); } + /// `y[i] = gelu_tanh(gate[i]) · up[i]`, elementwise over `n` values: the + /// rule of [`crate::gelu`] on the gate value, then one rounded multiply by + /// the up value (llama.cpp's `unary_gated_op_kernel` at `op_gelu`). + #[kernel] + #[launch_bounds(256)] + #[launch_contract( + domain = 1, + block = (256, 1, 1), + requires = (gate.len() >= n, up.len() >= n, y.len() >= n) + )] + pub fn geglu_tanh(gate: &[f32], up: &[f32], n: u32, mut y: DisjointSlice) { + let i = thread::index_1d().get(); + if i >= n as usize { + return; + } + // SAFETY: the `requires` above were launcher-checked, `n` is this + // kernel's parameter and `i` this thread's `index_1d` past the `i < n` + // guard, and the launch is `domain = 1` with the contract's exact 1-D + // block. + let e = unsafe { Elem2::new(gate, up, &mut y, i) }; + e.map(|g, u| mul_rn_f32(gelu_tanh_dev(g), u)); + } + /// `y[i] = a[i] + b[i]`, elementwise over `n` values — exact; the body /// is one add, so nothing is extracted into a core. #[kernel] @@ -2146,6 +2172,35 @@ impl ElemKernels { Ok(()) } + /// Enqueue `y = gelu_tanh(gate) · up` over `n` values (flat; token-major + /// spans of any width). Asynchronous, allocation-free, capturable. + pub fn enqueue_geglu( + &self, + stream: &CudaStream, + gate: &DeviceBuffer, + up: &DeviceBuffer, + n: usize, + y: &mut DeviceBuffer, + ) -> Result<(), GpuError> { + if n == 0 || gate.len() < n || up.len() < n || y.len() < n { + return Err(GpuError::shape( + "enqueue_geglu", + format!( + "n={n}, gate.len() {}, up.len() {}, y.len() {}", + gate.len(), + up.len(), + y.len() + ), + )); + } + let n = launch_u32("enqueue_geglu", "n", n)?; + let prep = self + .module + .prepare_geglu_tanh(LaunchConfig1D::new(n.div_ceil(256), 256, 0))?; + self.module.geglu_tanh(stream, &prep, gate, up, n, y)?; + Ok(()) + } + /// Enqueue `y = a + b` over `n` values (flat). Exact. Asynchronous, /// allocation-free, capturable. pub fn enqueue_add( diff --git a/crates/gpu/src/gelu.rs b/crates/gpu/src/gelu.rs new file mode 100644 index 000000000..84419bc44 --- /dev/null +++ b/crates/gpu/src/gelu.rs @@ -0,0 +1,171 @@ +//! The tanh GELU (`ggml_gelu`): `0.5·x·(1 + tanh(√(2/π)·x·(1 + 0.044715·x²)))`, one rule on one +//! f32 for every kernel that applies it, and its host twin. The GELU-gated linear unit over f32 +//! rows (`ggml_geglu_split`, a GEGLU feed-forward's activation), `y = gelu_tanh(gate) · up`, is +//! `crate::elem`'s `geglu_tanh` ([`crate::elem::ElemKernels::enqueue_geglu`]). +//! +//! Numeric rule: the expression of llama.cpp's CUDA `op_gelu` in f32 in its op order — the +//! polynomial `1 + (0.044715·x)·x` contracted to one fused multiply-add as nvcc contracts it, +//! every other op rounded on its own — with libdevice's `tanhf`. [`gelu_tanh_f32`] is the rule on +//! the host with the host's `tanh`, which differs from the device's by ulps. + +use cuda_device::float::{add_rn_f32, fma_rn_f32, mul_rn_f32}; + +/// `√(2/π)` as the f32 llama.cpp's `SQRT_2_OVER_PI` rounds to. +const SQRT_2_OVER_PI: f32 = 0.797_884_6; +/// llama.cpp's `GELU_COEF_A`. +const COEF_A: f32 = 0.044_715; + +/// The rule on one f32 on the host. +#[must_use] +pub fn gelu_tanh_f32(x: f32) -> f32 { + let poly = (COEF_A * x).mul_add(x, 1.0); + let t = (SQRT_2_OVER_PI * x * poly).tanh(); + (0.5 * x) * (1.0 + t) +} + +/// The rule on one f32 on the device: a core for kernels to call (docs/gpu-design.md decision 6). +#[inline(always)] +pub fn gelu_tanh_dev(v: f32) -> f32 { + let poly = fma_rn_f32(mul_rn_f32(COEF_A, v), v, 1.0); + let t = mul_rn_f32(mul_rn_f32(SQRT_2_OVER_PI, v), poly).tanh(); + mul_rn_f32(mul_rn_f32(0.5, v), add_rn_f32(1.0, t)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::Gpu; + use cuda_core::DeviceBuffer; + + /// f32's unit roundoff. + const U: f64 = 1.0 / 16_777_216.0; + + fn gamma(n: u32) -> f64 { + f64::from(n) * U / (1.0 - f64::from(n) * U) + } + + /// The exact unit on one pair, in f64 at the rule's f32 constants, and the device rule's + /// first-order distance to it. With `X` the exact tanh argument, `T = tanh X`, `S = 1 + T`: + /// the polynomial's two roundings and the argument's two put `X` off by `γ(4)·|X|`, which + /// `tanh` carries scaled by `1 − T²`; libdevice's `tanhf` adds 2 ulp, at most `4u·|T|` + /// (CUDA Math API, `tanhf`); `1 + t` adds `u·|S|`; `0.5·g` is exact, and the two products add + /// `γ(2)·|y|`. The first three move `y` scaled by `|0.5·g·up|`, not by `|y|`: where `S` + /// cancels toward zero the distance does not shrink with it. + fn exact(g: f32, u: f32) -> (f64, f64) { + let (g, u) = (f64::from(g), f64::from(u)); + let x = f64::from(SQRT_2_OVER_PI) * g * (1.0 + f64::from(COEF_A) * g * g); + let t = x.tanh(); + // 1 + tanh(x) without the cancellation at large negative x. + let s = 2.0 / (1.0 + (-2.0 * x).exp()); + let y = 0.5 * g * s * u; + let bound = (0.5 * g * u).abs() + * (gamma(4) * x.abs() * (1.0 - t * t) + 4.0 * U * t.abs() + U * s) + + gamma(2) * y.abs(); + (y, bound) + } + + /// Gate values: every multiple of 1/256 in [−16, 16] and a few magnitudes past where `tanh` + /// saturates; up values seeded in [−2, 2], every eighth one exactly 1. + fn inputs() -> (Vec, Vec) { + let mut gate: Vec = (-4096..=4096).map(|i| i as f32 / 256.0).collect(); + for m in [20.0f32, 37.5, 100.0, 1e3, 1e4] { + gate.extend([m, -m]); + } + let mut s = 0x9e37_79b9u32; + let up = (0..gate.len()) + .map(|i| { + s = s.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + if i.is_multiple_of(8) { + 1.0 + } else { + (s >> 8) as f32 / 4_194_304.0 - 2.0 + } + }) + .collect(); + (gate, up) + } + + /// The gated unit on the card: every value within its bound of the exact unit, and an + /// untouched output tail; a NaN gate or up value NaN + /// at its own index and nowhere else; an empty launch and each short buffer refused by name. + #[test] + #[ignore = "needs a CUDA device; `just gate-gpu-lib` runs it on the box"] + fn hw_geglu_is_the_rule_within_its_bound() { + let gpu = Gpu::with_device(0).expect("a Gpu on device 0"); + let (k, stream) = (gpu.elem(), gpu.stream()); + let run = |gate: &[f32], up: &[f32]| -> Vec { + const GUARD: f32 = 12_345.0; + let n = gate.len(); + let padded = (n + 1).next_multiple_of(256); + let (mut g, mut u) = (gate.to_vec(), up.to_vec()); + g.resize(padded, 1.0); + u.resize(padded, 1.0); + let g = DeviceBuffer::from_host(stream, &g).expect("gate"); + let u = DeviceBuffer::from_host(stream, &u).expect("up"); + let mut y = DeviceBuffer::from_host(stream, &vec![GUARD; padded]).expect("y"); + k.enqueue_geglu(stream, &g, &u, n, &mut y).expect("geglu"); + stream.synchronize().expect("sync"); + let mut y = y.to_host_vec(stream).expect("y back"); + assert!( + y[n..].iter().all(|v| v.to_bits() == GUARD.to_bits()), + "geglu: output past n must stay untouched" + ); + y.truncate(n); + y + }; + let (gate, up) = inputs(); + let y = run(&gate, &up); + let (mut over, mut worst, mut host_equal) = (0usize, 0.0f64, 0usize); + for (i, (&g, &u)) in gate.iter().zip(&up).enumerate() { + assert!(y[i].is_finite(), "geglu: gate {g:e} up {u:e}: {}", y[i]); + let (want, bound) = exact(g, u); + let e = (f64::from(y[i]) - want).abs(); + if e > bound { + over += 1; + if over <= 8 { + eprintln!( + "geglu: gate {g:e} up {u:e}: {} off {want:e} by {e:e} > {bound:e}", + y[i] + ); + } + } + if bound > 0.0 { + worst = worst.max(e / bound); + } + host_equal += usize::from(y[i].to_bits() == (gelu_tanh_f32(g) * u).to_bits()); + } + println!( + "geglu: {} values, {over} past their bound, worst error/bound {worst:.3}, {host_equal} \ + bit for bit the host rule", + gate.len() + ); + assert_eq!(over, 0, "geglu: values past their bound"); + + // A NaN gate at −1.61, NaN up values at gates −16 (saturated) and 0.73. + let (mut g2, mut u2) = (gate[3584..4608].to_vec(), up[3584..4608].to_vec()); + g2[0] = -16.0; + u2[0] = f32::NAN; + g2[100] = f32::NAN; + u2[700] = f32::NAN; + let y2 = run(&g2, &u2); + let nan: Vec = (0..y2.len()).filter(|&i| y2[i].is_nan()).collect(); + assert_eq!( + nan, + [0, 100, 700], + "geglu: a NaN input is NaN at its index alone" + ); + + for (gate_len, up_len, y_len, n, what) in [ + (9, 9, 9, 0, "an empty launch"), + (8, 9, 9, 9, "a short gate buffer"), + (9, 8, 9, 9, "a short up buffer"), + (9, 9, 8, 9, "a short y buffer"), + ] { + let g = DeviceBuffer::from_host(stream, &gate[..gate_len]).expect("gate"); + let u = DeviceBuffer::from_host(stream, &up[..up_len]).expect("up"); + let mut y = DeviceBuffer::::zeroed(stream, y_len).expect("y"); + let e = k.enqueue_geglu(stream, &g, &u, n, &mut y).expect_err(what); + assert!(e.to_string().contains("enqueue_geglu"), "{what}: {e}"); + } + } +} diff --git a/crates/gpu/src/lib.rs b/crates/gpu/src/lib.rs index 4ee46ab8f..816d7c91c 100644 --- a/crates/gpu/src/lib.rs +++ b/crates/gpu/src/lib.rs @@ -43,6 +43,7 @@ pub mod flash_gqa; pub mod flash_gqa_prefill; pub mod fused; pub mod gated_quant; +pub mod gelu; pub mod gemm; pub(crate) mod graph; pub mod hc_gated; diff --git a/tools/gate-paths.tsv b/tools/gate-paths.tsv index 28ce77fd9..c05d72cab 100644 --- a/tools/gate-paths.tsv +++ b/tools/gate-paths.tsv @@ -160,6 +160,7 @@ crates/engram/src/rows.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4e crates/hf/src/fetch.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-fixture gate-ds41-bind gate-ds41-place gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media the fetch Event that model_file.rs reports; the fixture fetch's range reads crates/decision/src/lib.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-decision-lev gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 the decision crate's Error: the decision gates crates/vision/src/lib.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-serve gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-vision gate-gpu-glm5next-serve gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-decision-clef gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-ds41-media gate-gpu-routed-legs VisionError, which the media and preprocess paths return +crates/gpu/src/gelu.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-mamba2 gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-gpu-clef-hidden gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-app gate-gpu-head gate-gpu-mcol gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-nvtier gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-vision gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-glm5v-tower gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen35moe-mtp the shared tanh GELU rule: the registered elementwise module and image towers that use it crates/gpu-vision/src/{aligner,attn,attn_group,chain,encoder,gelu_tanh,gemm_bf16,glm5v,layer_norm,mean_groups,mlp,norm,pos_bilinear,qwen3vl,rope2d}.rs gate-gpu-clef-serve gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-glm5v-tower gate-gpu-qwen3-serve gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-gpu-vision gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-lev-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media the image towers' card kernels: the tower, serve and media-seat gates that run them crates/gpu-vision/src/lib.rs gate-gpu-clef-serve gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-glm5v-tower gate-gpu-qwen3-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-gpu-vision gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-lev-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media the tower crate's root, which the Qwen3.5 media gates also link crates/vision/src/arch/glm5v/{card,hparams,mod,names,tensors}.rs gate-ds41-bind gate-ds41-place gate-gpu-clef-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-serve gate-gpu-glm5next-stagger gate-gpu-glm5v-tower gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-gpu-vision gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 the GLM-5V family's hparams, tensor names and card homes diff --git a/tools/ref/ptx-shapes.tsv b/tools/ref/ptx-shapes.tsv index e386a1e16..c862f8719 100644 --- a/tools/ref/ptx-shapes.tsv +++ b/tools/ref/ptx-shapes.tsv @@ -125,6 +125,8 @@ generate_ds41 gdn_delta 0 0 generate_ds41 gdn_delta_lanes 0 0 generate_ds41 gdn_norm_gate 0 0 generate_ds41 gdn_norm_gate_sigmoid 0 0 +# jit_local is ptxas 13.3.73's sm_86 stack frame, not read on a cc 8.6 card; the two agree on every row of this table and of both cc 12.1 tables. +generate_ds41 geglu_tanh 0 0 generate_ds41 gemm_iq3xxs 0 0 generate_ds41 gemm_iq4nl 0 0 generate_ds41 gemm_iq4xs 0 0 @@ -374,6 +376,8 @@ gate_e2e gdn_delta 0 0 gate_e2e gdn_delta_lanes 0 0 gate_e2e gdn_norm_gate 0 0 gate_e2e gdn_norm_gate_sigmoid 0 0 +# jit_local is ptxas 13.3.73's sm_86 stack frame, not read on a cc 8.6 card; the two agree on every row of this table and of both cc 12.1 tables. +gate_e2e geglu_tanh 0 0 gate_e2e gemm_iq3xxs 0 0 gate_e2e gemm_iq4nl 0 0 gate_e2e gemm_iq4xs 0 0 diff --git a/tools/ref/ptx-shapes/sm_121-cc12.1.tsv b/tools/ref/ptx-shapes/sm_121-cc12.1.tsv index 49d4ab6f7..4323c3ddd 100644 --- a/tools/ref/ptx-shapes/sm_121-cc12.1.tsv +++ b/tools/ref/ptx-shapes/sm_121-cc12.1.tsv @@ -128,6 +128,7 @@ generate_ds41 gdn_delta 0 0 generate_ds41 gdn_delta_lanes 0 0 generate_ds41 gdn_norm_gate 0 0 generate_ds41 gdn_norm_gate_sigmoid 0 0 +generate_ds41 geglu_tanh 0 0 generate_ds41 gemm_iq3xxs 0 0 generate_ds41 gemm_iq4nl 0 0 generate_ds41 gemm_iq4xs 0 0 @@ -381,6 +382,7 @@ gate_e2e gdn_delta 0 0 gate_e2e gdn_delta_lanes 0 0 gate_e2e gdn_norm_gate 0 0 gate_e2e gdn_norm_gate_sigmoid 0 0 +gate_e2e geglu_tanh 0 0 gate_e2e gemm_iq3xxs 0 0 gate_e2e gemm_iq4nl 0 0 gate_e2e gemm_iq4xs 0 0 diff --git a/tools/ref/ptx-shapes/sm_86-cc12.1.tsv b/tools/ref/ptx-shapes/sm_86-cc12.1.tsv index 620bdf738..e5c18ef89 100644 --- a/tools/ref/ptx-shapes/sm_86-cc12.1.tsv +++ b/tools/ref/ptx-shapes/sm_86-cc12.1.tsv @@ -128,6 +128,7 @@ generate_ds41 gdn_delta 0 0 generate_ds41 gdn_delta_lanes 0 0 generate_ds41 gdn_norm_gate 0 0 generate_ds41 gdn_norm_gate_sigmoid 0 0 +generate_ds41 geglu_tanh 0 0 generate_ds41 gemm_iq3xxs 0 0 generate_ds41 gemm_iq4nl 0 0 generate_ds41 gemm_iq4xs 0 0 @@ -381,6 +382,7 @@ gate_e2e gdn_delta 0 0 gate_e2e gdn_delta_lanes 0 0 gate_e2e gdn_norm_gate 0 0 gate_e2e gdn_norm_gate_sigmoid 0 0 +gate_e2e geglu_tanh 0 0 gate_e2e gemm_iq3xxs 0 0 gate_e2e gemm_iq4nl 0 0 gate_e2e gemm_iq4xs 0 0 diff --git a/tools/unsafe-ratchet.txt b/tools/unsafe-ratchet.txt index 621a6660e..f5ca6bce5 100644 --- a/tools/unsafe-ratchet.txt +++ b/tools/unsafe-ratchet.txt @@ -229,7 +229,9 @@ gguf 9 2 0 # embedding gather: its position words, the code and scale words, the output row) and gqa_flash_seg_g5 (the head-128 # group-5 flash entry) read and write through raw pointers, and seg_scalar_g (the scalar core that entry calls) is one # unsafe fn. The group-8 entry keeps its own body. Each block with its SAFETY line. -gpu 1787 196 10 +# PIN(2026-10-15): gpu 1787 -> 1788 blocks, a new kernel entry, not old code relaxed: geglu_tanh's elementwise access +# (Elem2::new), with its SAFETY line. +gpu 1788 196 10 # PIN(2026-10-01): gpu-deepseek41 459 -> 464 blocks, 51 -> 52 fns, a new kernel entry, not old code relaxed: # ds41_ffn_card_acc_8_tier (the tier form of the card sum at 8 slots, chain/ffn/batch.rs) adds its two blocks and # card_acc_tier_at (the one unsafe fn) with its three, each with its SAFETY line.