diff --git a/README.md b/README.md index adacd0c0b..b129dd748 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,7 @@ bloomery-serve --hf unsloth/Qwen3-30B-A3B-Instruct-2507-GGUF:Q4_K_M --port 8080 | Cards | One card, or one card plus one expert-tier card (`--place bp`). With `--place` unset and two cards visible, Qwen3.8 and GLM-5.3 add the second card when their plan finds it pays | | Host RAM under the model | Runs on Qwen3.8, GLM-5.3 and V4.1: the routed experts the RAM cannot hold are paged from the model file on NVMe through a RAM arena. Tokens equal the all-RAM load. A plan keeps 512 MiB under the free RAM it reads, and a paged load shrinks its arena when it finds less RAM than planned. A paged Qwen3.8 load serves one slot by default with the MTP draft off; a set `BLOOMERY_DRAFT=mtp` runs, and a set `--parallel` is served unless the room's floor refuses it by name. Adaptive residency runs on it. In 0.3.1 a set `--parallel` of 2 or more on a paged Qwen3.8 plan failed at load. On WSL2 with the model on a Windows drive, set `BLOOMERY_NVTIER_READ=buffered` | | MiMo-V2.6-Flash | Its hot routed experts sit on the card, the rest on the host. After a 5,620-token prompt it decodes at 34.0 tok/s (0.2.10: 19.1), first token at 52.9 s (RTX A6000, `--place` unset, 0.3.0's release check) | +| Text embeddings | EmbeddingGemma 2 Q8_0 beside a chat model: `--embd-model FILE` or `--embd-hf ggml-org/embeddinggemma-2-GGUF:Q8_0`. `/v1/embeddings` and `/embedding` return mean-pooled vectors; `dimensions` selects the 128, 256 or 512 cuts below the file's `embedding_length_out`, or that full width (768 for this Q8_0 file), normalized after the cut. Up to 8192 tokens per input; the side seat reserves its bytes before the chat plan places experts. See [serving](docs/serving.md) | | Image input | V4.1, Qwen3.6, MiniCPM-V 4.7, GLM-5.3 and the Clef decide seat, with `--mmproj ` (llama-server's flag). Under `--hf` with no `--mmproj`, a seat takes the projector in the model's repo when it costs the text model no card experts or context; otherwise it prints the projector's bytes and serves text (`--no-mmproj` skips it). Images go in as base64 (`data:` URLs), not fetched URLs. A request with an image to a seat started without an encoder is refused by name | ### Known issues and limits (0.3.3) diff --git a/crates/gpu-gates/Cargo.toml b/crates/gpu-gates/Cargo.toml index deec778e4..dbb138857 100644 --- a/crates/gpu-gates/Cargo.toml +++ b/crates/gpu-gates/Cargo.toml @@ -135,17 +135,17 @@ required-features = ["deepseek41"] [[bin]] name = "bloomery-serve" path = "src/bin/bloomery_serve.rs" -required-features = ["glm5next", "clef", "mimo2"] +required-features = ["glm5next", "clef", "mimo2", "gemma-embedding2"] [[bin]] name = "bloomery-serve-ds41" path = "src/bin/bloomery_serve_ds41.rs" -required-features = ["deepseek41"] +required-features = ["deepseek41", "gemma-embedding2"] [[bin]] name = "bloomery-serve-qwen38" path = "src/bin/bloomery_serve_qwen38.rs" -required-features = ["deepseek41"] +required-features = ["deepseek41", "gemma-embedding2"] [[bin]] name = "gate_qwen38_serve" @@ -543,6 +543,10 @@ required-features = ["gpu"] name = "gate_gemma_embedding2_e2e" required-features = ["gemma-embedding2"] +[[bin]] +name = "gate_gemma_embedding2_serve" +required-features = ["deepseek41", "gemma-embedding2"] + [[bin]] name = "gate_qwen3moe_router" required-features = ["gpu"] diff --git a/crates/gpu-gates/src/arch/embedding.rs b/crates/gpu-gates/src/arch/embedding.rs new file mode 100644 index 000000000..b3fb0d920 --- /dev/null +++ b/crates/gpu-gates/src/arch/embedding.rs @@ -0,0 +1,35 @@ +//! The embedding architectures a side seat serves, with each model's input bound. + +use crate::side_seat::Row; + +#[cfg(feature = "gemma-embedding2")] +use crate::side_seat::gemma_embedding2::{bytes, open}; +#[cfg(not(feature = "gemma-embedding2"))] +use crate::side_seat::{no_encoder_bytes as bytes, no_encoder_open as open}; + +/// The architectures a side seat serves. +pub const ROWS: &[Row] = &[ + // The file's context_length is 262144; the model card names 8192 tokens. + Row { + arch: model::arch::GEMMA_EMBEDDING2, + ctx: model::arch::gemma_embedding2::hparams::CARD_CTX, + bytes, + open, + }, +]; + +#[cfg(feature = "gemma-embedding2")] +pub(super) fn gemma_embedding2_dims(split: &gguf::Split) -> Result, crate::GateError> { + const MRL: [usize; 3] = [128, 256, 512]; + let n_out = model::arch::gemma_embedding2::hparams::Hparams::read(split)?.n_out; + if n_out < MRL[0] { + return Err(format!( + "embedding model: embedding_length_out {n_out} cannot serve the {}-wide MRL cut", + MRL[0] + ) + .into()); + } + let mut dims: Vec<_> = MRL.into_iter().filter(|&width| width < n_out).collect(); + dims.push(n_out); + Ok(dims) +} diff --git a/crates/gpu-gates/src/beside.rs b/crates/gpu-gates/src/beside.rs new file mode 100644 index 000000000..b1bfd730d --- /dev/null +++ b/crates/gpu-gates/src/beside.rs @@ -0,0 +1,184 @@ +//! What a plan holds beside its own tensors: a model the process serves beside the planned one (the +//! image encoder's tower, the embedding model of a side seat) that sits on the stage card, as a +//! named reserve of fixed bytes the plan's fit counts ([`model::placement::Card::reserves`]). One +//! function enters it into a placed plan's machine and a whole-fit probe's ([`reserve_on`]), one +//! reads it back ([`reserve_in`]), and a plan's refusal ([`refusal`]) and a verdict's line +//! ([`note`]) name it. Free of the encoder crates, so a placement module every build includes can +//! take it. + +use std::fmt::Display; + +use model::placement::{Card, Machine}; + +/// What the plan's reserve for the image encoder's tower is called, as the plan's terms name it +/// (beside `workstation`'s draft and tier-batch reserves). +pub const IMAGE_ENCODER: &str = "image encoder"; + +/// What the plan's reserve for a side seat's embedding model is called ([`crate::side_seat`]). +pub const EMBEDDING: &str = "embedding model"; + +/// The bytes a plan reserves on its stage card beside its own tensors, by what they are for; +/// `None` for one that sits off the plan or is not served. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Beside { + /// The image encoder's tower. + pub image: Option, + /// The embedding model a side seat serves. + pub embedding: Option, +} + +impl Beside { + /// Each reserve by its name, in the order the stage card takes them. + pub fn named(self) -> impl Iterator { + [(IMAGE_ENCODER, self.image), (EMBEDDING, self.embedding)] + .into_iter() + .filter_map(|(name, bytes)| Some((name, bytes?))) + } +} + +/// `machine` with `beside`'s reserves on its stage card. The one place they enter a machine: a +/// placed plan's builder and a whole-fit probe both call it, so both fit the same bytes. +pub fn reserve_on(machine: &mut Machine, beside: Beside) { + for (name, bytes) in beside.named() { + machine.cards[0].reserves.push((name.to_owned(), bytes)); + } +} + +/// The reserves [`reserve_on`] put on `machine`'s stage card: what a plan built on it refuses +/// beside ([`refusal`]). +#[must_use] +pub fn reserve_in(machine: &Machine) -> Beside { + let held = |name: &str| { + machine + .cards + .first()? + .reserves + .iter() + .find(|(n, _)| n == name) + .map(|&(_, bytes)| bytes) + }; + Beside { + image: held(IMAGE_ENCODER), + embedding: held(EMBEDDING), + } +} + +/// A machine builder (`layers` to its [`Machine`]) with [`reserve_on`] applied to what it builds. +pub fn with_reserve(plain: impl Fn(usize) -> Machine, beside: Beside) -> impl Fn(usize) -> Machine { + move |layers| { + let mut machine = plain(layers); + reserve_on(&mut machine, beside); + machine + } +} + +/// What a verdict's line adds when its card carries reserves [`reserve_on`] puts there: each by +/// name and bytes, in [`Beside`]'s order, beside the reserves term the verdict sums them in. +/// Nothing for a card that carries none, so a load with nothing beside it prints its line as it +/// did. +#[must_use] +pub fn note(card: &Card) -> String { + [IMAGE_ENCODER, EMBEDDING] + .iter() + .filter_map(|&want| card.reserves.iter().find(|(name, _)| name == want)) + .map(|(name, bytes)| format!("; the reserves term holds the {name}'s {bytes} B")) + .collect() +} + +/// A plan's refusal, beside the reserves that were in it: with none the refusal as it is. +pub fn refusal(beside: Beside, e: impl Display) -> String { + let held: Vec = beside + .named() + .map(|(name, bytes)| format!("the {name}'s {bytes} B as the stage card's {name} reserve")) + .collect(); + if held.is_empty() { + return e.to_string(); + } + format!("the plan holds {}: {e}", held.join(" and ")) +} + +#[cfg(test)] +mod tests { + use model::placement::workstation::plan_a; + + use super::*; + + const BYTES: u64 = 1_476_000_000; + + /// The whole-fit probe and the placed plan take the one reserve: the machines both build carry + /// the same named bytes, the card sets aside exactly that much more, and none changes nothing. + #[test] + fn the_probe_and_the_placed_plan_see_the_same_reserve() { + let image = Beside { + image: Some(BYTES), + embedding: None, + }; + let plain = plan_a(40); + let mut probe = plan_a(40); + reserve_on(&mut probe, image); + let placed = with_reserve(plan_a, image)(40); + assert_eq!(probe, placed); + assert_eq!( + probe.cards[0].reserves.last(), + Some(&(IMAGE_ENCODER.to_owned(), BYTES)) + ); + assert_eq!( + probe.cards[0].reserve_bytes(), + plain.cards[0].reserve_bytes() + BYTES + ); + assert_eq!( + probe.cards[0].set_aside_bytes(), + plain.cards[0].set_aside_bytes() + BYTES + ); + assert_eq!(reserve_in(&probe), image); + assert_eq!(reserve_in(&plain), Beside::default()); + let mut none = plan_a(40); + reserve_on(&mut none, Beside::default()); + assert_eq!(none, plain); + assert_eq!(with_reserve(plan_a, Beside::default())(40), plain); + assert_eq!(note(&plain.cards[0]), ""); + assert_eq!( + note(&probe.cards[0]), + "; the reserves term holds the image encoder's 1476000000 B" + ); + assert_eq!(refusal(Beside::default(), "no fit"), "no fit"); + assert_eq!( + refusal(image, "no fit"), + "the plan holds the image encoder's 1476000000 B as the stage card's image encoder \ + reserve: no fit" + ); + } + + /// Both reserves enter the stage card in order and come back, and the note and the refusal + /// name each. + #[test] + fn the_embedding_model_is_a_second_reserve() { + let both = Beside { + image: Some(BYTES), + embedding: Some(7), + }; + let mut machine = plan_a(40); + reserve_on(&mut machine, both); + assert_eq!(reserve_in(&machine), both); + assert_eq!( + machine.cards[0].reserve_bytes(), + plan_a(40).cards[0].reserve_bytes() + BYTES + 7 + ); + assert_eq!( + note(&machine.cards[0]), + "; the reserves term holds the image encoder's 1476000000 B; the reserves term holds \ + the embedding model's 7 B" + ); + assert_eq!( + refusal( + Beside { + image: None, + embedding: Some(7), + }, + "no fit" + ), + "the plan holds the embedding model's 7 B as the stage card's embedding model \ + reserve: no fit" + ); + } +} diff --git a/crates/gpu-gates/src/bin/bloomery_serve.rs b/crates/gpu-gates/src/bin/bloomery_serve.rs index eadec6f79..30b54ccc9 100644 --- a/crates/gpu-gates/src/bin/bloomery_serve.rs +++ b/crates/gpu-gates/src/bin/bloomery_serve.rs @@ -48,6 +48,13 @@ //! it); a file whose architecture does not match an explicit //! `--model` is refused by name before any load. //! `--version` prints this crate's version and the build's commit and exits. +//! `--embd-model FILE` or `--embd-hf [:]` names an embedding +//! model the server holds beside a generative seat's model for its embeddings routes +//! (`bloomery_gpu_gates::side_seat`): read and seated before the chat model's plan, which holds +//! its bytes when it sits on the stage card, and loaded after the chat model. A file of an +//! embedding architecture given as `-m` is refused by name, and so is an embedding model beside +//! the decide and nemotron seats. EmbeddingGemma 2's row opens the encoder on the seat card; +//! its pass owns the output projection and mean pooling, and the HTTP layer cuts and normalizes. //! //! Two checks run before anything is fetched or loaded, each a named refusal //! (`serve::preflight`): first of all, on a CPU that does not run the @@ -117,6 +124,7 @@ mod place; mod drive { use std::process::ExitCode; + use bloomery_gpu_gates::side_seat; use bloomery_gpu_gates::{GateError, exit_with, model_file, ref_model_path}; use cuda_core::sys; use gguf::Split; @@ -131,7 +139,8 @@ mod drive { const USAGE: &str = "usage: bloomery-serve [--model ds41|qwen38|glm|qwen3|mimo2|nemotron|decide] \ [-m PATH | --model-file PATH | --hf [:]] [--head \ - [--head-config ]] ]] [--embd-model FILE | --embd-hf \ + [:]] | --version"; @@ -290,6 +299,14 @@ models (the --model word is optional with a model file: the file's architecture let path = ref_model_path()?; let split = Split::open(&path).map_err(|e| format!("open {}: {e}", path.display()))?; let (seat, arch) = match split.architecture() { + Some(a) if side_seat::row(a).is_some() => { + return Err(format!( + "{} is a {a} file, an embedding model: serve it beside a chat model with \ + --embd-model", + path.display() + ) + .into()); + } Some(a) if decide::ROWS.iter().any(|r| r.backbones.contains(&a)) => { return Err(format!( "{}: {}", @@ -442,7 +459,19 @@ models (the --model word is optional with a model file: the file's architecture Ok(v) => v, Err(e) => return exit_with(NAME, Err(e)), }; + // The side model's flags come out before the seat reads its own; its file is read in the + // seat, after the levers. + let (side, rest) = match side_seat::Flags::take(&rest) { + Ok(v) => v, + Err(e) => return exit_with(NAME, Err(e)), + }; if let Some(from) = from { + if side.names() { + return exit_with( + NAME, + Err("--embd-model: the decide seat serves no embedding model".into()), + ); + } return ended(decide::run(&rest, from, repo.as_deref())); } let (seat, arch) = match seat_of_file() { @@ -463,12 +492,18 @@ models (the --model word is optional with a model file: the file's architecture ); } let r = match seat { - Model::Ds41 => ds41::run(&rest), - Model::Qwen38 => qwen38::run(&rest), - Model::Glm => glm::run(&rest), - Model::Qwen3 => qwen3::run(&rest), - Model::Mimo2 => mimo2::run(&rest), - Model::Nemotron => nemotron::run(&rest), + Model::Ds41 => ds41::run(&rest, &side), + Model::Qwen38 => qwen38::run(&rest, &side), + Model::Glm => glm::run(&rest, &side), + Model::Qwen3 => qwen3::run(&rest, &side), + Model::Mimo2 => mimo2::run(&rest, &side), + Model::Nemotron => { + if side.names() { + Err("--embd-model: the nemotron seat serves no embedding model".into()) + } else { + nemotron::run(&rest) + } + } }; ended(r) } diff --git a/crates/gpu-gates/src/bin/bloomery_serve_ds41.rs b/crates/gpu-gates/src/bin/bloomery_serve_ds41.rs index e41964993..cbc149194 100644 --- a/crates/gpu-gates/src/bin/bloomery_serve_ds41.rs +++ b/crates/gpu-gates/src/bin/bloomery_serve_ds41.rs @@ -6,7 +6,8 @@ #[cfg(feature = "deepseek41")] fn main() -> std::process::ExitCode { let args: Vec = std::env::args().skip(1).collect(); - match ds41::run(&args) { + let side = bloomery_gpu_gates::side_seat::Flags::take(&args); + match side.and_then(|(side, rest)| ds41::run(&rest, &side)) { // EX_SOFTWARE: the engine, not the listener or the load, ended the run. Ok(serve::ServeError::Engine(f)) => { eprintln!("bloomery-serve-ds41: {f}"); diff --git a/crates/gpu-gates/src/bin/bloomery_serve_qwen38.rs b/crates/gpu-gates/src/bin/bloomery_serve_qwen38.rs index 95671c063..f858ecee0 100644 --- a/crates/gpu-gates/src/bin/bloomery_serve_qwen38.rs +++ b/crates/gpu-gates/src/bin/bloomery_serve_qwen38.rs @@ -6,7 +6,8 @@ #[cfg(feature = "gpu")] fn main() -> std::process::ExitCode { let args: Vec = std::env::args().skip(1).collect(); - match qwen38::run(&args) { + let side = bloomery_gpu_gates::side_seat::Flags::take(&args); + match side.and_then(|(side, rest)| qwen38::run(&rest, &side)) { // EX_SOFTWARE: the engine, not the listener or the load, ended the run. Ok(serve::ServeError::Engine(f)) => { eprintln!("bloomery-serve-qwen38: {f}"); diff --git a/crates/gpu-gates/src/bin/gate_clef_hidden.rs b/crates/gpu-gates/src/bin/gate_clef_hidden.rs index 0dfe696ed..a5b29e4a6 100644 --- a/crates/gpu-gates/src/bin/gate_clef_hidden.rs +++ b/crates/gpu-gates/src/bin/gate_clef_hidden.rs @@ -60,7 +60,8 @@ mod gate { use bloomery_gpu::Gpu; use bloomery_gpu::arch::qwen3moe::{KvQ8, Open35, PrefillPath, Qwen35moeModel}; use bloomery_gpu_gates::{ - GateError, RefManifest, checks_failed, data_dir, load_ref_in, ref_model_path, verdict, + GateError, RefManifest, checks_failed, data_dir, load_ref_in, ref_model_path, sha256, + verdict, }; use gguf::Split; use refset::arch::qwen35::{ @@ -235,16 +236,6 @@ mod gate { quantile(v, 0.5) } - /// The sha256 of `path`, by `sha256sum`. - fn sha256(path: &Path) -> Result { - let out = std::process::Command::new("sha256sum").arg(path).output()?; - let text = String::from_utf8_lossy(&out.stdout); - match text.split_whitespace().next() { - Some(d) if out.status.success() && d.len() == 64 => Ok(d.to_string()), - _ => Err(format!("sha256sum {}: {:?}", path.display(), text.trim()).into()), - } - } - /// One set: its ids, read from the tokens file its header names and held /// to the header's digest and count, and mainline's rows. fn open_set( diff --git a/crates/gpu-gates/src/bin/gate_ds41_serve.rs b/crates/gpu-gates/src/bin/gate_ds41_serve.rs index 3bb553733..db071436f 100644 --- a/crates/gpu-gates/src/bin/gate_ds41_serve.rs +++ b/crates/gpu-gates/src/bin/gate_ds41_serve.rs @@ -306,6 +306,7 @@ mod gate { use std::thread::JoinHandle; use std::time::Duration; + use bloomery_gpu_gates::beside::Beside; use bloomery_gpu_gates::bind::nvidia_smi_index; use bloomery_gpu_gates::ctx_ladder::{self, ctx_line, default_line_held, rung_above}; use bloomery_gpu_gates::generate::Place; @@ -1924,11 +1925,15 @@ mod gate { let tier = crate::ds41_place::tier_batch(Place::Bp, &inputs.hp); let slots = NonZeroUsize::MIN; let ask = |vision: Option| { + let held = Beside { + image: vision, + embedding: None, + }; crate::ds41_place::ctx_choice( &split, &inputs, Place::Bp, - (None, tier, vision), + (None, tier, held), (None, slots), levers, ) diff --git a/crates/gpu-gates/src/bin/gate_gemma_embedding2_e2e.rs b/crates/gpu-gates/src/bin/gate_gemma_embedding2_e2e.rs index a7d305710..f2cbf939b 100644 --- a/crates/gpu-gates/src/bin/gate_gemma_embedding2_e2e.rs +++ b/crates/gpu-gates/src/bin/gate_gemma_embedding2_e2e.rs @@ -50,23 +50,14 @@ mod gate { use bloomery_gpu_gemma_embedding2::{Encoder, Pass, card_bytes}; use gguf::Split; use model::arch::gemma_embedding2::hparams::{CARD_CTX, Hparams}; - use refset::arch::gemma_embedding2::{IDS_SET, MODEL, MODEL_SHA256, TEXT, TEXT_CPU}; + use refset::arch::gemma_embedding2::{ + IDS_SET, MODEL, MODEL_SHA256, REL_FLOOR as FLOOR, TEXT, TEXT_CPU, TWIN, + }; use refset::emb2::Emb2Set; use refset::family::Family; use refset::ik::RefHeader; use refset::tokenizer::read_ids; - /// A row's band: this many times the CPU twin's distance from the card - /// set (`shared/glm5next_media.rs`'s `TWIN`, the Qwen seats' rule). - const TWIN: f64 = 2.5; - - /// The least band. - // PIN(2026-10-15): 1e-2 over the distances whose twin band is smaller, - // measured on a DGX Spark: `inp_per_layer` 1.344e-3 to 1.594e-3 against - // twins of 1.066e-7 to 2.410e-7 in every case, and the shortest cases' - // `l_out` up to 9.046e-3 at layer 0. - const FLOOR: f64 = 1.0e-2; - /// One case of the set: its name and its ids. struct Case { id: String, diff --git a/crates/gpu-gates/src/bin/gate_gemma_embedding2_serve.rs b/crates/gpu-gates/src/bin/gate_gemma_embedding2_serve.rs new file mode 100644 index 000000000..ff6e35f33 --- /dev/null +++ b/crates/gpu-gates/src/bin/gate_gemma_embedding2_serve.rs @@ -0,0 +1,555 @@ +//! EmbeddingGemma 2 beside Qwen3-30B-A3B-Instruct-2507 Q4_K_M: the server vectors, +//! the reserve's expert cost and unified-pool term, greedy tokens alone and during +//! embedding calls, and named refusals. One visible card keeps the side seat on the stage. +//! The plain 12 GiB plan leaves host experts; adding the reserve to the side run's +//! budget restores that split. At the unchanged budget the side run must seat fewer experts. + +#[cfg(feature = "gemma-embedding2")] +fn main() -> std::process::ExitCode { + bloomery_gpu_gates::exit_with("gate_gemma_embedding2_serve", gate::run()) +} + +#[cfg(feature = "gemma-embedding2")] +#[path = "shared/qwen3moe_place.rs"] +#[allow( + dead_code, + reason = "the gate uses the seat's planner, not its loaders" +)] +mod q3place; + +#[cfg(feature = "gemma-embedding2")] +mod gate { + use std::path::{Path, PathBuf}; + use std::process::Command; + use std::time::Duration; + + use bloomery_gpu::arch::qwen3moe::KvQ8; + use bloomery_gpu_gates::beside::{self, Beside}; + use bloomery_gpu_gates::generate::Place; + use bloomery_gpu_gates::record::{self, Log}; + use bloomery_gpu_gates::serve_client::{ + Served, check, curl, ids_of, json_of, metric, plan_run, serve_cmd, server_log, tokenized, + }; + use bloomery_gpu_gates::{ + GateError, RefManifest, checks_failed, data_dir, gpu_census, ref_model_path, + ref_tensor_logical_in, sha256, + }; + use gguf::Split; + use model::placement::{Plan, PlanLevers, UnifiedPool}; + use refset::arch::gemma_embedding2::{ + IDS_SET, MODEL, MODEL_SHA256, REL_FLOOR, SERVER, TEXT_CPU, TWIN, + }; + use refset::emb2::Emb2Set; + use serde_json::{Value, json}; + + use crate::q3place::PlaceQ3; + + const CTX: usize = 4096; + const BUDGET: u64 = 12 * 1024 * 1024 * 1024; + const N_PREDICT: usize = 512; + const PROMPT: &str = "The capital of France is"; + + struct Case { + name: String, + text: String, + tokens: u64, + } + + fn cases(man: &RefManifest) -> Result, GateError> { + let mut cases = Vec::new(); + for line in &man.header.other { + let Some(rest) = line.strip_prefix("# tokenize\t") else { + continue; + }; + let fields: Vec<_> = rest.split('\t').collect(); + let [name, "equal", count] = fields[..] else { + return Err(format!("server set: malformed tokenization identity {line:?}").into()); + }; + let tokens = count + .strip_prefix("prompt_tokens ") + .ok_or_else(|| format!("server set: missing prompt_tokens in {line:?}"))? + .parse()?; + cases.push(Case { + name: name.to_owned(), + text: std::fs::read_to_string( + data_dir().join(IDS_SET).join(format!("{name}.txt")), + )?, + tokens, + }); + } + if cases.is_empty() { + return Err("server set: no text cases".into()); + } + cases.sort_by_key(|c| c.tokens); + Ok(cases) + } + + fn tap(man: &RefManifest, name: &str) -> Result, GateError> { + Ok(ref_tensor_logical_in(&man.dir, man.tensor(name, 0)?)?) + } + + fn vector(v: &Value) -> Result, GateError> { + v["data"][0]["embedding"] + .as_array() + .ok_or("no embedding array")? + .iter() + .map(|x| { + let x = x.as_f64().ok_or("an embedding value is not numeric")? as f32; + if !x.is_finite() { + return Err("an embedding value is not finite".into()); + } + Ok(x) + }) + .collect() + } + + fn norm(v: &[f32]) -> f64 { + v.iter().map(|&x| f64::from(x).powi(2)).sum::().sqrt() + } + + fn distances(a: &[f32], b: &[f32]) -> (f64, f64) { + if a.len() != b.len() || a.is_empty() || a.iter().chain(b).any(|x| !x.is_finite()) { + return (f64::INFINITY, f64::INFINITY); + } + let dot: f64 = a + .iter() + .zip(b) + .map(|(&x, &y)| f64::from(x) * f64::from(y)) + .sum(); + let cosine = (1.0 - dot / (norm(a) * norm(b))).abs(); + let max = a + .iter() + .zip(b) + .map(|(&x, &y)| (f64::from(x) - f64::from(y)).abs()) + .fold(0.0, f64::max); + (cosine, max) + } + + fn within(ours: &[f32], want: &[f32], raw: &[f32], twin: &[f32], normalized: bool) -> bool { + let delta: Vec<_> = raw.iter().zip(twin).map(|(&x, &y)| x - y).collect(); + // The encoder gate's pooled floor, before normalization: the bands below derive from it, + // not from measured serving distances. + let relative = REL_FLOOR.max(TWIN * norm(&delta) / norm(raw)); + let absolute = relative * norm(raw); + // Cutting can reduce the norm. Normalization magnifies an absolute error E by at most + // 2 E / (norm - E); cosine distance between unit vectors is half their squared L2 distance. + let cut_norm = norm(&raw[..want.len()]); + let unit = 2.0 * absolute / (cut_norm - absolute); + let max_band = if normalized { unit } else { absolute }; + let cosine_band = unit * unit / 2.0; + let (cosine, max) = distances(ours, want); + println!( + "cosine {cosine:.6e} band {cosine_band:.6e}; max_abs {max:.6e} band {max_band:.6e} [derived bands]" + ); + relative.is_finite() + && cut_norm > absolute + && cosine.is_finite() + && max.is_finite() + && cosine <= cosine_band + && max <= max_band + } + + fn embedding(url: &dyn Fn(&str) -> String, body: Value) -> Result { + let (status, text) = curl(&url("/v1/embeddings"), Some(&body), false)?; + json_of("/v1/embeddings", status, &text) + } + + fn vectors( + url: &dyn Fn(&str) -> String, + cases: &[Case], + server: &RefManifest, + cpu: &RefManifest, + ) -> Result { + let mut ok = true; + let name = Path::new(MODEL) + .file_name() + .ok_or("embedding file name")? + .to_string_lossy(); + for (index, c) in cases.iter().enumerate() { + let raw = tap(server, &format!("{}/raw", c.name))?; + let twin = tap(cpu, &format!("{}/pooled", c.name))?; + if raw.len() != 768 || twin.len() != raw.len() { + return Err(format!("{}: reference vector widths are not 768", c.name).into()); + } + for (width, normalized, row) in [ + (768, false, "raw"), + (768, true, "embd"), + (128, true, "mrl128"), + (256, true, "mrl256"), + (512, true, "mrl512"), + ] { + if index != 0 && width != 768 { + continue; + } + let body = json!({"input": c.text, "model": name, + "dimensions": width, "embd_normalize": if normalized { 2 } else { -1 }}); + let v = embedding(url, body)?; + let ours = vector(&v)?; + let want = tap(server, &format!("{}/{row}", c.name))?; + if want.len() != width { + return Err(format!("{}: {row} width", c.name).into()); + } + check( + &mut ok, + &format!("vectors_{}_{row}", c.name), + v["model"] == name.as_ref() + && v["usage"]["prompt_tokens"] == c.tokens + && within(&ours, &want, &raw, &twin, normalized), + ); + } + } + Ok(ok) + } + + fn cmd(budget: u64) -> Result { + let mut cmd = serve_cmd()?; + cmd.env(bloomery_levers::CARD_BUDGET, budget.to_string()) + .env(bloomery_levers::DRAFT, "off"); + Ok(cmd) + } + + fn args(model: &str, side: bool) -> Vec<&str> { + let mut args = vec![ + "-m", + model, + "--port", + "0", + "--place", + "a", + "--ctx", + "4096", + "--parallel", + "1", + ]; + if side { + args.extend(["--embd-model", MODEL]); + } + args + } + + fn log(path: &Path) -> Result { + server_log(path, &[&record::PLAN38, &record::EMBEDDING_SEAT]) + } + + fn split(log: &Log) -> Result<(u64, u64), GateError> { + let plan = log.first(&record::PLAN38)?.ok_or("missing plan record")?; + Ok((plan.u64("card_experts")?, plan.u64("host_experts")?)) + } + + fn plan(model: &str, dir: &Path, side: bool, budget: u64) -> Result { + std::fs::create_dir_all(dir)?; + let err = dir.join("plan.err"); + let (status, _) = plan_run(cmd(budget)?, &args(model, side), &err)?; + if !status.success() { + return Err(format!("plan failed ({status}): {}", err.display()).into()); + } + log(&err) + } + + fn pool(model: &Path, reserve: u64) -> Result { + let split = Split::open(model)?; + let census = gpu_census::census()?; + let q = PlaceQ3::qwen3_on( + &split, + Place::A, + &census, + CTX, + KvQ8::F16, + Beside { + image: None, + embedding: Some(reserve), + }, + )?; + let p = q.plan( + CTX, + &PlanLevers { + card_budget_bytes: Some(BUDGET + reserve), + }, + )?; + let mut held = p.machine.clone(); + // Exercise the pool formula on discrete cards too, keeping the real plan's rows and totals. + if held.nodes[0].unified.is_none() { + held.nodes[0].unified = Some(UnifiedPool { + bytes: held.host().usable_bytes, + }); + } + let mut plain = held.clone(); + plain.cards[0] + .reserves + .retain(|(name, _)| name != beside::EMBEDDING); + let with = Plan { + machine: &held, + ..p.clone() + } + .pool_need() + .ok_or("no pool term")?; + let without = Plan { + machine: &plain, + ..p.clone() + } + .pool_need() + .ok_or("no pool term")?; + println!( + "pool cards with {} B, without {} B, reserve {reserve} B", + with.cards, without.cards + ); + Ok(beside::reserve_in(p.machine).embedding == Some(reserve) + && with.cards.checked_sub(without.cards) == Some(reserve) + && with.host == without.host) + } + + fn chat(url: &dyn Fn(&str) -> String, ids: &[u32]) -> Result, GateError> { + let body = json!({"prompt": ids, "n_predict": N_PREDICT, "temperature": 0, + "ignore_eos": true, "return_tokens": true, "cache_prompt": false}); + let (status, text) = curl(&url("/completion"), Some(&body), false)?; + Ok(ids_of(&json_of("/completion", status, &text)?["tokens"])) + } + + fn refusal( + url: &dyn Fn(&str) -> String, + body: Value, + status: u16, + kind: &str, + ) -> Result { + let (got, text) = curl(&url("/v1/embeddings"), Some(&body), false)?; + let v: Value = serde_json::from_str(&text)?; + Ok(got == status && v["error"]["type"] == kind && v["error"]["message"].is_string()) + } + + fn concurrent( + url: &dyn Fn(&str) -> String, + ids: &[u32], + c: &Case, + want: &[u32], + server: &RefManifest, + cpu: &RefManifest, + ) -> Result { + let completion_url = url("/completion"); + let ids = ids.to_vec(); + let raw = tap(server, &format!("{}/raw", c.name))?; + let twin = tap(cpu, &format!("{}/pooled", c.name))?; + let oracle = tap(server, &format!("{}/embd", c.name))?; + let start = metric(url, "n_decode_total")?.ok_or("missing n_decode_total")?; + std::thread::scope(|scope| { + let generating = + scope.spawn(|| chat(&|_| completion_url.clone(), &ids).map_err(|e| e.to_string())); + let mut active = false; + for _ in 0..200 { + if generating.is_finished() { + break; + } + if metric(url, "n_decode_total")?.unwrap_or(start) > start { + active = true; + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + let mut decode_steps = 0.0_f64; + for _ in 0..32 { + if !active || generating.is_finished() { + break; + } + let before = metric(url, "n_decode_total")?.ok_or("missing n_decode_total")?; + let v = embedding(url, json!({"input": c.text}))?; + let after = metric(url, "n_decode_total")?.ok_or("missing n_decode_total")?; + if !within(&vector(&v)?, &oracle, &raw, &twin, true) { + return Err("concurrent embedding is outside the server vector band".into()); + } + decode_steps = decode_steps.max(after - before); + if decode_steps >= 2.0 { + break; + } + } + let got = generating + .join() + .map_err(|_| "chat request panicked")? + .map_err(|e| format!("chat request: {e}"))?; + println!( + "concurrent decode observed {decode_steps} steps during one {}-token embedding; {} greedy tokens", + c.tokens, + got.len() + ); + Ok(decode_steps >= 2.0 && got == want) + }) + } + + pub fn run() -> Result<(), GateError> { + bloomery_levers::at_main(&[])?; + record::at_main( + "gate_gemma_embedding2_serve", + record::GATE_GEMMA_EMBEDDING2_SERVE, + ); + let mut args_it = std::env::args().skip(1); + let mut dir = PathBuf::from("target/embedding2-serve-gate"); + while let Some(a) = args_it.next() { + if a != "--dir" { + return Err(format!("unknown argument {a}; takes --dir DIR").into()); + } + dir = args_it.next().ok_or("--dir needs DIR")?.into(); + } + std::fs::create_dir_all(&dir)?; + if gpu_census::census()?.len() != 1 { + return Err( + "serve gate needs one visible card, so the side seat reserves the stage card" + .into(), + ); + } + if sha256(Path::new(MODEL))? != MODEL_SHA256 { + return Err("embedding model sha256 differs from the server set".into()); + } + let model_path = ref_model_path()?; + let model = model_path.to_str().ok_or("chat model path is not UTF-8")?; + if Split::open(model)?.architecture() != Some("qwen3moe") { + return Err("serve gate needs Qwen3-30B-A3B-Instruct-2507 Q4_K_M".into()); + } + let server = Emb2Set::open(&SERVER.path(SERVER.sets[0]), &SERVER)?.man; + let cpu = Emb2Set::open(&TEXT_CPU.path(TEXT_CPU.sets[0]), &TEXT_CPU)?.man; + let cases = cases(&server)?; + let longest = cases.last().ok_or("server set: no text cases")?; + let plain = plan(model, &dir.join("plan-plain"), false, BUDGET)?; + let reserved = plan(model, &dir.join("plan-reserved"), true, BUDGET)?; + let seat = reserved + .first(&record::EMBEDDING_SEAT)? + .ok_or("no embedding_seat record")?; + let reserve = seat.u64("reserve")?; + let compensated = plan(model, &dir.join("plan-compensated"), true, BUDGET + reserve)?; + let (base_card, base_host) = split(&plain)?; + let (less_card, more_host) = split(&reserved)?; + println!( + "plain split {base_card}/{base_host}, reserved {less_card}/{more_host}, reserve {reserve} B" + ); + let mut ok = true; + check( + &mut ok, + "plan_expert_cost", + base_card > 0 + && base_host > 0 + && reserve > 0 + && less_card < base_card + && more_host > base_host, + ); + check( + &mut ok, + "plan_same_split", + split(&plain)? == split(&compensated)?, + ); + check( + &mut ok, + "reserve_record", + seat.word("arch")? == "gemma-embedding2" + && reserve == seat.u64("bytes")? + && seat.u64("ctx")? == 8192 + && reserve + == bloomery_gpu_gemma_embedding2::card_bytes(&Split::open(MODEL)?, 8192)?, + ); + check( + &mut ok, + "pool_cards_count_reserve", + pool(&model_path, reserve)?, + ); + + let mut baseline = Vec::new(); + for (side, budget, arm) in [(false, BUDGET, "plain"), (true, BUDGET + reserve, "side")] { + let work = dir.join(arm); + std::fs::create_dir_all(&work)?; + let err = work.join("server.err"); + let mut served = Served::spawn_cmd(cmd(budget)?, &args(model, side), &work)?; + let addr = served.address(&err, 600, Duration::from_secs(1))?; + let url = |path: &str| format!("http://{addr}{path}"); + let ids = tokenized(&url, PROMPT)?; + let tokens = chat(&url, &ids)?; + std::fs::write(work.join("greedy.json"), serde_json::to_string(&tokens)?)?; + check( + &mut ok, + &format!("{arm}_greedy_count"), + tokens.len() == N_PREDICT, + ); + check( + &mut ok, + &format!("{arm}_loaded_split"), + split(&log(&err)?)? == split(&plain)?, + ); + if !side { + baseline = tokens; + check( + &mut ok, + "no_side_501", + refusal(&url, json!({"input":"a"}), 501, "not_supported_error")?, + ); + } else { + check(&mut ok, "idle_greedy_same", tokens == baseline); + ok &= vectors(&url, &cases, &server, &cpu)?; + let (status, text) = curl(&url("/v1/models"), None, false)?; + let models = json_of("/v1/models", status, &text)?; + let chat_name = models["data"][0]["id"].as_str().ok_or("no chat model id")?; + check( + &mut ok, + "embedding_model_listed", + models["data"][1]["id"] + == Path::new(MODEL) + .file_name() + .ok_or("embedding file name")? + .to_string_lossy() + .as_ref() + && models["data"][1]["meta"]["n_embd"] == 768 + && models["data"][1]["meta"]["n_ctx"] == 8192, + ); + check( + &mut ok, + "chat_model_refused", + refusal( + &url, + json!({"input":"a", "model": chat_name}), + 400, + "invalid_request_error", + )?, + ); + check( + &mut ok, + "unknown_model_refused", + refusal( + &url, + json!({"input":"a", "model":"not-a-served-model"}), + 400, + "invalid_request_error", + )?, + ); + check( + &mut ok, + "dimensions_refused", + refusal( + &url, + json!({"input":"a", "dimensions":129}), + 400, + "invalid_request_error", + )?, + ); + check( + &mut ok, + "context_refused", + refusal( + &url, + json!({"input":vec![1;8193]}), + 400, + "exceed_context_size_error", + )?, + ); + check( + &mut ok, + "concurrent_greedy_same", + concurrent(&url, &ids, longest, &baseline, &server, &cpu)?, + ); + } + println!("{arm} server stopped {}", served.stop()?); + } + let err = dir.join("embedding-as-chat.err"); + let (status, lines) = plan_run(cmd(BUDGET)?, &["-m", MODEL], &err)?; + let text = lines.join("\n"); + check( + &mut ok, + "embedding_as_chat_refused", + !status.success() && text.contains("embedding model") && text.contains("--embd-model"), + ); + if ok { Ok(()) } else { Err(checks_failed()) } + } +} diff --git a/crates/gpu-gates/src/bin/gate_qwen35_media.rs b/crates/gpu-gates/src/bin/gate_qwen35_media.rs index 50113a917..4e165183c 100644 --- a/crates/gpu-gates/src/bin/gate_qwen35_media.rs +++ b/crates/gpu-gates/src/bin/gate_qwen35_media.rs @@ -82,7 +82,7 @@ mod gate { use bloomery_gpu::model::StepMode; use bloomery_gpu::weights::DevWeight; use bloomery_gpu::{Gpu, GpuError, capture_template}; - use bloomery_gpu_gates::{GateError, checks_failed, ref_model_path, verdict}; + use bloomery_gpu_gates::{GateError, checks_failed, ref_model_path, sha256, verdict}; use bloomery_gpu_vision::f32_bf16; use gguf::Split; use refset::arch::qwen35::MODEL_FLASH_Q8; @@ -283,15 +283,6 @@ mod gate { c2p: Vec, } - fn sha256(path: &Path) -> Result { - let out = std::process::Command::new("sha256sum").arg(path).output()?; - let text = String::from_utf8_lossy(&out.stdout); - match text.split_whitespace().next() { - Some(d) if out.status.success() && d.len() == 64 => Ok(d.to_string()), - _ => Err(format!("sha256sum {}: {:?}", path.display(), text.trim()).into()), - } - } - impl Case { /// Prompt `n` (0-based) of the seat's set C: its ids read from the tokens file the set names, held to the /// header's digest and count, the image-pad run of each span checked, and the rows of each span's image diff --git a/crates/gpu-gates/src/bin/gate_qwen3_serve.rs b/crates/gpu-gates/src/bin/gate_qwen3_serve.rs index 1cc25822e..4ad22e36c 100644 --- a/crates/gpu-gates/src/bin/gate_qwen3_serve.rs +++ b/crates/gpu-gates/src/bin/gate_qwen3_serve.rs @@ -448,18 +448,19 @@ mod gate { use bloomery_gpu::arch::qwen3moe::ubatch::ubatch_size; use bloomery_gpu::arch::qwen3moe::{KvQ8, Open35, Qwen35moeModel, checkpoint_budget35}; use bloomery_gpu::linear::{KHeadMap, LinearShape}; + use bloomery_gpu_gates::beside; use bloomery_gpu_gates::ctx_ladder::{ ctx_line, default_line_held, field as field_u64, rung_above, rungs, target_of, }; use bloomery_gpu_gates::generate::Place; use bloomery_gpu_gates::record::{self, Fields}; use bloomery_gpu_gates::serve_client::{ - self, Answer, PrefixTurns, Served, agree, beside, chat_is_those_ids, check, curl, + self, Answer, PrefixTurns, Served, agree, chat_is_those_ids, check, curl, edit_and_extension, ids_of, json_of, metric, parse_ids, rendered, serve_cmd, server_log, tokenized, }; use bloomery_gpu_gates::vision_seat::{self, EncoderSeat}; - use bloomery_gpu_gates::{GateError, checks_failed}; + use bloomery_gpu_gates::{GateError, checks_failed, sha256}; use gguf::{Gguf, Split}; use model::placement::WholeLoad; use model::placement::workstation::{A6000, DeviceInfo, GRANULE, RTX_3090}; @@ -3077,16 +3078,6 @@ mod gate { agree: usize, } - /// The sha256 of the file at `path`, as `sha256sum` prints it. - fn sha256(path: &Path) -> Result { - let out = Command::new("sha256sum").arg(path).output()?; - let text = String::from_utf8_lossy(&out.stdout).into_owned(); - match text.split_whitespace().next() { - Some(h) if out.status.success() => Ok(h.to_owned()), - _ => Err(format!("sha256sum {}: {:?}", path.display(), text.trim()).into()), - } - } - /// The set of `family` whose name ends in `suffix`, opened as a set of that family. fn open_set(family: &'static Family, suffix: &str) -> Result { let name = family @@ -3643,7 +3634,7 @@ mod gate { let seat = &img.seat; let named = format!( "; the reserves term holds the {}'s {} B", - vision_seat::RESERVE, + beside::IMAGE_ENCODER, seat.reserve.unwrap_or(0) ); check( @@ -3651,7 +3642,7 @@ mod gate { "mmproj_whole_fit_names_the_reserve", match seat.reserve { Some(r) => r == img.bytes.total() && fit.ends_with(&named), - None => !fit.contains(vision_seat::RESERVE), + None => !fit.contains(beside::IMAGE_ENCODER), }, ); check( @@ -3862,7 +3853,7 @@ mod gate { println!("{line}"); let counted = format!( ", the {} reserve of {} B counted", - vision_seat::RESERVE, + beside::IMAGE_ENCODER, img.seat.reserve.unwrap_or(0) ); check( @@ -3870,7 +3861,7 @@ mod gate { "mmproj_ctx_default_names_the_reserve", match img.seat.reserve { Some(_) => line.contains(&counted), - None => !line.contains(vision_seat::RESERVE), + None => !line.contains(beside::IMAGE_ENCODER), }, ); Ok(()) diff --git a/crates/gpu-gates/src/bin/gate_serve_start.rs b/crates/gpu-gates/src/bin/gate_serve_start.rs index 0da8132f6..d11f1d00d 100644 --- a/crates/gpu-gates/src/bin/gate_serve_start.rs +++ b/crates/gpu-gates/src/bin/gate_serve_start.rs @@ -365,10 +365,7 @@ fn hash_in_background(files: Vec) -> Hashed { } fn sha256(p: &Path) -> Result { - let size = std::fs::metadata(p) - .map_err(|e| format!("{}: {e}", p.display()))? - .len(); - hf::fetch::digest(p, hf::Check::Sha256, size).map_err(|e| format!("{}: {e}", p.display())) + bloomery_gpu_gates::sha256(p).map_err(|e| e.to_string()) } /// `messages` of the text chat. diff --git a/crates/gpu-gates/src/bin/generate_qwen3moe.rs b/crates/gpu-gates/src/bin/generate_qwen3moe.rs index 52b819746..dc74c5c73 100644 --- a/crates/gpu-gates/src/bin/generate_qwen3moe.rs +++ b/crates/gpu-gates/src/bin/generate_qwen3moe.rs @@ -1148,8 +1148,11 @@ mod cli { experts, &PlanLevers::from_levers(levers)?, mtp.as_ref(), - // The CLI's load leaves checkpoints off (`set_checkpoints`). - Seqs::none(if mtp.is_some() { 1 } else { slots }), + q38place::Residents { + // The CLI's load leaves checkpoints off (`set_checkpoints`). + seqs: Seqs::none(if mtp.is_some() { 1 } else { slots }), + held: bloomery_gpu_gates::beside::Beside::default(), + }, ) })?; chosen.record().print(); diff --git a/crates/gpu-gates/src/bin/shared/ds41_place.rs b/crates/gpu-gates/src/bin/shared/ds41_place.rs index bb3d7fb65..a83a953b5 100644 --- a/crates/gpu-gates/src/bin/shared/ds41_place.rs +++ b/crates/gpu-gates/src/bin/shared/ds41_place.rs @@ -10,11 +10,11 @@ use std::num::NonZeroUsize; use bloomery_gpu_deepseek41::body::Deepseek41Model; use bloomery_gpu_gates::GateError; +use bloomery_gpu_gates::beside::{self, Beside}; use bloomery_gpu_gates::generate::{Place, PlaceWhy, TierRule, choose_ctx}; use bloomery_gpu_gates::gpu_census; use bloomery_gpu_gates::record::Record; use bloomery_gpu_gates::residency38::{Seqs, reserve_checkpoints}; -use bloomery_gpu_gates::vision_seat; use gguf::Split; use model::arch::deepseek41::hparams::Hparams; use model::arch::deepseek41::place::PlanInputs; @@ -81,7 +81,7 @@ pub fn with_cards( /// (`generate::choose_ctx`): the limit is the file's trained context within /// [`KERNEL_POSITIONS`], and a probe is the load's plan of every slot at /// a slot context on `place`'s machine with `reserve` (the DSpark draft's), -/// `tier_batch` and `vision` (the encoder's stage-card reserve), its refusal +/// `tier_batch` and `held` (the stage card's reserves), its refusal /// read for the memory it was short of. `set` is the total the slots split. #[allow( dead_code, @@ -91,7 +91,7 @@ pub fn ctx_choice( split: &Split, inputs: &PlanInputs, place: Place, - (reserve, tier_batch, vision): (Option, Option, Option), + (reserve, tier_batch, held): (Option, Option, Beside), (set, slots): (Option, NonZeroUsize), levers: &PlanLevers, ) -> Result, GateError> { @@ -105,7 +105,7 @@ pub fn ctx_choice( (Some((KERNEL_POSITIONS, "KERNEL_POSITIONS")), None), |c| -> Result, GateError> { let mut machine = place.machine(reserve, tier_batch)?(inputs.model.layers); - vision_seat::reserve_on(&mut machine, vision); + beside::reserve_on(&mut machine, held); reserve_checkpoints(&mut machine, Seqs::none(slots.get()), c)?; match inputs.plan_with_slots(&machine, c, levers, slots) { Ok(_) => Ok(Probe::Holds(())), diff --git a/crates/gpu-gates/src/bin/shared/glm5next_place.rs b/crates/gpu-gates/src/bin/shared/glm5next_place.rs index ba6c01017..ac6a3dd03 100644 --- a/crates/gpu-gates/src/bin/shared/glm5next_place.rs +++ b/crates/gpu-gates/src/bin/shared/glm5next_place.rs @@ -6,9 +6,9 @@ //! them here. use bloomery_gpu_gates::GateError; +use bloomery_gpu_gates::beside::{self, Beside}; use bloomery_gpu_gates::generate::{Place, TierRule, choose_ctx_of}; use bloomery_gpu_gates::residency38::glm_machine; -use bloomery_gpu_gates::vision_seat; use bloomery_gpu_glm5next::checkpoint_price; use model::arch::glm5next::hparams::Hparams; use model::arch::glm5next::place::{ @@ -110,15 +110,15 @@ pub fn plan_of<'a>( } /// A plan refusal on `machine` as the seat names it: the image encoder's -/// reserve named when the machine carries one ([`vision_seat::refusal`]). +/// reserve named when the machine carries one ([`beside::refusal`]). #[allow( dead_code, reason = "the serve seat names its refusals; the CLI and the routed legs read them bare" )] pub fn named(machine: &Machine, e: PlaceError) -> GateError { - match vision_seat::reserve_in(machine) { - Some(bytes) => GateError::from(vision_seat::refusal(Some(bytes), e)), - None => GateError::from(e), + match beside::reserve_in(machine) { + held if held == Beside::default() => GateError::from(e), + held => GateError::from(beside::refusal(held, e)), } } diff --git a/crates/gpu-gates/src/bin/shared/qwen35_open.rs b/crates/gpu-gates/src/bin/shared/qwen35_open.rs index 22f16b13e..42326038a 100644 --- a/crates/gpu-gates/src/bin/shared/qwen35_open.rs +++ b/crates/gpu-gates/src/bin/shared/qwen35_open.rs @@ -6,8 +6,8 @@ use bloomery_gpu::Gpu; use bloomery_gpu::arch::qwen3moe::{KvQ8, Open35, Qwen35moeModel}; use bloomery_gpu_gates::GateError; +use bloomery_gpu_gates::beside; use bloomery_gpu_gates::generate::Place; -use bloomery_gpu_gates::image_plan; use gguf::Split; use model::arch::qwen3moe::place as q3; use model::arch::qwen35moe::place as q35; @@ -77,7 +77,7 @@ pub fn open_with( /// [`open_with`] whose whole-fit machine goes through `seat` before the verdict: a seat that holds /// bytes of the card for something else (the image encoder on the stage card, -/// `vision_seat::reserve_on`) puts them there, so the verdict counts them. +/// `beside::reserve_on`) puts them there, so the verdict counts them. pub fn open_seated( path: &Path, ctx: usize, @@ -168,7 +168,7 @@ fn refuse_unless_whole_fits( seat: &dyn Fn(&mut Machine), ) -> Result<(), GateError> { let need = whole_verdict(split, o, card, seat)?; - let held = image_plan::note(&need.card); + let held = beside::note(&need.card); eprintln!("whole fit at ctx {}: {need}{held}", o.ctx); if need.fits() { return Ok(()); diff --git a/crates/gpu-gates/src/bin/shared/qwen38_place.rs b/crates/gpu-gates/src/bin/shared/qwen38_place.rs index 911e228fe..982d8372e 100644 --- a/crates/gpu-gates/src/bin/shared/qwen38_place.rs +++ b/crates/gpu-gates/src/bin/shared/qwen38_place.rs @@ -6,6 +6,7 @@ //! them here. use bloomery_gpu_gates::GateError; +use bloomery_gpu_gates::beside::{self, Beside}; use bloomery_gpu_gates::generate::{Place, PlaceWhy, TierRule, choose_ctx_of, short_in}; use bloomery_gpu_gates::residency38::{Seqs, reserve_checkpoints}; use model::arch::qwen35moe::place::{ @@ -39,6 +40,12 @@ pub fn choose( Place::choose(flag, census, Q38_RULE, tier_of) } +/// The resident sequences and the named reserves held beside a tier plan. +pub(crate) struct Residents { + pub(crate) seqs: Seqs, + pub(crate) held: Beside, +} + /// The experts `cards`' plan (b′) holds on its tier card at `ctx` positions /// and ubatches of `ub` under `experts` and `levers` — the load `mtp` names /// planning the file's MTP draft beside the target, else the plain plan of @@ -49,7 +56,8 @@ pub fn choose( /// `PlanInputs::plan_with_slots`/`plan_mtp_with_slots`); a plan that refuses /// (a host expert rule on a tier machine, an idle tier card, a context past /// the card's, a host room under the NVMe tier's floor) is the caller's -/// error, which `Place::choose` answers with `a`. +/// error, which `Place::choose` answers with `a`. The stage card takes `held`'s named +/// reserves before the tier count is chosen, just as the serving plan does. pub fn tier_experts( inputs: &PlanInputs, cards: (CardSpec, CardSpec), @@ -57,7 +65,7 @@ pub fn tier_experts( experts: Experts, levers: &PlanLevers, mtp: Option<&MtpInputs>, - seqs: Seqs, + Residents { seqs, held }: Residents, ) -> Result { let slots = seqs.slots; let mut machine = machine_bp_on( @@ -67,6 +75,7 @@ pub fn tier_experts( mtp.map(|m| m.card_bytes_of(ctx, slots)).transpose()?, tier_batch(&inputs.hp, ub), ); + beside::reserve_on(&mut machine, held); reserve_checkpoints(&mut machine, seqs, ctx)?; let plan = match mtp { None => inputs.plan_with_slots(&machine, ctx, levers, experts, slots)?, diff --git a/crates/gpu-gates/src/bin/shared/qwen3moe_place.rs b/crates/gpu-gates/src/bin/shared/qwen3moe_place.rs index c3dd2201f..e224ba9d5 100644 --- a/crates/gpu-gates/src/bin/shared/qwen3moe_place.rs +++ b/crates/gpu-gates/src/bin/shared/qwen3moe_place.rs @@ -36,6 +36,7 @@ use bloomery_gpu::model::{GpuModel, Slots}; use bloomery_gpu::rope_neox::q8_plane_lens; use bloomery_gpu::{Gpu, Qwen3moeModel}; use bloomery_gpu_gates::GateError; +use bloomery_gpu_gates::beside::{self, Beside}; use bloomery_gpu_gates::generate::{Place, PlaceWhy, choose_ctx, short_in}; use bloomery_gpu_gates::image_plan; use bloomery_gpu_gates::record::{self, Record}; @@ -239,7 +240,7 @@ fn laid( )); } reserve_checkpoints(&mut machine, seqs, slot_ctx)?; - image_plan::reserve_on(&mut machine, media.encoder); + beside::reserve_on(&mut machine, media.beside); Ok(machine) } @@ -249,14 +250,15 @@ fn slot_ctx(ctx: usize, slots: usize) -> usize { ctx / slots.max(1) } -/// What a media load adds to the machines and verdicts of a load ([`laid`], [`whole_on`]): the -/// image encoder's bytes when its tower sits on the stage card -/// (`vision_seat::EncoderSeat::reserve`, entered by [`image_plan::reserve_on`]), and the width -/// in f32 of the rotation table every sequence holds on a media load (`Open35::media`; -/// [`image_plan::table_bytes_per_position`] a position). A text load adds neither. +/// What a load adds to its machines and verdicts beside its own rows ([`laid`], [`whole_on`]): +/// the reserves of the models the process serves beside it that sit on the stage card +/// (`vision_seat::EncoderSeat::reserve`, entered by [`beside::reserve_on`]), and the width in +/// f32 of the rotation table every sequence holds on a media load (`Open35::media`; +/// [`image_plan::table_bytes_per_position`] a position). A text load with nothing beside it adds +/// neither. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] struct Media { - encoder: Option, + beside: Beside, tables: Option, } @@ -287,16 +289,17 @@ impl PlaceQ3 { reason = "the e2e gates' load; the serve seat and the routed legs plan on one census (qwen3_on)" )] pub fn qwen3(file: &Split, place: Place, ctx: usize, kv: KvQ8) -> Result { - PlaceQ3::qwen3_on(file, place, &census()?, ctx, kv) + PlaceQ3::qwen3_on(file, place, &census()?, ctx, kv, Beside::default()) } - /// [`PlaceQ3::qwen3`] resolved against `census`. + /// [`PlaceQ3::qwen3`] resolved against `census`, its machine carrying `beside`'s reserves. pub fn qwen3_on( file: &Split, place: Place, census: &[DeviceInfo], ctx: usize, kv: KvQ8, + beside: Beside, ) -> Result { let place = PlaceQ3::resolve(place, census)?; let mut inputs = q3::PlanInputs::read(file)?; @@ -305,7 +308,10 @@ impl PlaceQ3 { } let inputs = Inputs::Qwen3(inputs); let arena = Qwen3moeModel::placed_arena_bytes(file, ctx)?; - let media = Media::default(); + let media = Media { + beside, + tables: None, + }; let machine = laid( place.card_specs()?[0], &inputs, @@ -333,14 +339,14 @@ impl PlaceQ3 { reason = "the CLI's and the e2e gates' load; the serve seat's holds its slots (qwen35_on)" )] pub fn qwen35(file: &Split, place: Place, o: Open35) -> Result { - PlaceQ3::qwen35_on(file, place, &census()?, o, Seqs::ONE, None) + PlaceQ3::qwen35_on(file, place, &census()?, o, Seqs::ONE, Beside::default()) } /// [`PlaceQ3::qwen35`] of a load that holds `seqs`, each of its slots /// opened under `o` (a slot's share of the total, [`share`]), its machine /// carrying their rows ([`laid`]) and, on a media load (`o.media`), the - /// tables of its sequences and the image encoder's `encoder` bytes when - /// the tower sits on this card, resolved against `census`: the serve + /// tables of its sequences, and `beside`'s reserves (the image encoder's + /// bytes when the tower sits on this card), resolved against `census`: the serve /// seat's, and one sequence's under [`PlaceQ3::qwen35`]. pub fn qwen35_on( file: &Split, @@ -348,7 +354,7 @@ impl PlaceQ3 { census: &[DeviceInfo], o: Open35, seqs: Seqs, - encoder: Option, + beside: Beside, ) -> Result { let place = PlaceQ3::resolve(place, census)?; let mut inputs = q35::PlanInputs::describe(file)?; @@ -356,7 +362,7 @@ impl PlaceQ3 { inputs.kv = inputs.kv.in_q8(); } let media = Media { - encoder, + beside, tables: o.media.then_some(inputs.hp.rope_dims), }; let inputs = Inputs::Qwen35(Box::new(inputs)); @@ -411,7 +417,7 @@ impl PlaceQ3 { ( Seqs { slots, ..self.seqs }, u64::try_from(slot_ctx(usize::try_from(ctx)?, slots))?, - self.media.encoder, + self.media.beside, ), load, ) @@ -618,7 +624,7 @@ impl Whole { fn line(&self, ctx: usize) -> String { match self { Whole::Fits(n) | Whole::Short(n) => { - format!("whole fit at ctx {ctx}: {n}{}", image_plan::note(&n.card)) + format!("whole fit at ctx {ctx}: {n}{}", beside::note(&n.card)) } Whole::Refused(why) => format!("whole fit at ctx {ctx}: {why}"), } @@ -639,9 +645,9 @@ impl Whole { /// layers, `kv` bytes of cache — on `spec`'s card: the card the plan lays /// out for the program ([`q3::machine`], its step arenas' scratch), on a /// unified machine with the checkpoints `seqs` pin at `slot_ctx` positions a -/// slot ([`reserve_checkpoints`], `Machine::whole_card`) and the image -/// `encoder`'s bytes when its tower sits on the card ([`image_plan::reserve_on`], -/// the reserve a placed plan carries), and what the program holds past it +/// slot ([`reserve_checkpoints`], `Machine::whole_card`) and `beside`'s +/// reserves ([`beside::reserve_on`], the reserves a placed plan carries), and +/// what the program holds past it /// (`load`, or /// the program's refusal), sized by `placement::whole_need`, the verdict's /// one owner. A tensor no card format loads is a refusal; any other planner @@ -651,7 +657,7 @@ fn whole_on( model: &ModelTensors, layers: usize, kv: u64, - (seqs, slot_ctx, encoder): (Seqs, u64, Option), + (seqs, slot_ctx, beside): (Seqs, u64, Beside), load: Result, ) -> Result { let refused = |why: &dyn std::fmt::Display| { @@ -663,7 +669,7 @@ fn whole_on( }; let mut machine = q3::machine(spec, layers, 0); reserve_checkpoints(&mut machine, seqs, slot_ctx)?; - image_plan::reserve_on(&mut machine, encoder); + beside::reserve_on(&mut machine, beside); let card = machine .whole_card() .ok_or_else(|| format!("the whole load on {}: a machine with no card", spec.name))?; @@ -751,7 +757,7 @@ impl PlaceQ3 { /// What a run with `--place` unset loads for one sequence /// ([`unplaced_qwen3_slots`]), as the CLI's own runs do. pub fn unplaced_qwen3(file: &Split, ctx: usize, kv: KvQ8) -> Result { - unplaced_qwen3_slots(file, &census()?, ctx, 1, kv) + unplaced_qwen3_slots(file, &census()?, ctx, 1, (kv, Beside::default())) } /// What a run with `--place` unset loads, decided before any load on one @@ -761,7 +767,7 @@ pub fn unplaced_qwen3(file: &Split, ctx: usize, kv: KvQ8) -> Result Result { - let probe = PlaceQ3::qwen3_on(file, PlaceQ3::unplaced_on(census)?, census, ctx, kv)?; + let probe = PlaceQ3::qwen3_on(file, PlaceQ3::unplaced_on(census)?, census, ctx, kv, beside)?; let whole = probe.whole_at(u64::try_from(ctx)?, slots, qwen3_whole(file))?; probe.unplaced(ctx, &whole) } @@ -782,7 +788,7 @@ pub fn unplaced_qwen3_slots( /// of a qwen35moe file under `o`: one sequence taking no checkpoints /// ([`unplaced_qwen35_slots`]), as the CLI's own runs do. pub fn unplaced_qwen35(file: &Split, o: &Open35) -> Result { - unplaced_qwen35_slots(file, &census()?, o, Seqs::ONE, None) + unplaced_qwen35_slots(file, &census()?, o, Seqs::ONE, Beside::default()) } /// What a run with `--place` unset loads, as [`unplaced_qwen3_slots`] @@ -791,14 +797,14 @@ pub fn unplaced_qwen35(file: &Split, o: &Open35) -> Result /// its share over the program's own arena and /// reserve as a slot's load gives them ([`qwen35_whole`] at [`share`]); the /// placed plan on `a`'s card, carrying the sequences' rows ([`laid`]), when -/// it does not take it; the image `encoder`'s bytes are in the verdict and in -/// the plan. +/// it does not take it; `beside`'s reserves are in the verdict and in the +/// plan. pub fn unplaced_qwen35_slots( file: &Split, census: &[DeviceInfo], o: &Open35, seqs: Seqs, - encoder: Option, + beside: Beside, ) -> Result { let at = share(o, o.ctx, seqs.slots); let probe = PlaceQ3::qwen35_on( @@ -807,7 +813,7 @@ pub fn unplaced_qwen35_slots( census, at, seqs, - encoder, + beside, )?; let whole = probe.whole_at(u64::try_from(o.ctx)?, seqs.slots, qwen35_whole(file, at))?; probe.unplaced(o.ctx, &whole) @@ -816,7 +822,9 @@ pub fn unplaced_qwen35_slots( /// What shapes the qwen3 serve seat's load beside its context and slots: the /// file's family, `--place`, the plan's levers, the cache planes, and under /// `--mmproj` the media flag (every sequence holds the rotation table an -/// image's positions rewrite) and the encoder's reserve on the stage card. +/// image's positions rewrite), and the reserves of what the process serves +/// beside the model on the stage card (the encoder's, a side seat's embedding +/// model's). #[allow( dead_code, reason = "the serve seat, the routed legs and the types gate read it; the CLI and the e2e gates include the planner without it" @@ -828,7 +836,7 @@ pub struct ServeOpts<'a> { pub levers: &'a PlanLevers, pub kv: KvQ8, pub media: bool, - pub encoder: Option, + pub beside: Beside, } /// What the serve seat's file loads as, decided once before the open at the @@ -895,7 +903,7 @@ pub fn serve_load( (Arch::Qwen35moe, Some(p)) => { let at = share(&serve_open35(total, o.kv, o.media)?, total, slots); let seqs = seqs_qwen35(split, at.ubatch, slots)?; - let q = PlaceQ3::qwen35_on(split, p, census, at, seqs, o.encoder)?; + let q = PlaceQ3::qwen35_on(split, p, census, at, seqs, o.beside)?; let plan = q.plan(total, o.levers)?; let whole = q.whole_card_qwen35(split, &plan, (total, slots), at)?; return Ok(ServeLoad::Placed { @@ -908,10 +916,10 @@ pub fn serve_load( (Arch::Qwen35moe, None) => { let open = serve_open35(total, o.kv, o.media)?; let seqs = seqs_qwen35(split, open.ubatch, slots)?; - unplaced_qwen35_slots(split, census, &open, seqs, o.encoder)? + unplaced_qwen35_slots(split, census, &open, seqs, o.beside)? } (_, Some(p)) => { - let q = PlaceQ3::qwen3_on(split, p, census, total, o.kv)?; + let q = PlaceQ3::qwen3_on(split, p, census, total, o.kv, o.beside)?; let plan = q.plan(total, o.levers)?; let whole = q.whole_card_qwen3(split, &plan, (total, slots))?; return Ok(ServeLoad::Placed { @@ -921,7 +929,7 @@ pub fn serve_load( whole, }); } - (_, None) => unplaced_qwen3_slots(split, census, total, slots, o.kv)?, + (_, None) => unplaced_qwen3_slots(split, census, total, slots, (o.kv, o.beside))?, }; Ok(match unplaced { Unplaced::Whole(card) => ServeLoad::Whole(card), diff --git a/crates/gpu-gates/src/bin/shared/qwen3moe_taps.rs b/crates/gpu-gates/src/bin/shared/qwen3moe_taps.rs index 693151ae8..671d2ce48 100644 --- a/crates/gpu-gates/src/bin/shared/qwen3moe_taps.rs +++ b/crates/gpu-gates/src/bin/shared/qwen3moe_taps.rs @@ -71,15 +71,7 @@ pub fn read_ids(path: &Path) -> Result, GateError> { .collect() } -/// The sha256 of `path`, by `sha256sum`. -pub fn sha256(path: &Path) -> Result { - let out = std::process::Command::new("sha256sum").arg(path).output()?; - let text = String::from_utf8_lossy(&out.stdout); - match text.split_whitespace().next() { - Some(d) if out.status.success() && d.len() == 64 => Ok(d.to_string()), - _ => Err(format!("sha256sum {}: {:?}", path.display(), text.trim()).into()), - } -} +pub use bloomery_gpu_gates::sha256; /// `count` prompts of `len` ids from the ids file `path`, window `k` starting /// at `k · ⌊n / count⌋` of its `n` ids: spread over the whole file and never diff --git a/crates/gpu-gates/src/bin/shared/qwen3moe_types.rs b/crates/gpu-gates/src/bin/shared/qwen3moe_types.rs index 692e2e4ab..1d9445acd 100644 --- a/crates/gpu-gates/src/bin/shared/qwen3moe_types.rs +++ b/crates/gpu-gates/src/bin/shared/qwen3moe_types.rs @@ -51,7 +51,7 @@ pub fn serve_default(split: &Split) -> Result { levers: &levers, kv: KvQ8::F16, media: false, - encoder: None, + beside: Default::default(), }; let (chosen, _) = q3place::serve_choice(split, (None, DEFAULT_SLOTS), opts) .map_err(|e| format!("the --ctx choice refused: {e}"))?; diff --git a/crates/gpu-gates/src/bin/shared/routed_legs/deepseek41.rs b/crates/gpu-gates/src/bin/shared/routed_legs/deepseek41.rs index 67eeb6360..fbe6feec3 100644 --- a/crates/gpu-gates/src/bin/shared/routed_legs/deepseek41.rs +++ b/crates/gpu-gates/src/bin/shared/routed_legs/deepseek41.rs @@ -21,6 +21,7 @@ use std::sync::Arc; use super::ds41place; use super::{Adapter, Arm, Outcome, census, floor_or, outcome_of, refused_of, view_of}; +use bloomery_gpu_gates::beside::Beside; use bloomery_gpu_gates::generate::Place; use gguf::{GgmlType, Split}; use model::arch::deepseek41::place::{PlaceError, PlanInputs}; @@ -84,7 +85,7 @@ fn arm(inputs: &Arc, name: &'static str, seat: Option<&Split>) -> Re split, inputs, place, - (None, tier_batch, None), + (None, tier_batch, Beside::default()), (None, n), &PlanLevers::default(), ) diff --git a/crates/gpu-gates/src/bin/shared/routed_legs/qwen35moe.rs b/crates/gpu-gates/src/bin/shared/routed_legs/qwen35moe.rs index 998ce64c1..47dbdbf21 100644 --- a/crates/gpu-gates/src/bin/shared/routed_legs/qwen35moe.rs +++ b/crates/gpu-gates/src/bin/shared/routed_legs/qwen35moe.rs @@ -50,7 +50,7 @@ fn serve_arm(split: &Split) -> Result { levers: &levers, kv: KvQ8::F16, media: false, - encoder: None, + beside: Default::default(), }; let total = floor_or( q3place::serve_choice(split, (None, DEFAULT_SLOTS), opts).map(|(chosen, _)| chosen.total), @@ -78,7 +78,7 @@ fn open35(ctx: usize) -> Result { /// reading (`q3place::PlaceQ3::qwen35_on` rebuilds it). fn arm(split: &Split, name: &'static str, o: &Open35, seqs: q3place::Seqs) -> Result { let census = q3place::census().map_err(|e| outcome_of(e.as_ref()))?; - match q3place::unplaced_qwen35_slots(split, &census, o, seqs, None) + match q3place::unplaced_qwen35_slots(split, &census, o, seqs, Default::default()) .map_err(|e| outcome_of(e.as_ref())) { Ok(q3place::Unplaced::Whole(_)) => { @@ -88,7 +88,7 @@ fn arm(split: &Split, name: &'static str, o: &Open35, seqs: q3place::Seqs) -> Re &census, q3place::share(o, o.ctx, seqs.slots), seqs, - None, + Default::default(), ) .map_err(|e| outcome_of(e.as_ref()))?; let ctx = o.ctx; diff --git a/crates/gpu-gates/src/bin/shared/routed_legs/qwen38.rs b/crates/gpu-gates/src/bin/shared/routed_legs/qwen38.rs index 62ead8299..cc9dd6fdd 100644 --- a/crates/gpu-gates/src/bin/shared/routed_legs/qwen38.rs +++ b/crates/gpu-gates/src/bin/shared/routed_legs/qwen38.rs @@ -145,7 +145,10 @@ fn choose(inputs: &PlanInputs, ctx: u64, seqs: Seqs) -> Result { // The default runs carry no MTP draft: the draft levers // unset. None, - seqs, + q38place::Residents { + seqs, + held: bloomery_gpu_gates::beside::Beside::default(), + }, ) }) .map(|chosen| chosen.place) diff --git a/crates/gpu-gates/src/bin/shared/routed_legs/qwen3moe.rs b/crates/gpu-gates/src/bin/shared/routed_legs/qwen3moe.rs index ce7425d9b..92a45619b 100644 --- a/crates/gpu-gates/src/bin/shared/routed_legs/qwen3moe.rs +++ b/crates/gpu-gates/src/bin/shared/routed_legs/qwen3moe.rs @@ -40,7 +40,7 @@ impl Adapter for Row { levers: &levers, kv: KvQ8::F16, media: false, - encoder: None, + beside: Default::default(), }; let serve = floor_or( q3place::serve_choice(split, (None, DEFAULT_SLOTS), opts) @@ -64,7 +64,7 @@ impl Adapter for Row { fn arm(split: &Split, name: &'static str, ctx: usize, slots: usize) -> Result { let kv = KvQ8::F16; let census = q3place::census().map_err(|e| outcome_of(e.as_ref()))?; - match q3place::unplaced_qwen3_slots(split, &census, ctx, slots, kv) + match q3place::unplaced_qwen3_slots(split, &census, ctx, slots, (kv, Default::default())) .map_err(|e| outcome_of(e.as_ref())) { Ok(q3place::Unplaced::Whole(_)) => { @@ -72,8 +72,9 @@ fn arm(split: &Split, name: &'static str, ctx: usize, slots: usize) -> Result, ) -> Result<(Box