serve, gpu-gates: /v1/embeddings over an EmbeddingGemma 2 model held beside the chat model (--embd-model), with its serve gate

README.md

@@ -57,6 +57,7 @@ bloomery-serve --hf unsloth/Qwen3-30B-A3B-Instruct-2507-GGUF:Q4_K_M --port 8080
 | Cards | One card, or one card plus one expert-tier card (`--place bp`). With `--place` unset and two cards visible, Qwen3.8 and GLM-5.3 add the second card when their plan finds it pays |
 | Host RAM under the model | Runs on Qwen3.8, GLM-5.3 and V4.1: the routed experts the RAM cannot hold are paged from the model file on NVMe through a RAM arena. Tokens equal the all-RAM load. A plan keeps 512 MiB under the free RAM it reads, and a paged load shrinks its arena when it finds less RAM than planned. A paged Qwen3.8 load serves one slot by default with the MTP draft off; a set `BLOOMERY_DRAFT=mtp` runs, and a set `--parallel` is served unless the room's floor refuses it by name. Adaptive residency runs on it. In 0.3.1 a set `--parallel` of 2 or more on a paged Qwen3.8 plan failed at load. On WSL2 with the model on a Windows drive, set `BLOOMERY_NVTIER_READ=buffered` |
 | MiMo-V2.6-Flash | Its hot routed experts sit on the card, the rest on the host. After a 5,620-token prompt it decodes at 34.0 tok/s (0.2.10: 19.1), first token at 52.9 s (RTX A6000, `--place` unset, 0.3.0's release check) |
+| Text embeddings | EmbeddingGemma 2 Q8_0 beside a chat model: `--embd-model FILE` or `--embd-hf ggml-org/embeddinggemma-2-GGUF:Q8_0`. `/v1/embeddings` and `/embedding` return mean-pooled vectors; `dimensions` selects the 128, 256 or 512 cuts below the file's `embedding_length_out`, or that full width (768 for this Q8_0 file), normalized after the cut. Up to 8192 tokens per input; the side seat reserves its bytes before the chat plan places experts. See [serving](docs/serving.md) |
 | Image input | V4.1, Qwen3.6, MiniCPM-V 4.7, GLM-5.3 and the Clef decide seat, with `--mmproj <file>` (llama-server's flag). Under `--hf` with no `--mmproj`, a seat takes the projector in the model's repo when it costs the text model no card experts or context; otherwise it prints the projector's bytes and serves text (`--no-mmproj` skips it). Images go in as base64 (`data:` URLs), not fetched URLs. A request with an image to a seat started without an encoder is refused by name |
 
 ### Known issues and limits (0.3.3)

crates/gpu-gates/Cargo.toml

@@ -135,17 +135,17 @@ required-features = ["deepseek41"]
 [[bin]]
 name = "bloomery-serve"
 path = "src/bin/bloomery_serve.rs"
-required-features = ["glm5next", "clef", "mimo2"]
+required-features = ["glm5next", "clef", "mimo2", "gemma-embedding2"]
 
 [[bin]]
 name = "bloomery-serve-ds41"
 path = "src/bin/bloomery_serve_ds41.rs"
-required-features = ["deepseek41"]
+required-features = ["deepseek41", "gemma-embedding2"]
 
 [[bin]]
 name = "bloomery-serve-qwen38"
 path = "src/bin/bloomery_serve_qwen38.rs"
-required-features = ["deepseek41"]
+required-features = ["deepseek41", "gemma-embedding2"]
 
 [[bin]]
 name = "gate_qwen38_serve"
@@ -543,6 +543,10 @@ required-features = ["gpu"]
 name = "gate_gemma_embedding2_e2e"
 required-features = ["gemma-embedding2"]
 
+[[bin]]
+name = "gate_gemma_embedding2_serve"
+required-features = ["deepseek41", "gemma-embedding2"]
+
 [[bin]]
 name = "gate_qwen3moe_router"
 required-features = ["gpu"]

crates/gpu-gates/src/arch/embedding.rs

@@ -0,0 +1,35 @@
+//! The embedding architectures a side seat serves, with each model's input bound.
+
+use crate::side_seat::Row;
+
+#[cfg(feature = "gemma-embedding2")]
+use crate::side_seat::gemma_embedding2::{bytes, open};
+#[cfg(not(feature = "gemma-embedding2"))]
+use crate::side_seat::{no_encoder_bytes as bytes, no_encoder_open as open};
+
+/// The architectures a side seat serves.
+pub const ROWS: &[Row] = &[
+    // The file's context_length is 262144; the model card names 8192 tokens.
+    Row {
+        arch: model::arch::GEMMA_EMBEDDING2,
+        ctx: model::arch::gemma_embedding2::hparams::CARD_CTX,
+        bytes,
+        open,
+    },
+];
+
+#[cfg(feature = "gemma-embedding2")]
+pub(super) fn gemma_embedding2_dims(split: &gguf::Split) -> Result<Vec<usize>, crate::GateError> {
+    const MRL: [usize; 3] = [128, 256, 512];
+    let n_out = model::arch::gemma_embedding2::hparams::Hparams::read(split)?.n_out;
+    if n_out < MRL[0] {
+        return Err(format!(
+            "embedding model: embedding_length_out {n_out} cannot serve the {}-wide MRL cut",
+            MRL[0]
+        )
+        .into());
+    }
+    let mut dims: Vec<_> = MRL.into_iter().filter(|&width| width < n_out).collect();
+    dims.push(n_out);
+    Ok(dims)
+}

crates/gpu-gates/src/beside.rs

@@ -0,0 +1,184 @@
+//! What a plan holds beside its own tensors: a model the process serves beside the planned one (the
+//! image encoder's tower, the embedding model of a side seat) that sits on the stage card, as a
+//! named reserve of fixed bytes the plan's fit counts ([`model::placement::Card::reserves`]). One
+//! function enters it into a placed plan's machine and a whole-fit probe's ([`reserve_on`]), one
+//! reads it back ([`reserve_in`]), and a plan's refusal ([`refusal`]) and a verdict's line
+//! ([`note`]) name it. Free of the encoder crates, so a placement module every build includes can
+//! take it.
+
+use std::fmt::Display;
+
+use model::placement::{Card, Machine};
+
+/// What the plan's reserve for the image encoder's tower is called, as the plan's terms name it
+/// (beside `workstation`'s draft and tier-batch reserves).
+pub const IMAGE_ENCODER: &str = "image encoder";
+
+/// What the plan's reserve for a side seat's embedding model is called ([`crate::side_seat`]).
+pub const EMBEDDING: &str = "embedding model";
+
+/// The bytes a plan reserves on its stage card beside its own tensors, by what they are for;
+/// `None` for one that sits off the plan or is not served.
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+pub struct Beside {
+    /// The image encoder's tower.
+    pub image: Option<u64>,
+    /// The embedding model a side seat serves.
+    pub embedding: Option<u64>,
+}
+
+impl Beside {
+    /// Each reserve by its name, in the order the stage card takes them.
+    pub fn named(self) -> impl Iterator<Item = (&'static str, u64)> {
+        [(IMAGE_ENCODER, self.image), (EMBEDDING, self.embedding)]
+            .into_iter()
+            .filter_map(|(name, bytes)| Some((name, bytes?)))
+    }
+}
+
+/// `machine` with `beside`'s reserves on its stage card. The one place they enter a machine: a
+/// placed plan's builder and a whole-fit probe both call it, so both fit the same bytes.
+pub fn reserve_on(machine: &mut Machine, beside: Beside) {
+    for (name, bytes) in beside.named() {
+        machine.cards[0].reserves.push((name.to_owned(), bytes));
+    }
+}
+
+/// The reserves [`reserve_on`] put on `machine`'s stage card: what a plan built on it refuses
+/// beside ([`refusal`]).
+#[must_use]
+pub fn reserve_in(machine: &Machine) -> Beside {
+    let held = |name: &str| {
+        machine
+            .cards
+            .first()?
+            .reserves
+            .iter()
+            .find(|(n, _)| n == name)
+            .map(|&(_, bytes)| bytes)
+    };
+    Beside {
+        image: held(IMAGE_ENCODER),
+        embedding: held(EMBEDDING),
+    }
+}
+
+/// A machine builder (`layers` to its [`Machine`]) with [`reserve_on`] applied to what it builds.
+pub fn with_reserve(plain: impl Fn(usize) -> Machine, beside: Beside) -> impl Fn(usize) -> Machine {
+    move |layers| {
+        let mut machine = plain(layers);
+        reserve_on(&mut machine, beside);
+        machine
+    }
+}
+
+/// What a verdict's line adds when its card carries reserves [`reserve_on`] puts there: each by
+/// name and bytes, in [`Beside`]'s order, beside the reserves term the verdict sums them in.
+/// Nothing for a card that carries none, so a load with nothing beside it prints its line as it
+/// did.
+#[must_use]
+pub fn note(card: &Card) -> String {
+    [IMAGE_ENCODER, EMBEDDING]
+        .iter()
+        .filter_map(|&want| card.reserves.iter().find(|(name, _)| name == want))
+        .map(|(name, bytes)| format!("; the reserves term holds the {name}'s {bytes} B"))
+        .collect()
+}
+
+/// A plan's refusal, beside the reserves that were in it: with none the refusal as it is.
+pub fn refusal(beside: Beside, e: impl Display) -> String {
+    let held: Vec<String> = beside
+        .named()
+        .map(|(name, bytes)| format!("the {name}'s {bytes} B as the stage card's {name} reserve"))
+        .collect();
+    if held.is_empty() {
+        return e.to_string();
+    }
+    format!("the plan holds {}: {e}", held.join(" and "))
+}
+
+#[cfg(test)]
+mod tests {
+    use model::placement::workstation::plan_a;
+
+    use super::*;
+
+    const BYTES: u64 = 1_476_000_000;
+
+    /// The whole-fit probe and the placed plan take the one reserve: the machines both build carry
+    /// the same named bytes, the card sets aside exactly that much more, and none changes nothing.
+    #[test]
+    fn the_probe_and_the_placed_plan_see_the_same_reserve() {
+        let image = Beside {
+            image: Some(BYTES),
+            embedding: None,
+        };
+        let plain = plan_a(40);
+        let mut probe = plan_a(40);
+        reserve_on(&mut probe, image);
+        let placed = with_reserve(plan_a, image)(40);
+        assert_eq!(probe, placed);
+        assert_eq!(
+            probe.cards[0].reserves.last(),
+            Some(&(IMAGE_ENCODER.to_owned(), BYTES))
+        );
+        assert_eq!(
+            probe.cards[0].reserve_bytes(),
+            plain.cards[0].reserve_bytes() + BYTES
+        );
+        assert_eq!(
+            probe.cards[0].set_aside_bytes(),
+            plain.cards[0].set_aside_bytes() + BYTES
+        );
+        assert_eq!(reserve_in(&probe), image);
+        assert_eq!(reserve_in(&plain), Beside::default());
+        let mut none = plan_a(40);
+        reserve_on(&mut none, Beside::default());
+        assert_eq!(none, plain);
+        assert_eq!(with_reserve(plan_a, Beside::default())(40), plain);
+        assert_eq!(note(&plain.cards[0]), "");
+        assert_eq!(
+            note(&probe.cards[0]),
+            "; the reserves term holds the image encoder's 1476000000 B"
+        );
+        assert_eq!(refusal(Beside::default(), "no fit"), "no fit");
+        assert_eq!(
+            refusal(image, "no fit"),
+            "the plan holds the image encoder's 1476000000 B as the stage card's image encoder \
+             reserve: no fit"
+        );
+    }
+
+    /// Both reserves enter the stage card in order and come back, and the note and the refusal
+    /// name each.
+    #[test]
+    fn the_embedding_model_is_a_second_reserve() {
+        let both = Beside {
+            image: Some(BYTES),
+            embedding: Some(7),
+        };
+        let mut machine = plan_a(40);
+        reserve_on(&mut machine, both);
+        assert_eq!(reserve_in(&machine), both);
+        assert_eq!(
+            machine.cards[0].reserve_bytes(),
+            plan_a(40).cards[0].reserve_bytes() + BYTES + 7
+        );
+        assert_eq!(
+            note(&machine.cards[0]),
+            "; the reserves term holds the image encoder's 1476000000 B; the reserves term holds \
+             the embedding model's 7 B"
+        );
+        assert_eq!(
+            refusal(
+                Beside {
+                    image: None,
+                    embedding: Some(7),
+                },
+                "no fit"
+            ),
+            "the plan holds the embedding model's 7 B as the stage card's embedding model \
+             reserve: no fit"
+        );
+    }
+}

crates/gpu-gates/src/bin/bloomery_serve.rs

@@ -48,6 +48,13 @@
 //! it); a file whose architecture does not match an explicit
 //! `--model` is refused by name before any load.
 //! `--version` prints this crate's version and the build's commit and exits.
+//! `--embd-model FILE` or `--embd-hf <repo>[:<quant>]` names an embedding
+//! model the server holds beside a generative seat's model for its embeddings routes
+//! (`bloomery_gpu_gates::side_seat`): read and seated before the chat model's plan, which holds
+//! its bytes when it sits on the stage card, and loaded after the chat model. A file of an
+//! embedding architecture given as `-m` is refused by name, and so is an embedding model beside
+//! the decide and nemotron seats. EmbeddingGemma 2's row opens the encoder on the seat card;
+//! its pass owns the output projection and mean pooling, and the HTTP layer cuts and normalizes.
 //!
 //! Two checks run before anything is fetched or loaded, each a named refusal
 //! (`serve::preflight`): first of all, on a CPU that does not run the
@@ -117,6 +124,7 @@ mod place;
 mod drive {
     use std::process::ExitCode;
 
+    use bloomery_gpu_gates::side_seat;
     use bloomery_gpu_gates::{GateError, exit_with, model_file, ref_model_path};
     use cuda_core::sys;
     use gguf::Split;
@@ -131,7 +139,8 @@ mod drive {
 
     const USAGE: &str = "usage: bloomery-serve [--model ds41|qwen38|glm|qwen3|mimo2|nemotron|decide] \
                          [-m PATH | --model-file PATH | --hf <repo>[:<quant>]] [--head <weights> \
-                         [--head-config <file>]] <the chosen seat's own flags, as \
+                         [--head-config <file>]] [--embd-model FILE | --embd-hf \
+                         <repo>[:<quant>]] <the chosen seat's own flags, as \
                          bloomery-serve-ds41, bloomery-serve-qwen38, the glm, the qwen3, the \
                          mimo2, the nemotron or the decide seat takes them> | --version";
 
@@ -290,6 +299,14 @@ models (the --model word is optional with a model file: the file's architecture
         let path = ref_model_path()?;
         let split = Split::open(&path).map_err(|e| format!("open {}: {e}", path.display()))?;
         let (seat, arch) = match split.architecture() {
+            Some(a) if side_seat::row(a).is_some() => {
+                return Err(format!(
+                    "{} is a {a} file, an embedding model: serve it beside a chat model with \
+                     --embd-model",
+                    path.display()
+                )
+                .into());
+            }
             Some(a) if decide::ROWS.iter().any(|r| r.backbones.contains(&a)) => {
                 return Err(format!(
                     "{}: {}",
@@ -442,7 +459,19 @@ models (the --model word is optional with a model file: the file's architecture
             Ok(v) => v,
             Err(e) => return exit_with(NAME, Err(e)),
         };
+        // The side model's flags come out before the seat reads its own; its file is read in the
+        // seat, after the levers.
+        let (side, rest) = match side_seat::Flags::take(&rest) {
+            Ok(v) => v,
+            Err(e) => return exit_with(NAME, Err(e)),
+        };
         if let Some(from) = from {
+            if side.names() {
+                return exit_with(
+                    NAME,
+                    Err("--embd-model: the decide seat serves no embedding model".into()),
+                );
+            }
             return ended(decide::run(&rest, from, repo.as_deref()));
         }
         let (seat, arch) = match seat_of_file() {
@@ -463,12 +492,18 @@ models (the --model word is optional with a model file: the file's architecture
             );
         }
         let r = match seat {
-            Model::Ds41 => ds41::run(&rest),
-            Model::Qwen38 => qwen38::run(&rest),
-            Model::Glm => glm::run(&rest),
-            Model::Qwen3 => qwen3::run(&rest),
-            Model::Mimo2 => mimo2::run(&rest),
-            Model::Nemotron => nemotron::run(&rest),
+            Model::Ds41 => ds41::run(&rest, &side),
+            Model::Qwen38 => qwen38::run(&rest, &side),
+            Model::Glm => glm::run(&rest, &side),
+            Model::Qwen3 => qwen3::run(&rest, &side),
+            Model::Mimo2 => mimo2::run(&rest, &side),
+            Model::Nemotron => {
+                if side.names() {
+                    Err("--embd-model: the nemotron seat serves no embedding model".into())
+                } else {
+                    nemotron::run(&rest)
+                }
+            }
         };
         ended(r)
     }

crates/gpu-gates/src/bin/bloomery_serve_ds41.rs

@@ -6,7 +6,8 @@
 #[cfg(feature = "deepseek41")]
 fn main() -> std::process::ExitCode {
     let args: Vec<String> = std::env::args().skip(1).collect();
-    match ds41::run(&args) {
+    let side = bloomery_gpu_gates::side_seat::Flags::take(&args);
+    match side.and_then(|(side, rest)| ds41::run(&rest, &side)) {
         // EX_SOFTWARE: the engine, not the listener or the load, ended the run.
         Ok(serve::ServeError::Engine(f)) => {
             eprintln!("bloomery-serve-ds41: {f}");

crates/gpu-gates/src/bin/bloomery_serve_qwen38.rs

@@ -6,7 +6,8 @@
 #[cfg(feature = "gpu")]
 fn main() -> std::process::ExitCode {
     let args: Vec<String> = std::env::args().skip(1).collect();
-    match qwen38::run(&args) {
+    let side = bloomery_gpu_gates::side_seat::Flags::take(&args);
+    match side.and_then(|(side, rest)| qwen38::run(&rest, &side)) {
         // EX_SOFTWARE: the engine, not the listener or the load, ended the run.
         Ok(serve::ServeError::Engine(f)) => {
             eprintln!("bloomery-serve-qwen38: {f}");

crates/gpu-gates/src/bin/gate_clef_hidden.rs

@@ -60,7 +60,8 @@ mod gate {
     use bloomery_gpu::Gpu;
     use bloomery_gpu::arch::qwen3moe::{KvQ8, Open35, PrefillPath, Qwen35moeModel};
     use bloomery_gpu_gates::{
-        GateError, RefManifest, checks_failed, data_dir, load_ref_in, ref_model_path, verdict,
+        GateError, RefManifest, checks_failed, data_dir, load_ref_in, ref_model_path, sha256,
+        verdict,
     };
     use gguf::Split;
     use refset::arch::qwen35::{
@@ -235,16 +236,6 @@ mod gate {
         quantile(v, 0.5)
     }
 
-    /// The sha256 of `path`, by `sha256sum`.
-    fn sha256(path: &Path) -> Result<String, GateError> {
-        let out = std::process::Command::new("sha256sum").arg(path).output()?;
-        let text = String::from_utf8_lossy(&out.stdout);
-        match text.split_whitespace().next() {
-            Some(d) if out.status.success() && d.len() == 64 => Ok(d.to_string()),
-            _ => Err(format!("sha256sum {}: {:?}", path.display(), text.trim()).into()),
-        }
-    }
-
     /// One set: its ids, read from the tokens file its header names and held
     /// to the header's digest and count, and mainline's rows.
     fn open_set(