From d3e248d0ef56454f5c923c0ebc16c86e5f15f327 Mon Sep 17 00:00:00 2001 From: juliankang4 <200967947+juliankang4@users.noreply.github.com> Date: Fri, 16 Oct 2026 18:30:41 +0900 Subject: [PATCH] serve: discovery names the embedding model beside the chat seat --- crates/gpu-gates/src/side_seat.rs | 21 ++++++- .../src/side_seat/gemma_embedding2.rs | 6 ++ crates/serve/src/api.rs | 8 ++- crates/serve/src/embed.rs | 2 + crates/serve/src/embeddings.rs | 60 ++++++++++++++++--- crates/serve/src/mock.rs | 4 ++ docs/serving.md | 4 +- 7 files changed, 92 insertions(+), 13 deletions(-) diff --git a/crates/gpu-gates/src/side_seat.rs b/crates/gpu-gates/src/side_seat.rs index 396e2f7cc..c2fbf75e7 100644 --- a/crates/gpu-gates/src/side_seat.rs +++ b/crates/gpu-gates/src/side_seat.rs @@ -289,6 +289,7 @@ pub struct SideEngine { thread: Option>, tokenizer: Arc, name: String, + path: String, ctx: usize, dims: Vec, media: Option, @@ -320,6 +321,7 @@ impl SideEngine { let facts = ( model.tokenizer(), model.name(), + model.path(), model.ctx(), model.dims(), model.media(), @@ -348,7 +350,7 @@ impl SideEngine { } }) .map_err(|e| format!("starting the embedding model's thread: {e}"))?; - let (tokenizer, name, ctx, dims, media) = match opened.recv() { + let (tokenizer, name, path, ctx, dims, media) = match opened.recv() { Ok(Ok(facts)) => facts, refused => { // The thread ended with its open; nothing is left on its card. @@ -364,6 +366,7 @@ impl SideEngine { thread: Some(thread), tokenizer, name, + path, ctx, dims, media, @@ -398,6 +401,10 @@ impl Embed for SideEngine { self.name.clone() } + fn path(&self) -> String { + self.path.clone() + } + fn ctx(&self) -> usize { self.ctx } @@ -590,6 +597,9 @@ mod tests { fn name(&self) -> String { "stub".to_owned() } + fn path(&self) -> String { + "/models/stub.gguf".to_owned() + } fn ctx(&self) -> usize { 8 } @@ -628,8 +638,13 @@ mod tests { }) .expect("open"); assert_eq!( - (engine.name(), engine.ctx(), engine.dims()), - ("stub".to_owned(), 8, vec![1]) + (engine.name(), engine.path(), engine.ctx(), engine.dims()), + ( + "stub".to_owned(), + "/models/stub.gguf".to_owned(), + 8, + vec![1] + ) ); assert_eq!(engine.embed(&[5, 6]).expect("a pass"), vec![2.0]); let e = engine.embed(&[0]).expect_err("the panic"); diff --git a/crates/gpu-gates/src/side_seat/gemma_embedding2.rs b/crates/gpu-gates/src/side_seat/gemma_embedding2.rs index 3d5a1c3ef..fd88ed35a 100644 --- a/crates/gpu-gates/src/side_seat/gemma_embedding2.rs +++ b/crates/gpu-gates/src/side_seat/gemma_embedding2.rs @@ -73,6 +73,7 @@ pub(super) fn open(side: &Side, seat: &EncoderSeat) -> Result, St vision, tok, name, + path: side.path.display().to_string(), dims, })) }; @@ -85,6 +86,7 @@ struct Model { vision: Option, tok: Arc, name: String, + path: String, dims: Vec, } @@ -108,6 +110,10 @@ impl Embed for Model { self.name.clone() } + fn path(&self) -> String { + self.path.clone() + } + fn ctx(&self) -> usize { self.encoder.ctx() } diff --git a/crates/serve/src/api.rs b/crates/serve/src/api.rs index 9ea005d57..8c5c666f0 100644 --- a/crates/serve/src/api.rs +++ b/crates/serve/src/api.rs @@ -1827,7 +1827,7 @@ fn models(state: &State) -> Value { } fn props(state: &State) -> Value { - json!({ + let mut props = json!({ "system_prompt": "", "model_alias": state.alias, "model_path": state.model_path, @@ -1843,7 +1843,11 @@ fn props(state: &State) -> Value { "n_ctx": state.info.ctx_max, "build_info": format!("bloomery-serve {VERSION}"), "engine": state.engine_props, - }) + }); + if let Some(side) = &state.side { + props["embedding_model"] = json!({ "alias": side.name, "path": side.path }); + } + props } /// The commit this build names, set by `build.rs`. `--version` reads it too. diff --git a/crates/serve/src/embed.rs b/crates/serve/src/embed.rs index f8f4a4e62..a597ae1d1 100644 --- a/crates/serve/src/embed.rs +++ b/crates/serve/src/embed.rs @@ -23,6 +23,8 @@ pub trait Embed: Send { fn tokenizer(&self) -> Arc; /// The model's name: its `/v1/models` id and the `model` every OpenAI answer carries. fn name(&self) -> String; + /// The model file, reported as `/props`' `embedding_model.path`. + fn path(&self) -> String; /// The most positions one input takes. fn ctx(&self) -> usize; /// The widths a request's `dimensions` may cut the pooled vector to, ascending; the last is diff --git a/crates/serve/src/embeddings.rs b/crates/serve/src/embeddings.rs index 0c43ada2b..06770e428 100644 --- a/crates/serve/src/embeddings.rs +++ b/crates/serve/src/embeddings.rs @@ -47,6 +47,7 @@ pub(super) struct Side { embed: Mutex>, tok: Arc, pub(super) name: String, + pub(super) path: String, pub(super) ctx: usize, dims: Vec, media: Option, @@ -65,6 +66,7 @@ impl Side { media: embed.media(), tok: embed.tokenizer(), name: embed.name(), + path: embed.path(), ctx: embed.ctx(), dims, embed: Mutex::new(embed), @@ -561,14 +563,58 @@ mod tests { &json!({ "input": "a", "encoding_format": "f16" }), ); assert_eq!(status, 400, "{v}"); + } - let (status, text) = roundtrip(addr, "GET", "/v1/models", &[], ""); - let v: Value = serde_json::from_str(&text).expect("json"); - assert_eq!(status, 200); - assert_eq!(v["data"][1]["id"], "mock-embd"); - assert_eq!(v["data"][1]["meta"]["n_embd"], 4); - assert_eq!(v["data"][1]["meta"]["n_ctx"], 32); - assert!(v["data"][1]["meta"].get("n_ctx_train").is_none()); + /// Discovery lists every seated model and reports the side model's file only when present. + #[test] + fn discovery_names_the_seated_models() { + for side in [None, Some(MockEmbed::new(32))] { + let has_side = side.is_some(); + let (addr, state, calls, _) = serve(side); + let mut data = vec![json!({ + "id": "mock", + "object": "model", + "created": state.start_unix, + "owned_by": "bloomery", + "meta": { "n_vocab": state.info.n_vocab, "n_ctx_train": 64 }, + "max_model_len": 64, + })]; + let mut names = vec![json!({ "name": "mock", "model": "mock" })]; + if has_side { + data.push(json!({ + "id": "mock-embd", + "object": "model", + "created": state.start_unix, + "owned_by": "bloomery", + "meta": { "n_ctx": 32, "n_embd": 4 }, + "max_model_len": 32, + })); + names.push(json!({ "name": "mock-embd", "model": "mock-embd" })); + } + for path in ["/v1/models", "/models"] { + let (status, body) = roundtrip(addr, "GET", path, &[], ""); + assert_eq!(status, 200, "{path}: {body}"); + let v: Value = serde_json::from_str(&body).expect("models JSON"); + assert_eq!( + v, + json!({ "object": "list", "data": data, "models": names }) + ); + } + let (status, body) = roundtrip(addr, "GET", "/props", &[], ""); + assert_eq!(status, 200, "{body}"); + let v: Value = serde_json::from_str(&body).expect("props JSON"); + assert_eq!(v["model_alias"], "mock"); + assert_eq!(v["model_path"], state.model_path); + if has_side { + assert_eq!( + v["embedding_model"], + json!({ "alias": "mock-embd", "path": "/models/mock-embd.gguf" }) + ); + } else { + assert!(v.get("embedding_model").is_none(), "{v}"); + } + assert!(calls.lock().expect("calls").is_empty()); + } } /// Which inputs a request holds and the ids each passes, as llama-server splits and diff --git a/crates/serve/src/mock.rs b/crates/serve/src/mock.rs index d7ce67cc3..3ef8a32d5 100644 --- a/crates/serve/src/mock.rs +++ b/crates/serve/src/mock.rs @@ -893,6 +893,10 @@ impl crate::embed::Embed for MockEmbed { "mock-embd".to_owned() } + fn path(&self) -> String { + "/models/mock-embd.gguf".to_owned() + } + fn ctx(&self) -> usize { self.ctx } diff --git a/docs/serving.md b/docs/serving.md index 7c75e4733..0855465ec 100644 --- a/docs/serving.md +++ b/docs/serving.md @@ -33,7 +33,9 @@ Your OpenAI client works unchanged. bloomery speaks llama-server's HTTP API and `--mmproj` and `--embd-model` on the same off-plan card do not count each other's bytes; the second open fails by name if the card is short. The response `model` is the embedding file name, also listed separately in `/v1/models` with - `meta.n_embd` and `meta.n_ctx`. On `/v1/embeddings`, a `model` of another type than a string or a name + `meta.n_embd` and `meta.n_ctx`. `/props` adds `embedding_model` with the side model's `alias` and + `path` when one is loaded; `model_alias` and `model_path` still name the chat model. + On `/v1/embeddings`, a `model` of another type than a string or a name other than the embedding model's is a 400 `invalid_request_error` and runs no pass; `null` is unset. `/embedding` and `/embeddings` ignore `model`. An input past the model's context is a 400 `exceed_context_size_error` naming both lengths. llama-server checks its physical batch first and answers an input