Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 18 additions & 3 deletions crates/gpu-gates/src/side_seat.rs
Original file line number Diff line number Diff line change
Expand Up @@ -289,6 +289,7 @@ pub struct SideEngine {
thread: Option<JoinHandle<()>>,
tokenizer: Arc<dyn Tokenizer>,
name: String,
path: String,
ctx: usize,
dims: Vec<usize>,
media: Option<serve::media::SharedMediaModel>,
Expand Down Expand Up @@ -320,6 +321,7 @@ impl SideEngine {
let facts = (
model.tokenizer(),
model.name(),
model.path(),
model.ctx(),
model.dims(),
model.media(),
Expand Down Expand Up @@ -348,7 +350,7 @@ impl SideEngine {
}
})
.map_err(|e| format!("starting the embedding model's thread: {e}"))?;
let (tokenizer, name, ctx, dims, media) = match opened.recv() {
let (tokenizer, name, path, ctx, dims, media) = match opened.recv() {
Ok(Ok(facts)) => facts,
refused => {
// The thread ended with its open; nothing is left on its card.
Expand All @@ -364,6 +366,7 @@ impl SideEngine {
thread: Some(thread),
tokenizer,
name,
path,
ctx,
dims,
media,
Expand Down Expand Up @@ -398,6 +401,10 @@ impl Embed for SideEngine {
self.name.clone()
}

fn path(&self) -> String {
self.path.clone()
}

fn ctx(&self) -> usize {
self.ctx
}
Expand Down Expand Up @@ -590,6 +597,9 @@ mod tests {
fn name(&self) -> String {
"stub".to_owned()
}
fn path(&self) -> String {
"/models/stub.gguf".to_owned()
}
fn ctx(&self) -> usize {
8
}
Expand Down Expand Up @@ -628,8 +638,13 @@ mod tests {
})
.expect("open");
assert_eq!(
(engine.name(), engine.ctx(), engine.dims()),
("stub".to_owned(), 8, vec![1])
(engine.name(), engine.path(), engine.ctx(), engine.dims()),
(
"stub".to_owned(),
"/models/stub.gguf".to_owned(),
8,
vec![1]
)
);
assert_eq!(engine.embed(&[5, 6]).expect("a pass"), vec![2.0]);
let e = engine.embed(&[0]).expect_err("the panic");
Expand Down
6 changes: 6 additions & 0 deletions crates/gpu-gates/src/side_seat/gemma_embedding2.rs
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ pub(super) fn open(side: &Side, seat: &EncoderSeat) -> Result<Box<dyn Embed>, St
vision,
tok,
name,
path: side.path.display().to_string(),
dims,
}))
};
Expand All @@ -85,6 +86,7 @@ struct Model {
vision: Option<Resident>,
tok: Arc<Vocab>,
name: String,
path: String,
dims: Vec<usize>,
}

Expand All @@ -108,6 +110,10 @@ impl Embed for Model {
self.name.clone()
}

fn path(&self) -> String {
self.path.clone()
}

fn ctx(&self) -> usize {
self.encoder.ctx()
}
Expand Down
8 changes: 6 additions & 2 deletions crates/serve/src/api.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1827,7 +1827,7 @@ fn models(state: &State) -> Value {
}

fn props(state: &State) -> Value {
json!({
let mut props = json!({
"system_prompt": "",
"model_alias": state.alias,
"model_path": state.model_path,
Expand All @@ -1843,7 +1843,11 @@ fn props(state: &State) -> Value {
"n_ctx": state.info.ctx_max,
"build_info": format!("bloomery-serve {VERSION}"),
"engine": state.engine_props,
})
});
if let Some(side) = &state.side {
props["embedding_model"] = json!({ "alias": side.name, "path": side.path });
}
props
}

/// The commit this build names, set by `build.rs`. `--version` reads it too.
Expand Down
2 changes: 2 additions & 0 deletions crates/serve/src/embed.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ pub trait Embed: Send {
fn tokenizer(&self) -> Arc<dyn Tokenizer>;
/// The model's name: its `/v1/models` id and the `model` every OpenAI answer carries.
fn name(&self) -> String;
/// The model file, reported as `/props`' `embedding_model.path`.
fn path(&self) -> String;
/// The most positions one input takes.
fn ctx(&self) -> usize;
/// The widths a request's `dimensions` may cut the pooled vector to, ascending; the last is
Expand Down
60 changes: 53 additions & 7 deletions crates/serve/src/embeddings.rs
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ pub(super) struct Side {
embed: Mutex<Box<dyn Embed>>,
tok: Arc<dyn Tokenizer>,
pub(super) name: String,
pub(super) path: String,
pub(super) ctx: usize,
dims: Vec<usize>,
media: Option<media::SharedMediaModel>,
Expand All @@ -65,6 +66,7 @@ impl Side {
media: embed.media(),
tok: embed.tokenizer(),
name: embed.name(),
path: embed.path(),
ctx: embed.ctx(),
dims,
embed: Mutex::new(embed),
Expand Down Expand Up @@ -561,14 +563,58 @@ mod tests {
&json!({ "input": "a", "encoding_format": "f16" }),
);
assert_eq!(status, 400, "{v}");
}

let (status, text) = roundtrip(addr, "GET", "/v1/models", &[], "");
let v: Value = serde_json::from_str(&text).expect("json");
assert_eq!(status, 200);
assert_eq!(v["data"][1]["id"], "mock-embd");
assert_eq!(v["data"][1]["meta"]["n_embd"], 4);
assert_eq!(v["data"][1]["meta"]["n_ctx"], 32);
assert!(v["data"][1]["meta"].get("n_ctx_train").is_none());
/// Discovery lists every seated model and reports the side model's file only when present.
#[test]
fn discovery_names_the_seated_models() {
for side in [None, Some(MockEmbed::new(32))] {
let has_side = side.is_some();
let (addr, state, calls, _) = serve(side);
let mut data = vec![json!({
"id": "mock",
"object": "model",
"created": state.start_unix,
"owned_by": "bloomery",
"meta": { "n_vocab": state.info.n_vocab, "n_ctx_train": 64 },
"max_model_len": 64,
})];
let mut names = vec![json!({ "name": "mock", "model": "mock" })];
if has_side {
data.push(json!({
"id": "mock-embd",
"object": "model",
"created": state.start_unix,
"owned_by": "bloomery",
"meta": { "n_ctx": 32, "n_embd": 4 },
"max_model_len": 32,
}));
names.push(json!({ "name": "mock-embd", "model": "mock-embd" }));
}
for path in ["/v1/models", "/models"] {
let (status, body) = roundtrip(addr, "GET", path, &[], "");
assert_eq!(status, 200, "{path}: {body}");
let v: Value = serde_json::from_str(&body).expect("models JSON");
assert_eq!(
v,
json!({ "object": "list", "data": data, "models": names })
);
}
let (status, body) = roundtrip(addr, "GET", "/props", &[], "");
assert_eq!(status, 200, "{body}");
let v: Value = serde_json::from_str(&body).expect("props JSON");
assert_eq!(v["model_alias"], "mock");
assert_eq!(v["model_path"], state.model_path);
if has_side {
assert_eq!(
v["embedding_model"],
json!({ "alias": "mock-embd", "path": "/models/mock-embd.gguf" })
);
} else {
assert!(v.get("embedding_model").is_none(), "{v}");
}
assert!(calls.lock().expect("calls").is_empty());
}
}

/// Which inputs a request holds and the ids each passes, as llama-server splits and
Expand Down
4 changes: 4 additions & 0 deletions crates/serve/src/mock.rs
Original file line number Diff line number Diff line change
Expand Up @@ -893,6 +893,10 @@ impl crate::embed::Embed for MockEmbed {
"mock-embd".to_owned()
}

fn path(&self) -> String {
"/models/mock-embd.gguf".to_owned()
}

fn ctx(&self) -> usize {
self.ctx
}
Expand Down
4 changes: 3 additions & 1 deletion docs/serving.md
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,9 @@ Your OpenAI client works unchanged. bloomery speaks llama-server's HTTP API and
`--mmproj` and `--embd-model` on the same off-plan card do not count each other's bytes; the second open
fails by name if the card is short.
The response `model` is the embedding file name, also listed separately in `/v1/models` with
`meta.n_embd` and `meta.n_ctx`. On `/v1/embeddings`, a `model` of another type than a string or a name
`meta.n_embd` and `meta.n_ctx`. `/props` adds `embedding_model` with the side model's `alias` and
`path` when one is loaded; `model_alias` and `model_path` still name the chat model.
On `/v1/embeddings`, a `model` of another type than a string or a name
other than the embedding model's is a 400 `invalid_request_error` and runs no pass; `null` is unset.
`/embedding` and `/embeddings` ignore `model`. An input past the model's context is a 400
`exceed_context_size_error` naming both lengths. llama-server checks its physical batch first and answers an input
Expand Down
Loading