diff --git a/src/http/README.md b/src/http/README.md index e4845f716..fbf9b53fe 100644 --- a/src/http/README.md +++ b/src/http/README.md @@ -39,6 +39,8 @@ Step labels in history (ATO-1382): `GET /api/sessions/{id}` adds `done_line` to [Local model recheck](route-local-model-recheck.ts) (ATO-873): `POST /api/local-model/recheck`, no body, reads `localModels.url` fresh from the config file, probes it the way saving it does and answers `{ url, verdict, ok, message, portHint }` (`ok`, `loading`, `ollama`, `atomic_chat`, `openai_compat`, `key_rejected`, `not_model_server`, `unreachable`). It never takes a URL from the client and writes nothing. A failed step's and turn's `error` frame carries `code` and plain `advice` for a local server ([fields](failure-reason-frame.ts)): `local_server_not_found` (404/405 at `localModels.url`, the case for a Reconnect button), `local_provider_not_found` (the same at another address), `local_context_exceeded`, `local_tools_unsupported` (a 400 saying the server takes no tool calls: llama-server without `--jinja`, an Ollama model without tool support), `local_request_invalid`, `atomic_chat_key_required` (the `atomic-chat` provider answered 401/403: its API key protection is on, ATO-1274; not reported to Sentry), `local_port_taken` (the managed engine's address answered 401 while our engine's process is not there: another program holds the port, ATO-1277); `local_engine_down` wins over them, and so does `atomic_chat_not_running` (ATO-1278: the Atomic Chat app refused the connection past a short start grace; the user opens it). +[Local model performance](route-local-model-performance.ts) (issue #996): `GET /api/local-model/performance` answers `{ local, url, capacity, samples, slots }` for the desktop's Performance pane. `samples` (oldest first, at most `capacity`, 300, kept in memory only) are the runtime's llama-server client's completed requests, one each: `at`, `modelId` (file name only), `streamed`, `promptTokens`, `cacheTokens`, `promptMs`, `promptPerSecond` (`null` for a fully cached prompt), `predictedTokens`, `predictedMs`, `generationPerSecond`, `ttftMs` (streams: call start to first delta, retries and queueing included), `contextTokens` and `concurrent` (most calls open at once during it). A failed or abandoned request records nothing; no prompt or reply text is kept. `local` and `url` come from the cached config the live turns use. Only when `local` is true does the route read the server's `/slots` once, bounded at 1.5 s: `slots` is `{ total, busy, contextPerSlot }`, `null` on any failure. Always `200`; an older agent answers `404`. + [Telegram pairing routes](route-telegram-pairing.ts) (ATO-859): `POST /api/telegram/pairing` opens a 5-minute window through `TelegramChannel.startPairing` and answers `{ status: "waiting", code, expiresAt, timeoutMs, botUsername, instruction }` with `cache-control: no-store`. That reply is the only place the one-time code leaves the process; it is never logged and never in the `GET`. While an owner is paired the POST answers `409 owner_already_paired` until it is repeated with `{ "replaceOwner": true }`; a channel that is not `up` answers `409 telegram_not_running`. `GET` reports `idle`, `waiting`, `claimed` (with `owner: { userId, username }` and `pairedLine`), `expired`, `cancelled` or `failed`; `DELETE` cancels the window. Bearer auth like every `/api` route. [Approval rules routes](route-approval-rules.ts) (ATO-1141): `GET /api/approval-rules?workingDir=|sessionId=` returns `{ rules, globalPath, projectPath, projectRules }`: the global file's rules and the folder's project-file ask / deny rules (read-only). `DELETE /api/approval-rules/{id}` removes a global rule (`{ removed: true }`, `404` when there is none). Contract: [approval README](../approval/README.md). diff --git a/src/http/route-local-model-performance.test.ts b/src/http/route-local-model-performance.test.ts new file mode 100644 index 000000000..aec689b3e --- /dev/null +++ b/src/http/route-local-model-performance.test.ts @@ -0,0 +1,211 @@ +import { createServer, type Server } from "node:http"; +import { readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; + +import { resetConfigCache } from "../config/index.js"; +import type { PerformanceSample } from "../llm/llama-performance-samples.js"; +import type { AgentRuntime } from "../runtime/bootstrap.js"; +import { startTestHarness, type Harness } from "./test-harness.js"; + +type SlotsReply = { status: number; body: string } | "hang"; + +/** A stand-in llama-server that answers `GET /slots` and counts the requests. */ +async function startSlotsServer( + reply: SlotsReply, +): Promise<{ url: string; hits: () => number; stop: () => Promise }> { + let hits = 0; + const srv: Server = createServer((req, res) => { + if (req.url !== "/slots") { + res.writeHead(404).end(); + return; + } + hits += 1; + if (reply === "hang") return; + res.writeHead(reply.status, { "content-type": "application/json" }).end(reply.body); + }); + await new Promise((resolve) => srv.listen(0, "127.0.0.1", resolve)); + const address = srv.address(); + const port = typeof address === "object" && address ? address.port : 0; + return { + url: `http://127.0.0.1:${port}`, + hits: () => hits, + stop: () => + new Promise((resolve) => { + srv.closeAllConnections(); + srv.close(() => resolve()); + }), + }; +} + +const SAMPLE: PerformanceSample = { + at: 1794329429000, + modelId: "Qwen3.6-35B-A3B-MXFP4_MOE.gguf", + streamed: true, + promptTokens: 2, + cacheTokens: 0, + promptMs: 32.969, + promptPerSecond: 60.663047104856076, + predictedTokens: 40, + predictedMs: 1158.29, + generationPerSecond: 33.67032435745798, + ttftMs: 412, + contextTokens: 42, + concurrent: 1, +}; + +/** Rewrite the harness's config file with an `llm` block and drop the cache. */ +function setProviders( + harness: Harness, + activeTextProvider: string, + providers: Array>, +): void { + const path = join(harness.stateDir, "config.json"); + const current = JSON.parse(readFileSync(path, "utf8")) as Record; + writeFileSync( + path, + JSON.stringify({ ...current, llm: { activeTextProvider, providers } }, null, 2), + "utf8", + ); + resetConfigCache(); +} + +async function getPerformance( + harness: Harness, + headers: Record = {}, +): Promise<{ status: number; body: Record }> { + const res = await fetch(`${harness.baseUrl}/api/local-model/performance`, { headers }); + return { status: res.status, body: (await res.json()) as Record }; +} + +describe("GET /api/local-model/performance (issue #996)", () => { + let harness: Harness | null = null; + let server: Awaited> | null = null; + + afterEach(async () => { + if (harness) await harness.cleanup(); + harness = null; + if (server) await server.stop(); + server = null; + }); + + async function withSlots(reply: SlotsReply): Promise { + server = await startSlotsServer(reply); + harness = await startTestHarness({ localModelsUrl: server.url }); + return harness; + } + + it("answers a fresh agent with no samples and the live slots", async () => { + const h = await withSlots({ + status: 200, + body: JSON.stringify([ + { id: 0, n_ctx: 262144, is_processing: true }, + { id: 1, n_ctx: 262144, is_processing: false }, + ]), + }); + const { status, body } = await getPerformance(h); + expect(status).toBe(200); + expect(body).toEqual({ + local: true, + url: server!.url, + capacity: 300, + samples: [], + slots: { total: 2, busy: 1, contextPerSlot: 262144 }, + }); + expect(server!.hits()).toBe(1); + }); + + it("reports slots as null when /slots hangs past its bound", async () => { + const h = await withSlots("hang"); + const started = Date.now(); + const { status, body } = await getPerformance(h); + expect(status).toBe(200); + expect(body.slots).toBeNull(); + expect(Date.now() - started).toBeGreaterThanOrEqual(1400); + expect(Date.now() - started).toBeLessThan(5000); + }); + + const unreadable: Array<[string, SlotsReply]> = [ + ["a 404", { status: 404, body: "{}" }], + ["a non-array body", { status: 200, body: JSON.stringify({ slots: [] }) }], + ["an array of no slots", { status: 200, body: JSON.stringify([null]) }], + ]; + for (const [name, reply] of unreadable) { + it(`reports slots as null for ${name}`, async () => { + const h = await withSlots(reply); + const { status, body } = await getPerformance(h); + expect(status).toBe(200); + expect(body.slots).toBeNull(); + expect(body.local).toBe(true); + }); + } + + it("keeps the load when a slot has no n_ctx", async () => { + const h = await withSlots({ + status: 200, + body: JSON.stringify([{ id: 0, is_processing: true }]), + }); + const { body } = await getPerformance(h); + expect(body.slots).toEqual({ total: 1, busy: 1, contextPerSlot: null }); + }); + + it("asks nothing of the local server while the chat is on a cloud model", async () => { + const h = await withSlots({ status: 200, body: JSON.stringify([{ id: 0, n_ctx: 8 }]) }); + let slotsCalls = 0; + const runtime = h.runtime as AgentRuntime; + runtime.localPerformance = () => ({ + capacity: 300, + samples: [SAMPLE], + fetchSlots: async () => { + slotsCalls += 1; + return []; + }, + }); + setProviders(h, "cloud", [ + { id: "local", kind: "llama-server" }, + { id: "cloud", kind: "openrouter" }, + ]); + const { status, body } = await getPerformance(h); + expect(status).toBe(200); + expect(body).toEqual({ + local: false, + url: server!.url, + capacity: 300, + samples: [SAMPLE], + slots: null, + }); + expect(slotsCalls).toBe(0); + expect(server!.hits()).toBe(0); + }); + + it("counts a custom provider of kind llama-server as local", async () => { + const h = await withSlots({ status: 200, body: JSON.stringify([{ id: 0, n_ctx: 4096 }]) }); + setProviders(h, "box", [ + { id: "box", kind: "llama-server" }, + { id: "cloud", kind: "openrouter" }, + ]); + const { body } = await getPerformance(h); + expect(body.local).toBe(true); + expect(body.slots).toEqual({ total: 1, busy: 0, contextPerSlot: 4096 }); + }); + + it("answers without samples or slots on a runtime that has no local performance", async () => { + const h = await withSlots({ status: 200, body: JSON.stringify([{ id: 0, n_ctx: 8 }]) }); + const runtime = h.runtime as AgentRuntime; + delete runtime.localPerformance; + const { status, body } = await getPerformance(h); + expect(status).toBe(200); + expect(body).toMatchObject({ local: true, capacity: 300, samples: [], slots: null }); + expect(server!.hits()).toBe(0); + }); + + it("sits behind the API key", async () => { + server = await startSlotsServer({ status: 200, body: "[]" }); + harness = await startTestHarness({ apiKey: "secret", localModelsUrl: server.url }); + const missing = await fetch(`${harness.baseUrl}/api/local-model/performance`); + expect(missing.status).toBe(401); + const { status, body } = await getPerformance(harness, { authorization: "Bearer secret" }); + expect(status).toBe(200); + expect(body.capacity).toBe(300); + }); +}); diff --git a/src/http/route-local-model-performance.ts b/src/http/route-local-model-performance.ts new file mode 100644 index 000000000..7fdc56ca9 --- /dev/null +++ b/src/http/route-local-model-performance.ts @@ -0,0 +1,79 @@ +import { getConfig } from "../config/index.js"; +import { + activeTextProviderIsLlamaServer, + resolveLlmConfig, +} from "../llm/provider/registry/index.js"; +import { PERFORMANCE_CAPACITY } from "../llm/llama-performance-samples.js"; +import { readSlotOccupancy } from "../tools/fusion/worker-result.js"; +import { sendJson, type HttpHandler } from "./request-context.js"; + +/** How long the route waits for the local server's `/slots`. */ +export const PERFORMANCE_SLOTS_TIMEOUT_MS = 1500; + +export interface PerformanceSlots { + total: number; + busy: number; + /** The first slot's `n_ctx`; `null` when the server does not report it. */ + contextPerSlot: number | null; +} + +/** + * `GET /api/local-model/performance` — the desktop's Performance pane + * (issue #996). Answers + * + * `{ local, url, capacity, samples, slots }` + * + * `samples` are the local model server's recent completed requests, + * oldest first: speeds, token counts, first-token wait and how many + * requests shared the server, never any prompt or reply text. `local` + * says whether the running agent's chat goes to llama-server at all; it + * and `url` come from the cached config the live turns read, which + * `POST /api/llm/active` resets. Only when `local` is true does the route + * ask the server's `/slots` (once, bounded at 1.5 s) for the live load + * and the context per slot; any failure there is `slots: null`, never an + * error status. Writes nothing. Behind the API key like every `/api` + * route. + */ +export function createLocalModelPerformanceHandler(): HttpHandler { + return async (_req, res, ctx) => { + const config = getConfig(); + const local = activeTextProviderIsLlamaServer(resolveLlmConfig(config)); + const source = ctx.runtime.localPerformance?.() ?? null; + const slots = local && source !== null ? await readSlots(source.fetchSlots) : null; + sendJson(res, 200, { + local, + url: config.localModels.url, + capacity: source?.capacity ?? PERFORMANCE_CAPACITY, + samples: source?.samples ?? [], + slots, + }); + }; +} + +async function readSlots( + fetchSlots: (timeoutMs: number) => Promise, +): Promise { + let body: unknown; + try { + body = await fetchSlots(PERFORMANCE_SLOTS_TIMEOUT_MS); + } catch { + return null; + } + return slotsFromBody(body); +} + +/** The slot table's load and context size, or `null` when it is not a slot table. */ +export function slotsFromBody(body: unknown): PerformanceSlots | null { + const occupancy = readSlotOccupancy(body); + if (occupancy === null || !Array.isArray(body)) return null; + const first = body.find( + (entry): entry is Record => entry !== null && typeof entry === "object", + ); + const nCtx = first?.n_ctx; + return { + total: occupancy.total, + busy: occupancy.busy, + contextPerSlot: + typeof nCtx === "number" && Number.isFinite(nCtx) && nCtx > 0 ? nCtx : null, + }; +} diff --git a/src/http/route-table.ts b/src/http/route-table.ts index ce36beee9..ec91b37dd 100644 --- a/src/http/route-table.ts +++ b/src/http/route-table.ts @@ -6,6 +6,7 @@ import { createChatCompletionsHandler } from "./openai-chat-completions.js"; import { createCancelCompletionHandler } from "./openai-completions-cancel.js"; import { createHealthHandler } from "./route-health.js"; import { createLocalModelRecheckHandler } from "./route-local-model-recheck.js"; +import { createLocalModelPerformanceHandler } from "./route-local-model-performance.js"; import { createModelTiersHandler } from "./route-model-tiers.js"; import { createModelsHandler } from "./route-models.js"; import { createCapabilitiesHandler } from "./route-capabilities.js"; @@ -226,6 +227,8 @@ export function buildRouteTable(): RouteDefinition[] { { method: "POST", path: "/api/context-preview", handler: createContextPreviewHandler() }, // ATO-873: the desktop's Reconnect button after a local server answered 404/405. { method: "POST", path: "/api/local-model/recheck", handler: createLocalModelRecheckHandler() }, + // Issue #996: the desktop's Performance pane; recent local request timings and the live slots. + { method: "GET", path: "/api/local-model/performance", handler: createLocalModelPerformanceHandler() }, // ATO-1021: the desktop's model pick, swapped live instead of restarting serve. { method: "POST", path: "/api/llm/active", handler: createSetActiveLlmHandler() }, // ATO-1024: the desktop starts serve first and attaches the embedding server once it is up. diff --git a/src/llm/llama-performance-samples.test.ts b/src/llm/llama-performance-samples.test.ts new file mode 100644 index 000000000..7ad900698 --- /dev/null +++ b/src/llm/llama-performance-samples.test.ts @@ -0,0 +1,235 @@ +import { describe, expect, it } from "vitest"; + +import { + PERFORMANCE_CAPACITY, + PerformanceRing, + count, + generationRate, + lenientNum, + num, + sampleFromPayload, + type PerformanceSample, +} from "./llama-performance-samples.js"; + +/** A real final `/completion` event from llama.cpp b11028, with a neutral path. */ +const FINAL_EVENT: Record = { + content: "", + stop: true, + timings: { + cache_n: 0, + prompt_n: 2, + prompt_ms: 32.969, + prompt_per_token_ms: 16.4845, + prompt_per_second: 60.663047104856076, + predicted_n: 40, + predicted_ms: 1158.29, + predicted_per_token_ms: 29.699743589743587, + predicted_per_second: 33.67032435745798, + draft_n: 96, + draft_n_accepted: 13, + }, + tokens_predicted: 40, + tokens_evaluated: 2, + tokens_cached: 41, + model: "/home/user/models/Qwen3.6-35B-A3B-MXFP4_MOE.gguf", +}; + +const CONTEXT = { streamed: true, ttftMs: 412, concurrent: 1, at: 1794329429000 }; + +function withTimings( + timings: Record, + extra: Record = {}, +): Record { + return { stop: true, timings, ...extra }; +} + +function sample(payload: Record): PerformanceSample { + const s = sampleFromPayload(payload, CONTEXT); + if (s === null) throw new Error("expected a sample"); + return s; +} + +describe("sampleFromPayload", () => { + it("maps llama.cpp's final event exactly", () => { + expect(sampleFromPayload(FINAL_EVENT, CONTEXT)).toEqual({ + at: 1794329429000, + modelId: "Qwen3.6-35B-A3B-MXFP4_MOE.gguf", + streamed: true, + promptTokens: 2, + cacheTokens: 0, + promptMs: 32.969, + promptPerSecond: 60.663047104856076, + predictedTokens: 40, + predictedMs: 1158.29, + generationPerSecond: 33.67032435745798, + ttftMs: 412, + contextTokens: 42, + concurrent: 1, + }); + }); + + it("passes the call's own facts through", () => { + const s = sampleFromPayload(FINAL_EVENT, { + streamed: false, + ttftMs: null, + concurrent: 3, + at: 5, + }); + expect(s).toMatchObject({ streamed: false, ttftMs: null, concurrent: 3, at: 5 }); + }); + + it("computes the prompt speed when the server does not state it", () => { + const s = sample(withTimings({ prompt_n: 100, prompt_ms: 400, cache_n: 8 })); + expect(s.promptPerSecond).toBe(250); + expect(s.cacheTokens).toBe(8); + }); + + it("gives a fully cached prompt no speed, whatever prompt_per_second says", () => { + const s = sample( + withTimings({ prompt_n: 0, prompt_ms: 1, prompt_per_second: 999, cache_n: 50 }), + ); + expect(s.promptTokens).toBe(0); + expect(s.promptPerSecond).toBeNull(); + expect(s.contextTokens).toBe(50); + }); + + it("leaves the prompt and context unknown without prompt_n", () => { + const s = sample(withTimings({ predicted_n: 4, cache_n: 9, prompt_per_second: 10 })); + expect(s.promptTokens).toBeNull(); + expect(s.promptPerSecond).toBeNull(); + expect(s.contextTokens).toBeNull(); + }); + + it("falls back to tokens_predicted for both the count and the context", () => { + const s = sample(withTimings({ prompt_n: 3, cache_n: 2 }, { tokens_predicted: 7 })); + expect(s.predictedTokens).toBe(7); + expect(s.contextTokens).toBe(12); + }); + + it("never reads top-level tokens_cached into the context", () => { + const s = sample(withTimings({ prompt_n: 3 }, { tokens_cached: 1000 })); + expect(s.cacheTokens).toBe(0); + expect(s.contextTokens).toBe(3); + }); + + it("turns NaN, negative and string values into null", () => { + const s = sample( + withTimings({ + prompt_n: Number.NaN, + cache_n: -4, + prompt_ms: "32", + prompt_per_second: "60", + predicted_n: -1, + predicted_ms: Number.POSITIVE_INFINITY, + predicted_per_second: "33", + }), + ); + expect(s).toMatchObject({ + promptTokens: null, + cacheTokens: 0, + promptMs: null, + promptPerSecond: null, + predictedTokens: 0, + predictedMs: null, + generationPerSecond: null, + contextTokens: null, + }); + expect( + sample(withTimings({ prompt_n: "5", predicted_n: "2" }, { tokens_predicted: "2" })), + ).toMatchObject({ promptTokens: null, predictedTokens: 0 }); + }); + + it("records nothing without a timings object", () => { + expect(sampleFromPayload({ stop: true }, CONTEXT)).toBeNull(); + expect(sampleFromPayload({ stop: true, timings: null }, CONTEXT)).toBeNull(); + expect(sampleFromPayload({ stop: true, timings: 5 }, CONTEXT)).toBeNull(); + }); + + it("strips a model path to its file name, on either separator", () => { + expect(sample(withTimings({}, { model: "/a/b/m.gguf" })).modelId).toBe("m.gguf"); + expect(sample(withTimings({}, { model: "C:\\models\\w.gguf" })).modelId).toBe("w.gguf"); + expect(sample(withTimings({}, { model: "bare-alias" })).modelId).toBe("bare-alias"); + expect(sample(withTimings({}, { model: "" })).modelId).toBeNull(); + expect(sample(withTimings({}, { model: 42 })).modelId).toBeNull(); + expect(sample(withTimings({})).modelId).toBeNull(); + }); +}); + +describe("num / count", () => { + it("accept only finite numbers, and counts only non-negative ones", () => { + expect(num(1.5)).toBe(1.5); + expect(num("1.5")).toBeNull(); + expect(num(Number.NaN)).toBeNull(); + expect(count(2.9)).toBe(2); + expect(count(0)).toBe(0); + expect(count(-1)).toBeNull(); + expect(count(undefined)).toBeNull(); + }); +}); + +describe("generationRate", () => { + // The arithmetic `recordThroughput` used before it called this helper, + // for numeric input: the stated rate when positive, else n / ms. + function before(t: Record): number | null { + const stated = typeof t.predicted_per_second === "number" ? t.predicted_per_second : Number.NaN; + if (Number.isFinite(stated) && stated > 0) return stated; + const n = typeof t.predicted_n === "number" && Number.isFinite(t.predicted_n) ? t.predicted_n : 0; + const ms = typeof t.predicted_ms === "number" && Number.isFinite(t.predicted_ms) ? t.predicted_ms : 0; + const rate = n > 0 && ms > 0 ? (n / ms) * 1000 : Number.NaN; + return Number.isFinite(rate) && rate > 0 ? rate : null; + } + + const cases: Array<[string, Record, number | null]> = [ + ["stated", { predicted_per_second: 33.67032435745798, predicted_n: 1, predicted_ms: 1 }, 33.67032435745798], + ["absent: from n and ms", { predicted_n: 100, predicted_ms: 10_000 }, 10], + ["zero", { predicted_per_second: 0, predicted_n: 0, predicted_ms: 0 }, null], + ["missing", {}, null], + ]; + + for (const [name, timings, expected] of cases) { + it(`matches the old arithmetic: ${name}`, () => { + expect(generationRate(timings)).toBe(expected); + expect(generationRate(timings)).toBe(before(timings)); + }); + } + + it("never returns a non-finite rate, whatever n / ms works out to", () => { + expect(generationRate({ predicted_n: 1, predicted_ms: 1e-320 })).toBeNull(); + expect(generationRate({ predicted_n: "1", predicted_ms: "1e-320" }, lenientNum)).toBeNull(); + }); + + it("reads numeric strings only when asked, as the throughput mean always has", () => { + const t = { predicted_per_second: "33.5", predicted_n: "10", predicted_ms: "1000" }; + expect(generationRate(t)).toBeNull(); + expect(generationRate(t, lenientNum)).toBe(33.5); + expect(generationRate({ predicted_n: "100", predicted_ms: "10000" }, lenientNum)).toBe(10); + expect(lenientNum("abc")).toBeNull(); + expect(lenientNum("")).toBeNull(); + }); +}); + +describe("PerformanceRing", () => { + function at(n: number): PerformanceSample { + return { ...sample(FINAL_EVENT), at: n }; + } + + it("keeps the newest 300, oldest first", () => { + const ring = new PerformanceRing(); + expect(ring.capacity).toBe(PERFORMANCE_CAPACITY); + expect(PERFORMANCE_CAPACITY).toBe(300); + for (let i = 1; i <= 305; i += 1) ring.push(at(i)); + const list = ring.list(); + expect(list).toHaveLength(300); + expect(list[0]?.at).toBe(6); + expect(list[299]?.at).toBe(305); + }); + + it("hands out a copy", () => { + const ring = new PerformanceRing(2); + ring.push(at(1)); + const list = ring.list(); + list.pop(); + list.push(at(99)); + expect(ring.list().map((s) => s.at)).toEqual([1]); + }); +}); diff --git a/src/llm/llama-performance-samples.ts b/src/llm/llama-performance-samples.ts new file mode 100644 index 000000000..1afa0fb92 --- /dev/null +++ b/src/llm/llama-performance-samples.ts @@ -0,0 +1,169 @@ +/** + * Per-request timings of the local model server, kept for the + * performance view (`GET /api/local-model/performance`, issue #996). + * + * One sample per completed llama-server completion: the speeds and + * token counts llama.cpp reports in the final event's `timings`, plus + * what only the client can see — how long the person waited for the + * first token and how many requests shared the server meanwhile. No + * prompt or reply text is ever kept. + */ + +/** How many samples a client keeps, newest last. */ +export const PERFORMANCE_CAPACITY = 300; + +export interface PerformanceSample { + /** `Date.now()` when the completion finished. */ + at: number; + /** The model file's name, without its directory; `null` when not reported. */ + modelId: string | null; + streamed: boolean; + /** Prompt tokens the server evaluated this time (`prompt_n`). */ + promptTokens: number | null; + /** Prompt tokens reused from the KV cache (`cache_n`). */ + cacheTokens: number; + promptMs: number | null; + /** `null` for a prompt served entirely from the cache. */ + promptPerSecond: number | null; + predictedTokens: number; + predictedMs: number | null; + generationPerSecond: number | null; + /** Wait from the call to its first content or reasoning delta; streaming only. */ + ttftMs: number | null; + /** `promptTokens + cacheTokens + predictedTokens`, `null` without `prompt_n`. */ + contextTokens: number | null; + /** Most completion calls open on this client at once during this one (≥ 1). */ + concurrent: number; +} + +/** `v` when it is a finite number, else `null`. Strings are not numbers here. */ +export function num(v: unknown): number | null { + return typeof v === "number" && Number.isFinite(v) ? v : null; +} + +/** + * `num`, but a numeric string counts too. The client's throughput mean + * (what Fusion sizes worker time limits from) has always read timings this + * way, so it keeps doing so through `generationRate(t, lenientNum)`; the + * performance samples read numbers only. + */ +export function lenientNum(v: unknown): number | null { + if (typeof v === "string" && v.length > 0) { + const parsed = Number.parseFloat(v); + return Number.isFinite(parsed) ? parsed : null; + } + return num(v); +} + +/** A whole, non-negative count, or `null`. */ +export function count(v: unknown): number | null { + const n = num(v); + return n !== null && n >= 0 ? Math.trunc(n) : null; +} + +/** + * Generation speed, tokens per second, from one completion's `timings`. + * llama.cpp states it directly (`predicted_per_second`); an older server + * without it still reports the count and the milliseconds it took. + * `null` when neither gives a positive speed. + */ +export function generationRate( + timings: Record, + read: (v: unknown) => number | null = num, +): number | null { + const stated = read(timings.predicted_per_second); + if (stated !== null && stated > 0) return stated; + const n = read(timings.predicted_n); + const ms = read(timings.predicted_ms); + if (n === null || ms === null || n <= 0 || ms <= 0) return null; + // A tiny `ms` can overflow the division; the mean never took Infinity. + const rate = (n / ms) * 1000; + return Number.isFinite(rate) && rate > 0 ? rate : null; +} + +/** The last segment of a path, so a local file path never leaves the process. */ +function modelName(model: unknown): string | null { + if (typeof model !== "string" || model.length === 0) return null; + const segments = model.split(/[/\\]/); + const last = segments[segments.length - 1] ?? ""; + return last.length > 0 ? last : null; +} + +export interface SampleContext { + streamed: boolean; + ttftMs: number | null; + concurrent: number; + at: number; +} + +/** + * Map one llama-server completion payload (the unary response, or the + * stream's final `stop: true` event) to a sample. `null` when the payload + * carries no `timings` object. + */ +export function sampleFromPayload( + payload: Record, + context: SampleContext, +): PerformanceSample | null { + const timings = payload.timings; + if (timings === null || typeof timings !== "object" || Array.isArray(timings)) { + return null; + } + const t = timings as Record; + const promptTokens = count(t.prompt_n); + const cacheTokens = count(t.cache_n) ?? 0; + const promptMs = num(t.prompt_ms); + const predictedTokens = count(t.predicted_n) ?? count(payload.tokens_predicted) ?? 0; + return { + at: context.at, + modelId: modelName(payload.model), + streamed: context.streamed, + promptTokens, + cacheTokens, + promptMs, + promptPerSecond: promptRate(t, promptTokens, promptMs), + predictedTokens, + predictedMs: num(t.predicted_ms), + generationPerSecond: generationRate(t), + ttftMs: context.ttftMs, + // `tokens_cached` is deliberately not read: see `resolvePromptUsage`. + contextTokens: + promptTokens === null ? null : promptTokens + cacheTokens + predictedTokens, + concurrent: context.concurrent, + }; +} + +/** + * Prompt speed. A prompt served wholly from the cache evaluated nothing, + * so it has no speed, whatever `prompt_per_second` claims. + */ +function promptRate( + t: Record, + promptTokens: number | null, + promptMs: number | null, +): number | null { + if (promptTokens === null || promptTokens === 0) return null; + const stated = num(t.prompt_per_second); + if (stated !== null && stated > 0) return stated; + if (promptMs !== null && promptMs > 0) return (promptTokens / promptMs) * 1000; + return null; +} + +/** A bounded list of samples: the newest `capacity`, oldest first. */ +export class PerformanceRing { + private readonly items: PerformanceSample[] = []; + + constructor(readonly capacity: number = PERFORMANCE_CAPACITY) {} + + push(sample: PerformanceSample): void { + this.items.push(sample); + if (this.items.length > this.capacity) { + this.items.splice(0, this.items.length - this.capacity); + } + } + + /** A copy, oldest first. */ + list(): PerformanceSample[] { + return this.items.slice(); + } +} diff --git a/src/llm/llama-server-client-performance.test.ts b/src/llm/llama-server-client-performance.test.ts new file mode 100644 index 000000000..8f3a28419 --- /dev/null +++ b/src/llm/llama-server-client-performance.test.ts @@ -0,0 +1,310 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { LlamaServerClient } from "./llama-server-client.js"; +import type { StreamChunk } from "./llama-server-client.js"; + +/** + * The client's per-request performance samples (issue #996): one per + * completed call, committed only when the call succeeds, with how many + * calls overlapped it and how long the first token took. + */ + +const TIMINGS = { + cache_n: 0, + prompt_n: 2, + prompt_ms: 32.969, + prompt_per_second: 60.663047104856076, + predicted_n: 40, + predicted_ms: 1158.29, + predicted_per_second: 33.67032435745798, +}; + +const FINAL = { + content: "", + stop: true, + timings: TIMINGS, + tokens_predicted: 40, + tokens_evaluated: 2, + model: "/home/user/models/Qwen3.6-35B-A3B-MXFP4_MOE.gguf", +}; + +const DELTA = 'data: {"content":"hi","stop":false}\n\n'; +const STOP = `data: ${JSON.stringify(FINAL)}\n\n`; + +type Reply = (init: RequestInit) => Response | Promise; + +/** A client whose fetch answers from `replies` in order, the last one repeating. */ +function clientWith( + replies: Reply[], + extra: { retries?: number; backoffMs?: number; sleep?: (ms: number) => Promise } = {}, +): { client: LlamaServerClient; calls: () => number } { + let n = 0; + const client = new LlamaServerClient({ + baseUrl: "http://127.0.0.1:9999", + progressWatch: false, + completionRetries: extra.retries ?? 1, + completionRetryBackoffMs: extra.backoffMs ?? 0, + sleep: extra.sleep ?? (async () => {}), + fetchImpl: (async (_input: string | URL | Request, init?: RequestInit) => { + const reply = replies[Math.min(n, replies.length - 1)]!; + n += 1; + return reply(init ?? {}); + }) as typeof fetch, + }); + return { client, calls: () => n }; +} + +function sse(chunks: string[], end: "close" | "error" = "close"): Response { + const encoder = new TextEncoder(); + let i = 0; + const stream = new ReadableStream({ + pull(controller) { + if (i < chunks.length) { + controller.enqueue(encoder.encode(chunks[i]!)); + i += 1; + return; + } + if (end === "close") controller.close(); + else controller.error(new TypeError("terminated")); + }, + }); + return new Response(stream, { + status: 200, + headers: { "content-type": "text/event-stream" }, + }); +} + +function json(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { "content-type": "application/json" }, + }); +} + +const ok: Reply = () => sse([DELTA, STOP]); +const unaryOk: Reply = () => json({ ...FINAL, content: "hi" }); +const unavailable: Reply = () => new Response("unavailable", { status: 503 }); + +async function drain(stream: AsyncGenerator): Promise { + let next = await stream.next(); + while (!next.done) next = await stream.next(); +} + +/** A fresh lone request on `client` records `concurrent: 1`: nothing was left open. */ +async function expectNothingLeftOpen(client: LlamaServerClient): Promise { + const before = client.performanceSamples().length; + await drain(client.completeStream({ prompt: "again" })); + const samples = client.performanceSamples(); + expect(samples).toHaveLength(before + 1); + expect(samples[samples.length - 1]?.concurrent).toBe(1); +} + +afterEach(() => { + vi.useRealTimers(); +}); + +describe("LlamaServerClient performance samples", () => { + it("records a streamed completion", async () => { + const { client } = clientWith([ok]); + expect(client.performanceCapacity).toBe(300); + expect(client.performanceSamples()).toEqual([]); + await drain(client.completeStream({ prompt: "p" })); + const samples = client.performanceSamples(); + expect(samples).toHaveLength(1); + const s = samples[0]!; + expect(s).toMatchObject({ + streamed: true, + concurrent: 1, + modelId: "Qwen3.6-35B-A3B-MXFP4_MOE.gguf", + generationPerSecond: 33.67032435745798, + promptPerSecond: 60.663047104856076, + contextTokens: 42, + }); + expect(Number.isFinite(s.ttftMs)).toBe(true); + expect(s.ttftMs).toBeGreaterThanOrEqual(0); + // The throughput mean still sees the same completion. + expect(client.measuredTokensPerSecond()).toBe(33.67032435745798); + }); + + it("records a unary completion without a first-token time", async () => { + const { client } = clientWith([unaryOk]); + await client.complete({ prompt: "p" }); + expect(client.performanceSamples()).toEqual([ + expect.objectContaining({ streamed: false, ttftMs: null, concurrent: 1 }), + ]); + }); + + it("hands out a copy of the samples", async () => { + const { client } = clientWith([unaryOk]); + await client.complete({ prompt: "p" }); + client.performanceSamples().pop(); + expect(client.performanceSamples()).toHaveLength(1); + }); + + it("records nothing for a completion without timings", async () => { + const { client } = clientWith([() => json({ content: "x", stop: true })]); + await client.complete({ prompt: "p" }); + expect(client.performanceSamples()).toEqual([]); + }); + + it("counts two overlapping streams as two", async () => { + const { client } = clientWith([ok]); + const a = client.completeStream({ prompt: "a" }); + const b = client.completeStream({ prompt: "b" }); + expect((await a.next()).value).toMatchObject({ delta: "hi" }); + expect((await b.next()).value).toMatchObject({ delta: "hi" }); + await drain(a); + await drain(b); + expect(client.performanceSamples().map((s) => s.concurrent)).toEqual([2, 2]); + await expectNothingLeftOpen(client); + }); + + it("counts a unary call that overlaps a stream on both", async () => { + const { client } = clientWith([ok, unaryOk, ok]); + const stream = client.completeStream({ prompt: "a" }); + expect((await stream.next()).value).toMatchObject({ delta: "hi" }); + await client.complete({ prompt: "b" }); + await drain(stream); + const samples = client.performanceSamples(); + expect(samples.map((s) => [s.streamed, s.concurrent])).toEqual([ + [false, 2], + [true, 2], + ]); + await expectNothingLeftOpen(client); + }); + + it("counts a retried stream once and includes the backoff in its first-token time", async () => { + vi.useFakeTimers({ toFake: ["Date"] }); + const slept: number[] = []; + const { client, calls } = clientWith([unavailable, ok], { + retries: 3, + backoffMs: 400, + sleep: async (ms) => { + slept.push(ms); + vi.setSystemTime(Date.now() + ms); + }, + }); + await drain(client.completeStream({ prompt: "p" })); + expect(calls()).toBe(2); + expect(slept).toHaveLength(1); + const samples = client.performanceSamples(); + expect(samples).toHaveLength(1); + expect(samples[0]?.concurrent).toBe(1); + expect(samples[0]?.ttftMs).toBeGreaterThanOrEqual(slept[0]!); + vi.useRealTimers(); + await expectNothingLeftOpen(client); + }); + + it("counts a retried unary call once", async () => { + const { client } = clientWith([unavailable, unaryOk], { retries: 3 }); + await client.complete({ prompt: "p" }); + expect(client.performanceSamples().map((s) => s.concurrent)).toEqual([1]); + }); + + it("records nothing when the retries run out", async () => { + const { client } = clientWith([unavailable, unavailable, ok], { retries: 2 }); + await expect(drain(client.completeStream({ prompt: "p" }))).rejects.toThrow(); + expect(client.performanceSamples()).toEqual([]); + await expectNothingLeftOpen(client); + }); + + it("records nothing for an HTTP 500, streamed or unary", async () => { + const { client } = clientWith([ + () => new Response("boom", { status: 500 }), + () => new Response("boom", { status: 500 }), + ok, + ]); + await expect(drain(client.completeStream({ prompt: "p" }))).rejects.toThrow(); + await expect(client.complete({ prompt: "p" })).rejects.toThrow(); + expect(client.performanceSamples()).toEqual([]); + await expectNothingLeftOpen(client); + }); + + it("records nothing for a call aborted before its first byte", async () => { + const hang: Reply = (init) => + new Promise((_resolve, reject) => { + init.signal?.addEventListener("abort", () => + reject(Object.assign(new Error("aborted"), { name: "AbortError" })), + ); + }); + const { client } = clientWith([hang, ok]); + const abort = new AbortController(); + const stream = client.completeStream({ prompt: "p", signal: abort.signal }); + const pending = stream.next(); + await Promise.resolve(); + abort.abort(); + await expect(pending).rejects.toThrow(); + expect(client.performanceSamples()).toEqual([]); + await expectNothingLeftOpen(client); + }); + + it("records nothing for a call aborted during the backoff", async () => { + const abort = new AbortController(); + const { client } = clientWith([unavailable, ok], { + retries: 3, + sleep: async () => abort.abort(), + }); + await expect( + drain(client.completeStream({ prompt: "p", signal: abort.signal })), + ).rejects.toThrow(); + const unary = new AbortController(); + const second = clientWith([unavailable, unaryOk], { + retries: 3, + sleep: async () => unary.abort(), + }); + await expect(second.client.complete({ prompt: "p", signal: unary.signal })).rejects.toThrow(); + expect(client.performanceSamples()).toEqual([]); + expect(second.client.performanceSamples()).toEqual([]); + await expectNothingLeftOpen(client); + }); + + it("records nothing when the transport fails after the stop event", async () => { + const { client } = clientWith([() => sse([DELTA, STOP], "error"), ok]); + const stream = client.completeStream({ prompt: "p" }); + let next = await stream.next(); + while (!next.done && !next.value.done) next = await stream.next(); + expect(next.value).toMatchObject({ done: true }); + await expect(stream.next()).rejects.toThrow(); + expect(client.performanceSamples()).toEqual([]); + await expectNothingLeftOpen(client); + }); + + it("records nothing when the consumer walks away after the done chunk", async () => { + const { client } = clientWith([ok]); + const stream = client.completeStream({ prompt: "p" }); + let next = await stream.next(); + while (!next.done && !next.value.done) next = await stream.next(); + expect(next.value).toMatchObject({ done: true }); + await stream.return(undefined as never); + expect(client.performanceSamples()).toEqual([]); + await expectNothingLeftOpen(client); + }); + + it("records nothing when the caller aborts after the done chunk", async () => { + const { client } = clientWith([ok]); + const abort = new AbortController(); + const stream = client.completeStream({ prompt: "p", signal: abort.signal }); + let next = await stream.next(); + while (!next.done && !next.value.done) next = await stream.next(); + expect(next.value).toMatchObject({ done: true }); + abort.abort(); + await stream.next().catch(() => undefined); + expect(client.performanceSamples()).toEqual([]); + await expectNothingLeftOpen(client); + }); + + it("records nothing for a stream that ends without a stop event", async () => { + const { client } = clientWith([() => sse([DELTA]), ok]); + await drain(client.completeStream({ prompt: "p" })); + expect(client.performanceSamples()).toEqual([]); + await expectNothingLeftOpen(client); + }); + + it("has no first-token time for a stream with no delta before its stop", async () => { + const { client } = clientWith([() => sse([STOP])]); + await drain(client.completeStream({ prompt: "p" })); + expect(client.performanceSamples()).toEqual([ + expect.objectContaining({ streamed: true, ttftMs: null, concurrent: 1 }), + ]); + }); +}); diff --git a/src/llm/llama-server-client.test.ts b/src/llm/llama-server-client.test.ts index 09d071a2b..ce5d82513 100644 --- a/src/llm/llama-server-client.test.ts +++ b/src/llm/llama-server-client.test.ts @@ -85,6 +85,27 @@ describe("LlamaServerClient.measuredTokensPerSecond (F19)", () => { expect(client.measuredTokensPerSecond()).toBe(20); }); + it("ignores a rate that overflows to Infinity, as it always has", async () => { + const client = clientWith([ + { timings: { predicted_per_second: 30 } }, + { timings: { predicted_n: 1, predicted_ms: 1e-320 } }, + ]); + await client.complete({ prompt: "p" }); + await client.complete({ prompt: "p" }); + expect(client.measuredTokensPerSecond()).toBe(30); + }); + + it("still reads numeric-string timings, as it always has", async () => { + const client = clientWith([ + { timings: { predicted_per_second: "40" } }, + { timings: { predicted_n: "100", predicted_ms: "10000" } }, + ]); + await client.complete({ prompt: "p" }); + expect(client.measuredTokensPerSecond()).toBe(40); + await client.complete({ prompt: "p" }); + expect(client.measuredTokensPerSecond()).toBe(25); + }); + it("keeps only the last eight completions, so it follows the load", async () => { const client = clientWith([{ timings: { predicted_per_second: 100 } }]); for (let i = 0; i < 3; i += 1) await client.complete({ prompt: "p" }); diff --git a/src/llm/llama-server-client.ts b/src/llm/llama-server-client.ts index ba75c6186..d9b6aac80 100644 --- a/src/llm/llama-server-client.ts +++ b/src/llm/llama-server-client.ts @@ -11,6 +11,14 @@ import { createStreamRepetitionWatcher } from "./reliability/stream-repetition-w import { drySamplerFields } from "./dry-sampler.js"; import { takePromptProgress } from "./llama-prompt-progress.js"; import type { StructuredLogger } from "../tracing/structured-logger.js"; +import { + PERFORMANCE_CAPACITY, + PerformanceRing, + generationRate, + lenientNum, + sampleFromPayload, + type PerformanceSample, +} from "./llama-performance-samples.js"; import type { CompletionRequest, CompletionResult, @@ -586,6 +594,16 @@ export interface LlamaServerProps { /** Completions the rolling throughput mean is taken over. */ const THROUGHPUT_WINDOW = 8; +/** + * One `complete()` / `completeStream()` call while it is open, retries + * and backoff included. `peak` is the most calls open on this client at + * any moment of its life, itself counted. + */ +interface OpenCall { + readonly openedAt: number; + peak: number; +} + export class LlamaServerClient { /** When set, this fixed base wins; otherwise each request reads `getConfig().llama.url`. */ private readonly baseUrlOverride: string | undefined; @@ -611,6 +629,12 @@ export class LlamaServerClient { private readonly logger: Pick | undefined; /** Recent generation speeds, tokens/s, newest last — see `measuredTokensPerSecond`. */ private readonly throughputSamples: number[] = []; + /** Completed calls' timings for the performance view — see `performanceSamples`. */ + private readonly performance = new PerformanceRing(); + /** The completion calls open right now — see `openCall`. */ + private readonly openCalls = new Set(); + /** How many samples `performanceSamples()` keeps. */ + readonly performanceCapacity = PERFORMANCE_CAPACITY; /** * The base URL whose `/slots` answered 404/405/501 — the endpoint is * not built into this llama.cpp, so the progress watch can never do @@ -742,20 +766,63 @@ export class LlamaServerClient { private recordThroughput(payload: Record): void { const timings = payload.timings; if (timings === null || typeof timings !== "object") return; - const t = timings as Record; - let perSecond = toNumber(t.predicted_per_second, Number.NaN); - if (!Number.isFinite(perSecond) || perSecond <= 0) { - const n = toNumber(t.predicted_n, 0); - const ms = toNumber(t.predicted_ms, 0); - perSecond = n > 0 && ms > 0 ? (n / ms) * 1000 : Number.NaN; - } - if (!Number.isFinite(perSecond) || perSecond <= 0) return; + const perSecond = generationRate(timings as Record, lenientNum); + if (perSecond === null) return; this.throughputSamples.push(perSecond); if (this.throughputSamples.length > THROUGHPUT_WINDOW) { this.throughputSamples.shift(); } } + /** + * The timings of this client's recent completed calls, oldest first, at + * most `performanceCapacity` (issue #996). A copy. A call is recorded + * only once it has succeeded: a stream that fails or is abandoned after + * its stop event leaves nothing behind. + */ + performanceSamples(): PerformanceSample[] { + return this.performance.list(); + } + + /** + * Open one call's registration: entered once per `complete()` / + * `completeStream()`, just before `runWithRetry`, so retries, backoff + * and a refreshed-key replay all happen inside it and never add to the + * count. Every open call's `peak` rises to the new number open, so two + * calls that overlap at any moment both report 2. Close it with + * `closeCall` on every exit. + */ + private openCall(): OpenCall { + const call: OpenCall = { openedAt: Date.now(), peak: 0 }; + this.openCalls.add(call); + const open = this.openCalls.size; + for (const other of this.openCalls) { + if (other.peak < open) other.peak = open; + } + return call; + } + + private closeCall(call: OpenCall): void { + this.openCalls.delete(call); + } + + /** Record a completed call's sample, from the payload staged where its timings arrived. */ + private commitSample( + payload: Record | null, + call: OpenCall, + streamed: boolean, + ttftMs: number | null, + ): void { + if (payload === null) return; + const sample = sampleFromPayload(payload, { + streamed, + ttftMs, + concurrent: call.peak, + at: Date.now(), + }); + if (sample !== null) this.performance.push(sample); + } + /** * Render `messages` through the server's chat template * (`POST /apply-template`, present in the bundled build). Returns the @@ -935,37 +1002,49 @@ export class LlamaServerClient { async complete(request: CompletionRequest): Promise { const { url, headers, body } = this.prepareRequest(request, false); - return this.runWithRetry( - url, - async () => { - const { controller, cleanup, timedOut } = this.createRequestController( - request.signal, - "total", - undefined, - request.requestTimeoutMs, - ); - try { - const response = await this.fetchImpl(url, { - method: "POST", - headers, - body, - signal: controller.signal, - }); - if (!response.ok) { - throw await buildHttpError(response, url); + // The payload of the attempt that answered, recorded once the result + // is in hand: a throw after it (normalisation) records nothing. + let staged = null as Record | null; + const call = this.openCall(); + try { + const result = await this.runWithRetry( + url, + async () => { + staged = null; + const { controller, cleanup, timedOut } = this.createRequestController( + request.signal, + "total", + undefined, + request.requestTimeoutMs, + ); + try { + const response = await this.fetchImpl(url, { + method: "POST", + headers, + body, + signal: controller.signal, + }); + if (!response.ok) { + throw await buildHttpError(response, url); + } + const json = (await response.json()) as Record; + this.recordThroughput(json); + staged = json; + return normaliseCompletionResponse(json); + } catch (err) { + throw this.wrapTransportError(err, url, timedOut(), request.requestTimeoutMs); + } finally { + cleanup(); } - const json = (await response.json()) as Record; - this.recordThroughput(json); - return normaliseCompletionResponse(json); - } catch (err) { - throw this.wrapTransportError(err, url, timedOut(), request.requestTimeoutMs); - } finally { - cleanup(); - } - }, - request.signal, - headers, - ); + }, + request.signal, + headers, + ); + this.commitSample(staged, call, false, null); + return result; + } finally { + this.closeCall(call); + } } async *completeStream( @@ -994,6 +1073,11 @@ export class LlamaServerClient { keepAlive: (next: Exclude) => void; startStreamDeadline: () => void; }; + // One registration for the whole call, retries included; closed in + // the `catch` just below or the generator's `finally`, whichever ends + // it. The first-token time counts from here: failed attempts, backoff + // and queueing are all part of the wait the person sees. + const call = this.openCall(); try { opened = await this.runWithRetry( url, @@ -1057,6 +1141,7 @@ export class LlamaServerClient { headers, ); } catch (err) { + this.closeCall(call); if (err instanceof LlamaServerError) throw err; const message = err instanceof Error ? err.message : String(err); throw new LlamaServerError( @@ -1133,6 +1218,10 @@ export class LlamaServerClient { let looped: CompletionResult | null = null; // A real (non-progress) event has arrived (ATO-1027). let replyStarted = false; + // The stop event's payload, recorded as a sample only if this + // generator then returns normally; and when the first delta came. + let staged: Record | null = null; + let firstDeltaAt: number | null = null; while (looped === null) { const { value, done } = await reader.read(); if (done) break; @@ -1201,6 +1290,7 @@ export class LlamaServerClient { ? parsed.reasoning_content : ""; if (delta.length > 0 || reasoningDelta.length > 0) { + firstDeltaAt ??= Date.now(); accumulated += delta; accumulatedReasoning += reasoningDelta; yield { delta, reasoningDelta, done: false }; @@ -1221,6 +1311,7 @@ export class LlamaServerClient { } if (parsed.stop) { this.recordThroughput(parsed); + staged = parsed; finalResult = normaliseCompletionResponse(parsed); if (finalResult.content.length === 0) { finalResult.content = accumulated; @@ -1243,9 +1334,20 @@ export class LlamaServerClient { controller.abort(); yield { delta: "", reasoningDelta: "", done: true }; outcome = "done"; + // `controller` was aborted on purpose just above; only the + // caller's own abort means this call did not complete. + if (!request.signal?.aborted) { + this.commitSample(staged, call, true, ttftOf(firstDeltaAt, call)); + } return looped; } outcome = "done"; + // A caller that aborted after the done chunk, or a deadline that + // fired, ends a call that did not complete: a body already closed + // will not throw on the next read, so ask the signals. + if (!controller.signal.aborted && !request.signal?.aborted) { + this.commitSample(staged, call, true, ttftOf(firstDeltaAt, call)); + } return finalResult; } catch (err) { outcome = "error"; @@ -1313,6 +1415,7 @@ export class LlamaServerClient { abortedBeforeRelease: abortedBefore, ...(failure === undefined ? {} : { error: failure }), }); + this.closeCall(call); } } @@ -1961,6 +2064,11 @@ function tryParseJson(text: string): unknown { } } +/** Wait from the call's start to its first delta; `null` without one. */ +function ttftOf(firstDeltaAt: number | null, call: OpenCall): number | null { + return firstDeltaAt === null ? null : Math.max(0, firstDeltaAt - call.openedAt); +} + function parseSseEvent(rawEvent: string): Record | null { const lines = rawEvent.split("\n"); const dataLines: string[] = []; diff --git a/src/runtime/bootstrap.ts b/src/runtime/bootstrap.ts index 5a608f80c..fe6537e23 100644 --- a/src/runtime/bootstrap.ts +++ b/src/runtime/bootstrap.ts @@ -915,6 +915,14 @@ export async function createAgentRuntime( await profileManager?.refresh(); }, reconnectMemoryEmbeddings: () => memoryEmbeddings.reconnect(), + // The same client instance the llama-server provider sends every + // local completion through (runtime-inference passes it as + // `llamaClient`), so its samples are the turns' own. + localPerformance: () => ({ + capacity: llama.performanceCapacity, + samples: llama.performanceSamples(), + fetchSlots: (timeoutMs: number) => llama.fetchSlots(timeoutMs), + }), setApprovalHandlerForSession: (sessionId, handler) => approvalRouter.setForSession(sessionId, handler), setAnalyticsEnabled, diff --git a/src/runtime/runtime-contract.ts b/src/runtime/runtime-contract.ts index 00c0a5597..5b1f36e36 100644 --- a/src/runtime/runtime-contract.ts +++ b/src/runtime/runtime-contract.ts @@ -14,6 +14,7 @@ import type { SwarmRegistry } from "../channels/swarm/index.js"; import type { BotFactory } from "../channels/telegram/index.js"; import type { McpManager } from "../mcp/index.js"; import type { CompletionResult, StreamChunk } from "../llm/llama-server-client.js"; +import type { PerformanceSample } from "../llm/llama-performance-samples.js"; import type { SlotManager } from "../llm/slot-manager.js"; import type { ApprovalGate } from "../approval/approval-gate.js"; import type { ApprovalLevel } from "../approval/approval-level.js"; @@ -177,6 +178,16 @@ export interface CreateAgentRuntimeOptions { } +/** What `AgentRuntime.localPerformance` hands the performance route. */ +export interface LocalPerformance { + /** The most samples kept. */ + capacity: number; + /** Completed local requests, oldest first. */ + samples: PerformanceSample[]; + /** `GET /slots` on the local server, aborted after `timeoutMs`; throws on any failure. */ + fetchSlots(timeoutMs: number): Promise; +} + /** Where memory's embedding attachment stands after `reconnectMemoryEmbeddings`. */ export interface MemoryEmbeddingReconnect { health: "ok" | "unreachable" | "disabled"; @@ -616,6 +627,14 @@ export interface AgentRuntime { * older hosts need not implement it. */ reconnectMemoryEmbeddings?(): Promise; + /** + * The local model server's recent per-request timings, from the one + * llama-server client the provider uses, and a bounded `/slots` read + * of the same server (issue #996). Read by + * `GET /api/local-model/performance`. Optional so test doubles and + * embedders need not implement it; the route then reports no samples. + */ + localPerformance?(): LocalPerformance; /** * Register `handler` as the approval sink for `sessionId`. Every * `ApprovalRequest` whose `sessionId` matches will be routed to