From ac0729381d9fa0b2728031c49941400f4e431f04 Mon Sep 17 00:00:00 2001 From: T-Crypt <132922589+T-Crypt@users.noreply.github.com> Date: Wed, 21 Oct 2026 23:54:02 -0600 Subject: [PATCH 1/6] test(engines): pin metrics_reported on every counter reader An engine that never answered its counters endpoint reads a zero rate, and the harbor draws that zero as measured. Add the field and assert what each reader must set; every true case fails until the readers fill it. --- internal/engines/engines_test.go | 47 ++++++++++++++++++++++++++++++++ internal/engines/more_test.go | 12 +++++--- internal/proto/proto.go | 4 +++ 3 files changed, 59 insertions(+), 4 deletions(-) diff --git a/internal/engines/engines_test.go b/internal/engines/engines_test.go index d984cc6..2eb5bd4 100644 --- a/internal/engines/engines_test.go +++ b/internal/engines/engines_test.go @@ -63,6 +63,41 @@ func TestLlamaSwapFollowsUpstreamsAndComputesRates(t *testing.T) { if e.DecodeTPS < 99.9 || e.DecodeTPS > 100.1 { t.Fatalf("decode = %v t/s, want 100", e.DecodeTPS) } + if !e.MetricsReported { + t.Fatal("metrics_reported = false with a serving upstream") + } +} + +// llama-swap starts llama-server without --metrics, so the proxy answers 501: +// the engine stays up with its model listed and the counters read unreported, +// never a measured zero. +func TestLlamaSwapWithoutMetrics(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/metrics" { + w.WriteHeader(http.StatusNotImplemented) + fmt.Fprint(w, "This server does not support metrics endpoint. Start it with --metrics") + return + } + http.NotFound(w, r) + })) + defer upstream.Close() + swap := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/running" { + http.NotFound(w, r) + return + } + fmt.Fprintf(w, `{"running":[{"model":"Qwen3-8B","state":"ready","proxy":%q}]}`, upstream.URL) + })) + defer swap.Close() + w := &Watcher{Engines: []config.Engine{{Name: "llama-swap", Type: "llama-swap", URL: swap.URL}}} + w.Poll(context.Background()) + e := w.Snapshot()[0] + if !e.Up || len(e.Models) != 1 || e.Models[0].Name != "Qwen3-8B" { + t.Fatalf("engine = %+v", e) + } + if e.Metrics.OutputTokens != 0 || e.MetricsReported { + t.Fatalf("metrics = %+v reported = %v, want zero counters unreported", e.Metrics, e.MetricsReported) + } } func TestOllamaReportsModelsAsEngineClaims(t *testing.T) { @@ -165,6 +200,9 @@ func TestCustomMap(t *testing.T) { if e.Metrics.OutputTokens != 42 || e.Metrics.Waiting != 3 { t.Fatalf("metrics = %+v", e.Metrics) } + if !e.MetricsReported { + t.Fatal("metrics_reported = false after a mapped metrics read") + } } // llamacppSlotsFake serves the llama.cpp family's three endpoints: the @@ -197,6 +235,9 @@ func TestLlamaCppReadsPropsAndSlots(t *testing.T) { if !e.Up || e.Error != "" { t.Fatalf("engine = %+v", e) } + if !e.MetricsReported { + t.Fatal("metrics_reported = false after /metrics answered") + } if e.Window != 8192 { t.Fatalf("window = %d, want 8192", e.Window) } @@ -400,6 +441,9 @@ func TestStrataReadsHealthAndMetrics(t *testing.T) { if e.Metrics.OutputTokens != 1500 || e.Metrics.Processing != 1 || e.Metrics.Waiting != 2 { t.Fatalf("metrics = %+v", e.Metrics) } + if !e.MetricsReported { + t.Fatal("metrics_reported = false with serving /metrics") + } if e.DecodeTPS < 19.9 || e.DecodeTPS > 20.1 { t.Fatalf("decode = %v t/s, want 20", e.DecodeTPS) } @@ -436,4 +480,7 @@ func TestStrataKeyedInstallStaysUp(t *testing.T) { if e.Window != 16384 || e.Models[0].Window != 16384 { t.Fatalf("windows: engine %d model %d, want 16384", e.Window, e.Models[0].Window) } + if e.MetricsReported { + t.Fatal("metrics_reported = true while /metrics is refused") + } } diff --git a/internal/engines/more_test.go b/internal/engines/more_test.go index 76e5e79..647c7f8 100644 --- a/internal/engines/more_test.go +++ b/internal/engines/more_test.go @@ -49,12 +49,13 @@ var engineCases = []struct { processing float64 waiting float64 output float64 + reported bool }{ {typ: "vllm", routes: map[string]string{ "/v1/models": `{"data":[{"id":"Qwen/Qwen3-8B","max_model_len":32768}]}`, "/metrics": "vllm:prompt_tokens_total 400\nvllm:generation_tokens_total 900\nvllm:num_requests_running 2\nvllm:num_requests_waiting 3\n", "/version": `{"version":"0.11.0"}`, - }, models: []string{"Qwen/Qwen3-8B"}, window: 32768, processing: 2, waiting: 3, output: 900}, + }, models: []string{"Qwen/Qwen3-8B"}, window: 32768, processing: 2, waiting: 3, output: 900, reported: true}, // vLLM with metrics disabled is still up. {typ: "vllm", routes: map[string]string{ "/v1/models": `{"data":[{"id":"m","max_model_len":8192}]}`, @@ -63,12 +64,12 @@ var engineCases = []struct { "/v1/models": `{"data":[{"id":"glm","context_length":65536}]}`, "/metrics": "sglang:generation_tokens_total 77\nsglang:num_requests_waiting 1\n", "/get_model_info": `{"model_path":"glm"}`, - }, models: []string{"glm"}, window: 65536, waiting: 1, output: 77}, + }, models: []string{"glm"}, window: 65536, waiting: 1, output: 77, reported: true}, {typ: "localai", routes: map[string]string{ "/v1/models": `{"data":[{"id":"phi"}]}`, "/metrics": "localai_completion_tokens_total 12\n", "/healthz": "204", - }, models: []string{"phi"}, output: 12}, + }, models: []string{"phi"}, output: 12, reported: true}, {typ: "openai", routes: map[string]string{ "/v1/models": `{"data":[{"id":"a"},{"id":"b","context_length":4096}]}`, }, models: []string{"a", "b"}, window: 4096}, @@ -90,7 +91,7 @@ var engineCases = []struct { {typ: "freetoken", routes: map[string]string{ "/v1/stats": `{"stats":{"tokens_generated":500,"prompt_tokens":40,"queue":2,"generating":true}}`, "/health": `{"ok":true}`, - }, processing: 1, waiting: 2, output: 500}, + }, processing: 1, waiting: 2, output: 500, reported: true}, {typ: "comfyui", routes: map[string]string{ "/queue": `{"queue_running":[[1,"a"]],"queue_pending":[[2,"b"],[3,"c"]]}`, "/system_stats": `{"system":{"comfyui_version":"0.3.60"},"devices":[]}`, @@ -119,6 +120,9 @@ func TestEveryEngineReader(t *testing.T) { t.Fatalf("window %d processing %g waiting %g output %g; want %d %g %g %g", e.Window, e.Metrics.Processing, e.Metrics.Waiting, e.Metrics.OutputTokens, c.window, c.processing, c.waiting, c.output) } + if e.MetricsReported != c.reported { + t.Fatalf("metrics_reported = %v, want %v", e.MetricsReported, c.reported) + } }) } } diff --git a/internal/proto/proto.go b/internal/proto/proto.go index 8f8714b..e8b84db 100644 --- a/internal/proto/proto.go +++ b/internal/proto/proto.go @@ -449,6 +449,10 @@ type Engine struct { Metrics EngineMetrics `json:"metrics"` DecodeTPS float64 `json:"decode_tps"` PrefillTPS float64 `json:"prefill_tps"` + // MetricsReported is true when this read got real counters from the + // engine. A zero rate beside it is an idle engine; without it the + // engine never answered its metrics endpoint and the rate is unknown. + MetricsReported bool `json:"metrics_reported"` // Window is the engine-level context budget: the /props n_ctx for the // llama.cpp family, the max model window for llama-swap. 0 = not reported. Window int `json:"window,omitempty"` From bd8d1f1a9805c7d94e49cf73c487c73cc79dcc5b Mon Sep 17 00:00:00 2001 From: T-Crypt <132922589+T-Crypt@users.noreply.github.com> Date: Wed, 21 Oct 2026 23:54:21 -0600 Subject: [PATCH 2/6] fix(engines): readers report when the counters endpoint answered The llama.cpp family, llama-swap upstreams, strata, the Prometheus readers and FreeToken now set metrics_reported on a read that actually got counters, so a zero rate is distinguishable from silence. --- internal/engines/engines.go | 4 ++++ internal/engines/more.go | 2 ++ 2 files changed, 6 insertions(+) diff --git a/internal/engines/engines.go b/internal/engines/engines.go index 4cebddc..376e9fd 100644 --- a/internal/engines/engines.go +++ b/internal/engines/engines.go @@ -256,6 +256,7 @@ func (w *Watcher) read(ctx context.Context, e config.Engine) proto.Engine { // A build without /props, or that 501s on /slots (KoboldCpp), stays // up with a zero window or no slots; a down engine fetches neither. if err == nil { + out.MetricsReported = true if b, perr := w.get(ctx, base+"/props"); perr == nil { var p struct { NCtx int `json:"n_ctx"` @@ -427,6 +428,7 @@ func (w *Watcher) custom(ctx context.Context, base string, e config.Engine, out PromptSeconds: f("prompt_seconds"), OutputSeconds: f("output_seconds"), Processing: f("processing"), Waiting: f("waiting"), } + out.MetricsReported = true } } if e.Models != "" { @@ -554,6 +556,7 @@ func (w *Watcher) llamaSwap(ctx context.Context, base string, out *proto.Engine) if err == nil { // an upstream without /metrics still counts as loaded out.Metrics = out.Metrics.Add(mm) + out.MetricsReported = true } // Best-effort per model: the proxy's context window and its live KV // slots, tagged with the model they sit on. A 404/501 on either @@ -689,6 +692,7 @@ func (w *Watcher) strata(ctx context.Context, base, keyEnv string, out *proto.En if err := json.Unmarshal(b, &m); err != nil { return nil } + out.MetricsReported = true switch { case m.Live.Running != nil: out.Metrics.Processing = float64(*m.Live.Running) diff --git a/internal/engines/more.go b/internal/engines/more.go index 1d5ceda..1f16c9f 100644 --- a/internal/engines/more.go +++ b/internal/engines/more.go @@ -111,6 +111,7 @@ func (w *Watcher) openAIWithProm(ctx context.Context, base, keyEnv, prefix strin } if m, ok := w.promMetrics(ctx, base+"/metrics", keyEnv, prefix, names); ok { out.Metrics = m + out.MetricsReported = true } return nil } @@ -240,6 +241,7 @@ func (w *Watcher) freetoken(ctx context.Context, base, keyEnv string, out *proto if err := json.Unmarshal(b, &body); err != nil { return fmt.Errorf("/v1/stats: %w", err) } + out.MetricsReported = true if inner, ok := body["stats"].(map[string]any); ok { body = inner } From b40524a93c69789c81d6744ff244b78265a650a4 Mon Sep 17 00:00:00 2001 From: T-Crypt <132922589+T-Crypt@users.noreply.github.com> Date: Wed, 21 Oct 2026 23:56:55 -0600 Subject: [PATCH 3/6] fix(harbor): engine card shows n/a for metrics it never got An engine that never answered its counters endpoint (llama-swap without --metrics on its llama-server) drew DECODE 0.0 t/s and zero token totals. With metrics_reported false, decode, prefill and the token totals render n/a, model rows show n/a where per-model VRAM is absent, and a reported zero still prints as 0.0. The demo fixture marks the llama-swap card as reporting so the card reads live. --- internal/harbor/web/demo.json | 1 + internal/harbor/web/engines.js | 13 ++++++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/internal/harbor/web/demo.json b/internal/harbor/web/demo.json index a255c66..2b232eb 100644 --- a/internal/harbor/web/demo.json +++ b/internal/harbor/web/demo.json @@ -196,6 +196,7 @@ }, "decode_tps": 125.4, "prefill_tps": 3410.6, + "metrics_reported": true, "state": "up", "at": "2026-10-09T22:53:58.104512-06:00", "age_seconds": 3.2, diff --git a/internal/harbor/web/engines.js b/internal/harbor/web/engines.js index b09819b..6030249 100644 --- a/internal/harbor/web/engines.js +++ b/internal/harbor/web/engines.js @@ -161,21 +161,24 @@ export class EngineView { c.models.textContent = ""; if (Array.isArray(eng.models) && eng.models.length > 0) { for (const m of eng.models) { - const vram = m.reported_vram ? " " + fmtGiB(m.reported_vram) + "G" : ""; + const vram = na(m.reported_vram, (v) => " " + fmtGiB(v) + "G"); c.models.append(el("div", "engine-model", el("span", null, m.name || "?"), el("span", "ms", (m.state || "") + vram))); } } else { c.models.append(el("div", "engine-models-note", "no models reported")); } - c.decode.num.textContent = (eng.decode_tps || 0).toFixed(1); - c.prefill.num.textContent = (eng.prefill_tps || 0).toFixed(1); + // metrics_reported separates a measured idle from an engine that never + // answered its counters endpoint; unreported figures read n/a, not 0.0 + c.decode.num.textContent = na(eng.metrics_reported ? eng.decode_tps : null, (v) => v.toFixed(1)); + c.prefill.num.textContent = na(eng.metrics_reported ? eng.prefill_tps : null, (v) => v.toFixed(1)); c.decode.spark.set(c.hist.decode); c.prefill.spark.set(c.hist.prefill); const m = eng.metrics || {}; + const tok = (v) => na(eng.metrics_reported ? v : null, fmtNum); c.totals.textContent = ""; c.totals.append( - el("span", null, "in " + fmtNum(m.prompt_tokens) + " tok"), - el("span", null, "out " + fmtNum(m.output_tokens) + " tok"), + el("span", null, "in " + tok(m.prompt_tokens) + " tok"), + el("span", null, "out " + tok(m.output_tokens) + " tok"), el("span", null, "queue " + (Math.round(m.processing || 0) + "+" + Math.round(m.waiting || 0))) ); c.extra.textContent = ""; From cca9e5769864743fb5d7b121580214145fb00f7b Mon Sep 17 00:00:00 2001 From: T-Crypt <132922589+T-Crypt@users.noreply.github.com> Date: Wed, 21 Oct 2026 23:56:55 -0600 Subject: [PATCH 4/6] fix(cli): status prints a dash for rates an engine never reported DECODE and PREFILL in the ENGINE table are unknown, not zero, when metrics_reported is false. --- internal/cli/status.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/internal/cli/status.go b/internal/cli/status.go index a5916fd..a6eb9ea 100644 --- a/internal/cli/status.go +++ b/internal/cli/status.go @@ -175,8 +175,13 @@ func (v statusView) Human(c *Context, ok bool) error { if b, ok := est[e.Name]; ok { estimate = server.MiB(b) } - fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%.0f t/s\t%.0f t/s\t%.0f\t%.0f\t%s\n", e.Name, e.Type, esc(up), strings.Join(names, ","), - e.DecodeTPS, e.PrefillTPS, e.Metrics.OutputTokens, e.Metrics.Processing+e.Metrics.Waiting, estimate) + // a rate with no metrics behind it is unknown, not a measured idle + decode, prefill := "-", "-" + if e.MetricsReported { + decode, prefill = fmt.Sprintf("%.0f t/s", e.DecodeTPS), fmt.Sprintf("%.0f t/s", e.PrefillTPS) + } + fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%s\t%.0f\t%.0f\t%s\n", e.Name, e.Type, esc(up), strings.Join(names, ","), + decode, prefill, e.Metrics.OutputTokens, e.Metrics.Processing+e.Metrics.Waiting, estimate) } } tw.Flush() From dfff910b5c76e4b6423ad9fa8cf75b300c6ce351 Mon Sep 17 00:00:00 2001 From: T-Crypt <132922589+T-Crypt@users.noreply.github.com> Date: Wed, 21 Oct 2026 23:56:55 -0600 Subject: [PATCH 5/6] docs(engines): llama-swap counters need --metrics on llama-server Without it the reader gets a 501 per upstream and the card shows n/a. CHANGELOG entry credits @azrael-229 and links #140. --- CHANGELOG.md | 4 ++++ contrib/INTEGRATION.md | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f52387a..81a2aac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -89,6 +89,10 @@ has a matching GitHub release whose body comes from its section below. (`[32m[INFO][0m`), and a progress bar's line as its last redraw. - Windows: a process the GPU counters report but nvidia-smi does not list, such as dwm.exe holding the card's memory, now shows its name instead of a blank one. +- The harbor engine cards and `berth status` draw an unreported rate as unknown, not zero. An + engine whose counters endpoint never answered (a llama-swap whose llama-server starts without + `--metrics`) shows n/a for decode, prefill and the token totals, and the ENGINE table prints a + dash. Reported by @azrael-229 ([#140](https://github.com/T-Crypt/berth/issues/140)). ## [v0.3.3] - 2026-10-21 diff --git a/contrib/INTEGRATION.md b/contrib/INTEGRATION.md index aa21636..8fea013 100644 --- a/contrib/INTEGRATION.md +++ b/contrib/INTEGRATION.md @@ -58,7 +58,9 @@ models: llama-swap stops a model with SIGTERM; `berth run` forwards it to the engine's process group. While Berth waits for the card, llama-swap's `healthCheckTimeout` keeps the request open. -Reading: `type = "llama-swap"` follows every running model's upstream metrics. +Reading: `type = "llama-swap"` follows every running model's upstream metrics, which exist only +when the model's `cmd` starts `llama-server` with `--metrics`; without it the engine stays up and +the harbor shows n/a for decode, prefill and token totals. ## llama.cpp (`llama-server`), NInfer, KoboldCpp From c6b443ec539dd8fedb9f300955b8fd9a96d73063 Mon Sep 17 00:00:00 2001 From: T-Crypt <132922589+T-Crypt@users.noreply.github.com> Date: Thu, 22 Oct 2026 00:03:24 -0600 Subject: [PATCH 6/6] fix(harbor): a model row without reported VRAM reads "ready n/a", not "readyn/a" --- internal/harbor/web/engines.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/harbor/web/engines.js b/internal/harbor/web/engines.js index 6030249..a2f73b5 100644 --- a/internal/harbor/web/engines.js +++ b/internal/harbor/web/engines.js @@ -161,7 +161,7 @@ export class EngineView { c.models.textContent = ""; if (Array.isArray(eng.models) && eng.models.length > 0) { for (const m of eng.models) { - const vram = na(m.reported_vram, (v) => " " + fmtGiB(v) + "G"); + const vram = " " + na(m.reported_vram, (v) => fmtGiB(v) + "G"); c.models.append(el("div", "engine-model", el("span", null, m.name || "?"), el("span", "ms", (m.state || "") + vram))); } } else {