Release 0.2.0

.env.example

@@ -16,6 +16,6 @@ ADMIN_PASSWORD=
 # Must start with llmb_ — e.g. echo "llmb_$(openssl rand -hex 24)"
 SERVICE_API_KEY=
 
-# LLMBENCH_VERSION=0.1.2       # the released images to run
+# LLMBENCH_VERSION=0.2.0       # the released images to run
 # LLMBENCH_PORT=8080
 # WORKER_PROCESSES=2

CHANGELOG.md

@@ -5,6 +5,30 @@ All notable changes to this project are documented here.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+
+## [0.2.0] - 2026-10-10
+
+Versioned with LLM AutoTune 0.2.0, which it is tested with.
+
+### Changed
+
+- `perf_guidellm` samples a fixed number of requests, like the sweep module:
+  it stops after concurrency × `requests_per_concurrency` (default 20)
+  requests, with `max_seconds` as the cap. A fixed sample keeps a result's
+  precision the same however fast the endpoint is. Clear
+  `requests_per_concurrency` for the old duration-only test.
+
+### Fixed
+
+- The chart's postgres starts on a freshly formatted block volume (Ceph RBD,
+  EBS, a persistent disk): it keeps its data in a `pgdata` subdirectory, as
+  initdb refuses a mount point holding `lost+found`. A database an earlier
+  chart initialised at the volume root stays where it is.
+- The install notes give the frontend's NodePort when
+  `service.frontend.type` is `NodePort`, instead of telling you to
+  port-forward.
+
 ## [0.1.2] - 2026-09-29
 
 Versioned with LLM AutoTune 0.1.2, which it is tested with; there is no 0.1.1.
@@ -52,5 +76,7 @@ Versioned with LLM AutoTune 0.1.2, which it is tested with; there is no 0.1.1.
   Hub, for amd64 and arm64.
 - Released under the Apache License 2.0.
 
+[Unreleased]: https://github.com/modelsphere/llm-bench/compare/v0.2.0...HEAD
+[0.2.0]: https://github.com/modelsphere/llm-bench/compare/v0.1.2...v0.2.0
 [0.1.2]: https://github.com/modelsphere/llm-bench/releases/tag/v0.1.2
 [0.1.0]: https://github.com/modelsphere/llm-bench/releases/tag/v0.1.0

bench/modules/perf_guidellm.py

@@ -41,9 +41,21 @@ class PerfGuidellmParams(BaseModel):
     concurrency: int = Field(default=5, ge=1, le=4096, description="Concurrent users / concurrency level")
     input_tokens: int = Field(default=50000, ge=1, le=1_000_000_000, description="Input tokens per request")
     output_tokens: int = Field(default=1500, ge=1, description="Max output tokens per request")
-    max_seconds: float = Field(default=300.0, ge=1.0, le=604800.0, description="Test duration in seconds")
+    max_seconds: float = Field(
+        default=300.0, ge=1.0, le=604800.0,
+        description="Duration cap in seconds; the whole test when requests_per_concurrency is empty",
+    )
     request_timeout: float = Field(default=120.0, ge=1.0, le=86400.0, description="Per-request timeout in seconds")
     warmup_seconds: float = Field(default=30.0, ge=0.0, le=86400.0, description="[guidellm] Warmup duration in seconds")
+    requests_per_concurrency: Optional[int] = Field(
+        default=20, ge=1, le=100_000,
+        description=(
+            "Target request count per concurrency slot: the test stops after "
+            "concurrency × this many requests, or at max_seconds, whichever comes "
+            "first. A fixed sample size keeps the measurement's precision the same "
+            "however fast the service is. Leave empty for a duration-only test."
+        ),
+    )
     dataset_path: Optional[str] = Field(
         default=None,
         description="[guidellm] Path to ShareGPT-format dataset file. Leave blank to use random tokens."
@@ -208,7 +220,9 @@ def run(
 
         progress_cb(0.05, f"Starting guidellm: concurrency={p.concurrency}, "
                          f"input={p.input_tokens}, output={p.output_tokens}, "
-                         f"duration={p.max_seconds}s")
+                         + (f"requests={p.concurrency * p.requests_per_concurrency} "
+                            f"(cap {p.max_seconds}s)" if p.requests_per_concurrency
+                            else f"duration={p.max_seconds}s"))
 
         metrics = self._run_guidellm(endpoint, p, output_dir, progress_cb, cancel_event)
 
@@ -243,6 +257,7 @@ def _run_guidellm(self, endpoint, p, output_dir, progress_cb, cancel_event):
             output_dir=output_dir,
             progress_cb=progress_cb,
             cancel_event=cancel_event,
+            requests_per_concurrency=getattr(p, "requests_per_concurrency", None),
             log_tag="perf_guidellm",
         )
 

bench/modules/perf_guidellm_sweep.py

@@ -230,8 +230,9 @@ class PerfGuidellmSweepParams(BaseModel):
     max_seconds: float = Field(
         default=300.0, ge=1.0, le=604800.0,
         description=(
-            "[grid] Test duration PER concurrency level, in seconds. "
-            "[auto] Duration of the final confirmation run at the discovered level."
+            "[grid] Duration cap PER concurrency level, in seconds; the level's "
+            "whole duration when requests_per_concurrency is empty. "
+            "[auto] The same, for the final confirmation run at the discovered level."
         ),
     )
     request_timeout: float = Field(default=120.0, ge=1.0, le=86400.0, description="Per-request timeout in seconds")

deploy/helm/llm-bench/Chart.yaml

@@ -4,8 +4,8 @@ description: LLM Benchmark Platform - FastAPI backend, Vue frontend, Dramatiq wo
 
 type: application
 
-version: 0.1.2
-appVersion: "0.1.2"
+version: 0.2.0
+appVersion: "0.2.0"
 
 dependencies:
   - name: postgresql

deploy/helm/llm-bench/templates/NOTES.txt

@@ -19,6 +19,12 @@ Ingress:
 {{- range .Values.ingress.hosts }}
   http{{ if $.Values.ingress.tls }}s{{ end }}://{{ .host }}
 {{- end }}
+{{- else if eq .Values.service.frontend.type "NodePort" }}
+
+The frontend is on a NodePort of every node:
+  kubectl get svc -n {{ .Release.Namespace }} {{ include "llm-bench.fullname" . }}-frontend -o jsonpath='{.spec.ports[0].nodePort}'
+
+Then open http://<node address>:<that port>
 {{- else }}
 
 No ingress enabled. Access the frontend via port-forward:

deploy/helm/llm-bench/templates/postgres.yaml

@@ -57,12 +57,21 @@ spec:
         - name: postgres
           image: "{{ .Values.postgres.image.registry }}/{{ .Values.postgres.image.repository }}:{{ .Values.postgres.image.tag }}"
           imagePullPolicy: {{ .Values.image.backend.pullPolicy }}
-          # Override CMD only (not the entrypoint), so docker-entrypoint.sh still
-          # runs initdb/startup, then execs the server with our tuning.
+          # The data lives one level below the mount point: a freshly formatted
+          # block volume (ext4 on Ceph RBD, EBS, a PD) has lost+found at its
+          # root, and initdb refuses a non-empty directory. A database already
+          # initialised at the root, by an earlier version of this chart, stays
+          # where it is. docker-entrypoint.sh still runs initdb/startup, then
+          # execs the server with our tuning.
+          command: ["sh", "-c"]
           args:
-            - "postgres"
-            - "-c"
-            - "max_connections={{ .Values.postgres.maxConnections }}"
+            - |
+              if [ -s /var/lib/postgresql/data/PG_VERSION ]; then
+                export PGDATA=/var/lib/postgresql/data
+              else
+                export PGDATA=/var/lib/postgresql/data/pgdata
+              fi
+              exec docker-entrypoint.sh postgres -c max_connections={{ .Values.postgres.maxConnections }}
           env:
             - name: POSTGRES_DB
               value: {{ .Values.postgres.database | quote }}

docker-compose.yml

@@ -15,7 +15,7 @@
 # For a cluster, use the Helm chart in deploy/helm/llm-bench instead.
 
 x-backend: &backend
-  image: docker.io/4pdosc/llm-bench-backend:${LLMBENCH_VERSION:-0.1.2}
+  image: docker.io/4pdosc/llm-bench-backend:${LLMBENCH_VERSION:-0.2.0}
   build:
     context: .
     dockerfile: platform/backend/Dockerfile
@@ -100,7 +100,7 @@ services:
     restart: unless-stopped
 
   frontend:
-    image: docker.io/4pdosc/llm-bench-frontend:${LLMBENCH_VERSION:-0.1.2}
+    image: docker.io/4pdosc/llm-bench-frontend:${LLMBENCH_VERSION:-0.2.0}
     build:
       context: platform/frontend
     ports:

platform/backend/app/main.py

@@ -214,7 +214,7 @@ async def _sync_modules() -> None:
 app = FastAPI(
     title="LLM Benchmark Platform API",
     description="Multi-tenant LLM benchmark leaderboard with modular test units",
-    version="0.1.2",
+    version="0.2.0",
     lifespan=lifespan,
     # `servers` makes Swagger "Try it out" target {prefix}/... (e.g. /api/auth/login)
     # so requests route back through nginx. Routing itself stays at bare paths.

platform/backend/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "llm-bench-backend"
-version = "0.1.2"
+version = "0.2.0"
 description = "LLMBench: the API, the worker, and the benchmark modules they run"
 requires-python = ">=3.12"
 dependencies = [

platform/frontend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "llm-bench-frontend",
-  "version": "0.1.2",
+  "version": "0.2.0",
   "private": true,
   "type": "module",
   "scripts": {

tests/modules/test_perf_guidellm.py

@@ -144,3 +144,27 @@ def test_sweep_only_metrics_stay_out_of_the_single_module():
     }
     # Everything else the sweep declares, the single module declares too
     assert single - sweep == set()
+
+
+@pytest.mark.parametrize("per_slot", [20, None])
+def test_a_level_stops_at_a_request_count_unless_cleared(monkeypatch, tmp_path, per_slot):
+    """Like the sweep: concurrency x requests_per_concurrency requests, with
+    max_seconds as the cap; cleared, the duration alone bounds the test."""
+    import bench.modules.perf_guidellm as perf
+
+    seen = {}
+
+    def fake_load(endpoint, **kwargs):
+        seen.update(kwargs)
+        return None
+
+    monkeypatch.setattr(perf, "run_guidellm_load", fake_load)
+    cls = get_module("perf_guidellm")
+    params = cls.ParamsSchema(concurrency=4, requests_per_concurrency=per_slot)
+    cls()._run_guidellm(None, params, str(tmp_path), lambda *a: None, None)
+    assert seen["requests_per_concurrency"] == per_slot
+    assert seen["max_seconds"] == params.max_seconds
+
+
+def test_requests_per_concurrency_defaults_to_a_count():
+    assert get_module("perf_guidellm").ParamsSchema().requests_per_concurrency == 20