diff --git a/.env.example b/.env.example index 096d404..db4f2d8 100644 --- a/.env.example +++ b/.env.example @@ -16,6 +16,6 @@ ADMIN_PASSWORD= # Must start with llmb_ — e.g. echo "llmb_$(openssl rand -hex 24)" SERVICE_API_KEY= -# LLMBENCH_VERSION=0.1.2 # the released images to run +# LLMBENCH_VERSION=0.2.0 # the released images to run # LLMBENCH_PORT=8080 # WORKER_PROCESSES=2 diff --git a/CHANGELOG.md b/CHANGELOG.md index a252706..91e82bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,30 @@ All notable changes to this project are documented here. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +## [0.2.0] - 2026-10-10 + +Versioned with LLM AutoTune 0.2.0, which it is tested with. + +### Changed + +- `perf_guidellm` samples a fixed number of requests, like the sweep module: + it stops after concurrency × `requests_per_concurrency` (default 20) + requests, with `max_seconds` as the cap. A fixed sample keeps a result's + precision the same however fast the endpoint is. Clear + `requests_per_concurrency` for the old duration-only test. + +### Fixed + +- The chart's postgres starts on a freshly formatted block volume (Ceph RBD, + EBS, a persistent disk): it keeps its data in a `pgdata` subdirectory, as + initdb refuses a mount point holding `lost+found`. A database an earlier + chart initialised at the volume root stays where it is. +- The install notes give the frontend's NodePort when + `service.frontend.type` is `NodePort`, instead of telling you to + port-forward. + ## [0.1.2] - 2026-09-29 Versioned with LLM AutoTune 0.1.2, which it is tested with; there is no 0.1.1. @@ -52,5 +76,7 @@ Versioned with LLM AutoTune 0.1.2, which it is tested with; there is no 0.1.1. Hub, for amd64 and arm64. - Released under the Apache License 2.0. +[Unreleased]: https://github.com/modelsphere/llm-bench/compare/v0.2.0...HEAD +[0.2.0]: https://github.com/modelsphere/llm-bench/compare/v0.1.2...v0.2.0 [0.1.2]: https://github.com/modelsphere/llm-bench/releases/tag/v0.1.2 [0.1.0]: https://github.com/modelsphere/llm-bench/releases/tag/v0.1.0 diff --git a/bench/modules/perf_guidellm.py b/bench/modules/perf_guidellm.py index 367afd6..66f3190 100644 --- a/bench/modules/perf_guidellm.py +++ b/bench/modules/perf_guidellm.py @@ -41,9 +41,21 @@ class PerfGuidellmParams(BaseModel): concurrency: int = Field(default=5, ge=1, le=4096, description="Concurrent users / concurrency level") input_tokens: int = Field(default=50000, ge=1, le=1_000_000_000, description="Input tokens per request") output_tokens: int = Field(default=1500, ge=1, description="Max output tokens per request") - max_seconds: float = Field(default=300.0, ge=1.0, le=604800.0, description="Test duration in seconds") + max_seconds: float = Field( + default=300.0, ge=1.0, le=604800.0, + description="Duration cap in seconds; the whole test when requests_per_concurrency is empty", + ) request_timeout: float = Field(default=120.0, ge=1.0, le=86400.0, description="Per-request timeout in seconds") warmup_seconds: float = Field(default=30.0, ge=0.0, le=86400.0, description="[guidellm] Warmup duration in seconds") + requests_per_concurrency: Optional[int] = Field( + default=20, ge=1, le=100_000, + description=( + "Target request count per concurrency slot: the test stops after " + "concurrency × this many requests, or at max_seconds, whichever comes " + "first. A fixed sample size keeps the measurement's precision the same " + "however fast the service is. Leave empty for a duration-only test." + ), + ) dataset_path: Optional[str] = Field( default=None, description="[guidellm] Path to ShareGPT-format dataset file. Leave blank to use random tokens." @@ -208,7 +220,9 @@ def run( progress_cb(0.05, f"Starting guidellm: concurrency={p.concurrency}, " f"input={p.input_tokens}, output={p.output_tokens}, " - f"duration={p.max_seconds}s") + + (f"requests={p.concurrency * p.requests_per_concurrency} " + f"(cap {p.max_seconds}s)" if p.requests_per_concurrency + else f"duration={p.max_seconds}s")) metrics = self._run_guidellm(endpoint, p, output_dir, progress_cb, cancel_event) @@ -243,6 +257,7 @@ def _run_guidellm(self, endpoint, p, output_dir, progress_cb, cancel_event): output_dir=output_dir, progress_cb=progress_cb, cancel_event=cancel_event, + requests_per_concurrency=getattr(p, "requests_per_concurrency", None), log_tag="perf_guidellm", ) diff --git a/bench/modules/perf_guidellm_sweep.py b/bench/modules/perf_guidellm_sweep.py index c45de3e..728065b 100644 --- a/bench/modules/perf_guidellm_sweep.py +++ b/bench/modules/perf_guidellm_sweep.py @@ -230,8 +230,9 @@ class PerfGuidellmSweepParams(BaseModel): max_seconds: float = Field( default=300.0, ge=1.0, le=604800.0, description=( - "[grid] Test duration PER concurrency level, in seconds. " - "[auto] Duration of the final confirmation run at the discovered level." + "[grid] Duration cap PER concurrency level, in seconds; the level's " + "whole duration when requests_per_concurrency is empty. " + "[auto] The same, for the final confirmation run at the discovered level." ), ) request_timeout: float = Field(default=120.0, ge=1.0, le=86400.0, description="Per-request timeout in seconds") diff --git a/deploy/helm/llm-bench/Chart.yaml b/deploy/helm/llm-bench/Chart.yaml index 1414473..c1ff12d 100644 --- a/deploy/helm/llm-bench/Chart.yaml +++ b/deploy/helm/llm-bench/Chart.yaml @@ -4,8 +4,8 @@ description: LLM Benchmark Platform - FastAPI backend, Vue frontend, Dramatiq wo type: application -version: 0.1.2 -appVersion: "0.1.2" +version: 0.2.0 +appVersion: "0.2.0" dependencies: - name: postgresql diff --git a/deploy/helm/llm-bench/templates/NOTES.txt b/deploy/helm/llm-bench/templates/NOTES.txt index edb216d..a687db6 100644 --- a/deploy/helm/llm-bench/templates/NOTES.txt +++ b/deploy/helm/llm-bench/templates/NOTES.txt @@ -19,6 +19,12 @@ Ingress: {{- range .Values.ingress.hosts }} http{{ if $.Values.ingress.tls }}s{{ end }}://{{ .host }} {{- end }} +{{- else if eq .Values.service.frontend.type "NodePort" }} + +The frontend is on a NodePort of every node: + kubectl get svc -n {{ .Release.Namespace }} {{ include "llm-bench.fullname" . }}-frontend -o jsonpath='{.spec.ports[0].nodePort}' + +Then open http://: {{- else }} No ingress enabled. Access the frontend via port-forward: diff --git a/deploy/helm/llm-bench/templates/postgres.yaml b/deploy/helm/llm-bench/templates/postgres.yaml index 3d769c0..1feb503 100644 --- a/deploy/helm/llm-bench/templates/postgres.yaml +++ b/deploy/helm/llm-bench/templates/postgres.yaml @@ -57,12 +57,21 @@ spec: - name: postgres image: "{{ .Values.postgres.image.registry }}/{{ .Values.postgres.image.repository }}:{{ .Values.postgres.image.tag }}" imagePullPolicy: {{ .Values.image.backend.pullPolicy }} - # Override CMD only (not the entrypoint), so docker-entrypoint.sh still - # runs initdb/startup, then execs the server with our tuning. + # The data lives one level below the mount point: a freshly formatted + # block volume (ext4 on Ceph RBD, EBS, a PD) has lost+found at its + # root, and initdb refuses a non-empty directory. A database already + # initialised at the root, by an earlier version of this chart, stays + # where it is. docker-entrypoint.sh still runs initdb/startup, then + # execs the server with our tuning. + command: ["sh", "-c"] args: - - "postgres" - - "-c" - - "max_connections={{ .Values.postgres.maxConnections }}" + - | + if [ -s /var/lib/postgresql/data/PG_VERSION ]; then + export PGDATA=/var/lib/postgresql/data + else + export PGDATA=/var/lib/postgresql/data/pgdata + fi + exec docker-entrypoint.sh postgres -c max_connections={{ .Values.postgres.maxConnections }} env: - name: POSTGRES_DB value: {{ .Values.postgres.database | quote }} diff --git a/docker-compose.yml b/docker-compose.yml index cbd33ee..b89ca4f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,7 +15,7 @@ # For a cluster, use the Helm chart in deploy/helm/llm-bench instead. x-backend: &backend - image: docker.io/4pdosc/llm-bench-backend:${LLMBENCH_VERSION:-0.1.2} + image: docker.io/4pdosc/llm-bench-backend:${LLMBENCH_VERSION:-0.2.0} build: context: . dockerfile: platform/backend/Dockerfile @@ -100,7 +100,7 @@ services: restart: unless-stopped frontend: - image: docker.io/4pdosc/llm-bench-frontend:${LLMBENCH_VERSION:-0.1.2} + image: docker.io/4pdosc/llm-bench-frontend:${LLMBENCH_VERSION:-0.2.0} build: context: platform/frontend ports: diff --git a/platform/backend/app/main.py b/platform/backend/app/main.py index c9afa65..270c3ac 100644 --- a/platform/backend/app/main.py +++ b/platform/backend/app/main.py @@ -214,7 +214,7 @@ async def _sync_modules() -> None: app = FastAPI( title="LLM Benchmark Platform API", description="Multi-tenant LLM benchmark leaderboard with modular test units", - version="0.1.2", + version="0.2.0", lifespan=lifespan, # `servers` makes Swagger "Try it out" target {prefix}/... (e.g. /api/auth/login) # so requests route back through nginx. Routing itself stays at bare paths. diff --git a/platform/backend/pyproject.toml b/platform/backend/pyproject.toml index ecfb36f..e4dff97 100644 --- a/platform/backend/pyproject.toml +++ b/platform/backend/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "llm-bench-backend" -version = "0.1.2" +version = "0.2.0" description = "LLMBench: the API, the worker, and the benchmark modules they run" requires-python = ">=3.12" dependencies = [ diff --git a/platform/backend/uv.lock b/platform/backend/uv.lock index 9a818c9..d1a8ef2 100644 --- a/platform/backend/uv.lock +++ b/platform/backend/uv.lock @@ -1029,7 +1029,7 @@ wheels = [ [[package]] name = "llm-bench-backend" -version = "0.1.2" +version = "0.2.0" source = { virtual = "." } dependencies = [ { name = "aiohttp" }, diff --git a/platform/frontend/package-lock.json b/platform/frontend/package-lock.json index 81924c8..7d12bef 100644 --- a/platform/frontend/package-lock.json +++ b/platform/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "llm-bench-frontend", - "version": "0.1.2", + "version": "0.2.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "llm-bench-frontend", - "version": "0.1.2", + "version": "0.2.0", "dependencies": { "axios": "^1.7.0", "chart.js": "^4.5.1", diff --git a/platform/frontend/package.json b/platform/frontend/package.json index 3a5ed2a..9b6b71c 100644 --- a/platform/frontend/package.json +++ b/platform/frontend/package.json @@ -1,6 +1,6 @@ { "name": "llm-bench-frontend", - "version": "0.1.2", + "version": "0.2.0", "private": true, "type": "module", "scripts": { diff --git a/tests/modules/test_perf_guidellm.py b/tests/modules/test_perf_guidellm.py index 2335c83..45fc98d 100644 --- a/tests/modules/test_perf_guidellm.py +++ b/tests/modules/test_perf_guidellm.py @@ -144,3 +144,27 @@ def test_sweep_only_metrics_stay_out_of_the_single_module(): } # Everything else the sweep declares, the single module declares too assert single - sweep == set() + + +@pytest.mark.parametrize("per_slot", [20, None]) +def test_a_level_stops_at_a_request_count_unless_cleared(monkeypatch, tmp_path, per_slot): + """Like the sweep: concurrency x requests_per_concurrency requests, with + max_seconds as the cap; cleared, the duration alone bounds the test.""" + import bench.modules.perf_guidellm as perf + + seen = {} + + def fake_load(endpoint, **kwargs): + seen.update(kwargs) + return None + + monkeypatch.setattr(perf, "run_guidellm_load", fake_load) + cls = get_module("perf_guidellm") + params = cls.ParamsSchema(concurrency=4, requests_per_concurrency=per_slot) + cls()._run_guidellm(None, params, str(tmp_path), lambda *a: None, None) + assert seen["requests_per_concurrency"] == per_slot + assert seen["max_seconds"] == params.max_seconds + + +def test_requests_per_concurrency_defaults_to_a_count(): + assert get_module("perf_guidellm").ParamsSchema().requests_per_concurrency == 20 diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..7518fc9 --- /dev/null +++ b/uv.lock @@ -0,0 +1,3 @@ +version = 1 +revision = 3 +requires-python = ">=3.12"