feat: add the quality, latency, and cost evaluation harness

README.md

@@ -163,6 +163,32 @@ dependent entry. Responses carry `X-Cache: miss | exact | semantic`.
 | Semantic | Disabled by default; requires `public` privacy, deterministic generation, and an extraction, classification, or summarization task. |
 | Router decision | Reuses stable task classification; cleared when the policy version changes. |
 
+## Evaluation
+
+Every optimization is graded against a committed dataset in
+[`benchmarks/datasets`](benchmarks/datasets), never against sampled production traffic.
+`execute()` runs each case through a caller-supplied transport and times it; `build_report()`
+turns the outcomes into one `EvaluationReport` that always pairs quality with latency and
+GPU cost, never one alone.
+
+```bash
+python -c "
+from llm_router.evaluation import build_report, execute, load_dataset
+cases = load_dataset('benchmarks/datasets/extraction-v1.jsonl')
+outcomes = execute(cases, lambda case: (case.expected, True, 0.05))
+print(build_report(outcomes, model_id='small-specialist', model_revision='rev-1').summary())
+"
+```
+
+- Constrained tasks (extraction, classification) are scored by exact match, generative tasks
+  by token overlap, and structured tasks score zero when the output is not valid JSON.
+- `compare()` rejects a variant that buys latency or throughput with a quality or
+  structured-validity regression, however small; `render_comparison()` prints the verdict
+  with every regression reason.
+- [`benchmarks/workloads`](benchmarks/workloads) holds reproducible k6 steady and burst load
+  definitions. The burst scenario asserts bounded queue behavior — an explicit `429`/`503`
+  rejection — rather than unbounded tail latency.
+
 ## Runtime settings
 
 All settings use the `ROUTER_` prefix.

benchmarks/datasets/extraction-v1.jsonl

@@ -0,0 +1,5 @@
+{"id": "extract-001", "task": "extraction", "prompt": "Extract the invoice number and total from: Invoice INV-4417, total 182.50 USD. Reply as JSON.", "expected": "{\"invoice_number\": \"INV-4417\", \"total\": \"182.50 USD\"}", "structured": true}
+{"id": "extract-002", "task": "extraction", "prompt": "Extract the claim id from: Claim CLM-9921 was filed on 2026-04-02. Reply as JSON.", "expected": "{\"claim_id\": \"CLM-9921\"}", "structured": true}
+{"id": "classify-001", "task": "classification", "prompt": "Classify this ticket as billing, technical, or other: my card was charged twice.", "expected": "billing"}
+{"id": "classify-002", "task": "classification", "prompt": "Classify this ticket as billing, technical, or other: the dashboard returns a 500 error.", "expected": "technical"}
+{"id": "summarize-001", "task": "summarization", "prompt": "Summarize: revenue grew in every region while support costs fell for the third quarter running.", "expected": "revenue grew in every region and support costs fell"}

benchmarks/workloads/burst.js

@@ -0,0 +1,49 @@
+// Burst load definition (section 16): verifies bounded queue behaviour and
+// predictable rejection rather than unbounded tail latency.
+//   k6 run -e BASE_URL=http://127.0.0.1:8000 -e API_KEY=dev-key benchmarks/workloads/burst.js
+import http from "k6/http";
+import { check } from "k6";
+
+export const options = {
+  scenarios: {
+    burst: {
+      executor: "ramping-arrival-rate",
+      startRate: 10,
+      timeUnit: "1s",
+      preAllocatedVUs: 100,
+      maxVUs: 400,
+      stages: [
+        { target: 10, duration: "1m" },
+        { target: 200, duration: "30s" },
+        { target: 200, duration: "1m" },
+        { target: 10, duration: "1m" },
+      ],
+    },
+  },
+  thresholds: {
+    // Under saturation the platform must reject predictably, not queue without bound.
+    "http_req_duration": ["p(99)<10000"],
+    "checks": ["rate>0.99"],
+  },
+};
+
+const BASE_URL = __ENV.BASE_URL || "http://127.0.0.1:8000";
+const API_KEY = __ENV.API_KEY || "dev-key";
+
+export default function () {
+  const response = http.post(
+    `${BASE_URL}/v1/chat/completions`,
+    JSON.stringify({
+      model: "auto",
+      messages: [{ role: "user", content: "Classify this burst probe ticket." }],
+      max_tokens: 64,
+      routing: { task: "classification", privacy: "private" },
+    }),
+    { headers: { "Content-Type": "application/json", Authorization: `Bearer ${API_KEY}` } },
+  );
+
+  check(response, {
+    "no unhandled failure": (r) => [200, 429, 503].includes(r.status),
+    "rejection is explicit": (r) => r.status === 200 || r.json("error.type") !== undefined,
+  });
+}

benchmarks/workloads/steady.js

@@ -0,0 +1,55 @@
+// Sustained load definition (section 16). Reproducible from this committed file:
+//   k6 run -e BASE_URL=http://127.0.0.1:8000 -e API_KEY=dev-key benchmarks/workloads/steady.js
+import http from "k6/http";
+import { check } from "k6";
+
+export const options = {
+  scenarios: {
+    steady: {
+      executor: "constant-arrival-rate",
+      rate: 20,
+      timeUnit: "1s",
+      duration: "5m",
+      preAllocatedVUs: 40,
+      maxVUs: 120,
+    },
+  },
+  thresholds: {
+    // Report quality and latency together; a passing run is not a quality claim.
+    "http_req_duration{expected_response:true}": ["p(95)<2000", "p(99)<5000"],
+    "http_req_failed": ["rate<0.01"],
+  },
+};
+
+const BASE_URL = __ENV.BASE_URL || "http://127.0.0.1:8000";
+const API_KEY = __ENV.API_KEY || "dev-key";
+
+// Prompt-length distribution: short classification, medium extraction, long RAG.
+const PROMPTS = [
+  { task: "classification", text: "Classify this ticket: my card was charged twice." },
+  { task: "extraction", text: "Extract the invoice number from: Invoice INV-4417, total 182.50 USD." },
+  {
+    task: "rag",
+    text: "According to the documents provided, summarize the retention policy. ".repeat(24),
+  },
+];
+
+export default function () {
+  const prompt = PROMPTS[Math.floor(Math.random() * PROMPTS.length)];
+  const response = http.post(
+    `${BASE_URL}/v1/chat/completions`,
+    JSON.stringify({
+      model: "auto",
+      messages: [{ role: "user", content: prompt.text }],
+      max_tokens: 128,
+      routing: { task: prompt.task, privacy: "private" },
+    }),
+    { headers: { "Content-Type": "application/json", Authorization: `Bearer ${API_KEY}` } },
+  );
+
+  check(response, {
+    "served or rejected predictably": (r) => r.status === 200 || r.status === 503 || r.status === 429,
+    "overload carries retry guidance": (r) => r.status !== 503 || r.headers["Retry-After"] !== undefined,
+    "route is attributed": (r) => r.status !== 200 || r.headers["X-Route-Model"] !== undefined,
+  });
+}