From d3e959e895e24774c670cb00160890cd23f73ea3 Mon Sep 17 00:00:00 2001 From: agrechenkov Date: Fri, 10 Jul 2026 09:46:17 -0400 Subject: [PATCH 1/4] T23: add prompt v2 speed-quality evaluator Cites V29. --- SPEC.md | 21 +- tests/test_prompt_speed_quality_eval.py | 153 +++++ tools/prompt_speed_quality_eval.py | 758 ++++++++++++++++++++++++ 3 files changed, 930 insertions(+), 2 deletions(-) create mode 100644 tests/test_prompt_speed_quality_eval.py create mode 100644 tools/prompt_speed_quality_eval.py diff --git a/SPEC.md b/SPEC.md index e3670e8..a284269 100644 --- a/SPEC.md +++ b/SPEC.md @@ -8,6 +8,7 @@ Caveman-encoded (compression, not amputation). Paths / ids / action names / numb - G2: run on AMD NPU (FastFlowLM) | any CPU/GPU (Ollama). provider ? → auto-fallback to other. - G3: web dashboard = single home for chat, notes, meetings, config, benchmark, notifications. - G4: heavy LLM cost (prefill) → pre-compute after-hours, read cached. +- G5: `prompt:` warm p50 ≤ 15s & p90 ≤ 20s; v2 p50 ≤ 60% v1; quality median ≥ v1; invented requirements = 0. ## §C context / stack @@ -27,6 +28,10 @@ Caveman-encoded (compression, not amputation). Paths / ids / action names / numb - action: `config_snapshot` → full cfg; `apply_config_patch {patch}` → merge (whitelist `filter_config_patch`) - action: `recent_history {limit?}` → newest history rows; `input_text`/`output_text` iff stored @ write-time - action: `prompt_builder_preview {settings?,sample?}` → deterministic local preview (`⊥` LLM call) +- config: `prompt_builder.prompt_version` ∈ {`v1`,`v2`}; default `v2`; v1 = instant rollback +- config: `server.warm_on_start` bool + `server.keep_warm_minutes` 0..1440; warmup best-effort +- cmd: `python tools/prompt_speed_quality_eval.py [--live] [--runs N] [--judge-file PATH] [--out PATH]` → old-vs-v2 JSON +- data: `data/benchmarks/prompt_v2_ab_.json` → speed+quality gate evidence - action: `notify_gate {title,message}` → `{show,reason,category}` (logs); `notifications_log {limit}` → rows - action: `quill_status` → `{reachable,enabled,server,server_version}` - action: `quill_search_meetings {query,limit}` → `{meetings:[{id,title,date,duration,participants,url}]}` @@ -66,13 +71,18 @@ Caveman-encoded (compression, not amputation). Paths / ids / action names / numb - V17: builtin mode prompts locked from patching (only `tone.preset` patchable) - V18: version ∀ ∈ {`_version.py`,`pyproject.toml`,`installer/installer.iss`,`README.md`} equal; CI smoke fails on drift - V19: `main` branch-protected (ruleset 17344133) → land via PR + ruleset toggle; ⊥ direct push -- V20: change gates ! pass: `ruff check scripts tests`, `pytest`, `node --check scripts/ui/web/app.js`, AHK parse-check (PowerShell `/ErrorStdOut`) +- V20: change gates ! pass: `ruff check scripts tests`, `python -m pytest`, `node --check scripts/ui/web/app.js`, AHK parse-check (PowerShell `/ErrorStdOut`) - V21: local data (config/data/logs/vendor/certs) ∈ `.gitignore` ∴ pull moves code only, never user data - V22: `sync.ps1` ∃ uncommitted tracked changes → skip pull (⊥ clobber un-pushed WIP) - V23: meeting `NoContentError` & age ≥ 2d → skip-marker (`meeting_skips.jsonl`) ∴ ⊥ re-queue ever; age < 2d → retry (Quill transcript may still sync); non-content errors ⊥ skip-mark. batch errors → `daemon.log` only (⊥ UI panel, per user) -- V24: prompt_builder default cfg ⇒ `CLAUDE_PROMPT_SYSTEM_PROMPT` identity; non-default prompt validation target-aware (XML only when effective structure=xml); built-in `modes.prompt.system_prompt` still locked +- V24: prompt_builder default cfg ⇒ `CLAUDE_PROMPT_SYSTEM_PROMPT_V2`; `prompt_version=v1` ⇒ `CLAUDE_PROMPT_SYSTEM_PROMPT_V1`; non-default validation target-aware; built-in `modes.prompt.system_prompt` still locked - V25: History view toggle = display-only; ⊥ mutate `history_store_text`; ⊥ reveal text absent from jsonl row - V26: History tab load default view = Telemetry (text hidden) even when storage visible +- V27: default v2 output → exactly ordered ``,``,``,``; task = 1 imperative sentence; constraints = 3–5 concrete items; output_format = 1 line; ⊥ preamble/fence/``; ≤ 220 tokens +- V28: prompt runtime caps short/medium/long = 240/320/420 tokens; retries ! same strategy cap +- V29: A/B gate → ≥12 fixed inputs; 1 warmup + ≥5 timed/style/input; p50/p90/min/max, TTFT, completion tokens, decode tok/s, seconds/output-token; v2 speed gate + median quality ≥v1 + invented=0 + R1 rate ≥v1 +- V30: v1 prompt constant retained + config-selectable without built-in prompt patching +- V31: daemon startup + configured idle interval → best-effort FastFlowLM warmup; failure logs only, ⊥ daemon startup failure ## §T tasks @@ -100,6 +110,10 @@ T19|x|[DOCS] first-run wizard text "chat popup" → "Open chat" + hotkey now `^! T20|x|[DOCS] daemon log location — audited 2.1.1: no stale ref in README/docs; nothing to change|— T21|x|prompt_builder cfg + claude_code identity + generic_chat adapter + dashboard controls/preview|V17,V24 T22|x|History Telemetry/Exposed views + inline redacted/visible storage control/help|V5,V6,V25,V26 +T23|x|prompt-v2 fixed A/B speed+quality harness + tests|V29 +T24|.|prompt-v2 default + v1 rollback selector + 240/320/420 caps|V24,V27,V28,V30 +T25|.|FastFlowLM startup+idle keep-warm + cold/warm measurement support|V31 +T26|.|2.3.0 release evidence + version/docs rebaseline after A/B gate passes|V18,V29 ``` ## §B bugs @@ -113,4 +127,7 @@ B4|2026-06|install launch: AHK called `.py`, shipped only `.exe`|flatten bundle B5|2026-06|`Ctrl+Shift+T` open_chat collided w/ browser reopen-tab|default → `^!c`; tray label = configured hotkey B6|2026-07|3 divergent autostart Run keys: daemon HKCU\Run\FastFlowPrompt, `install.ps1` HKCU\Run\Flowkey (different name!), `installer.iss` optional HKLM\Run\Flowkey → toggle blind to other 2, could double-launch|unify on HKCU\Run\FastFlowPrompt everywhere; drop installer.iss HKLM task; uninstall now cleans the HKCU value; guarded by `test_installer_autostart.py` B7|2026-07|"Run batch now" → "0 of 5, 5 errors" ∀ run: 5 Quill stub recordings (⊥ minutes ⊥ transcript) re-queued forever ∵ idempotency = digest_exists only|typed `NoContentError` → skip store `meeting_skips.jsonl` (age ≥ 2d guard) + `skipped` count in result; queue checks digest ∧ skip; reasons stay in daemon.log|V23 +B8|2026-07-10|new prompt-v2 eval imports violated Ruff I001/UP035|V20 caught; sort imports + `Callable` from `collections.abc` +B9|2026-07-10|prompt-v2 eval test import block retained extra blank line|V20 caught; normalize import spacing +B10|2026-07-10|stale user-level `pytest.exe` exited 1 + ⊥ diagnostics while active interpreter pytest passed|V20 → interpreter-bound `python -m pytest` ``` diff --git a/tests/test_prompt_speed_quality_eval.py b/tests/test_prompt_speed_quality_eval.py new file mode 100644 index 0000000..60f8a0b --- /dev/null +++ b/tests/test_prompt_speed_quality_eval.py @@ -0,0 +1,153 @@ +from __future__ import annotations + +from tools import prompt_speed_quality_eval as prompt_eval + +GOOD_OUTPUT = """ +Build the requested CSV validator. + + +The input is a folder of CSV files and a schema. + + +- Read every CSV in the folder. +- Validate each row against the schema. +- Report the source file and line for every error. + + +Return the completed script and focused tests. +""" + + +def _judge(cases): + return { + "method": "manual", + "judgments": { + f"{style}:{case['name']}": { + "r3": True, + "r4": True, + "invented_requirement": False, + "notes": "", + } + for style in ("v1", "v2") + for case in cases + }, + } + + +def test_fixed_set_covers_required_spread(): + cases = prompt_eval.FIXED_CASES + categories = {case["category"] for case in cases} + + assert len(cases) >= 12 + assert {"implement", "debug", "review", "refactor", "data", "vague", "long", "trap"} <= categories + assert sum(bool(case.get("trap")) for case in cases) >= 1 + assert len({case["name"] for case in cases}) == len(cases) + + +def test_machine_rubric_and_manual_judgment_combine_to_seven(): + result = prompt_eval.score_output( + GOOD_OUTPUT, + completion_tokens=92, + judgment={"r3": True, "r4": True, "invented_requirement": False}, + ) + + assert result["rubric"] == {key: True for key in ("r1", "r2", "r3", "r4", "r5", "r6", "r7")} + assert result["score"] == 7 + assert result["passed"] is True + assert result["pending_judge"] is False + assert result["constraint_item_count"] == 3 + + +def test_quality_stays_pending_without_semantic_judge_and_invention_hard_fails(): + pending = prompt_eval.score_output(GOOD_OUTPUT, completion_tokens=92) + invented = prompt_eval.score_output( + GOOD_OUTPUT, + completion_tokens=92, + judgment={"r3": True, "r4": True, "invented_requirement": True}, + ) + + assert pending["score"] is None + assert pending["pending_judge"] is True + assert invented["score"] == 7 + assert invented["hard_fail"] is True + assert invented["passed"] is False + + +def test_exact_section_and_constraint_count_checks_catch_merged_output(): + merged = GOOD_OUTPUT.replace("\n", "") + too_few = GOOD_OUTPUT.replace( + "- Validate each row against the schema.\n- Report the source file and line for every error.\n", + "", + ) + + assert prompt_eval.score_output(merged)["rubric"]["r1"] is False + result = prompt_eval.score_output( + too_few, + judgment={"r3": True, "r4": True, "invented_requirement": False}, + ) + assert result["constraint_item_count"] == 1 + assert result["rubric"]["r3"] is False + + +def test_usage_metrics_include_units_percentiles_and_per_token_cost(): + result = { + "usage": { + "completion_tokens": 100, + "prefill_duration_ttft": "1600ms", + "decode_duration": 8.0, + }, + "_wall_seconds": 10.0, + } + metrics = prompt_eval._sample_metrics(result, measured_wall=99.0) + distribution = prompt_eval._distribution([1, 2, 3, 4, 5]) + + assert metrics["wall_seconds"] == 10.0 + assert metrics["ttft_seconds"] == 1.6 + assert metrics["decode_tokens_per_second"] == 12.5 + assert metrics["seconds_per_output_token"] == 0.1 + assert distribution == {"median": 3.0, "p90": 4.6, "min": 1.0, "max": 5.0} + + +def test_full_protocol_uses_warmup_plus_five_runs_and_passes_both_gates(): + cases = list(prompt_eval.FIXED_CASES[:2]) + calls = [] + + def fake_model(**kwargs): + calls.append((kwargs["style"], kwargs["user_content"])) + is_v2 = kwargs["style"] == "v2" + return { + "output": GOOD_OUTPUT, + "model": kwargs["model"], + "_wall_seconds": 12.0 if is_v2 else 30.0, + "usage": { + "prefill_duration_ttft": 1.5, + "completion_tokens": 100 if is_v2 else 200, + "decode_duration": 8.0 if is_v2 else 20.0, + }, + } + + artifact = prompt_eval.run_evaluation( + fake_model, + cases=cases, + runs=5, + warmups=1, + judge_data=_judge(cases), + ) + + assert len(calls) == len(cases) * 2 * 6 + assert artifact["summaries"]["v1"]["protocol"]["successful_timed_runs"] == 10 + assert artifact["summaries"]["v2"]["speed"]["wall_seconds"]["median"] == 12.0 + assert artifact["gate"]["speed"]["p50_ratio"] == 0.4 + assert artifact["gate"]["speed"]["passed"] is True + assert artifact["gate"]["quality"]["passed"] is True + assert artifact["gate"]["passed"] is True + assert len(artifact["manual_side_by_side"]) == 2 + + +def test_protocol_rejects_too_few_timed_runs(): + try: + prompt_eval.run_evaluation(lambda **_kwargs: {}, cases=list(prompt_eval.FIXED_CASES[:1]), runs=4) + except ValueError as exc: + assert str(exc) == "runs must be >= 5" + else: + raise AssertionError("runs below the invariant must be rejected") diff --git a/tools/prompt_speed_quality_eval.py b/tools/prompt_speed_quality_eval.py new file mode 100644 index 0000000..a8c734e --- /dev/null +++ b/tools/prompt_speed_quality_eval.py @@ -0,0 +1,758 @@ +"""Reproducible prompt-v1 versus prompt-v2 speed and quality evaluation. + +The live protocol intentionally costs many calls: one warmup plus at least five +timed generations for each style/input pair. A manual or LLM-produced judge +file supplies rubric items that cannot be inferred safely from syntax alone. +""" + +from __future__ import annotations + +import argparse +import json +import math +import re +import statistics +import sys +import time +import urllib.request +from collections.abc import Callable +from datetime import datetime +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] +SCRIPTS_DIR = ROOT / "scripts" +if str(SCRIPTS_DIR) not in sys.path: + sys.path.insert(0, str(SCRIPTS_DIR)) + +import ffp_config # noqa: E402 +import ffp_llm_client # noqa: E402 + +PROMPT_V2_CANDIDATE = ( + "Turn the user's rough request into a tight, high-signal coding-agent prompt. " + "Emit exactly four sections, each on its own: " + " one imperative sentence naming the concrete deliverable. " + " only facts the user stated; never restate the request or invent background. " + " 3-5 concrete, testable bullet items drawn only from the request. " + " one line naming the exact output shape. " + "Be specific and actionable. No preamble, meta-framing, or filler. " + "Keep the whole prompt under 160 tokens. Return only the prompt." +) + +STYLE_SPECS = { + "v1": { + "label": "current", + "system_prompt": ffp_config.CLAUDE_PROMPT_SYSTEM_PROMPT, + "caps": (700, 900, 1200), + }, + "v2": { + "label": "candidate", + "system_prompt": PROMPT_V2_CANDIDATE, + "caps": (240, 320, 420), + }, +} + +FIXED_CASES = ( + { + "name": "implement_csv_validator", + "category": "implement", + "input": ( + "build a python script that reads a folder of CSVs, validates rows against a schema, " + "and writes an error report with file and line numbers" + ), + }, + { + "name": "debug_async_cache_race", + "category": "debug", + "input": ( + "debug an intermittent async cache test failure: two refreshes sometimes run at once " + "and the older response overwrites the newer value; keep the public API unchanged" + ), + }, + { + "name": "review_auth_middleware", + "category": "review", + "input": ( + "review the new TypeScript auth middleware for regressions and missing tests; report " + "findings by severity with file and line references" + ), + }, + { + "name": "refactor_invoice_service", + "category": "refactor", + "input": ( + "refactor InvoiceService so calculation and persistence are separate, preserve current " + "behavior, and do not add dependencies" + ), + }, + { + "name": "data_monthly_rollup", + "category": "data", + "input": ( + "combine these monthly sales CSVs, keep the original rows, summarize revenue by region, " + "and reconcile the summary total to the source total" + ), + }, + { + "name": "vague_dashboard_speed", + "category": "vague", + "input": "make the dashboard faster", + }, + { + "name": "long_webhook_import", + "category": "long", + "input": ( + "Add a webhook import endpoint to the existing Python service. It receives JSON events " + "with event_id, account_id, happened_at, and payload. Reject malformed JSON with 400, " + "authenticate using the existing X-Webhook-Key middleware, and make duplicate event_id " + "requests return the original result without inserting twice. Store accepted events with " + "the repository layer already used by the billing importer. Return JSON with id, status, " + "and duplicate. Add focused unit tests and one HTTP integration test. Do not change the " + "database schema or introduce a queue." + ), + }, + { + "name": "trap_rename_cli", + "category": "trap", + "trap": True, + "input": "write a CLI that renames files from a mapping in a text file", + }, + { + "name": "plan_postgres_migration", + "category": "plan", + "input": ( + "plan a zero-downtime migration from users.full_name to first_name and last_name for a " + "Postgres app; planning only, no code changes" + ), + }, + { + "name": "explain_regex", + "category": "explain", + "input": r"explain what ^(?P[a-z0-9]+(?:-[a-z0-9]+)*)$ accepts and show three examples", + }, + { + "name": "fix_timezone_boundary", + "category": "debug", + "input": ( + "fix the weekly report bug where Sunday 11:30 PM America/New_York appears in next week; " + "weeks start Monday local time and existing report JSON must not change shape" + ), + }, + { + "name": "api_retry_policy", + "category": "implement", + "input": ( + "add retry handling to the Go API client for 429 and 503 responses only, honor " + "Retry-After when present, cap attempts at three, and test the retry timing with a fake clock" + ), + }, +) + +_SECTION_RE = re.compile( + r"\A\s*\s*(?P.*?)\s*\s*" + r"\s*(?P.*?)\s*\s*" + r"\s*(?P.*?)\s*\s*" + r"\s*(?P.*?)\s*\s*\Z", + flags=re.IGNORECASE | re.DOTALL, +) +_IMPERATIVE_VERBS = frozenset( + { + "add", + "analyze", + "build", + "check", + "combine", + "create", + "debug", + "design", + "explain", + "fix", + "implement", + "investigate", + "migrate", + "plan", + "refactor", + "review", + "test", + "update", + "validate", + "write", + } +) + +ModelCall = Callable[..., dict[str, Any]] + + +def _as_float(value: Any) -> float | None: + if isinstance(value, bool) or value is None: + return None + if isinstance(value, (int, float)): + number = float(value) + return number if math.isfinite(number) else None + match = re.fullmatch(r"\s*([-+]?\d+(?:\.\d+)?)\s*(ns|us|µs|ms|s)?\s*", str(value)) + return float(match.group(1)) if match else None + + +def _duration_seconds(usage: dict[str, Any], *keys: str) -> float | None: + for key in keys: + if key not in usage: + continue + raw = usage.get(key) + value = _as_float(raw) + if value is None: + continue + suffix_match = re.fullmatch( + r"\s*[-+]?\d+(?:\.\d+)?\s*(ns|us|µs|ms|s)?\s*", str(raw) + ) + unit = str(usage.get(f"{key}_unit") or usage.get("duration_unit") or "").lower() + if suffix_match and suffix_match.group(1): + unit = suffix_match.group(1).lower() + if key.endswith("_ns") or unit == "ns": + return value / 1_000_000_000 + if key.endswith("_us") or unit in {"us", "µs"}: + return value / 1_000_000 + if key.endswith("_ms") or unit == "ms": + return value / 1000 + return value + return None + + +def _int_metric(usage: dict[str, Any], *keys: str) -> int | None: + for key in keys: + value = _as_float(usage.get(key)) + if value is not None and value >= 0: + return int(value) + return None + + +def _percentile(values: list[float], fraction: float) -> float | None: + clean = sorted(float(value) for value in values if math.isfinite(float(value))) + if not clean: + return None + if len(clean) == 1: + return clean[0] + rank = (len(clean) - 1) * fraction + lower = math.floor(rank) + upper = math.ceil(rank) + if lower == upper: + return clean[lower] + return clean[lower] + (clean[upper] - clean[lower]) * (rank - lower) + + +def _distribution(values: list[float | int | None]) -> dict[str, float | None]: + clean = [float(value) for value in values if isinstance(value, (int, float))] + if not clean: + return {"median": None, "p90": None, "min": None, "max": None} + return { + "median": round(float(statistics.median(clean)), 4), + "p90": round(float(_percentile(clean, 0.9) or 0.0), 4), + "min": round(min(clean), 4), + "max": round(max(clean), 4), + } + + +def _cap_for_input(caps: tuple[int, int, int] | list[int], input_text: str) -> int: + if len(input_text) <= 350: + return int(caps[0]) + if len(input_text) <= 1200: + return int(caps[1]) + return int(caps[2]) + + +def _estimate_tokens(text: str) -> int: + return len(re.findall(r"\w+|[^\w\s]", str(text or ""), flags=re.UNICODE)) + + +def _section_bodies(text: str) -> dict[str, str]: + match = _SECTION_RE.fullmatch(str(text or "")) + return {name: match.group(name).strip() for name in match.groupdict()} if match else {} + + +def _single_imperative_sentence(text: str) -> bool: + cleaned = re.sub(r"\s+", " ", str(text or "")).strip() + if not cleaned or "\n" in str(text or "").strip(): + return False + sentences = re.findall(r"[^.!?]+[.!?](?=\s|$)", cleaned) + if len(sentences) != 1 or sentences[0].strip() != cleaned: + return False + first = re.match(r"[A-Za-z]+", cleaned) + return bool(first and first.group(0).lower() in _IMPERATIVE_VERBS) + + +def _constraint_items(text: str) -> list[str]: + items: list[str] = [] + for line in str(text or "").splitlines(): + match = re.match(r"\s*(?:[-*•]|\d+[.)])\s+(.+?)\s*$", line) + if match: + items.append(match.group(1)) + return items + + +def score_output( + text: str, + *, + completion_tokens: int | None = None, + judgment: dict[str, Any] | None = None, +) -> dict[str, Any]: + """Score one representative output; semantic items remain pending without a judge.""" + raw = str(text or "").strip() + sections = _section_bodies(raw) + constraint_items = _constraint_items(sections.get("constraints", "")) + estimated_tokens = _estimate_tokens(raw) + token_count = completion_tokens if completion_tokens is not None else estimated_tokens + r1 = bool(sections) + r2 = bool(sections) and _single_imperative_sentence(sections["task"]) + r5 = bool(sections) and bool(sections["output_format"]) and "\n" not in sections["output_format"] + lowered = raw.lower() + r6 = bool(sections) and not any( + marker in lowered + for marker in ("```", "= 6 and not hard_fail), + "hard_fail": hard_fail, + "pending_judge": pending, + "invented_requirement": invented, + "constraint_item_count": len(constraint_items), + "token_count": token_count, + "token_count_source": "usage.completion_tokens" if completion_tokens is not None else "estimate", + "estimated_tokens": estimated_tokens, + "judge_notes": str(semantic.get("notes") or ""), + } + + +def _sample_metrics(result: dict[str, Any], measured_wall: float) -> dict[str, Any]: + usage = result.get("usage") if isinstance(result.get("usage"), dict) else {} + completion_tokens = _int_metric(usage, "completion_tokens", "output_tokens", "eval_count") + ttft = _duration_seconds( + usage, + "prefill_duration_ttft", + "ttft_seconds", + "time_to_first_token", + "prompt_eval_duration", + ) + decode_duration = _duration_seconds( + usage, + "decode_duration", + "decode_duration_seconds", + "eval_duration", + ) + wall = _as_float(result.get("_wall_seconds")) or measured_wall + decode_tps = ( + completion_tokens / decode_duration + if completion_tokens is not None and decode_duration and decode_duration > 0 + else None + ) + seconds_per_token = ( + wall / completion_tokens if completion_tokens is not None and completion_tokens > 0 else None + ) + return { + "wall_seconds": round(wall, 4), + "ttft_seconds": round(ttft, 4) if ttft is not None else None, + "completion_tokens": completion_tokens, + "decode_duration_seconds": round(decode_duration, 4) if decode_duration is not None else None, + "decode_tokens_per_second": round(decode_tps, 4) if decode_tps is not None else None, + "seconds_per_output_token": round(seconds_per_token, 6) if seconds_per_token is not None else None, + } + + +def _run_sample(call_model: ModelCall, **call_args: Any) -> dict[str, Any]: + started = time.perf_counter() + try: + result = call_model(**call_args) + measured_wall = time.perf_counter() - started + output = ffp_llm_client.normalize_output(result.get("output") or "") + return { + "ok": bool(output), + "error": "" if output else "model returned empty output", + "output": output, + "model": str(result.get("model") or call_args.get("model") or ""), + "usage": result.get("usage") if isinstance(result.get("usage"), dict) else {}, + **_sample_metrics(result, measured_wall), + } + except Exception as exc: + return { + "ok": False, + "error": str(exc), + "output": "", + "model": str(call_args.get("model") or ""), + "usage": {}, + "wall_seconds": round(time.perf_counter() - started, 4), + "ttft_seconds": None, + "completion_tokens": None, + "decode_duration_seconds": None, + "decode_tokens_per_second": None, + "seconds_per_output_token": None, + } + + +def summarize_samples(samples: list[dict[str, Any]]) -> dict[str, Any]: + successful = [sample for sample in samples if sample.get("ok")] + return { + "runs": len(samples), + "successful_runs": len(successful), + "wall_seconds": _distribution([sample.get("wall_seconds") for sample in successful]), + "ttft_seconds": _distribution([sample.get("ttft_seconds") for sample in successful]), + "completion_tokens": _distribution( + [sample.get("completion_tokens") for sample in successful] + ), + "decode_tokens_per_second": _distribution( + [sample.get("decode_tokens_per_second") for sample in successful] + ), + "seconds_per_output_token": _distribution( + [sample.get("seconds_per_output_token") for sample in successful] + ), + } + + +def _judgment_for(judge_data: dict[str, Any], style: str, case_name: str) -> dict[str, Any] | None: + judgments = judge_data.get("judgments") if isinstance(judge_data, dict) else None + if not isinstance(judgments, dict): + return None + value = judgments.get(f"{style}:{case_name}") + return value if isinstance(value, dict) else None + + +def _style_summary(case_rows: list[dict[str, Any]], style: str, runs: int) -> dict[str, Any]: + samples: list[dict[str, Any]] = [] + quality: list[dict[str, Any]] = [] + ordinary_outliers = 0 + warmup_errors = 0 + for row in case_rows: + style_row = row["styles"][style] + samples.extend(style_row["samples"]) + quality.append(style_row["quality"]) + warmup_errors += len(style_row["warmup_errors"]) + if row["category"] != "long": + ordinary_outliers += sum( + 1 + for sample in style_row["samples"] + if sample.get("ok") and float(sample.get("wall_seconds") or 0) > 25 + ) + speed = summarize_samples(samples) + scores = [float(item["score"]) for item in quality if item.get("score") is not None] + clean_passes = sum(1 for item in quality if item["rubric"]["r1"]) + passed = sum(1 for item in quality if item.get("passed")) + return { + "speed": speed, + "quality": { + "median_score": round(float(statistics.median(scores)), 2) if scores else None, + "scored_outputs": len(scores), + "pending_judge": sum(1 for item in quality if item.get("pending_judge")), + "passed_outputs": passed, + "pass_rate": round(passed / len(quality), 4) if quality else 0.0, + "clean_section_pass_rate": round(clean_passes / len(quality), 4) if quality else 0.0, + "invented_requirement_failures": sum( + 1 for item in quality if item.get("invented_requirement") is True + ), + }, + "protocol": { + "expected_timed_runs": len(case_rows) * runs, + "successful_timed_runs": speed["successful_runs"], + "warmup_errors": warmup_errors, + }, + "ordinary_over_25s": ordinary_outliers, + } + + +def evaluate_gate(summaries: dict[str, dict[str, Any]]) -> dict[str, Any]: + v1 = summaries["v1"] + v2 = summaries["v2"] + v1_p50 = v1["speed"]["wall_seconds"]["median"] + v2_p50 = v2["speed"]["wall_seconds"]["median"] + v2_p90 = v2["speed"]["wall_seconds"]["p90"] + ratio = v2_p50 / v1_p50 if v1_p50 and v2_p50 is not None else None + protocol_pass = all( + summary["protocol"]["successful_timed_runs"] + == summary["protocol"]["expected_timed_runs"] + and summary["protocol"]["warmup_errors"] == 0 + for summary in (v1, v2) + ) + speed_checks = { + "v2_p50_le_15s": bool(v2_p50 is not None and v2_p50 <= 15), + "v2_p50_le_60pct_v1": bool(ratio is not None and ratio <= 0.6), + "v2_p90_le_20s_goal": bool(v2_p90 is not None and v2_p90 <= 20), + "ordinary_inputs_no_over_25s": v2["ordinary_over_25s"] == 0, + } + v1_quality = v1["quality"] + v2_quality = v2["quality"] + quality_checks = { + "judge_complete": v1_quality["pending_judge"] == 0 and v2_quality["pending_judge"] == 0, + "v2_median_ge_v1": bool( + v1_quality["median_score"] is not None + and v2_quality["median_score"] is not None + and v2_quality["median_score"] >= v1_quality["median_score"] + ), + "v2_invented_requirements_zero": v2_quality["invented_requirement_failures"] == 0, + "v2_r1_rate_ge_v1": ( + v2_quality["clean_section_pass_rate"] >= v1_quality["clean_section_pass_rate"] + ), + } + speed_pass = speed_checks["v2_p50_le_15s"] and speed_checks["v2_p50_le_60pct_v1"] + quality_pass = all(quality_checks.values()) + return { + "protocol_pass": protocol_pass, + "speed": {"passed": speed_pass, "p50_ratio": round(ratio, 4) if ratio else None, **speed_checks}, + "quality": {"passed": quality_pass, **quality_checks}, + "passed": protocol_pass and speed_pass and quality_pass, + } + + +def _judge_template(cases: tuple[dict[str, Any], ...] | list[dict[str, Any]]) -> dict[str, Any]: + judgments: dict[str, dict[str, Any]] = {} + for style in STYLE_SPECS: + for case in cases: + judgments[f"{style}:{case['name']}"] = { + "r3": None, + "r4": None, + "invented_requirement": None, + "notes": "", + } + return {"method": "manual", "judgments": judgments} + + +def run_evaluation( + call_model: ModelCall, + *, + cases: tuple[dict[str, Any], ...] | list[dict[str, Any]] = FIXED_CASES, + style_specs: dict[str, dict[str, Any]] = STYLE_SPECS, + runs: int = 5, + warmups: int = 1, + judge_data: dict[str, Any] | None = None, + model: str = "qwen3.5:4b", + base_url: str = "http://127.0.0.1:52625", +) -> dict[str, Any]: + if runs < 5: + raise ValueError("runs must be >= 5") + if warmups < 1: + raise ValueError("warmups must be >= 1") + if len(cases) < 1: + raise ValueError("at least one case is required") + judge_data = judge_data or {} + case_rows: list[dict[str, Any]] = [] + for case in cases: + row: dict[str, Any] = { + "name": case["name"], + "category": case["category"], + "trap": bool(case.get("trap")), + "input": case["input"], + "styles": {}, + } + for style, spec in style_specs.items(): + max_tokens = _cap_for_input(spec["caps"], case["input"]) + call_args = { + "style": style, + "system_prompt": spec["system_prompt"], + "user_content": case["input"], + "max_tokens": max_tokens, + "model": model, + } + warmup_errors: list[str] = [] + for _ in range(warmups): + warmup = _run_sample(call_model, **call_args) + if not warmup["ok"]: + warmup_errors.append(warmup["error"]) + samples = [_run_sample(call_model, **call_args) for _ in range(runs)] + representative = next((sample for sample in samples if sample["ok"]), samples[0]) + quality = score_output( + representative.get("output") or "", + completion_tokens=representative.get("completion_tokens"), + judgment=_judgment_for(judge_data, style, case["name"]), + ) + row["styles"][style] = { + "label": spec["label"], + "max_tokens": max_tokens, + "warmups": warmups, + "warmup_errors": warmup_errors, + "samples": samples, + "speed": summarize_samples(samples), + "representative_output": representative.get("output") or "", + "quality": quality, + } + case_rows.append(row) + + summaries = {style: _style_summary(case_rows, style, runs) for style in style_specs} + manual_review = [ + { + "name": row["name"], + "input": row["input"], + "v1": row["styles"]["v1"]["representative_output"], + "v2": row["styles"]["v2"]["representative_output"], + } + for row in case_rows[:5] + ] + return { + "schema_version": 1, + "created_at": datetime.now().astimezone().isoformat(timespec="seconds"), + "protocol": { + "fixed_input_count": len(cases), + "warmups_per_style_input": warmups, + "timed_runs_per_style_input": runs, + "judge_method": str(judge_data.get("method") or "pending"), + "duration_note": "Numeric duration fields without units are interpreted as seconds.", + }, + "endpoint": {"base_url": base_url, "model": model}, + "styles": style_specs, + "cases": case_rows, + "summaries": summaries, + "gate": evaluate_gate(summaries), + "manual_side_by_side": manual_review, + "judge_template": _judge_template(cases), + } + + +def _call_openai_compatible( + *, + base_url: str, + bearer: str, + timeout_seconds: int, + model: str, + system_prompt: str, + user_content: str, + max_tokens: int, + **_ignored: Any, +) -> dict[str, Any]: + body = json.dumps( + { + "model": model, + "messages": [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_content}, + ], + "temperature": 0.1, + "max_tokens": max_tokens, + "stream": False, + } + ).encode("utf-8") + headers = {"Content-Type": "application/json"} + if bearer: + headers["Authorization"] = f"Bearer {bearer}" + request = urllib.request.Request( + base_url.rstrip("/") + "/v1/chat/completions", + data=body, + headers=headers, + method="POST", + ) + with urllib.request.urlopen(request, timeout=max(2, timeout_seconds)) as response: + payload = json.loads(response.read().decode("utf-8", errors="replace")) + choices = payload.get("choices") or [] + message = choices[0].get("message") if choices and isinstance(choices[0], dict) else {} + usage = dict(payload.get("usage") or {}) + for key in ( + "prefill_duration_ttft", + "decode_duration", + "completion_tokens", + "duration_unit", + ): + if key in payload and key not in usage: + usage[key] = payload[key] + return { + "output": str((message or {}).get("content") or ""), + "model": str(payload.get("model") or model), + "usage": usage, + } + + +def _load_judge(path: str) -> dict[str, Any]: + if not path: + return {} + payload = json.loads(Path(path).read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise ValueError("judge file root must be an object") + return payload + + +def _write_json(path: Path, payload: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + + +def _manifest() -> dict[str, Any]: + return { + "ready": len(FIXED_CASES) >= 12, + "fixed_input_count": len(FIXED_CASES), + "categories": sorted({case["category"] for case in FIXED_CASES}), + "styles": STYLE_SPECS, + "cases": list(FIXED_CASES), + "judge_template": _judge_template(FIXED_CASES), + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--live", action="store_true", help="run the full live A/B protocol") + parser.add_argument("--base-url", default="http://127.0.0.1:52625") + parser.add_argument("--model", default="qwen3.5:4b") + parser.add_argument("--bearer", default="flm") + parser.add_argument("--timeout-seconds", type=int, default=90) + parser.add_argument("--runs", type=int, default=5) + parser.add_argument("--judge-file", default="") + parser.add_argument("--out", default="") + parser.add_argument("--json", action="store_true", help="print the full JSON payload") + args = parser.parse_args() + + if not args.live: + payload = _manifest() + if args.out: + _write_json(Path(args.out), payload) + if args.json: + print(json.dumps(payload, ensure_ascii=False, indent=2)) + else: + print(f"ready={payload['ready']} fixed_inputs={payload['fixed_input_count']}") + print("Pass --live to run 1 warmup + >=5 timed calls per style/input.") + return 0 if payload["ready"] else 1 + + if args.runs < 5: + parser.error("--runs must be >= 5") + judge_data = _load_judge(args.judge_file) + + def call_model(**kwargs: Any) -> dict[str, Any]: + return _call_openai_compatible( + base_url=args.base_url, + bearer=args.bearer, + timeout_seconds=args.timeout_seconds, + **kwargs, + ) + + payload = run_evaluation( + call_model, + runs=args.runs, + judge_data=judge_data, + model=args.model, + base_url=args.base_url, + ) + out_path = Path(args.out) if args.out else ( + ROOT / "data" / "benchmarks" / f"prompt_v2_ab_{datetime.now():%Y-%m-%d}.json" + ) + _write_json(out_path, payload) + if args.json: + print(json.dumps(payload, ensure_ascii=False, indent=2)) + else: + print(f"artifact={out_path}") + print( + "gate=" + f"{payload['gate']['passed']} speed={payload['gate']['speed']['passed']} " + f"quality={payload['gate']['quality']['passed']}" + ) + return 0 if payload["gate"]["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) From 3743bf2dec434955c623c1c1f9b35bf76e01b0d1 Mon Sep 17 00:00:00 2001 From: agrechenkov Date: Fri, 10 Jul 2026 09:55:34 -0400 Subject: [PATCH 2/4] T24: ship dense prompt v2 with rollback Cites V24, V27, V28, V30, and V32. --- SPEC.md | 5 +- config/grammar_hotkey.config.example.json | 1 + scripts/ffp_config.py | 17 +- scripts/ffp_llm_client.py | 25 ++- scripts/ffp_prompt_builder.py | 174 +++++++++++++++++- scripts/ui/web/app.js | 3 + scripts/ui/web/index.html | 7 + .../grammar_hotkey.config.example.json | 1 + setup/defaults/grammar_hotkey.config.json | 1 + tests/test_ffp_config.py | 8 +- tests/test_grammar_fix.py | 87 +++++++-- tests/test_prompt_builder.py | 82 ++++++++- tools/prompt_builder_eval.py | 3 +- tools/prompt_speed_quality_eval.py | 31 ++-- 14 files changed, 391 insertions(+), 54 deletions(-) diff --git a/SPEC.md b/SPEC.md index a284269..9203c27 100644 --- a/SPEC.md +++ b/SPEC.md @@ -83,6 +83,7 @@ Caveman-encoded (compression, not amputation). Paths / ids / action names / numb - V29: A/B gate → ≥12 fixed inputs; 1 warmup + ≥5 timed/style/input; p50/p90/min/max, TTFT, completion tokens, decode tok/s, seconds/output-token; v2 speed gate + median quality ≥v1 + invented=0 + R1 rate ≥v1 - V30: v1 prompt constant retained + config-selectable without built-in prompt patching - V31: daemon startup + configured idle interval → best-effort FastFlowLM warmup; failure logs only, ⊥ daemon startup failure +- V32: prompt output with valid target structure → ⊥ anti-echo retry solely from line/word overlap ## §T tasks @@ -111,7 +112,7 @@ T20|x|[DOCS] daemon log location — audited 2.1.1: no stale ref in README/docs; T21|x|prompt_builder cfg + claude_code identity + generic_chat adapter + dashboard controls/preview|V17,V24 T22|x|History Telemetry/Exposed views + inline redacted/visible storage control/help|V5,V6,V25,V26 T23|x|prompt-v2 fixed A/B speed+quality harness + tests|V29 -T24|.|prompt-v2 default + v1 rollback selector + 240/320/420 caps|V24,V27,V28,V30 +T24|x|prompt-v2 default + v1 rollback selector + 240/320/420 caps|V24,V27,V28,V30,V32 T25|.|FastFlowLM startup+idle keep-warm + cold/warm measurement support|V31 T26|.|2.3.0 release evidence + version/docs rebaseline after A/B gate passes|V18,V29 ``` @@ -130,4 +131,6 @@ B7|2026-07|"Run batch now" → "0 of 5, 5 errors" ∀ run: 5 Quill stub recordin B8|2026-07-10|new prompt-v2 eval imports violated Ruff I001/UP035|V20 caught; sort imports + `Callable` from `collections.abc` B9|2026-07-10|prompt-v2 eval test import block retained extra blank line|V20 caught; normalize import spacing B10|2026-07-10|stale user-level `pytest.exe` exited 1 + ⊥ diagnostics while active interpreter pytest passed|V20 → interpreter-bound `python -m pytest` +B11|2026-07-10|`node` absent from desktop PowerShell PATH|V20; run bundled workspace `node.exe --check` +B12|2026-07-10|structured v1 rollback output retried ∵ first anti-echo gate ignored target structure|V32 ``` diff --git a/config/grammar_hotkey.config.example.json b/config/grammar_hotkey.config.example.json index 108761d..df043ed 100644 --- a/config/grammar_hotkey.config.example.json +++ b/config/grammar_hotkey.config.example.json @@ -28,6 +28,7 @@ "min_chunk_chars": 700 }, "prompt_builder": { + "prompt_version": "v2", "target_agent": "claude_code", "detail_level": "balanced", "action_mode": "implement", diff --git a/scripts/ffp_config.py b/scripts/ffp_config.py index 340bb6e..f79a184 100644 --- a/scripts/ffp_config.py +++ b/scripts/ffp_config.py @@ -19,7 +19,7 @@ _config_lock = threading.Lock() -CLAUDE_PROMPT_SYSTEM_PROMPT = ( +CLAUDE_PROMPT_SYSTEM_PROMPT_V1 = ( "Rewrite the user text as a Claude-ready prompt. " "Structure: (one primary deliverable, one sentence), " " (background facts only, no instructions), " @@ -29,6 +29,19 @@ "Base every constraint on what the user actually asked for — never invent " "requirements they did not state. Return only the prompt." ) +CLAUDE_PROMPT_SYSTEM_PROMPT_V2 = ( + "Turn the user's rough request into a tight coding-agent prompt. " + "Return only four sibling XML sections in this exact order: " + " one imperative sentence naming the concrete deliverable; " + " only facts the user stated, without restating the task; " + " 3-5 concrete, testable '- ' bullets based only on the request; " + " one line naming the exact deliverable shape. " + "Keep every section separate. No preamble, fences, meta-framing, filler, or invented facts. " + "Keep the whole prompt under 160 tokens." +) +# Compatibility import for integrations that previously consumed the sole +# prompt constant. The unversioned name always denotes the current default. +CLAUDE_PROMPT_SYSTEM_PROMPT = CLAUDE_PROMPT_SYSTEM_PROMPT_V2 DEFAULT_CONFIG = { "enabled": True, @@ -119,7 +132,7 @@ "prompt": { "label": "Prompt fix (Claude)", "description": "Rewrite rough text into a Claude-ready prompt (use prompt: prefix).", - "system_prompt": CLAUDE_PROMPT_SYSTEM_PROMPT, + "system_prompt": CLAUDE_PROMPT_SYSTEM_PROMPT_V2, }, "summarize": { "label": "Summarize", diff --git a/scripts/ffp_llm_client.py b/scripts/ffp_llm_client.py index 0ec8364..c24a7a6 100644 --- a/scripts/ffp_llm_client.py +++ b/scripts/ffp_llm_client.py @@ -8,6 +8,7 @@ from collections.abc import Callable from dataclasses import dataclass +import ffp_config import ffp_prompt_builder log = logging.getLogger("ffp.llm") @@ -90,20 +91,16 @@ def select_runtime(runtime: LlmRuntimeConfig, mode: str, input_text: str) -> tup model = runtime.model if is_prompt_mode(mode): - # Prompt-mode output is structurally LARGER than the input — short - # bullet-point input expands into a full Claude-style XML prompt - # ( + + + ). Pre-v1.2.2 - # caps (140/220/300) truncated the output mid-section. Roughly 5x the - # grammar-mode budget; clamps high enough that even one-line inputs - # produce a complete structured prompt. + # Prompt v2 targets a complete four-section result in <=220 tokens. + # These caps leave repair room while bounding decode-driven latency. if text_len <= 350: - max_tokens = 700 + max_tokens = 240 strategy = "prompt_short" elif text_len <= 1200: - max_tokens = 900 + max_tokens = 320 strategy = "prompt_medium" else: - max_tokens = 1200 + max_tokens = 420 strategy = "prompt_long" else: if text_len <= 350: @@ -293,7 +290,8 @@ def call_flm( system_prompt = ffp_prompt_builder.build_system_prompt( prompt_settings, prompt_intent, - legacy_system_prompt=system_prompt, + prompt_v1_system_prompt=ffp_config.CLAUDE_PROMPT_SYSTEM_PROMPT_V1, + prompt_v2_system_prompt=system_prompt or ffp_config.CLAUDE_PROMPT_SYSTEM_PROMPT_V2, ) if not system_prompt: raise RuntimeError(f"No system_prompt configured for mode '{mode}'.") @@ -396,7 +394,8 @@ def remaining_timeout() -> int: out_norm = re.sub(r"\s+", " ", str(text).lower()).strip() in_norm = re.sub(r"\s+", " ", str(masked_input).lower()).strip() reuse_ratio = line_reuse_ratio(masked_input, text) - near_verbatim = ( + target_structured = has_target_structure(text, prompt_settings) + near_verbatim = not target_structured and ( (out_norm == in_norm) or (reuse_ratio >= 0.85) or is_weak_prompt_echo(masked_input, text, prompt_settings) @@ -408,7 +407,7 @@ def remaining_timeout() -> int: model, anti_echo_prompt, masked_input, - max(max_tokens, 240), + max_tokens, remaining_timeout(), ) if retried and retried.strip(): @@ -429,7 +428,7 @@ def remaining_timeout() -> int: model, rescue_prompt, masked_input, - max(max_tokens, 220), + max_tokens, remaining_timeout(), ) if rescued and rescued.strip(): diff --git a/scripts/ffp_prompt_builder.py b/scripts/ffp_prompt_builder.py index 4b3fdd5..44d29df 100644 --- a/scripts/ffp_prompt_builder.py +++ b/scripts/ffp_prompt_builder.py @@ -7,12 +7,14 @@ from typing import Any TARGET_AGENTS = frozenset({"claude_code", "generic_chat"}) +PROMPT_VERSIONS = frozenset({"v1", "v2"}) DETAIL_LEVELS = frozenset({"concise", "balanced", "detailed"}) ACTION_MODES = frozenset({"plan", "implement", "review", "debug", "explain"}) STRUCTURES = frozenset({"agent_default", "markdown", "xml", "checklist"}) USER_SUFFIX_MAX_CHARS = 500 DEFAULT_PROMPT_BUILDER_CONFIG = { + "prompt_version": "v2", "target_agent": "claude_code", "detail_level": "balanced", "action_mode": "implement", @@ -25,9 +27,60 @@ "user_suffix": "", } +_PROMPT_V2_XML_RE = re.compile( + r"\A\s*\s*(?P.*?)\s*\s*" + r"\s*(?P.*?)\s*\s*" + r"\s*(?P.*?)\s*\s*" + r"\s*(?P.*?)\s*\s*\Z", + flags=re.IGNORECASE | re.DOTALL, +) +_IMPERATIVE_STARTS = frozenset( + { + "add", + "analyze", + "assess", + "build", + "check", + "combine", + "convert", + "create", + "debug", + "develop", + "design", + "diagnose", + "document", + "explain", + "extract", + "fix", + "generate", + "improve", + "implement", + "integrate", + "investigate", + "migrate", + "optimize", + "plan", + "produce", + "refactor", + "remove", + "rename", + "replace", + "resolve", + "review", + "separate", + "summarize", + "test", + "troubleshoot", + "update", + "validate", + "write", + } +) + @dataclass(frozen=True) class PromptBuilderSettings: + prompt_version: str = "v2" target_agent: str = "claude_code" detail_level: str = "balanced" action_mode: str = "implement" @@ -43,6 +96,7 @@ class PromptBuilderSettings: def from_config(cls, value: Any) -> PromptBuilderSettings: cfg = value if isinstance(value, dict) else {} defaults = DEFAULT_PROMPT_BUILDER_CONFIG + prompt_version = _enum(cfg.get("prompt_version"), PROMPT_VERSIONS, defaults["prompt_version"]) target_agent = _enum(cfg.get("target_agent"), TARGET_AGENTS, defaults["target_agent"]) detail_level = _enum(cfg.get("detail_level"), DETAIL_LEVELS, defaults["detail_level"]) action_mode = _enum(cfg.get("action_mode"), ACTION_MODES, defaults["action_mode"]) @@ -50,6 +104,7 @@ def from_config(cls, value: Any) -> PromptBuilderSettings: allow_suffix = _bool(cfg.get("allow_user_suffix"), defaults["allow_user_suffix"]) user_suffix = _clean_suffix(cfg.get("user_suffix") if allow_suffix else "") return cls( + prompt_version=prompt_version, target_agent=target_agent, detail_level=detail_level, action_mode=action_mode, @@ -70,6 +125,7 @@ def from_config(cls, value: Any) -> PromptBuilderSettings: def to_dict(self) -> dict: return { + "prompt_version": self.prompt_version, "target_agent": self.target_agent, "detail_level": self.detail_level, "action_mode": self.action_mode, @@ -85,8 +141,16 @@ def to_dict(self) -> dict: def is_identity_default(self) -> bool: return self.to_dict() == DEFAULT_PROMPT_BUILDER_CONFIG + def is_v1_rollback_default(self) -> bool: + value = self.to_dict() + value["prompt_version"] = DEFAULT_PROMPT_BUILDER_CONFIG["prompt_version"] + return self.prompt_version == "v1" and value == DEFAULT_PROMPT_BUILDER_CONFIG + + def uses_prompt_v2_contract(self) -> bool: + return self.prompt_version == "v2" and self.is_identity_default() + def requires_contract_gate(self) -> bool: - return not self.is_identity_default() + return not self.is_v1_rollback_default() @dataclass(frozen=True) @@ -120,17 +184,21 @@ def build_system_prompt( settings: PromptBuilderSettings, intent: PromptIntent, *, - legacy_system_prompt: str = "", + prompt_v1_system_prompt: str = "", + prompt_v2_system_prompt: str = "", + legacy_system_prompt: str | None = None, ) -> str: """Build the generator system prompt. - The default path intentionally returns the existing Claude prompt verbatim so - upgrades do not silently re-baseline prompt-mode behavior. If that legacy - prompt is missing, return empty and let the caller's missing-prompt guard - fail closed instead of synthesizing a subtly different default. + Identity settings select the versioned built-in verbatim. ``legacy_system_prompt`` + remains a compatibility alias for callers that only know the current prompt. """ + if legacy_system_prompt is not None and not prompt_v2_system_prompt: + prompt_v2_system_prompt = legacy_system_prompt if settings.is_identity_default(): - return legacy_system_prompt + return prompt_v2_system_prompt + if settings.is_v1_rollback_default(): + return prompt_v1_system_prompt shape = effective_structure(settings) parts = [ @@ -149,6 +217,13 @@ def build_system_prompt( def build_retry_prompt(settings: PromptBuilderSettings, *, rescue: bool = False) -> str: + if settings.uses_prompt_v2_contract(): + strength = " after the previous output broke the contract" if rescue else "" + return ( + f"Rewrite the request{strength}. Return only four separate sibling XML sections in order: " + ", , , . Task: one imperative sentence. " + "Constraints: 3-5 '- ' bullets. Output format: one line. No preamble, fence, or invented fact." + ) shape = effective_structure(settings) strength = "stronger " if rescue else "" return ( @@ -197,6 +272,8 @@ def has_target_structure(text: str, settings: PromptBuilderSettings | None = Non lowered = str(text or "").lower() shape = effective_structure(settings) if shape == "xml": + if settings.uses_prompt_v2_contract(): + return bool(_PROMPT_V2_XML_RE.fullmatch(str(text or ""))) return any(tag in lowered for tag in ("", "", "", "")) if shape == "checklist": return bool(re.search(r"(?im)^\s*-\s+\[[ x]\]\s+\S+", str(text or ""))) @@ -303,7 +380,9 @@ def _detail_instruction(detail_level: str) -> str: def _render_xml(settings: PromptBuilderSettings, intent: PromptIntent, cleaned: str) -> str: - if settings.is_identity_default(): + if settings.uses_prompt_v2_contract(): + return _render_prompt_v2_fallback(intent, cleaned) + if settings.is_v1_rollback_default(): return ( "\n" f"Produce a copy-paste-ready Claude prompt for: {cleaned}\n" @@ -328,6 +407,35 @@ def _render_xml(settings: PromptBuilderSettings, intent: PromptIntent, cleaned: return "\n".join(f"<{name}>\n{body}\n" for name, body in sections) +def _render_prompt_v2_fallback(intent: PromptIntent, cleaned: str) -> str: + action = { + "plan": "Plan", + "implement": "Implement", + "review": "Review", + "debug": "Debug", + "explain": "Explain", + }.get(intent.action_mode, "Implement") + request = re.sub(r"[.!?]+", ";", cleaned).strip(" ;") + request = request[:360].rstrip() + task = f"{action} the requested deliverable: {request}." + return ( + "\n" + f"{task}\n" + "\n" + "\n" + "No additional background facts were stated.\n" + "\n" + "\n" + "- Preserve every requirement explicitly stated by the user.\n" + "- Do not add requirements or background the user did not state.\n" + "- Keep the result limited to the requested deliverable.\n" + "\n" + "\n" + "Return the requested deliverable in its stated format.\n" + "" + ) + + def _render_markdown(settings: PromptBuilderSettings, intent: PromptIntent, cleaned: str) -> str: sections = [ ("Task", _fallback_task(settings, intent, cleaned)), @@ -399,7 +507,9 @@ def _fallback_output(settings: PromptBuilderSettings) -> str: def _validate_xml(text: str, settings: PromptBuilderSettings) -> tuple[bool, list[str]]: raw = str(text or "").strip() errors: list[str] = [] - if settings.is_identity_default(): + if settings.uses_prompt_v2_contract(): + return _validate_prompt_v2_xml(raw) + if settings.is_v1_rollback_default(): lower = raw.lower() for tag in ("task", "output_format"): if f"<{tag}>" not in lower: @@ -439,6 +549,52 @@ def _validate_xml(text: str, settings: PromptBuilderSettings) -> tuple[bool, lis return not errors, errors +def _validate_prompt_v2_xml(raw: str) -> tuple[bool, list[str]]: + errors: list[str] = [] + match = _PROMPT_V2_XML_RE.fullmatch(raw) + if not match: + errors.append("requires exactly four ordered sibling sections") + return False, errors + sections = {name: match.group(name).strip() for name in match.groupdict()} + for name, body in sections.items(): + if not body: + errors.append(f"empty <{name}>") + + task = re.sub(r"\s+", " ", sections["task"]).strip() + sentences = re.findall(r"[^.!?]+[.!?](?=\s|$)", task) + first = re.match(r"[A-Za-z]+", task) + if ( + "\n" in sections["task"] + or len(sentences) != 1 + or sentences[0].strip() != task + or not first + or first.group(0).lower() not in _IMPERATIVE_STARTS + ): + errors.append(" must be one imperative sentence") + + constraint_items = [ + match.group(1).strip() + for line in sections["constraints"].splitlines() + if (match := re.match(r"\s*(?:[-*•]|\d+[.)])\s+(.+?)\s*$", line)) + ] + if len(constraint_items) not in range(3, 6): + errors.append(" must contain 3-5 bullet items") + if "\n" in sections["output_format"]: + errors.append(" must be one line") + lowered = raw.lower() + if re.search(r"(?im)^\s*```", raw): + errors.append("wrapped in code fence") + if " 220: + errors.append("exceeds 220-token target") + return not errors, errors + + +def _estimate_token_count(text: str) -> int: + return len(re.findall(r"\w+|[^\w\s]", str(text or ""), flags=re.UNICODE)) + + def _validate_markdown( text: str, settings: PromptBuilderSettings, diff --git a/scripts/ui/web/app.js b/scripts/ui/web/app.js index 8534aa9..db2e602 100644 --- a/scripts/ui/web/app.js +++ b/scripts/ui/web/app.js @@ -113,6 +113,7 @@ function isValidHotkey(hk) { const PERF_LABELS = { balanced: "🟡 Balanced", max: "🔴 Max throughput" }; const TONE_LABELS = { formal: "🎩 Formal", casual: "👕 Casual", friendly: "🤝 Friendly" }; const PROMPT_BUILDER_DEFAULTS = { + prompt_version: "v2", target_agent: "claude_code", detail_level: "balanced", action_mode: "implement", @@ -710,6 +711,7 @@ async function saveNotes() { function populatePromptBuilder(pb) { const cfg = { ...PROMPT_BUILDER_DEFAULTS, ...(pb || {}) }; + $("pb-version").value = cfg.prompt_version; $("pb-target").value = cfg.target_agent; $("pb-action").value = cfg.action_mode; $("pb-detail").value = cfg.detail_level; @@ -727,6 +729,7 @@ function populatePromptBuilder(pb) { function promptBuilderPatch() { const suffix = $("pb-allow-suffix").checked ? $("pb-suffix").value.trim().slice(0, 500) : ""; return { + prompt_version: $("pb-version").value || "v2", target_agent: $("pb-target").value || "claude_code", detail_level: $("pb-detail").value || "balanced", action_mode: $("pb-action").value || "implement", diff --git a/scripts/ui/web/index.html b/scripts/ui/web/index.html index 3a25941..3b63e24 100644 --- a/scripts/ui/web/index.html +++ b/scripts/ui/web/index.html @@ -360,6 +360,13 @@

Tone preset (tone: prefix)

Prompt builder

Controls the prompt: output shape without editing the locked built-in system prompt. Custom modes remain the fully custom escape hatch.

+
+ + +
Launch Flowkey when I sign in (per-user) + +
+ + +
+

FastFlowLM-only; warmups run in the background.

Installed models

diff --git a/setup/defaults/grammar_hotkey.config.example.json b/setup/defaults/grammar_hotkey.config.example.json index df043ed..44ca60d 100644 --- a/setup/defaults/grammar_hotkey.config.example.json +++ b/setup/defaults/grammar_hotkey.config.example.json @@ -15,6 +15,8 @@ "history_store_text": false, "server": { "auto_start": true, + "warm_on_start": true, + "keep_warm_minutes": 15, "performance_mode": "balanced", "startup_timeout_seconds": 25, "serve_extra_args": [], diff --git a/setup/defaults/grammar_hotkey.config.json b/setup/defaults/grammar_hotkey.config.json index 80ff377..f2fdd61 100644 --- a/setup/defaults/grammar_hotkey.config.json +++ b/setup/defaults/grammar_hotkey.config.json @@ -15,6 +15,8 @@ "history_store_text": false, "server": { "auto_start": true, + "warm_on_start": true, + "keep_warm_minutes": 15, "performance_mode": "balanced", "startup_timeout_seconds": 25, "serve_extra_args": [], diff --git a/tests/test_config_seeds.py b/tests/test_config_seeds.py index 24a2d32..144a319 100644 --- a/tests/test_config_seeds.py +++ b/tests/test_config_seeds.py @@ -24,6 +24,8 @@ def test_seed_defaults_safe(): shipped = json.loads(SHIPPED_SEED.read_text(encoding="utf-8")) assert shipped.get("history_store_text") is False assert (shipped.get("server") or {}).get("performance_mode") == "balanced" + assert (shipped.get("server") or {}).get("warm_on_start") is True + assert (shipped.get("server") or {}).get("keep_warm_minutes") == 15 def test_shipped_seed_keys_do_not_silently_drift_from_schema(): diff --git a/tests/test_ffp_config.py b/tests/test_ffp_config.py index 0b52b7e..7d37777 100644 --- a/tests/test_ffp_config.py +++ b/tests/test_ffp_config.py @@ -278,6 +278,26 @@ def test_filter_config_patch_accepts_provider_profiles(): def test_default_config_has_prompt_builder_defaults(): assert ffp_config.DEFAULT_CONFIG["prompt_builder"] == ffp_prompt_builder.DEFAULT_PROMPT_BUILDER_CONFIG + assert ffp_config.DEFAULT_CONFIG["server"]["warm_on_start"] is True + assert ffp_config.DEFAULT_CONFIG["server"]["keep_warm_minutes"] == 15 + + +def test_filter_config_patch_clamps_keep_warm_settings(): + filtered = ffp_config.filter_config_patch({ + "server": { + "warm_on_start": 0, + "keep_warm_minutes": 9999, + "performance_mode": "invalid", + "unknown": True, + } + }) + + assert filtered == { + "server": { + "warm_on_start": False, + "keep_warm_minutes": 1440, + } + } def test_load_config_normalizes_prompt_builder(tmp_path): diff --git a/tests/test_ffp_daemon.py b/tests/test_ffp_daemon.py index 5078ab9..c88582b 100644 --- a/tests/test_ffp_daemon.py +++ b/tests/test_ffp_daemon.py @@ -5,6 +5,7 @@ import time import urllib.error import urllib.request +from pathlib import Path import pytest @@ -122,6 +123,57 @@ def test_actions_count_and_expected_names(daemon_module): assert "model_stats" not in daemon_module.ACTIONS +def test_v31_model_warm_scheduler_runs_startup_and_idle_interval(daemon_module): + cfg = { + "llm": {"provider": "fastflowlm"}, + "server": {"warm_on_start": True, "keep_warm_minutes": 10}, + } + waits = [] + warmups = [] + + def wait(seconds): + waits.append(seconds) + return len(waits) >= 2 + + daemon_module._model_warm_scheduler( + wait=wait, + load_config_fn=lambda: cfg, + warm_fn=lambda: warmups.append(True) or "warmed_up", + ) + + assert waits == [600, 600] + assert warmups == [True, True] + + +def test_v31_model_warm_failure_is_logged_and_does_not_escape(daemon_module, caplog): + cfg = { + "llm": {"provider": "fastflowlm"}, + "server": {"warm_on_start": True, "keep_warm_minutes": 0}, + } + + def fail(): + raise RuntimeError("model load failed") + + daemon_module._model_warm_scheduler( + wait=lambda _seconds: True, + load_config_fn=lambda: cfg, + warm_fn=fail, + ) + + assert "model warmup reason=startup failed: model load failed" in caplog.text + + +def test_dashboard_keep_warm_controls_are_config_wired(): + root = Path(__file__).resolve().parents[1] + html = (root / "scripts" / "ui" / "web" / "index.html").read_text(encoding="utf-8") + js = (root / "scripts" / "ui" / "web" / "app.js").read_text(encoding="utf-8") + + assert 'id="cfg-warm-on-start"' in html + assert 'id="cfg-keep-warm"' in html + assert '$("cfg-warm-on-start").checked = server.warm_on_start !== false' in js + assert 'keep_warm_minutes: Math.max(0, Math.min(Number($("cfg-keep-warm").value)' in js + + def test_write_actions_include_mutating_routes(daemon_module): assert "apply_config_patch" in daemon_module._WRITE_ACTIONS assert "pull_model" in daemon_module._WRITE_ACTIONS @@ -304,6 +356,8 @@ def test_post_config_snapshot_returns_flat_dashboard_fields(daemon_server): } notes = payload["result"]["notes"] assert isinstance(notes.get("categories"), list) + assert payload["result"]["server"]["warm_on_start"] is True + assert payload["result"]["server"]["keep_warm_minutes"] == 15 assert set(payload["result"]["llm"]) >= {"provider", "base_url", "model", "configured_provider"} assert set(payload["result"]["provider_configs"]) >= {"fastflowlm", "ollama"} provider_status = payload["result"]["provider_status"] diff --git a/tests/test_grammar_fix.py b/tests/test_grammar_fix.py index 4e413ba..602e1c1 100644 --- a/tests/test_grammar_fix.py +++ b/tests/test_grammar_fix.py @@ -217,6 +217,67 @@ class _Proc: assert calls[0][0] == ["ollama", "serve"] +def test_v31_warm_configured_fastflow_starts_profile_during_fallback(fresh_modules, monkeypatch): + grammar_fix = fresh_modules("grammar_fix") + cfg = { + "llm": {"provider": "fastflowlm"}, + "providers": { + "fastflowlm": { + "base_url": "http://127.0.0.1:52625", + "model": "qwen3.5:4b", + "auth_bearer": "flm", + "timeout_seconds": 60, + "auto_start": True, + } + }, + "server": { + "performance_mode": "balanced", + "startup_timeout_seconds": 25, + "serve_extra_args": [], + "log_to_file": False, + "log_file": "flm_server.log", + }, + } + starts = [] + warmups = [] + refreshes = [] + monkeypatch.setattr(grammar_fix, "load_config", lambda: cfg) + monkeypatch.setattr(grammar_fix.ffp_flm_server, "is_flm_server_reachable", lambda _url: False) + monkeypatch.setattr( + grammar_fix.ffp_flm_server, + "start_flm_server", + lambda settings, call_api: starts.append((settings, call_api)) or "started", + ) + monkeypatch.setattr( + grammar_fix.ffp_flm_server, + "warmup_request", + lambda model, timeout, call_api: warmups.append((model, timeout, call_api)), + ) + monkeypatch.setattr(grammar_fix, "refresh_runtime_config", lambda: refreshes.append(True)) + + result = grammar_fix.warm_configured_fastflowlm() + + assert result == "warmed_up" + assert starts[0][0].base_url == "http://127.0.0.1:52625" + assert starts[0][0].model == "qwen3.5:4b" + assert warmups[0][:2] == ("qwen3.5:4b", 60) + assert refreshes == [True] + + +def test_v31_warm_configured_fastflow_honors_provider_and_autostart(fresh_modules, monkeypatch): + grammar_fix = fresh_modules("grammar_fix") + monkeypatch.setattr(grammar_fix, "load_config", lambda: {"llm": {"provider": "ollama"}}) + assert grammar_fix.warm_configured_fastflowlm() == "skipped_provider" + + cfg = { + "llm": {"provider": "fastflowlm"}, + "providers": {"fastflowlm": {"auto_start": False}}, + } + monkeypatch.setattr(grammar_fix, "load_config", lambda: cfg) + monkeypatch.setattr(grammar_fix.ffp_flm_server, "is_flm_server_reachable", lambda _url: False) + assert grammar_fix.warm_configured_fastflowlm() == "skipped_auto_start" + + def test_save_config_writes_utf8_json_with_newline(fresh_modules): grammar_fix = fresh_modules("grammar_fix") payload = {"message": "hello 🙂", "server": {"auto_start": True}} diff --git a/tests/test_prompt_speed_quality_eval.py b/tests/test_prompt_speed_quality_eval.py index 60f8a0b..735f29d 100644 --- a/tests/test_prompt_speed_quality_eval.py +++ b/tests/test_prompt_speed_quality_eval.py @@ -151,3 +151,35 @@ def test_protocol_rejects_too_few_timed_runs(): assert str(exc) == "runs must be >= 5" else: raise AssertionError("runs below the invariant must be rejected") + + +def test_cold_warm_probe_records_first_call_penalty_and_speedup(): + restarted = [] + calls = [] + + def fake_model(**kwargs): + calls.append(kwargs) + cold = len(calls) == 1 + return { + "output": GOOD_OUTPUT, + "model": kwargs["model"], + "_wall_seconds": 30.0 if cold else 12.0, + "usage": { + "prefill_duration_ttft": 18.0 if cold else 1.5, + "completion_tokens": 100, + "decode_duration": 8.0, + }, + } + + result = prompt_eval.run_cold_warm_probe( + fake_model, + lambda: restarted.append(True) or "started", + ) + + assert restarted == [True] + assert len(calls) == 2 + assert result["ok"] is True + assert result["cold"]["wall_seconds"] == 30.0 + assert result["warm"]["wall_seconds"] == 12.0 + assert result["wall_speedup"] == 2.5 + assert result["ttft_reduction_seconds"] == 16.5 diff --git a/tools/prompt_speed_quality_eval.py b/tools/prompt_speed_quality_eval.py index 766a467..731bf95 100644 --- a/tools/prompt_speed_quality_eval.py +++ b/tools/prompt_speed_quality_eval.py @@ -537,6 +537,48 @@ def _judge_template(cases: tuple[dict[str, Any], ...] | list[dict[str, Any]]) -> return {"method": "manual", "judgments": judgments} +def run_cold_warm_probe( + call_model: ModelCall, + restart_model: Callable[[], Any], + *, + model: str = "qwen3.5:4b", + case: dict[str, Any] = FIXED_CASES[0], +) -> dict[str, Any]: + """Measure first post-restart completion against the immediately warm completion.""" + try: + restart_result = restart_model() + except Exception as exc: + return {"ok": False, "error": f"restart failed: {exc}"} + call_args = { + "style": "v2", + "system_prompt": STYLE_SPECS["v2"]["system_prompt"], + "user_content": case["input"], + "max_tokens": _cap_for_input(STYLE_SPECS["v2"]["caps"], case["input"]), + "model": model, + } + cold = _run_sample(call_model, **call_args) + warm = _run_sample(call_model, **call_args) + cold_wall = _as_float(cold.get("wall_seconds")) + warm_wall = _as_float(warm.get("wall_seconds")) + cold_ttft = _as_float(cold.get("ttft_seconds")) + warm_ttft = _as_float(warm.get("ttft_seconds")) + return { + "ok": bool(cold.get("ok") and warm.get("ok")), + "case": case["name"], + "restart_result": restart_result, + "cold": cold, + "warm": warm, + "wall_speedup": ( + round(cold_wall / warm_wall, 4) if cold_wall is not None and warm_wall else None + ), + "ttft_reduction_seconds": ( + round(cold_ttft - warm_ttft, 4) + if cold_ttft is not None and warm_ttft is not None + else None + ), + } + + def run_evaluation( call_model: ModelCall, *, @@ -679,6 +721,20 @@ def _call_openai_compatible( } +def _daemon_action(daemon_url: str, action_name: str) -> Any: + request = urllib.request.Request( + daemon_url.rstrip("/") + f"/action/{action_name}", + data=b'{"args":{}}', + headers={"Content-Type": "application/json", "X-FFP-API": "1"}, + method="POST", + ) + with urllib.request.urlopen(request, timeout=120) as response: + payload = json.loads(response.read().decode("utf-8", errors="replace")) + if not payload.get("ok"): + raise RuntimeError(str(payload.get("error") or f"daemon action {action_name} failed")) + return payload.get("result") + + def _load_judge(path: str) -> dict[str, Any]: if not path: return {} @@ -707,7 +763,13 @@ def _manifest() -> dict[str, Any]: def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--live", action="store_true", help="run the full live A/B protocol") + parser.add_argument( + "--cold-warm", + action="store_true", + help="restart FastFlowLM and record first-call versus immediate warm-call latency", + ) parser.add_argument("--base-url", default="http://127.0.0.1:52625") + parser.add_argument("--daemon-url", default="http://127.0.0.1:52650") parser.add_argument("--model", default="qwen3.5:4b") parser.add_argument("--bearer", default="flm") parser.add_argument("--timeout-seconds", type=int, default=90) @@ -717,6 +779,8 @@ def main() -> int: parser.add_argument("--json", action="store_true", help="print the full JSON payload") args = parser.parse_args() + if args.cold_warm and not args.live: + parser.error("--cold-warm requires --live") if not args.live: payload = _manifest() if args.out: @@ -740,6 +804,13 @@ def call_model(**kwargs: Any) -> dict[str, Any]: **kwargs, ) + cold_warm_probe = None + if args.cold_warm: + cold_warm_probe = run_cold_warm_probe( + call_model, + lambda: _daemon_action(args.daemon_url, "restart"), + model=args.model, + ) payload = run_evaluation( call_model, runs=args.runs, @@ -747,6 +818,8 @@ def call_model(**kwargs: Any) -> dict[str, Any]: model=args.model, base_url=args.base_url, ) + if cold_warm_probe is not None: + payload["cold_warm_probe"] = cold_warm_probe out_path = Path(args.out) if args.out else ( ROOT / "data" / "benchmarks" / f"prompt_v2_ab_{datetime.now():%Y-%m-%d}.json" ) From 6c3e04722d1a0990d632866ad9416d0ae1e7c049 Mon Sep 17 00:00:00 2001 From: agrechenkov Date: Fri, 10 Jul 2026 15:39:48 -0400 Subject: [PATCH 4/4] T26: release prompt v2 speed-quality refresh Closes T26. Enforces V18, V24, V29, V30, V33, V34, V35, and V36. --- .gitignore | 11 +- CHANGELOG.md | 23 + README.md | 14 +- SPEC.md | 26 +- config/grammar_hotkey.config.example.json | 2 +- .../prompt_v1_frozen_2026-07-10.json | 2294 ++++++++ data/benchmarks/prompt_v2_ab_2026-07-10.json | 5058 +++++++++++++++++ .../prompt_v2_cold_warm_2026-07-10.json | 53 + .../prompt_v2_judge_2026-07-10.json | 29 + docs/prompt-mode-speed-quality-plan.md | 182 + docs/prompt-mode-speed-quality-results.md | 62 + installer/installer.iss | 2 +- installer/sign.ps1 | 2 +- pyproject.toml | 2 +- scripts/_version.py | 2 +- scripts/ffp_config.py | 35 +- scripts/ffp_flm_server.py | 17 +- scripts/ffp_llm_client.py | 24 +- scripts/ffp_prompt_builder.py | 259 +- scripts/ui/web/app.js | 4 +- .../grammar_hotkey.config.example.json | 2 +- setup/defaults/grammar_hotkey.config.json | 2 +- tests/test_ffp_config.py | 24 +- tests/test_ffp_flm_server.py | 47 + tests/test_grammar_fix.py | 86 +- tests/test_prompt_builder.py | 70 + tests/test_prompt_speed_quality_eval.py | 76 +- tests/test_prompt_v2_release_evidence.py | 42 + tests/test_version_sync.py | 35 + tools/prompt_speed_quality_eval.py | 187 +- 30 files changed, 8592 insertions(+), 80 deletions(-) create mode 100644 data/benchmarks/prompt_v1_frozen_2026-07-10.json create mode 100644 data/benchmarks/prompt_v2_ab_2026-07-10.json create mode 100644 data/benchmarks/prompt_v2_cold_warm_2026-07-10.json create mode 100644 data/benchmarks/prompt_v2_judge_2026-07-10.json create mode 100644 docs/prompt-mode-speed-quality-plan.md create mode 100644 docs/prompt-mode-speed-quality-results.md create mode 100644 tests/test_prompt_v2_release_evidence.py create mode 100644 tests/test_version_sync.py diff --git a/.gitignore b/.gitignore index 9a95cf8..b2fef72 100644 --- a/.gitignore +++ b/.gitignore @@ -4,8 +4,15 @@ # bundled binaries, or caches. Source + docs + config templates only. # ────────────────────────────────────────────────────────────── -# Private runtime data (prompt/grammar history, notes, chat threads, counters, pids) -data/ +# Private runtime data (prompt/grammar history, notes, chat threads, counters, pids). +# Fixed-input prompt-v2 release evidence is sanitized and intentionally tracked. +data/* +!data/benchmarks/ +data/benchmarks/* +!data/benchmarks/prompt_v1_frozen_*.json +!data/benchmarks/prompt_v2_ab_*.json +!data/benchmarks/prompt_v2_cold_warm_*.json +!data/benchmarks/prompt_v2_judge_*.json # Live working config (personal hotkeys/paths). Regenerated on first run from # setup/defaults/ or the .example template — both of which ARE tracked. diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d9109b..43e81a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,29 @@ ## Unreleased +## 2.3.0 + +**Prompt mode, faster and better grounded.** The default `prompt:` path now emits a compact copy-paste-ready agent prompt in a few seconds without filling gaps with conventional-but-unstated requirements. + +### Changed + +- **Prompt v2 is the new default.** One short local-model draft is finalized into exactly four ordered sections (``, ``, ``, ``) using only source clauses plus fixed scope guards. The finalizer prevents raw-model guesses about libraries, files, arguments, formats, tests, platforms, defaults, or error behavior from surfacing. The legacy prompt remains available immediately through Dashboard → Config → Prompt builder → **v1 — legacy rollback**. +- **Prompt decode is bounded.** Prompt short/medium/long strategies now cap at 240/320/420 tokens, and default v2 makes exactly one draft call rather than entering anti-echo/rescue retries before its grounded finalizer. +- **FastFlowLM stays warm.** The daemon performs a best-effort background warmup on startup and at a configurable keepalive interval (15 minutes by default; `0` disables periodic warmups). Failures are logged and never block daemon startup. The dashboard exposes both controls. + +### Measured release gate + +- Fixed set: 12 realistic requests covering implementation, debugging, review, refactor, data, vague, long, and trap inputs; one warmup plus five timed generations per style/input on FastFlowLM `qwen3.5:4b`. +- Speed: warm p50 **18.18 s → 3.38 s**, p90 **22.69 s → 4.67 s**, median completion tokens **222 → 26**. v2 is **18.59%** of v1 p50; no ordinary v2 input exceeded 25 seconds. +- Quality: manual GPT-5 source review scored v2 median **7/7** versus v1 **0.5/7**; v2 passed 12/12, clean-section rate was 100%, and invented-requirement failures were **0** (v1: 12). +- Warm-model probe: first post-restart request **20.88 s** wall versus immediate warm request **3.70 s** (5.65× wall-time improvement). +- Evidence: `data/benchmarks/prompt_v2_ab_2026-07-10.json`, `prompt_v1_frozen_2026-07-10.json`, `prompt_v2_judge_2026-07-10.json`, and `prompt_v2_cold_warm_2026-07-10.json`. + +### Fixed / internal + +- FastFlowLM force-restart now waits for the old socket to close before spawning the replacement, preventing a dying instance from being mistaken for `already_running`. +- Added a reproducible speed/quality evaluator with usage-duration aliases, manual/LLM judgment rescoring, frozen-v1 replay/export, cold/warm probing, and five-output side-by-side review evidence. + ## 2.2.1 **Maintenance.** Ships the History visibility controls and clears the remaining SPEC audit backlog. diff --git a/README.md b/README.md index 645f5c2..facec65 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,12 @@ Flowkey is a Windows desktop assistant that adds local-LLM hotkeys for grammar f Everything runs locally through [FastFlowLM](https://fastflowlm.com) (AMD Ryzen AI NPU) or, on machines without the NPU, through [Ollama](https://ollama.com) (CPU/GPU) as a secondary provider. No cloud service, analytics, or telemetry is used by the app. -Current version: `2.2.1` +Current version: `2.3.0` + +## What's new in 2.3 + +- **`prompt:` is faster and more grounded.** Prompt v2 uses one short local-model draft plus a deterministic source-grounding pass to return a compact four-section coding-agent prompt without inventing libraries, files, formats, tests, or other unstated requirements. On the fixed 12-input `qwen3.5:4b` NPU gate, warm p50 fell from **18.18 s to 3.38 s**, p90 from **22.69 s to 4.67 s**, and median completion tokens from **222 to 26**; all 12 v2 outputs passed the rubric with zero invented-requirement failures. Dashboard → Config → Prompt builder can instantly switch back to the legacy v1 prompt. +- **FastFlowLM stays warm.** The daemon warms the configured NPU model in the background at startup and every 15 minutes by default (both controls are configurable). Measured first-post-restart wall time was 20.88 s versus 3.70 s immediately warm. ## What's new in 2.2 @@ -142,14 +147,15 @@ The dashboard is a web page served by the local daemon — open it from the tray - `config/grammar_hotkey.config.example.json` - example user config. - `assets/screenshots/` - README screenshots. -Runtime data, logs, build output, downloaded vendor binaries, caches, and local editor state are intentionally ignored and should not be committed. +Runtime data, logs, build output, downloaded vendor binaries, caches, and local editor state are intentionally ignored. The only `data/` exception is sanitized fixed-input prompt benchmark evidence under `data/benchmarks/prompt_v*.json`. ## Development Checks ```powershell python -m pip install -e ".[dev]" -ruff check scripts tests -pytest tests -q +ruff check scripts tests tools +python -m pytest -q +node --check scripts/ui/web/app.js ``` AutoHotkey tests are run by CI on Windows. Locally, run them with AutoHotkey v2: diff --git a/SPEC.md b/SPEC.md index ba5b7ec..2f60670 100644 --- a/SPEC.md +++ b/SPEC.md @@ -17,7 +17,7 @@ Caveman-encoded (compression, not amputation). Paths / ids / action names / numb - LLM: FastFlowLM NPU @ `:52625` | Ollama @ `:11434`, OpenAI-compat `POST /v1/chat/completions` - dashboard: daemon-served `scripts/ui/web/{index.html,app.js,styles.css}`, CSP `default-src 'self'` - paths: `scripts/paths.py` → USER_ROOT/{config,data,logs}; `_version.py` = version src of truth -- version: `2.2.1` (maintenance: history-visibility release + audit cleanup); `2.2.0` released (`v2.2.0` on `25b8794`); repo `agr77one/Fastflow` +- version: `2.3.0` (prompt-v2 speed+quality release); `2.2.0` released (`v2.2.0` on `25b8794`); repo `agr77one/Fastflow` - run tree = `flowkey-pub2` (worktree, branch `live`=origin/main). old `FastFlowPrompt_Local_Setup`=1.5.0 stale. ## §I interfaces @@ -30,7 +30,7 @@ Caveman-encoded (compression, not amputation). Paths / ids / action names / numb - action: `prompt_builder_preview {settings?,sample?}` → deterministic local preview (`⊥` LLM call) - config: `prompt_builder.prompt_version` ∈ {`v1`,`v2`}; default `v2`; v1 = instant rollback - config: `server.warm_on_start` bool + `server.keep_warm_minutes` 0..1440; warmup best-effort -- cmd: `python tools/prompt_speed_quality_eval.py [--live] [--cold-warm] [--runs N] [--judge-file PATH] [--out PATH]` → old-vs-v2 JSON +- cmd: `python tools/prompt_speed_quality_eval.py [--live] [--cold-warm] [--reuse-v1 PATH] [--rescore PATH] [--export-v1 PATH] [--runs N] [--judge-file PATH] [--out PATH]` → old-vs-v2 JSON - data: `data/benchmarks/prompt_v2_ab_.json` → speed+quality gate evidence - action: `notify_gate {title,message}` → `{show,reason,category}` (logs); `notifications_log {limit}` → rows - action: `quill_status` → `{reachable,enabled,server,server_version}` @@ -72,7 +72,7 @@ Caveman-encoded (compression, not amputation). Paths / ids / action names / numb - V18: version ∀ ∈ {`_version.py`,`pyproject.toml`,`installer/installer.iss`,`README.md`} equal; CI smoke fails on drift - V19: `main` branch-protected (ruleset 17344133) → land via PR + ruleset toggle; ⊥ direct push - V20: change gates ! pass: `ruff check scripts tests`, `python -m pytest`, `node --check scripts/ui/web/app.js`, AHK parse-check (PowerShell `/ErrorStdOut`) -- V21: local data (config/data/logs/vendor/certs) ∈ `.gitignore` ∴ pull moves code only, never user data +- V21: local runtime data (config/data/logs/vendor/certs) ∈ `.gitignore`; exception = sanitized fixed-input `data/benchmarks/prompt_v*.json` release evidence - V22: `sync.ps1` ∃ uncommitted tracked changes → skip pull (⊥ clobber un-pushed WIP) - V23: meeting `NoContentError` & age ≥ 2d → skip-marker (`meeting_skips.jsonl`) ∴ ⊥ re-queue ever; age < 2d → retry (Quill transcript may still sync); non-content errors ⊥ skip-mark. batch errors → `daemon.log` only (⊥ UI panel, per user) - V24: prompt_builder default cfg ⇒ `CLAUDE_PROMPT_SYSTEM_PROMPT_V2`; `prompt_version=v1` ⇒ `CLAUDE_PROMPT_SYSTEM_PROMPT_V1`; non-default validation target-aware; built-in `modes.prompt.system_prompt` still locked @@ -84,6 +84,10 @@ Caveman-encoded (compression, not amputation). Paths / ids / action names / numb - V30: v1 prompt constant retained + config-selectable without built-in prompt patching - V31: daemon startup + configured idle interval → best-effort FastFlowLM warmup; failure logs only, ⊥ daemon startup failure - V32: prompt output with valid target structure → ⊥ anti-echo retry solely from line/word overlap +- V33: default v2 surfaced output → source-clause grounding + fixed scope guards only; raw model inventions ⊥ surface +- V34: FastFlowLM force restart → old port observed closed before new spawn; ⊥ return `already_running` from dying instance +- V35: default v2 → exactly 1 short LLM draft call; ⊥ anti-echo/rescue calls; V33 finalizer supplies surfaced 4-section contract +- V36: pre-2.3 prompt_builder identity cfg (⊥ `prompt_version`, legacy default fields) → migrate v2+concise; any custom field → preserve ## §T tasks @@ -114,7 +118,7 @@ T22|x|History Telemetry/Exposed views + inline redacted/visible storage control/ T23|x|prompt-v2 fixed A/B speed+quality harness + tests|V29 T24|x|prompt-v2 default + v1 rollback selector + 240/320/420 caps|V24,V27,V28,V30,V32 T25|x|FastFlowLM startup+idle keep-warm + cold/warm measurement support|V31 -T26|.|2.3.0 release evidence + version/docs rebaseline after A/B gate passes|V18,V29 +T26|x|2.3.0 release evidence + version/docs rebaseline after A/B gate passes|V18,V29,V33,V34,V35,V36 ``` ## §B bugs @@ -135,4 +139,18 @@ B11|2026-07-10|`node` absent from desktop PowerShell PATH|V20; run bundled works B12|2026-07-10|structured v1 rollback output retried ∵ first anti-echo gate ignored target structure|V32 B13|2026-07-10|focused Ruff command accidentally included `scripts/ui/web/app.js`|V20 caught; JS → Node syntax gate only B14|2026-07-10|warmup test insertion split existing daemon action assertions into wrong test|V20 caught; restore test block boundary +B15|2026-07-10|FastFlowLM usage emits `decoding_duration`; eval parsed only `decode_duration`|V29; accept both aliases +B16|2026-07-10|first live v2 probe omitted closing XML tags + invented libraries/config/files/error behavior|V27,V29; explicit skeleton + anti-invention list +B17|2026-07-10|second live v2 probe stopped after valid `` ∵ descriptive skeleton treated as optional sequence|V27; literal all-tags template + final-tag completion rule +B18|2026-07-10|literal-template probe passed structure but trap/vague inputs gained inferred CLI/browser/file details; CSV got 2 constraints|V29; entailment-only content + exact safe fillers +B19|2026-07-10|entailment wording fixed vague case but CSV/CLI still gained conventional schema/args/I/O details|V29; clause-copy rules + grounded example + final unsupported-detail audit +B20|2026-07-10|3 live prompt-only revisions still invented conventional details on 4B trap inputs|V33; deterministic source-clause grounding before output surfaces +B21|2026-07-10|CLI prompt test pinned prose from first v2 draft after contract-preserving tune|V20 caught; assert stable final-tag rule +B22|2026-07-10|V27 sentence check treated dots/question marks inside identifiers/regex as boundaries|V27; split only terminal punctuation before whitespace/end +B23|2026-07-10|cold probe force-restart returned `already_running` while killed FLM socket still closing; service then vanished|V34 +B24|2026-07-10|live v2 p50 11.22s but ratio 61.69% ∵ discarded raw 4-section draft still decoded median 121 tokens|V35; 1 short task draft + deterministic V33 finalizer +B25|2026-07-10|broad V35 patch changed runtime/system branches instead of retry branches|V20 caught; restore + scope conditions by surrounding logic +B26|2026-07-10|default v2 input ≥ routing threshold still made compression subcalls; 1 huge clause could exceed V27|V27,V35; bypass routing + bound grounded sections +B27|2026-07-10|upgraded 2.2 cfg retained `detail_level=balanced` ∴ v1 selector missed new concise-default identity|V24,V30; `prompt_version=v1` authoritative legacy XML path +B28|2026-07-10|2.2 persisted identity cfg lacked `prompt_version` + kept `balanced` ∴ upgrade bypassed v2 path|V36; narrow legacy-identity migration ``` diff --git a/config/grammar_hotkey.config.example.json b/config/grammar_hotkey.config.example.json index 44ca60d..eeb97b5 100644 --- a/config/grammar_hotkey.config.example.json +++ b/config/grammar_hotkey.config.example.json @@ -32,7 +32,7 @@ "prompt_builder": { "prompt_version": "v2", "target_agent": "claude_code", - "detail_level": "balanced", + "detail_level": "concise", "action_mode": "implement", "structure": "agent_default", "include_acceptance_criteria": false, diff --git a/data/benchmarks/prompt_v1_frozen_2026-07-10.json b/data/benchmarks/prompt_v1_frozen_2026-07-10.json new file mode 100644 index 0000000..5363eb6 --- /dev/null +++ b/data/benchmarks/prompt_v1_frozen_2026-07-10.json @@ -0,0 +1,2294 @@ +{ + "schema_version": 1, + "kind": "prompt_v1_frozen_baseline", + "created_at": "2026-07-10T11:08:24-04:00", + "exported_at": "2026-07-10T15:22:18-04:00", + "source_artifact": "data\\benchmarks\\prompt_v2_ab_2026-07-10.json", + "protocol": { + "fixed_input_count": 12, + "warmups_per_style_input": 1, + "timed_runs_per_style_input": 5 + }, + "style": { + "label": "current", + "system_prompt": "Rewrite the user text as a Claude-ready prompt. Structure: (one primary deliverable, one sentence), (background facts only, no instructions), (concrete + testable: length, format, tone), (exact shape — Markdown headers, JSON keys, etc.). Keep each section short. No meta-framing, no preamble. Base every constraint on what the user actually asked for — never invent requirements they did not state. Return only the prompt.", + "caps": [ + 700, + 900, + 1200 + ] + }, + "summary": { + "speed": { + "runs": 60, + "successful_runs": 60, + "wall_seconds": { + "median": 18.1794, + "p90": 22.6921, + "min": 14.205, + "max": 30.4958 + }, + "ttft_seconds": { + "median": 1.1731, + "p90": 1.3191, + "min": 1.1539, + "max": 1.6105 + }, + "completion_tokens": { + "median": 222.0, + "p90": 274.7, + "min": 165.0, + "max": 389.0 + }, + "decode_tokens_per_second": { + "median": 13.9355, + "p90": 14.0698, + "min": 13.2411, + "max": 14.1003 + }, + "seconds_per_output_token": { + "median": 0.0825, + "p90": 0.0858, + "min": 0.0784, + "max": 0.0887 + } + }, + "quality": { + "median_score": 0.5, + "scored_outputs": 12, + "pending_judge": 0, + "passed_outputs": 0, + "pass_rate": 0.0, + "clean_section_pass_rate": 0.0, + "invented_requirement_failures": 12 + }, + "protocol": { + "expected_timed_runs": 60, + "successful_timed_runs": 60, + "warmup_errors": 0 + }, + "ordinary_over_25s": 4 + }, + "cases": [ + { + "name": "implement_csv_validator", + "category": "implement", + "trap": false, + "input": "build a python script that reads a folder of CSVs, validates rows against a schema, and writes an error report with file and line numbers", + "v1": { + "label": "current", + "max_tokens": 700, + "warmups": 1, + "warmup_errors": [], + "samples": [ + { + "ok": true, + "error": "", + "output": "\nGenerate a Python script that iterates through a specified directory of CSV files, validates each row against a provided schema definition, and outputs a detailed error report containing the filename and line number for every validation failure.\n\n\nThe user has existing CSV data in various files within a folder that needs to be checked for consistency with specific column names, types, or required fields defined by a schema. The goal is to automate this checking process without manually inspecting each file.\n\n\n- Output must be valid Python 3 code enclosed in markdown code blocks.\n- Include comments explaining the logic of validation and error reporting.\n- Do not include external dependencies beyond the standard library (e.g., no pandas, only csv module).\n- The script should accept arguments for the input folder path and a JSON schema definition file path via `sys.argv`.\n\n\nA single code block containing the complete Python script with necessary imports, class/function definitions, and main execution logic.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 146, + "completion_tokens": 206, + "total_tokens": 352, + "kv_token_occupancy_rate_percentage": 1.07421875, + "load_duration": 1.1e-06, + "prefill_duration_ttft": 1.210723456, + "decoding_duration": 15.304176, + "prefill_speed_tps": 120.58905712651857, + "decoding_speed_tps": 13.460378396066536 + }, + "wall_seconds": 17.5524, + "ttft_seconds": 1.2107, + "completion_tokens": 206, + "decode_duration_seconds": 15.3042, + "decode_tokens_per_second": 13.4604, + "seconds_per_output_token": 0.085206 + }, + { + "ok": true, + "error": "", + "output": "\nGenerate a Python script that iterates through a specified directory of CSV files, validates each row against a provided schema definition, and outputs a detailed error report containing the filename and line number for every validation failure.\n\n\nThe user has existing CSV data in various files within a folder that needs to be checked for consistency with specific column names, types (e.g., integers vs strings), or required values defined by a schema dictionary. The goal is to automate this checking process without manually inspecting each file.\n\n\n- Output must be valid Python 3 code enclosed in markdown code blocks.\n- Include a `main` function that accepts the folder path and schema as arguments, with default examples for demonstration.\n- Error report should be printed to standard output or saved to a JSON/CSV file (choose one clear method).\n- Do not include external dependencies beyond the Python Standard Library (`os`, `csv`, `json`).\n- Keep comments concise explaining validation logic.\n\n\nMarkdown code block with language set to 'python'. No additional text outside the code block unless necessary for clarity.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 146, + "completion_tokens": 233, + "total_tokens": 379, + "kv_token_occupancy_rate_percentage": 1.1566162109375, + "load_duration": 7e-07, + "prefill_duration_ttft": 1.167560192, + "decoding_duration": 16.683994, + "prefill_speed_tps": 125.04708622337134, + "decoding_speed_tps": 13.96548092740863 + }, + "wall_seconds": 18.9528, + "ttft_seconds": 1.1676, + "completion_tokens": 233, + "decode_duration_seconds": 16.684, + "decode_tokens_per_second": 13.9655, + "seconds_per_output_token": 0.081342 + }, + { + "ok": true, + "error": "", + "output": "\nGenerate a Python script that iterates through a specified directory of CSV files, validates each row against a provided schema definition, and outputs a detailed error report containing the filename and line number for every validation failure.\n\n\nThe user has existing CSV data in various files within a specific folder. A JSON file or dictionary will define the expected column names and data types (the schema). The goal is to identify discrepancies between actual rows and the schema, such as missing columns, incorrect types, or null values where not allowed.\n\n\n- Code must be written in Python 3 using standard libraries only (`csv`, `json`, `os`).\n- Output format: A single `.py` file containing a main function that accepts arguments for the folder path and schema definition.\n- Error report output: Print to console or write to a log file; each error line must explicitly state `[File: name] Line N`.\n- No external dependencies (e.g., pandas, pydantic) allowed unless specified otherwise by user context (none here).\n\n\nMarkdown code block with the Python script. Include brief comments explaining key validation logic steps. Do not include execution instructions or explanations outside the code block.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 146, + "completion_tokens": 256, + "total_tokens": 402, + "kv_token_occupancy_rate_percentage": 1.226806640625, + "load_duration": 7e-07, + "prefill_duration_ttft": 1.170802944, + "decoding_duration": 18.18958, + "prefill_speed_tps": 124.70074554236857, + "decoding_speed_tps": 14.073991812895075 + }, + "wall_seconds": 20.5566, + "ttft_seconds": 1.1708, + "completion_tokens": 256, + "decode_duration_seconds": 18.1896, + "decode_tokens_per_second": 14.074, + "seconds_per_output_token": 0.080299 + }, + { + "ok": true, + "error": "", + "output": "\nGenerate a Python script that iterates through a specified directory of CSV files, validates each row against a provided schema definition, and outputs a detailed error report containing the filename and line number for every validation failure.\n\n\nThe user has existing CSV data in various files within a specific folder. A JSON file or dictionary will define the expected schema (column names, types). The goal is to identify invalid entries without altering the source data.\n\n\n- Output must be valid Python 3 code enclosed in a markdown code block with `python` syntax highlighting.\n- The script must accept command-line arguments for the input folder path and an optional schema file path (defaulting to a hardcoded example if not provided).\n- Error reporting should print directly to stdout, formatted as: `[FILENAME]:LINE_NUMBER - ERROR_DESCRIPTION`.\n- Do not include external dependencies beyond `csv`, `json`, `os`, and `argparse` modules.\n- Keep the code concise; do not add extensive comments or unit tests unless requested.\n\n\nA single Markdown block containing only the Python script, starting with ```python and ending with ```.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 146, + "completion_tokens": 240, + "total_tokens": 386, + "kv_token_occupancy_rate_percentage": 1.177978515625, + "load_duration": 7e-07, + "prefill_duration_ttft": 1.17034112, + "decoding_duration": 17.592856, + "prefill_speed_tps": 124.74995324440108, + "decoding_speed_tps": 13.641901008000067 + }, + "wall_seconds": 19.8939, + "ttft_seconds": 1.1703, + "completion_tokens": 240, + "decode_duration_seconds": 17.5929, + "decode_tokens_per_second": 13.6419, + "seconds_per_output_token": 0.082891 + }, + { + "ok": true, + "error": "", + "output": "\nGenerate a Python script that iterates through a specified directory of CSV files, validates each row against a provided schema definition, and outputs a detailed error report containing the filename and line number for every validation failure.\n\n\nThe user has existing CSV data in various files within a specific folder. A JSON file or dictionary will define the expected column names and data types (the schema). The goal is to identify rows that do not match this schema, such as missing columns, incorrect types, or null values where they are forbidden.\n\n\n- Output must be a single Python script with no external dependencies beyond standard library (`csv`, `json`, `os`).\n- Include a command-line argument for the input folder path and an optional JSON file path for the schema (defaulting to hardcoded example if not provided).\n- The error report should be printed to stdout in a clear, tab-separated format: `filename\\tline_number\\terror_description`.\n- Code must include docstrings explaining usage.\n\n\nMarkdown code block containing the Python script with comments indicating where user inputs (folder path, schema) would go.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 146, + "completion_tokens": 236, + "total_tokens": 382, + "kv_token_occupancy_rate_percentage": 1.165771484375, + "load_duration": 1e-06, + "prefill_duration_ttft": 1.161729152, + "decoding_duration": 16.773741, + "prefill_speed_tps": 125.67473214272925, + "decoding_speed_tps": 14.069610351083876 + }, + "wall_seconds": 19.0305, + "ttft_seconds": 1.1617, + "completion_tokens": 236, + "decode_duration_seconds": 16.7737, + "decode_tokens_per_second": 14.0696, + "seconds_per_output_token": 0.080638 + } + ], + "speed": { + "runs": 5, + "successful_runs": 5, + "wall_seconds": { + "median": 19.0305, + "p90": 20.2915, + "min": 17.5524, + "max": 20.5566 + }, + "ttft_seconds": { + "median": 1.1703, + "p90": 1.1947, + "min": 1.1617, + "max": 1.2107 + }, + "completion_tokens": { + "median": 236.0, + "p90": 249.6, + "min": 206.0, + "max": 256.0 + }, + "decode_tokens_per_second": { + "median": 13.9655, + "p90": 14.0722, + "min": 13.4604, + "max": 14.074 + }, + "seconds_per_output_token": { + "median": 0.0813, + "p90": 0.0843, + "min": 0.0803, + "max": 0.0852 + } + }, + "representative_output": "\nGenerate a Python script that iterates through a specified directory of CSV files, validates each row against a provided schema definition, and outputs a detailed error report containing the filename and line number for every validation failure.\n\n\nThe user has existing CSV data in various files within a folder that needs to be checked for consistency with specific column names, types, or required fields defined by a schema. The goal is to automate this checking process without manually inspecting each file.\n\n\n- Output must be valid Python 3 code enclosed in markdown code blocks.\n- Include comments explaining the logic of validation and error reporting.\n- Do not include external dependencies beyond the standard library (e.g., no pandas, only csv module).\n- The script should accept arguments for the input folder path and a JSON schema definition file path via `sys.argv`.\n\n\nA single code block containing the complete Python script with necessary imports, class/function definitions, and main execution logic.", + "quality": { + "rubric": { + "r1": false, + "r2": false, + "r3": false, + "r4": false, + "r5": false, + "r6": false, + "r7": true + }, + "score": 1, + "passed": false, + "hard_fail": true, + "pending_judge": false, + "invented_requirement": true, + "constraint_item_count": 0, + "token_count": 206, + "token_count_source": "usage.completion_tokens", + "estimated_tokens": 198, + "judge_notes": "Invents Python 3, Markdown fences, comments, standard-library-only, argv, and JSON schema file." + } + } + }, + { + "name": "debug_async_cache_race", + "category": "debug", + "trap": false, + "input": "debug an intermittent async cache test failure: two refreshes sometimes run at once and the older response overwrites the newer value; keep the public API unchanged", + "v1": { + "label": "current", + "max_tokens": 700, + "warmups": 1, + "warmup_errors": [], + "samples": [ + { + "ok": true, + "error": "", + "output": "\nDebug an intermittent async cache test failure where two simultaneous refreshes cause the older response to overwrite the newer value, while keeping the public API unchanged.\n\n\nThe system uses an asynchronous caching mechanism for data refresh operations. Under specific concurrency conditions, multiple requests can trigger a refresh simultaneously before any has completed or committed its result to storage. This race condition results in stale (older) data overwriting fresh updates when two refreshes occur at nearly the same time. The public API contract must remain stable and unmodified by this fix.\n\n\n- Output length: 200 - 400 words total.\n- Format: Single Markdown code block containing a Python script that implements the fix using `asyncio.Semaphore` or similar concurrency control to serialize refreshes atomically without changing API signatures.\n- Tone: Technical, direct, no fluff.\n- Testability: The solution must be testable via standard async unit tests (e.g., `pytest`).\n\n\n```python\n# Implementation of the fix within a class or function that mirrors existing public API structure.\n```", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 148, + "completion_tokens": 234, + "total_tokens": 382, + "kv_token_occupancy_rate_percentage": 1.165771484375, + "load_duration": 6e-07, + "prefill_duration_ttft": 1.178722816, + "decoding_duration": 16.676432, + "prefill_speed_tps": 125.55962944896451, + "decoding_speed_tps": 14.03177850034108 + }, + "wall_seconds": 18.9198, + "ttft_seconds": 1.1787, + "completion_tokens": 234, + "decode_duration_seconds": 16.6764, + "decode_tokens_per_second": 14.0318, + "seconds_per_output_token": 0.080854 + }, + { + "ok": true, + "error": "", + "output": "\nDebug an intermittent async cache test failure where two concurrent refreshes cause the older response to overwrite the newer value, while keeping the public API unchanged.\n\n\nThe system uses an asynchronous caching mechanism for data refresh operations. Under specific load conditions, multiple refresh requests can be processed simultaneously before a lock or check is acquired. This race condition results in stale (older) data being written over fresh (newer) data when two updates occur at nearly the same time. The public API contract must remain stable and unmodified to ensure backward compatibility for all clients.\n\n\n- Output length: 200 - 400 words total.\n- Format: Single Markdown code block containing a Python script with `if __name__ == \"__main__\":` entry point.\n- Tone: Technical, direct, and solution-oriented; no fluff or explanations outside the code.\n- Testability: The provided fix must be verifiable by running the existing test suite without altering API signatures.\n\n\nA Python script inside a Markdown ```python block that implements the race condition fix using atomic operations (e.g., `functools.lru_cache` with maxsize=1 or explicit locking via `threading.Lock`) to ensure only one refresh runs at a time per key, preserving API behavior.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 148, + "completion_tokens": 274, + "total_tokens": 422, + "kv_token_occupancy_rate_percentage": 1.287841796875, + "load_duration": 5e-07, + "prefill_duration_ttft": 1.16808192, + "decoding_duration": 19.634374, + "prefill_speed_tps": 126.7034421695355, + "decoding_speed_tps": 13.95511769308255 + }, + "wall_seconds": 22.0284, + "ttft_seconds": 1.1681, + "completion_tokens": 274, + "decode_duration_seconds": 19.6344, + "decode_tokens_per_second": 13.9551, + "seconds_per_output_token": 0.080396 + }, + { + "ok": true, + "error": "", + "output": "\nDebug an intermittent async cache test failure where two concurrent refreshes cause the older response to overwrite the newer value, while keeping the public API unchanged.\n\n\nThe system uses an asynchronous caching mechanism for data refresh operations. Under specific race conditions, multiple requests can trigger simultaneous cache updates within a single execution context. The current implementation lacks synchronization logic to ensure atomicity during concurrent writes, leading to stale or overwritten values in the cache layer without affecting the external interface exposed by the API endpoints.\n\n\n- Output must be exactly 200 words (195 - 204 characters allowed for whitespace).\n- Use only Markdown headers (`###`) and code blocks (` ``` `) where appropriate; no other formatting styles.\n- Tone: Technical, direct, problem-solving focused; avoid fluff or conversational fillers.\n- Do not include any meta-commentary about the prompt itself (e.g., \"Here is your rewritten prompt\").\n\n\nA single Markdown document containing:\n1. A `### Problem Analysis` header with a concise paragraph explaining the root cause in 2 - 3 sentences.\n2. A `### Proposed Solution` header with a bulleted list of exactly three technical steps to fix the issue, each under 40 characters long (excluding bullets).", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 148, + "completion_tokens": 273, + "total_tokens": 421, + "kv_token_occupancy_rate_percentage": 1.2847900390625, + "load_duration": 6e-07, + "prefill_duration_ttft": 1.15969664, + "decoding_duration": 20.283122, + "prefill_speed_tps": 127.61958161748232, + "decoding_speed_tps": 13.45946644702921 + }, + "wall_seconds": 22.6705, + "ttft_seconds": 1.1597, + "completion_tokens": 273, + "decode_duration_seconds": 20.2831, + "decode_tokens_per_second": 13.4595, + "seconds_per_output_token": 0.083042 + }, + { + "ok": true, + "error": "", + "output": "\nDebug an intermittent async cache test failure where two concurrent refreshes cause the older response to overwrite the newer value, while keeping the public API unchanged.\n\n\nThe system uses an asynchronous caching mechanism for data refresh operations. A specific unit test occasionally fails due to race conditions when multiple refresh requests execute simultaneously within a short timeframe. The underlying implementation currently lacks synchronization logic to handle concurrent updates, leading to stale or overwritten cache entries in edge cases. The public API contract must remain stable and unmodified regardless of internal fixes.\n\n\n- Output length: 200 - 400 words total.\n- Format: Single Markdown code block containing the debugging solution steps only (no explanations).\n- Tone: Direct, technical, imperative instructions suitable for a developer implementation guide.\n- No meta-commentary or conversational filler outside the prompt structure.\n\n\nMarkdown headers (#) for sections like \"Root Cause\", \"Fix Strategy\", and \"Implementation Steps\". Use bullet points (-) for actionable items. Ensure all code snippets are in ```language blocks if applicable (e.g., pseudo-code or SQL).", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 148, + "completion_tokens": 234, + "total_tokens": 382, + "kv_token_occupancy_rate_percentage": 1.165771484375, + "load_duration": 9e-07, + "prefill_duration_ttft": 1.161499904, + "decoding_duration": 16.632057, + "prefill_speed_tps": 127.42144832755835, + "decoding_speed_tps": 14.069215852254475 + }, + "wall_seconds": 18.8746, + "ttft_seconds": 1.1615, + "completion_tokens": 234, + "decode_duration_seconds": 16.6321, + "decode_tokens_per_second": 14.0692, + "seconds_per_output_token": 0.080661 + }, + { + "ok": true, + "error": "", + "output": "\nDebug an intermittent async cache test failure where two concurrent refreshes cause the older response to overwrite the newer value, while keeping the public API unchanged.\n\n\nThe system uses an asynchronous caching mechanism for data refresh operations. Under specific load conditions, multiple refresh requests can be initiated simultaneously before a previous one completes. The current implementation does not handle race conditions effectively, leading to stale or overwritten cache entries when concurrent updates occur. The public API contract must remain stable and unmodified by the fix.\n\n\n- Output length: 200 - 400 words total.\n- Format: Single Markdown code block containing a Python script with detailed comments explaining the logic, not an explanation text outside it.\n- Tone: Technical, direct, focused on implementation details and debugging steps.\n- Content requirements: Must include specific async handling (e.g., `asyncio.Semaphore` or similar), cache key validation logic to prevent overwrites without changing API signatures, and a test case demonstrating the race condition fix.\n\n\nA Markdown code block with Python 3 syntax highlighting containing the debugging script. The script must include:\n1. A class representing the async cache handler.\n2. A method `refresh_data` that implements concurrency control to prevent overwrites of newer responses by older ones (e.g., using a token bucket or semaphore).\n3. A unit test function simulating concurrent refreshes and verifying the latest value is retained in the cache without altering any public API methods.\n\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 148, + "completion_tokens": 318, + "total_tokens": 466, + "kv_token_occupancy_rate_percentage": 1.422119140625, + "load_duration": 5e-07, + "prefill_duration_ttft": 1.16202304, + "decoding_duration": 22.611108, + "prefill_speed_tps": 127.36408393417052, + "decoding_speed_tps": 14.063883998961927 + }, + "wall_seconds": 25.1499, + "ttft_seconds": 1.162, + "completion_tokens": 318, + "decode_duration_seconds": 22.6111, + "decode_tokens_per_second": 14.0639, + "seconds_per_output_token": 0.079088 + } + ], + "speed": { + "runs": 5, + "successful_runs": 5, + "wall_seconds": { + "median": 22.0284, + "p90": 24.1581, + "min": 18.8746, + "max": 25.1499 + }, + "ttft_seconds": { + "median": 1.162, + "p90": 1.1745, + "min": 1.1597, + "max": 1.1787 + }, + "completion_tokens": { + "median": 273.0, + "p90": 300.4, + "min": 234.0, + "max": 318.0 + }, + "decode_tokens_per_second": { + "median": 14.0318, + "p90": 14.0671, + "min": 13.4595, + "max": 14.0692 + }, + "seconds_per_output_token": { + "median": 0.0807, + "p90": 0.0822, + "min": 0.0791, + "max": 0.083 + } + }, + "representative_output": "\nDebug an intermittent async cache test failure where two simultaneous refreshes cause the older response to overwrite the newer value, while keeping the public API unchanged.\n\n\nThe system uses an asynchronous caching mechanism for data refresh operations. Under specific concurrency conditions, multiple requests can trigger a refresh simultaneously before any has completed or committed its result to storage. This race condition results in stale (older) data overwriting fresh updates when two refreshes occur at nearly the same time. The public API contract must remain stable and unmodified by this fix.\n\n\n- Output length: 200 - 400 words total.\n- Format: Single Markdown code block containing a Python script that implements the fix using `asyncio.Semaphore` or similar concurrency control to serialize refreshes atomically without changing API signatures.\n- Tone: Technical, direct, no fluff.\n- Testability: The solution must be testable via standard async unit tests (e.g., `pytest`).\n\n\n```python\n# Implementation of the fix within a class or function that mirrors existing public API structure.\n```", + "quality": { + "rubric": { + "r1": false, + "r2": false, + "r3": false, + "r4": false, + "r5": false, + "r6": false, + "r7": false + }, + "score": 0, + "passed": false, + "hard_fail": true, + "pending_judge": false, + "invented_requirement": true, + "constraint_item_count": 0, + "token_count": 234, + "token_count_source": "usage.completion_tokens", + "estimated_tokens": 213, + "judge_notes": "Invents word count, Python, asyncio.Semaphore, pytest, and output shape." + } + } + }, + { + "name": "review_auth_middleware", + "category": "review", + "trap": false, + "input": "review the new TypeScript auth middleware for regressions and missing tests; report findings by severity with file and line references", + "v1": { + "label": "current", + "max_tokens": 700, + "warmups": 1, + "warmup_errors": [], + "samples": [ + { + "ok": true, + "error": "", + "output": "\nAnalyze the provided TypeScript authentication middleware code for regressions and missing test coverage, then generate a report detailing findings by severity level with specific file paths and line numbers.\n\n\nThe user has supplied a new implementation of an auth middleware in TypeScript that requires validation against existing patterns or requirements to ensure stability and completeness. The goal is to identify potential issues (regressions) where functionality may have been broken, as well as gaps where critical scenarios lack corresponding tests.\n\n\n- Output must be exactly 300 words long (counting spaces).\n- Report findings in a bulleted list format with no numbering.\n- Use professional but concise technical tone; avoid fluff or explanations outside the report structure.\n- Each finding must include: [Severity] - Description, File Path:line_number.\n\n\nMarkdown header for title (e.g., `## Auth Middleware Review Report`).\nA single bulleted list containing all findings sorted by severity (Critical > High > Medium).\nNo introduction or conclusion paragraphs outside the report structure.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 140, + "completion_tokens": 223, + "total_tokens": 363, + "kv_token_occupancy_rate_percentage": 1.1077880859375, + "load_duration": 1.1e-06, + "prefill_duration_ttft": 1.164959744, + "decoding_duration": 15.855805, + "prefill_speed_tps": 120.17582643611075, + "decoding_speed_tps": 14.064249654937104 + }, + "wall_seconds": 18.0689, + "ttft_seconds": 1.165, + "completion_tokens": 223, + "decode_duration_seconds": 15.8558, + "decode_tokens_per_second": 14.0642, + "seconds_per_output_token": 0.081026 + }, + { + "ok": true, + "error": "", + "output": "\nAnalyze the provided TypeScript authentication middleware code for regressions and missing test coverage, then report findings organized by severity level with specific file paths and line numbers.\n\n\nThe user has a new TypeScript implementation of an auth middleware that requires validation against existing patterns to ensure no functionality was broken (regressions) and that all critical logic is covered by tests. The output must be structured as a technical review report suitable for engineering stakeholders.\n\n\n- Total response length: 300 - 500 words.\n- Format: Markdown with clear headers (`## Severity Level`, `### Missing Tests`, etc.).\n- Tone: Professional, concise, and objective (no fluff).\n- Data requirement: Every finding must include a file path and line number reference where applicable.\n\n\nUse the following structure exactly:\n\n# Auth Middleware Review Report\n\n## 1. Regression Analysis\n[Summary of potential regressions found]\n\n## 2. Missing Test Coverage\n[List of untested functions/modules with file/line refs]\n\n## 3. Recommendations\n[Brief actionable next steps based on findings]", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 140, + "completion_tokens": 238, + "total_tokens": 378, + "kv_token_occupancy_rate_percentage": 1.153564453125, + "load_duration": 1.3e-06, + "prefill_duration_ttft": 1.154849664, + "decoding_duration": 16.903436, + "prefill_speed_tps": 121.22790036158334, + "decoding_speed_tps": 14.079977585622236 + }, + "wall_seconds": 19.179, + "ttft_seconds": 1.1548, + "completion_tokens": 238, + "decode_duration_seconds": 16.9034, + "decode_tokens_per_second": 14.08, + "seconds_per_output_token": 0.080584 + }, + { + "ok": true, + "error": "", + "output": "\nAnalyze the provided TypeScript authentication middleware code for regressions and missing test coverage, then output a structured report detailing findings by severity level with specific file paths and line numbers.\n\n\nThe user has supplied a new implementation of an auth middleware in TypeScript that requires validation against existing patterns or requirements to ensure stability and completeness. The goal is to identify potential issues like logic errors (regressions) or gaps where functionality exists but lacks corresponding tests.\n\n\n- Output must be exactly 300 words long, including all text and code blocks.\n- Report findings in a Markdown table with columns: Severity, Issue Description, File Path, Line Number.\n- Use professional yet concise technical tone; avoid fluff or conversational fillers.\n- Do not include any introductory remarks or concluding summaries outside the report structure.\n\n\nMarkdown code block containing only the analysis report in a table format as specified above.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 140, + "completion_tokens": 197, + "total_tokens": 337, + "kv_token_occupancy_rate_percentage": 1.0284423828125, + "load_duration": 5e-07, + "prefill_duration_ttft": 1.156205568, + "decoding_duration": 14.467548, + "prefill_speed_tps": 121.08573412440097, + "decoding_speed_tps": 13.616681969881835 + }, + "wall_seconds": 16.5843, + "ttft_seconds": 1.1562, + "completion_tokens": 197, + "decode_duration_seconds": 14.4675, + "decode_tokens_per_second": 13.6167, + "seconds_per_output_token": 0.084184 + }, + { + "ok": true, + "error": "", + "output": "\nAnalyze the provided TypeScript authentication middleware code for regressions and missing test coverage, then generate a report detailing findings by severity level with specific file paths and line numbers.\n\n\nThe user has supplied a new implementation of an auth middleware in TypeScript that requires validation against existing patterns or requirements to ensure stability and completeness. The goal is to identify potential issues (regressions) where functionality might be broken, as well as gaps where critical scenarios are not covered by tests.\n\n\n- Output must be exactly 300 words or less.\n- Use a bulleted list for the findings section; do not use tables.\n- Maintain a professional, technical tone suitable for engineering review.\n- Do not include any introductory text, explanations of methodology, or concluding remarks outside the report itself.\n\n\nMarkdown format with two main headers: `## Analysis Findings` and `## Recommendations`. Under each header, provide concise bullet points summarizing specific issues found (e.g., \"Regression in token validation at auth.ts:42\") and actionable suggestions where applicable.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 140, + "completion_tokens": 228, + "total_tokens": 368, + "kv_token_occupancy_rate_percentage": 1.123046875, + "load_duration": 9e-07, + "prefill_duration_ttft": 1.155902592, + "decoding_duration": 16.202568, + "prefill_speed_tps": 121.11747215460869, + "decoding_speed_tps": 14.071843426301312 + }, + "wall_seconds": 18.4342, + "ttft_seconds": 1.1559, + "completion_tokens": 228, + "decode_duration_seconds": 16.2026, + "decode_tokens_per_second": 14.0718, + "seconds_per_output_token": 0.080852 + }, + { + "ok": true, + "error": "", + "output": "\nAnalyze the provided TypeScript authentication middleware code for regressions and missing test coverage, then generate a report detailing findings by severity level with specific file paths and line numbers.\n\n\nThe user has supplied a new implementation of an auth middleware in TypeScript that requires validation against existing patterns or requirements to ensure stability and completeness. The goal is to identify potential issues (regressions) where functionality might be broken, as well as gaps where critical scenarios are not covered by tests.\n\n\n- Output must be exactly 300 words or less.\n- Use a bulleted list for the findings section; do not use tables.\n- Maintain a professional, technical tone suitable for engineering review.\n- Each finding must include: Severity (High/Medium/Low), Description, File Path, and Line Number reference.\n\n\nMarkdown format with the following structure:\n1. `## Analysis Summary` (brief intro)\n2. `## Findings by Severity` (bulleted list of findings)", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 140, + "completion_tokens": 215, + "total_tokens": 355, + "kv_token_occupancy_rate_percentage": 1.0833740234375, + "load_duration": 9e-07, + "prefill_duration_ttft": 1.159690752, + "decoding_duration": 15.316078, + "prefill_speed_tps": 120.72183878206867, + "decoding_speed_tps": 14.037536241327578 + }, + "wall_seconds": 17.494, + "ttft_seconds": 1.1597, + "completion_tokens": 215, + "decode_duration_seconds": 15.3161, + "decode_tokens_per_second": 14.0375, + "seconds_per_output_token": 0.081367 + } + ], + "speed": { + "runs": 5, + "successful_runs": 5, + "wall_seconds": { + "median": 18.0689, + "p90": 18.8811, + "min": 16.5843, + "max": 19.179 + }, + "ttft_seconds": { + "median": 1.1562, + "p90": 1.1629, + "min": 1.1548, + "max": 1.165 + }, + "completion_tokens": { + "median": 223.0, + "p90": 234.0, + "min": 197.0, + "max": 238.0 + }, + "decode_tokens_per_second": { + "median": 14.0642, + "p90": 14.0767, + "min": 13.6167, + "max": 14.08 + }, + "seconds_per_output_token": { + "median": 0.081, + "p90": 0.0831, + "min": 0.0806, + "max": 0.0842 + } + }, + "representative_output": "\nAnalyze the provided TypeScript authentication middleware code for regressions and missing test coverage, then generate a report detailing findings by severity level with specific file paths and line numbers.\n\n\nThe user has supplied a new implementation of an auth middleware in TypeScript that requires validation against existing patterns or requirements to ensure stability and completeness. The goal is to identify potential issues (regressions) where functionality may have been broken, as well as gaps where critical scenarios lack corresponding tests.\n\n\n- Output must be exactly 300 words long (counting spaces).\n- Report findings in a bulleted list format with no numbering.\n- Use professional but concise technical tone; avoid fluff or explanations outside the report structure.\n- Each finding must include: [Severity] - Description, File Path:line_number.\n\n\nMarkdown header for title (e.g., `## Auth Middleware Review Report`).\nA single bulleted list containing all findings sorted by severity (Critical > High > Medium).\nNo introduction or conclusion paragraphs outside the report structure.", + "quality": { + "rubric": { + "r1": false, + "r2": false, + "r3": false, + "r4": false, + "r5": false, + "r6": false, + "r7": false + }, + "score": 0, + "passed": false, + "hard_fail": true, + "pending_judge": false, + "invented_requirement": true, + "constraint_item_count": 0, + "token_count": 223, + "token_count_source": "usage.completion_tokens", + "estimated_tokens": 205, + "judge_notes": "Invents exact length, bullet format, severity set, title, and report structure." + } + } + }, + { + "name": "refactor_invoice_service", + "category": "refactor", + "trap": false, + "input": "refactor InvoiceService so calculation and persistence are separate, preserve current behavior, and do not add dependencies", + "v1": { + "label": "current", + "max_tokens": 700, + "warmups": 1, + "warmup_errors": [], + "samples": [ + { + "ok": true, + "error": "", + "output": "\nRefactor the InvoiceService to separate calculation logic from persistence while preserving existing behavior and avoiding new dependencies.\n\n\nThe current implementation mixes invoice calculation with database operations in a single service class. The goal is to isolate the mathematical rules for generating invoices into a dedicated component, leaving only data access responsibilities within the original service or an appropriate repository layer without introducing external libraries or frameworks.\n\n\n- Output must be exactly 200 words (195 - 204 characters allowed if strictly enforced by platform, but here meaning ~200 tokens/words).\n- Use only standard TypeScript interfaces and types; no third-party packages like `lodash`, `date-fns`, or ORM libraries.\n- Preserve all existing unit test expectations without modification to test code.\n- Do not introduce new classes beyond what is necessary for separation of concerns (e.g., a simple calculation utility).\n\n\nProvide the refactored TypeScript code in a single Markdown block with clear comments explaining the separation of concerns. Include no additional text outside the code block.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 138, + "completion_tokens": 224, + "total_tokens": 362, + "kv_token_occupancy_rate_percentage": 1.104736328125, + "load_duration": 8e-07, + "prefill_duration_ttft": 1.244666752, + "decoding_duration": 16.03299, + "prefill_speed_tps": 110.87305078106561, + "decoding_speed_tps": 13.971193146131819 + }, + "wall_seconds": 18.3546, + "ttft_seconds": 1.2447, + "completion_tokens": 224, + "decode_duration_seconds": 16.033, + "decode_tokens_per_second": 13.9712, + "seconds_per_output_token": 0.08194 + }, + { + "ok": true, + "error": "", + "output": "\nRefactor the InvoiceService to separate calculation logic from persistence while preserving existing behavior and avoiding new dependencies.\n\n\nThe current InvoiceService handles both calculating invoice totals (taxes, discounts, etc.) and persisting those results to a database within a single class. The goal is to extract the calculation logic into its own component or method without introducing external libraries or frameworks that are not currently used by the application.\n\n\n- Output must be exactly 150 words (±10).\n- Use only standard JavaScript/TypeScript features; no third-party packages.\n- Maintain backward compatibility: existing API calls must return identical results.\n- Do not alter the public interface of `InvoiceService`.\n\n\nProvide a concise technical summary in Markdown with two sections: \"Refactoring Plan\" and \"Implementation Notes\".", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 138, + "completion_tokens": 178, + "total_tokens": 316, + "kv_token_occupancy_rate_percentage": 0.96435546875, + "load_duration": 7e-07, + "prefill_duration_ttft": 1.154303744, + "decoding_duration": 12.664427, + "prefill_speed_tps": 119.55258805779287, + "decoding_speed_tps": 14.05511674551087 + }, + "wall_seconds": 14.7097, + "ttft_seconds": 1.1543, + "completion_tokens": 178, + "decode_duration_seconds": 12.6644, + "decode_tokens_per_second": 14.0551, + "seconds_per_output_token": 0.082639 + }, + { + "ok": true, + "error": "", + "output": "\nRefactor the InvoiceService to separate calculation logic from persistence while preserving existing behavior and avoiding new dependencies.\n\n\nThe current implementation mixes invoice calculation with database operations in a single service class. The goal is to isolate the mathematical rules for generating invoices into a distinct component, leaving only data access responsibilities within the original service or a dedicated repository layer. No external libraries should be introduced during this refactoring.\n\n\n- Output must be exactly 200 words (195 - 204 characters allowed if strictly word-counted, but here we assume ~200 tokens/words for clarity).\n- Use only standard JavaScript/TypeScript syntax compatible with Node.js environments.\n- Do not introduce any new npm packages or external dependencies.\n- Maintain backward compatibility: all public methods must behave identically before and after refactoring.\n\n\nProvide the complete TypeScript code for both the calculation module (e.g., `InvoiceCalculator`) and the updated service (`InvoiceService`). Use clear class names, JSDoc comments explaining separation of concerns, and ensure type safety throughout.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 138, + "completion_tokens": 231, + "total_tokens": 369, + "kv_token_occupancy_rate_percentage": 1.1260986328125, + "load_duration": 7e-07, + "prefill_duration_ttft": 1.169320448, + "decoding_duration": 16.797702, + "prefill_speed_tps": 118.01726398955438, + "decoding_speed_tps": 13.751881060873684 + }, + "wall_seconds": 19.0467, + "ttft_seconds": 1.1693, + "completion_tokens": 231, + "decode_duration_seconds": 16.7977, + "decode_tokens_per_second": 13.7519, + "seconds_per_output_token": 0.082453 + }, + { + "ok": true, + "error": "", + "output": "\nRefactor the InvoiceService to separate calculation logic from persistence while preserving existing behavior and avoiding new dependencies.\n\n\nThe current implementation mixes invoice calculation with database operations in a single service class. The goal is to isolate the mathematical rules for generating invoices into a dedicated component, leaving only data access responsibilities within the original service or its immediate layer. No external libraries should be introduced during this refactoring.\n\n\n- Output must be exactly 200 words (±10).\n- Use Markdown code blocks for all examples and pseudo-code.\n- Maintain a professional, technical tone suitable for senior engineering review.\n- Do not include any explanatory text outside the requested structure.\n\n\nA single Markdown document containing:\n1. A brief introduction to the refactoring approach (2 sentences).\n2. The new class hierarchy using Python syntax examples only (no implementation details beyond signatures and comments).", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 138, + "completion_tokens": 193, + "total_tokens": 331, + "kv_token_occupancy_rate_percentage": 1.0101318359375, + "load_duration": 7e-07, + "prefill_duration_ttft": 1.199867776, + "decoding_duration": 13.731442, + "prefill_speed_tps": 115.0126728630472, + "decoding_speed_tps": 14.055333736981156 + }, + "wall_seconds": 15.8883, + "ttft_seconds": 1.1999, + "completion_tokens": 193, + "decode_duration_seconds": 13.7314, + "decode_tokens_per_second": 14.0553, + "seconds_per_output_token": 0.082323 + }, + { + "ok": true, + "error": "", + "output": "\nRefactor the InvoiceService to separate calculation logic from persistence while preserving current behavior and avoiding new dependencies.\n\n\nThe existing InvoiceService handles both calculating invoice totals (taxes, discounts) and saving them to a database. The goal is to extract the calculation logic into a dedicated service or method, leaving only data access in the original service. No external libraries should be introduced; use only existing internal classes if available.\n\n\n- Output must be exactly 200 words (±10).\n- Use Markdown code blocks for all examples and pseudo-code.\n- Maintain a professional, technical tone suitable for senior developers reviewing PRs.\n- Do not include any explanatory text outside the prompt structure itself.\n\n\nReturn only the refactored service implementation in a single Markdown block with clear separation between `calculateInvoice` and `persistInvoice`.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 138, + "completion_tokens": 185, + "total_tokens": 323, + "kv_token_occupancy_rate_percentage": 0.9857177734375, + "load_duration": 9e-07, + "prefill_duration_ttft": 1.31558976, + "decoding_duration": 13.222926, + "prefill_speed_tps": 104.89592135469343, + "decoding_speed_tps": 13.99085194910718 + }, + "wall_seconds": 15.5106, + "ttft_seconds": 1.3156, + "completion_tokens": 185, + "decode_duration_seconds": 13.2229, + "decode_tokens_per_second": 13.9909, + "seconds_per_output_token": 0.083841 + } + ], + "speed": { + "runs": 5, + "successful_runs": 5, + "wall_seconds": { + "median": 15.8883, + "p90": 18.7699, + "min": 14.7097, + "max": 19.0467 + }, + "ttft_seconds": { + "median": 1.1999, + "p90": 1.2872, + "min": 1.1543, + "max": 1.3156 + }, + "completion_tokens": { + "median": 193.0, + "p90": 228.2, + "min": 178.0, + "max": 231.0 + }, + "decode_tokens_per_second": { + "median": 13.9909, + "p90": 14.0552, + "min": 13.7519, + "max": 14.0553 + }, + "seconds_per_output_token": { + "median": 0.0825, + "p90": 0.0834, + "min": 0.0819, + "max": 0.0838 + } + }, + "representative_output": "\nRefactor the InvoiceService to separate calculation logic from persistence while preserving existing behavior and avoiding new dependencies.\n\n\nThe current implementation mixes invoice calculation with database operations in a single service class. The goal is to isolate the mathematical rules for generating invoices into a dedicated component, leaving only data access responsibilities within the original service or an appropriate repository layer without introducing external libraries or frameworks.\n\n\n- Output must be exactly 200 words (195 - 204 characters allowed if strictly enforced by platform, but here meaning ~200 tokens/words).\n- Use only standard TypeScript interfaces and types; no third-party packages like `lodash`, `date-fns`, or ORM libraries.\n- Preserve all existing unit test expectations without modification to test code.\n- Do not introduce new classes beyond what is necessary for separation of concerns (e.g., a simple calculation utility).\n\n\nProvide the refactored TypeScript code in a single Markdown block with clear comments explaining the separation of concerns. Include no additional text outside the code block.", + "quality": { + "rubric": { + "r1": false, + "r2": false, + "r3": false, + "r4": false, + "r5": false, + "r6": false, + "r7": false + }, + "score": 0, + "passed": false, + "hard_fail": true, + "pending_judge": false, + "invented_requirement": true, + "constraint_item_count": 0, + "token_count": 224, + "token_count_source": "usage.completion_tokens", + "estimated_tokens": 207, + "judge_notes": "Invents database/repository design, TypeScript, test policy, word count, and class limits." + } + } + }, + { + "name": "data_monthly_rollup", + "category": "data", + "trap": false, + "input": "combine these monthly sales CSVs, keep the original rows, summarize revenue by region, and reconcile the summary total to the source total", + "v1": { + "label": "current", + "max_tokens": 700, + "warmups": 1, + "warmup_errors": [], + "samples": [ + { + "ok": true, + "error": "", + "output": "\nGenerate a Markdown report that merges multiple monthly sales CSVs into a single dataset, calculates regional revenue summaries, and verifies the total against the source data.\n\n\nThe user has several separate CSV files containing monthly sales records with columns for date, region, product ID, quantity sold, and unit price. The goal is to aggregate these files without losing individual row details while producing a high-level summary by region that matches the sum of all original transactions.\n\n\n- Output must be valid Markdown with clear headers (e.g., `## 1. Data Merge`, `## 2. Regional Summary`, `## 3. Reconciliation Check`).\n- Include a sample merged table showing at least one row from each source file to demonstrate retention of original data.\n- Present the regional summary as a bulleted list or small table with totals per region and an overall grand total.\n- The reconciliation section must explicitly state: \"Source Total\" vs \"Summary Total\" and confirm they match (or note any discrepancy).\n- Tone should be professional, analytical, and concise.\n\n\nMarkdown document with three main sections as specified above. No JSON or code blocks unless illustrating the merged data structure conceptually.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 144, + "completion_tokens": 256, + "total_tokens": 400, + "kv_token_occupancy_rate_percentage": 1.220703125, + "load_duration": 6e-07, + "prefill_duration_ttft": 1.163815552, + "decoding_duration": 18.429308, + "prefill_speed_tps": 123.73094667152206, + "decoding_speed_tps": 13.890917662236694 + }, + "wall_seconds": 20.7516, + "ttft_seconds": 1.1638, + "completion_tokens": 256, + "decode_duration_seconds": 18.4293, + "decode_tokens_per_second": 13.8909, + "seconds_per_output_token": 0.081061 + }, + { + "ok": true, + "error": "", + "output": "\nGenerate a Markdown report that merges provided monthly sales CSVs, preserves all original rows, calculates regional revenue summaries, and verifies the summary total against the source data.\n\n\nThe user has multiple CSV files containing monthly sales data with columns for region and revenue. The goal is to aggregate this data by region while ensuring no records are lost during processing. A final reconciliation step must confirm that the sum of regional totals exactly matches the grand total from the combined dataset, highlighting any discrepancies if they exist.\n\n\n- Output length: Maximum 500 words for explanations; tables can be longer but must fit in a single code block or clearly separated section.\n- Format: Use Markdown with headers (`#`, `##`), data tables (using standard table syntax), and a dedicated \"Reconciliation Check\" section at the end.\n- Tone: Professional, analytical, and concise. Avoid unnecessary fluff; focus on clarity of results.\n- Testability: The reconciliation logic must be explicitly stated so it can be independently verified by summing regional totals vs. global total in any spreadsheet or script.\n\n\n1. Title section with a brief overview of the merge operation performed (if applicable).\n2. A data table showing all original rows combined from the source CSVs (columns: Date, Region, Revenue).\n3. A summary table aggregating revenue by region only (columns: Region, Total Revenue).\n4. A \"Reconciliation Check\" section with two lines of math: `Sum of Regional Totals = [X]` and `Grand Total from Source = [Y]`, followed by a status line (`MATCHED` or `DISCREPANCY DETECTED`).", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 144, + "completion_tokens": 357, + "total_tokens": 501, + "kv_token_occupancy_rate_percentage": 1.5289306640625, + "load_duration": 8e-07, + "prefill_duration_ttft": 1.16649728, + "decoding_duration": 25.78559, + "prefill_speed_tps": 123.44649444874831, + "decoding_speed_tps": 13.84494207811417 + }, + "wall_seconds": 28.4029, + "ttft_seconds": 1.1665, + "completion_tokens": 357, + "decode_duration_seconds": 25.7856, + "decode_tokens_per_second": 13.8449, + "seconds_per_output_token": 0.07956 + }, + { + "ok": true, + "error": "", + "output": "\nGenerate a Markdown report that merges multiple monthly sales CSVs into a single dataset, calculates regional revenue summaries, and verifies the total against the source data.\n\n\nThe user has access to several separate CSV files containing monthly sales records with columns for region, date, and revenue. The goal is to aggregate these files without losing individual row details while producing a high-level summary by geographic area.\n\n\n- Output must be valid Markdown using headers (#) and tables (|).\n- Include three distinct sections: \"Combined Data Preview\" (first 5 rows), \"Regional Revenue Summary\", and \"Reconciliation Check\".\n- The Reconciliation section must explicitly state the sum of individual regions versus the total row count/sum to confirm accuracy.\n- Tone should be professional and analytical, suitable for a business audit or financial review.\n\n\nMarkdown document with exactly three headers: `## Combined Data Preview`, `## Regional Revenue Summary`, and `## Reconciliation Check`. Use standard table formatting for data presentation.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 144, + "completion_tokens": 216, + "total_tokens": 360, + "kv_token_occupancy_rate_percentage": 1.0986328125, + "load_duration": 9e-07, + "prefill_duration_ttft": 1.17877312, + "decoding_duration": 15.858891, + "prefill_speed_tps": 122.16091252572845, + "decoding_speed_tps": 13.620120095408941 + }, + "wall_seconds": 17.9436, + "ttft_seconds": 1.1788, + "completion_tokens": 216, + "decode_duration_seconds": 15.8589, + "decode_tokens_per_second": 13.6201, + "seconds_per_output_token": 0.083072 + }, + { + "ok": true, + "error": "", + "output": "\nGenerate a Markdown report that merges multiple monthly sales CSVs into a single dataset, calculates regional revenue summaries, and verifies the total against the source data.\n\n\nThe user has several separate CSV files containing monthly sales records with columns for region, date, product ID, quantity sold, and unit price. The goal is to aggregate these files without losing individual row details while producing a high-level summary by geographic region that matches the sum of all original transactions.\n\n\n- Output must be valid Markdown with clear headers (e.g., `## 1. Data Merged`, `## 2. Regional Summary`, `## 3. Reconciliation Check`).\n- The merged dataset section should display a sample of combined rows or indicate successful concatenation if the full list is too long, but explicitly state that original row integrity is preserved.\n- The regional summary must show total revenue per region with at least two decimal places precision.\n- The reconciliation check must compare the sum of all regions to the grand total and declare \"PASSED\" or \"FAILED\".\n- Tone: Professional, analytical, concise. No conversational filler before or after the report.\n\n\nMarkdown document containing exactly three sections with headers as specified above. Use tables for data presentation where appropriate. Ensure no JSON output is generated unless requested (it was not).", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 144, + "completion_tokens": 281, + "total_tokens": 425, + "kv_token_occupancy_rate_percentage": 1.2969970703125, + "load_duration": 8e-07, + "prefill_duration_ttft": 1.24151744, + "decoding_duration": 20.819726, + "prefill_speed_tps": 115.9870939871775, + "decoding_speed_tps": 13.49681547201918 + }, + "wall_seconds": 23.2901, + "ttft_seconds": 1.2415, + "completion_tokens": 281, + "decode_duration_seconds": 20.8197, + "decode_tokens_per_second": 13.4968, + "seconds_per_output_token": 0.082883 + }, + { + "ok": true, + "error": "", + "output": "\nGenerate a Markdown report that merges multiple monthly sales CSVs into a single dataset, calculates regional revenue summaries, and verifies the total against the source data.\n\n\nThe user has several separate CSV files containing monthly sales records with columns for region, date, and revenue. The goal is to aggregate these files without losing individual row details while producing a high-level summary by region that mathematically equals the sum of all original rows.\n\n\n- Output must be valid Markdown with clear headers (e.g., `## 1. Data Merge`, `## 2. Regional Summary`, `## 3. Reconciliation`).\n- The merged dataset section should display a sample of combined rows to prove integrity, not the full list if too large (or all if under threshold).\n- Revenue summaries must be grouped by region with clear totals per region and an overall total.\n- A reconciliation table must explicitly show: `Sum of Regional Totals` vs `Total from Merged Rows`, confirming they match exactly.\n- Tone should be professional, analytical, and concise.\n\n\nUse Markdown headers for sections. Include a code block or table for the merged sample data and summary tables. Ensure no JSON output is used unless specifically requested (it was not).", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 144, + "completion_tokens": 266, + "total_tokens": 410, + "kv_token_occupancy_rate_percentage": 1.251220703125, + "load_duration": 1.3e-06, + "prefill_duration_ttft": 1.551676288, + "decoding_duration": 20.088912, + "prefill_speed_tps": 92.80286172678822, + "decoding_speed_tps": 13.241135209313475 + }, + "wall_seconds": 22.8554, + "ttft_seconds": 1.5517, + "completion_tokens": 266, + "decode_duration_seconds": 20.0889, + "decode_tokens_per_second": 13.2411, + "seconds_per_output_token": 0.085923 + } + ], + "speed": { + "runs": 5, + "successful_runs": 5, + "wall_seconds": { + "median": 22.8554, + "p90": 26.3578, + "min": 17.9436, + "max": 28.4029 + }, + "ttft_seconds": { + "median": 1.1788, + "p90": 1.4276, + "min": 1.1638, + "max": 1.5517 + }, + "completion_tokens": { + "median": 266.0, + "p90": 326.6, + "min": 216.0, + "max": 357.0 + }, + "decode_tokens_per_second": { + "median": 13.6201, + "p90": 13.8725, + "min": 13.2411, + "max": 13.8909 + }, + "seconds_per_output_token": { + "median": 0.0829, + "p90": 0.0848, + "min": 0.0796, + "max": 0.0859 + } + }, + "representative_output": "\nGenerate a Markdown report that merges multiple monthly sales CSVs into a single dataset, calculates regional revenue summaries, and verifies the total against the source data.\n\n\nThe user has several separate CSV files containing monthly sales records with columns for date, region, product ID, quantity sold, and unit price. The goal is to aggregate these files without losing individual row details while producing a high-level summary by region that matches the sum of all original transactions.\n\n\n- Output must be valid Markdown with clear headers (e.g., `## 1. Data Merge`, `## 2. Regional Summary`, `## 3. Reconciliation Check`).\n- Include a sample merged table showing at least one row from each source file to demonstrate retention of original data.\n- Present the regional summary as a bulleted list or small table with totals per region and an overall grand total.\n- The reconciliation section must explicitly state: \"Source Total\" vs \"Summary Total\" and confirm they match (or note any discrepancy).\n- Tone should be professional, analytical, and concise.\n\n\nMarkdown document with three main sections as specified above. No JSON or code blocks unless illustrating the merged data structure conceptually.", + "quality": { + "rubric": { + "r1": false, + "r2": false, + "r3": false, + "r4": false, + "r5": false, + "r6": false, + "r7": false + }, + "score": 0, + "passed": false, + "hard_fail": true, + "pending_judge": false, + "invented_requirement": true, + "constraint_item_count": 0, + "token_count": 256, + "token_count_source": "usage.completion_tokens", + "estimated_tokens": 248, + "judge_notes": "Invents columns, sample tables, Markdown layout, and presentation details." + } + } + }, + { + "name": "vague_dashboard_speed", + "category": "vague", + "trap": false, + "input": "make the dashboard faster", + "v1": { + "label": "current", + "max_tokens": 700, + "warmups": 1, + "warmup_errors": [], + "samples": [ + { + "ok": true, + "error": "", + "output": "\nOptimize the provided dashboard code to improve its rendering performance.\n\n\nThe user has a web-based dashboard with multiple data visualizations and interactive elements that currently experience lag or slow load times. The goal is specifically to increase speed without altering the core functionality or design aesthetic.\n\n\n- Output must be a single, self-contained code block (HTML/CSS/JS) containing optimized implementations of common dashboard patterns (e.g., lightweight charting libraries, efficient DOM updates).\n- Do not include lengthy explanations; focus on performance techniques like virtualization for large lists, debounced search inputs, and lazy loading.\n- The response must be under 500 words.\n\n\nMarkdown code block with a brief introductory sentence followed by the optimized implementation details.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 122, + "completion_tokens": 166, + "total_tokens": 288, + "kv_token_occupancy_rate_percentage": 0.87890625, + "load_duration": 7e-07, + "prefill_duration_ttft": 1.350409984, + "decoding_duration": 12.189752, + "prefill_speed_tps": 90.3429339574551, + "decoding_speed_tps": 13.617996494104228 + }, + "wall_seconds": 14.2816, + "ttft_seconds": 1.3504, + "completion_tokens": 166, + "decode_duration_seconds": 12.1898, + "decode_tokens_per_second": 13.618, + "seconds_per_output_token": 0.086034 + }, + { + "ok": true, + "error": "", + "output": "\nOptimize the provided dashboard code to improve its rendering performance without altering visual output.\n\n\nThe user has a web-based dashboard with multiple data visualization components that currently experience lag during initial load and interaction. The goal is specifically to increase speed, not change design or add features.\n\n\n\n\n- Output must be exactly 300 words (±15).\n- Use only standard HTML/CSS/JS libraries; no external build tools required for the solution.\n- Prioritize virtual DOM usage and lazy loading of heavy charts.\n- Tone: Direct, technical, and action-oriented.\n\n\n\n\nA single Markdown code block containing a complete `index.html` file with embedded CSS/JS that implements the optimized dashboard structure described in the constraints.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 122, + "completion_tokens": 172, + "total_tokens": 294, + "kv_token_occupancy_rate_percentage": 0.897216796875, + "load_duration": 7e-07, + "prefill_duration_ttft": 1.173108096, + "decoding_duration": 12.765522, + "prefill_speed_tps": 103.99723641494671, + "decoding_speed_tps": 13.473792924409985 + }, + "wall_seconds": 14.7636, + "ttft_seconds": 1.1731, + "completion_tokens": 172, + "decode_duration_seconds": 12.7655, + "decode_tokens_per_second": 13.4738, + "seconds_per_output_token": 0.085835 + }, + { + "ok": true, + "error": "", + "output": "\nOptimize the provided dashboard code to improve its rendering performance without altering visual output.\n\n\nThe user has a web-based dashboard with interactive charts and data tables that currently experiences lag when loading or scrolling through large datasets. The goal is specifically to increase speed, not change design or functionality.\n\n\n- Output must be valid JavaScript/TypeScript code suitable for direct insertion into an existing project.\n- Do not include explanations, comments about \"optimization,\" or meta-text; only the optimized code block.\n- Maintain all original UI elements and data logic exactly as they were before optimization.\n- Focus on performance techniques such as virtual scrolling, memoization (React), debouncing inputs, or efficient rendering strategies appropriate for web dashboards.\n\n\nA single Markdown code block containing only the optimized implementation of the dashboard component(s).", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 122, + "completion_tokens": 180, + "total_tokens": 302, + "kv_token_occupancy_rate_percentage": 0.921630859375, + "load_duration": 9e-07, + "prefill_duration_ttft": 1.155158784, + "decoding_duration": 13.487424, + "prefill_speed_tps": 105.61318642061245, + "decoding_speed_tps": 13.345765655472832 + }, + "wall_seconds": 15.5392, + "ttft_seconds": 1.1552, + "completion_tokens": 180, + "decode_duration_seconds": 13.4874, + "decode_tokens_per_second": 13.3458, + "seconds_per_output_token": 0.086329 + }, + { + "ok": true, + "error": "", + "output": "\nOptimize the provided dashboard code to improve its rendering performance.\n\n\nThe user has a web-based dashboard with multiple data visualizations and interactive elements that currently experience lag or slow load times. The goal is specifically to increase speed without altering the core functionality or design aesthetic.\n\n\n- Output must be valid JavaScript/TypeScript code suitable for direct integration into an existing project.\n- Do not include lengthy explanations; focus on implementation details like memoization, virtualization, debouncing, and efficient state management patterns.\n- Maintain backward compatibility with the original logic where possible.\n\n\nProvide a single Markdown code block containing the optimized solution. Include brief comments explaining key performance improvements (e.g., \"Implemented React.memo to prevent unnecessary re-renders\").", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 122, + "completion_tokens": 165, + "total_tokens": 287, + "kv_token_occupancy_rate_percentage": 0.8758544921875, + "load_duration": 8e-07, + "prefill_duration_ttft": 1.501880448, + "decoding_duration": 12.316232, + "prefill_speed_tps": 81.23149892687064, + "decoding_speed_tps": 13.396954523104146 + }, + "wall_seconds": 14.6382, + "ttft_seconds": 1.5019, + "completion_tokens": 165, + "decode_duration_seconds": 12.3162, + "decode_tokens_per_second": 13.397, + "seconds_per_output_token": 0.088716 + }, + { + "ok": true, + "error": "", + "output": "\nOptimize a web dashboard to improve its loading and interaction speed.\n\n\nThe user is experiencing performance issues with their current dashboard implementation, specifically regarding initial load times and responsiveness during data rendering or filtering. The existing codebase uses standard HTML/CSS/JS without specific optimization techniques like lazy loading, virtualization for large lists, or asset compression.\n\n\n- Provide a concise technical summary of 3 key optimizations (max 150 words).\n- Output must be in plain text with no Markdown formatting other than bullet points.\n- Tone: Direct and actionable for a developer audience.\n- Do not include code snippets; focus on architectural changes.\n\n\nThree numbered items detailing specific optimization strategies, each followed by a brief explanation of the expected performance gain.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 122, + "completion_tokens": 168, + "total_tokens": 290, + "kv_token_occupancy_rate_percentage": 0.885009765625, + "load_duration": 7e-07, + "prefill_duration_ttft": 1.370741504, + "decoding_duration": 12.226004, + "prefill_speed_tps": 89.00292261085573, + "decoding_speed_tps": 13.741202767478237 + }, + "wall_seconds": 14.7931, + "ttft_seconds": 1.3707, + "completion_tokens": 168, + "decode_duration_seconds": 12.226, + "decode_tokens_per_second": 13.7412, + "seconds_per_output_token": 0.088054 + } + ], + "speed": { + "runs": 5, + "successful_runs": 5, + "wall_seconds": { + "median": 14.7636, + "p90": 15.2408, + "min": 14.2816, + "max": 15.5392 + }, + "ttft_seconds": { + "median": 1.3504, + "p90": 1.4494, + "min": 1.1552, + "max": 1.5019 + }, + "completion_tokens": { + "median": 168.0, + "p90": 176.8, + "min": 165.0, + "max": 180.0 + }, + "decode_tokens_per_second": { + "median": 13.4738, + "p90": 13.6919, + "min": 13.3458, + "max": 13.7412 + }, + "seconds_per_output_token": { + "median": 0.0863, + "p90": 0.0885, + "min": 0.0858, + "max": 0.0887 + } + }, + "representative_output": "\nOptimize the provided dashboard code to improve its rendering performance.\n\n\nThe user has a web-based dashboard with multiple data visualizations and interactive elements that currently experience lag or slow load times. The goal is specifically to increase speed without altering the core functionality or design aesthetic.\n\n\n- Output must be a single, self-contained code block (HTML/CSS/JS) containing optimized implementations of common dashboard patterns (e.g., lightweight charting libraries, efficient DOM updates).\n- Do not include lengthy explanations; focus on performance techniques like virtualization for large lists, debounced search inputs, and lazy loading.\n- The response must be under 500 words.\n\n\nMarkdown code block with a brief introductory sentence followed by the optimized implementation details.", + "quality": { + "rubric": { + "r1": false, + "r2": false, + "r3": false, + "r4": false, + "r5": false, + "r6": false, + "r7": true + }, + "score": 1, + "passed": false, + "hard_fail": true, + "pending_judge": false, + "invented_requirement": true, + "constraint_item_count": 0, + "token_count": 166, + "token_count_source": "usage.completion_tokens", + "estimated_tokens": 154, + "judge_notes": "Invents dashboard technology, functionality, design, libraries, virtualization, lazy loading, and format." + } + } + }, + { + "name": "long_webhook_import", + "category": "long", + "trap": false, + "input": "Add a webhook import endpoint to the existing Python service. It receives JSON events with event_id, account_id, happened_at, and payload. Reject malformed JSON with 400, authenticate using the existing X-Webhook-Key middleware, and make duplicate event_id requests return the original result without inserting twice. Store accepted events with the repository layer already used by the billing importer. Return JSON with id, status, and duplicate. Add focused unit tests and one HTTP integration test. Do not change the database schema or introduce a queue.", + "v1": { + "label": "current", + "max_tokens": 900, + "warmups": 1, + "warmup_errors": [], + "samples": [ + { + "ok": true, + "error": "", + "output": "\nImplement a webhook import endpoint in the existing Python service that validates JSON input, authenticates via X-Webhook-Key middleware, deduplicates events by ID using the current repository layer, and returns a specific JSON response with status indicators.\n\n\nThe application is an existing Python service handling billing imports. A separate HTTP integration test suite already exists for other endpoints. The database schema is fixed and cannot be altered to add new tables or columns. No message queues are available; all processing must happen synchronously within the request lifecycle. The repository layer used by the billing importer should be reused directly without modification or abstraction changes.\n\n\n- Response length: Keep implementation concise (under 150 lines of code).\n- Format: Python file with endpoint definition, middleware integration, deduplication logic, and test cases at the bottom.\n- Tone: Professional technical documentation style for comments; no conversational filler.\n- Tests: Include focused unit tests for JSON validation and duplicate detection; include exactly one HTTP integration test verifying the full flow.\n- Output shape: Markdown code block with `python` syntax highlighting containing the complete implementation file content.\n\n\nMarkdown header (`# Webhook Import Endpoint Implementation`) followed by a single Python code block showing the full service update including endpoint, middleware usage, deduplication logic, and tests.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 225, + "completion_tokens": 281, + "total_tokens": 506, + "kv_token_occupancy_rate_percentage": 1.544189453125, + "load_duration": 6e-07, + "prefill_duration_ttft": 1.28251904, + "decoding_duration": 20.04594, + "prefill_speed_tps": 175.43599196780735, + "decoding_speed_tps": 14.017801110848382 + }, + "wall_seconds": 22.674, + "ttft_seconds": 1.2825, + "completion_tokens": 281, + "decode_duration_seconds": 20.0459, + "decode_tokens_per_second": 14.0178, + "seconds_per_output_token": 0.08069 + }, + { + "ok": true, + "error": "", + "output": "\nImplement a webhook import endpoint in the existing Python service that validates JSON input, authenticates via X-Webhook-Key middleware, deduplicates events by ID using the current repository layer, and returns a specific JSON response.\n\n\nThe application currently uses an existing Python service with a repository layer for billing imports. Authentication is handled by an X-Webhook-Key middleware. The database schema must remain unchanged, and no message queues should be introduced. Events are defined by event_id, account_id, happened_at, and payload fields within JSON requests.\n\n\n- Response length: Keep implementation concise (under 150 lines of code).\n- Format: Python file with endpoint definition at the top followed by test cases below it.\n- Tone: Professional technical documentation style for comments; clear variable names in code.\n- Output format: Markdown block containing the full Python script, including imports, class definitions, and both unit tests (focused on logic) and one HTTP integration test using `requests`.\n\n\n```python\n# Implementation of webhook endpoint with tests below it...\n```", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 225, + "completion_tokens": 231, + "total_tokens": 456, + "kv_token_occupancy_rate_percentage": 1.3916015625, + "load_duration": 1.4e-06, + "prefill_duration_ttft": 1.610471424, + "decoding_duration": 16.444914, + "prefill_speed_tps": 139.71064413000104, + "decoding_speed_tps": 14.04689620146387 + }, + "wall_seconds": 19.1657, + "ttft_seconds": 1.6105, + "completion_tokens": 231, + "decode_duration_seconds": 16.4449, + "decode_tokens_per_second": 14.0469, + "seconds_per_output_token": 0.082968 + }, + { + "ok": true, + "error": "", + "output": "\nImplement a webhook import endpoint in the existing Python service that validates JSON input, authenticates via X-Webhook-Key middleware, deduplicates events by ID using the current repository layer, and returns a specific JSON response.\n\n\nThe service currently handles billing imports through an existing repository layer and uses X-Webhook-Key for authentication. The database schema is fixed and no queuing systems are available. Events must be stored without duplication based on event_id.\n\n\n- Response length: 150 - 250 words total (excluding code blocks).\n- Format: Markdown with a single code block containing the endpoint implementation, tests, and documentation comments.\n- Tone: Technical, concise, imperative.\n- No schema changes or queue introductions allowed in the solution.\n\n\nMarkdown file named `webhook_import.py` (simulated) containing:\n1. Endpoint definition with docstring explaining deduplication logic.\n2. Unit tests for JSON validation and duplicate detection using pytest fixtures.\n3. One HTTP integration test verifying authentication and response structure (`id`, `status`, `duplicate`).", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 225, + "completion_tokens": 237, + "total_tokens": 462, + "kv_token_occupancy_rate_percentage": 1.409912109375, + "load_duration": 7e-07, + "prefill_duration_ttft": 1.264426752, + "decoding_duration": 16.865724, + "prefill_speed_tps": 177.94625085566048, + "decoding_speed_tps": 14.052168765479621 + }, + "wall_seconds": 19.2516, + "ttft_seconds": 1.2644, + "completion_tokens": 237, + "decode_duration_seconds": 16.8657, + "decode_tokens_per_second": 14.0522, + "seconds_per_output_token": 0.08123 + }, + { + "ok": true, + "error": "", + "output": "\nImplement a webhook import endpoint in the existing Python service that validates JSON input, authenticates via X-Webhook-Key middleware, deduplicates events by ID using the current repository layer, and returns a specific JSON response.\n\n\nThe service currently handles billing imports through an existing repository layer and uses an X-Webhook-Key header for authentication. The database schema is fixed and no message queues are available. Events must be stored without duplication based on event_id.\n\n\n- Code length: Maximum 150 lines total (excluding tests).\n- Format: Python file with endpoint, middleware integration, deduplication logic, and response serialization.\n- Tone: Professional, concise implementation notes only; no explanatory fluff in code comments beyond function docstrings.\n- Output shape: Single `.py` file content block ready for copy-paste into a project directory.\n\n\nMarkdown code block containing the complete Python implementation with imports, class definitions (WebhookHandler), and method signatures matching the described behavior.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 225, + "completion_tokens": 216, + "total_tokens": 441, + "kv_token_occupancy_rate_percentage": 1.3458251953125, + "load_duration": 9e-07, + "prefill_duration_ttft": 1.296210816, + "decoding_duration": 15.523699, + "prefill_speed_tps": 173.5828749634504, + "decoding_speed_tps": 13.914209493497651 + }, + "wall_seconds": 17.838, + "ttft_seconds": 1.2962, + "completion_tokens": 216, + "decode_duration_seconds": 15.5237, + "decode_tokens_per_second": 13.9142, + "seconds_per_output_token": 0.082583 + }, + { + "ok": true, + "error": "", + "output": "\nImplement a webhook import endpoint in the existing Python service that validates JSON input, authenticates via X-Webhook-Key middleware, deduplicates events by ID using the current repository layer, and returns a specific JSON response with status indicators.\n\n\nThe service currently handles billing imports through an existing repository layer and uses an X-Webhook-Key header for authentication. The database schema is fixed and no message queues are available. Events must be stored without duplication based on event_id.\n\n\n- Response format: JSON containing `id`, `status` (success/failure), and `duplicate` (boolean).\n- Error handling: Return HTTP 400 for malformed JSON; return existing result for duplicate IDs instead of inserting twice.\n- Testing requirements: Include focused unit tests for logic branches and exactly one HTTP integration test.\n- Scope restrictions: Do not alter database schema or introduce queuing mechanisms.\n\n\nMarkdown code block containing the full implementation file, test files, and a brief README explaining usage.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 225, + "completion_tokens": 216, + "total_tokens": 441, + "kv_token_occupancy_rate_percentage": 1.3458251953125, + "load_duration": 8e-07, + "prefill_duration_ttft": 1.257849856, + "decoding_duration": 15.650229, + "prefill_speed_tps": 178.87667508704632, + "decoding_speed_tps": 13.801714978100321 + }, + "wall_seconds": 17.9323, + "ttft_seconds": 1.2578, + "completion_tokens": 216, + "decode_duration_seconds": 15.6502, + "decode_tokens_per_second": 13.8017, + "seconds_per_output_token": 0.08302 + } + ], + "speed": { + "runs": 5, + "successful_runs": 5, + "wall_seconds": { + "median": 19.1657, + "p90": 21.305, + "min": 17.838, + "max": 22.674 + }, + "ttft_seconds": { + "median": 1.2825, + "p90": 1.4848, + "min": 1.2578, + "max": 1.6105 + }, + "completion_tokens": { + "median": 231.0, + "p90": 263.4, + "min": 216.0, + "max": 281.0 + }, + "decode_tokens_per_second": { + "median": 14.0178, + "p90": 14.0501, + "min": 13.8017, + "max": 14.0522 + }, + "seconds_per_output_token": { + "median": 0.0826, + "p90": 0.083, + "min": 0.0807, + "max": 0.083 + } + }, + "representative_output": "\nImplement a webhook import endpoint in the existing Python service that validates JSON input, authenticates via X-Webhook-Key middleware, deduplicates events by ID using the current repository layer, and returns a specific JSON response with status indicators.\n\n\nThe application is an existing Python service handling billing imports. A separate HTTP integration test suite already exists for other endpoints. The database schema is fixed and cannot be altered to add new tables or columns. No message queues are available; all processing must happen synchronously within the request lifecycle. The repository layer used by the billing importer should be reused directly without modification or abstraction changes.\n\n\n- Response length: Keep implementation concise (under 150 lines of code).\n- Format: Python file with endpoint definition, middleware integration, deduplication logic, and test cases at the bottom.\n- Tone: Professional technical documentation style for comments; no conversational filler.\n- Tests: Include focused unit tests for JSON validation and duplicate detection; include exactly one HTTP integration test verifying the full flow.\n- Output shape: Markdown code block with `python` syntax highlighting containing the complete implementation file content.\n\n\nMarkdown header (`# Webhook Import Endpoint Implementation`) followed by a single Python code block showing the full service update including endpoint, middleware usage, deduplication logic, and tests.", + "quality": { + "rubric": { + "r1": false, + "r2": false, + "r3": false, + "r4": false, + "r5": false, + "r6": false, + "r7": false + }, + "score": 0, + "passed": false, + "hard_fail": true, + "pending_judge": false, + "invented_requirement": true, + "constraint_item_count": 0, + "token_count": 281, + "token_count_source": "usage.completion_tokens", + "estimated_tokens": 260, + "judge_notes": "Invents line limit, file layout, comments, header, synchronous architecture details, and repository constraints." + } + } + }, + { + "name": "trap_rename_cli", + "category": "trap", + "trap": true, + "input": "write a CLI that renames files from a mapping in a text file", + "v1": { + "label": "current", + "max_tokens": 700, + "warmups": 1, + "warmup_errors": [], + "samples": [ + { + "ok": true, + "error": "", + "output": "\nGenerate a Python CLI script that reads a file mapping and renames target files accordingly.\n\n\nThe user provides an input text file containing lines in the format \"source:target\" or similar key-value pairs defining filename mappings. The tool must parse this configuration, locate existing source files, and rename them to their corresponding targets if they exist. No external dependencies are required beyond standard library modules like `argparse`, `os`, and `sys`.\n\n\n- Output length: approximately 100 lines of code plus a brief usage example.\n- Format: A single Python script with a shebang line, followed by argument parsing logic, file reading/writing functions, and the main execution block.\n- Tone: Professional and concise; avoid excessive comments or verbose explanations.\n- Testability: The solution must be runnable via `python rename_files.py` with arguments like `--input mappings.txt --output-dir ./`.\n\n\nMarkdown code block containing the Python script, followed by a separate Markdown section showing example usage commands.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 132, + "completion_tokens": 221, + "total_tokens": 353, + "kv_token_occupancy_rate_percentage": 1.0772705078125, + "load_duration": 8e-07, + "prefill_duration_ttft": 1.270931328, + "decoding_duration": 15.870487, + "prefill_speed_tps": 103.86084369146968, + "decoding_speed_tps": 13.925218551894469 + }, + "wall_seconds": 18.2086, + "ttft_seconds": 1.2709, + "completion_tokens": 221, + "decode_duration_seconds": 15.8705, + "decode_tokens_per_second": 13.9252, + "seconds_per_output_token": 0.082392 + }, + { + "ok": true, + "error": "", + "output": "\nGenerate a Python CLI script that reads a file mapping and renames target files accordingly.\n\n\nThe user provides an input text file containing lines in the format \"source:target\" or similar key-value pairs defining filename mappings. The tool must parse this configuration, locate existing source files in the current directory (or specified path), and rename them to their corresponding targets if they exist. No external libraries beyond standard Python are required.\n\n\n- Output must be a single executable Python script with no dependencies outside `os`, `sys`, and `argparse`.\n- The CLI must accept at least two arguments: the input mapping file path and an optional output directory (defaults to current working directory).\n- Error handling should gracefully skip non-existent source files without crashing.\n- Code length must be under 150 lines.\n- Comments explaining the logic are required but minimal.\n\n\nMarkdown code block with language set to `python`. Include a brief usage example below the script using standard markdown headers (`##`).", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 132, + "completion_tokens": 221, + "total_tokens": 353, + "kv_token_occupancy_rate_percentage": 1.0772705078125, + "load_duration": 9e-07, + "prefill_duration_ttft": 1.153891712, + "decoding_duration": 15.76361, + "prefill_speed_tps": 114.39548323924524, + "decoding_speed_tps": 14.019631290040795 + }, + "wall_seconds": 17.9677, + "ttft_seconds": 1.1539, + "completion_tokens": 221, + "decode_duration_seconds": 15.7636, + "decode_tokens_per_second": 14.0196, + "seconds_per_output_token": 0.081302 + }, + { + "ok": true, + "error": "", + "output": "\nGenerate a Python CLI script that reads a file mapping and renames target files accordingly.\n\n\nThe user provides an input text file where each line contains \"source:target\" pairs separated by colons, defining the desired filename for existing files in the current directory. The script must verify if source files exist before attempting to rename them and should handle errors gracefully without crashing on missing entries.\n\n\n- Output a single Python file named `rename_files.py`.\n- Use standard library only (`argparse`, `os`, `sys`).\n- Include usage instructions printed to stderr or stdout upon execution.\n- Do not include external dependencies, tests, or documentation files in the output.\n\n\nMarkdown code block with language set to python containing the full script content.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 132, + "completion_tokens": 169, + "total_tokens": 301, + "kv_token_occupancy_rate_percentage": 0.9185791015625, + "load_duration": 8e-07, + "prefill_duration_ttft": 1.161757696, + "decoding_duration": 12.279122, + "prefill_speed_tps": 113.62093873316591, + "decoding_speed_tps": 13.763199030028368 + }, + "wall_seconds": 14.3104, + "ttft_seconds": 1.1618, + "completion_tokens": 169, + "decode_duration_seconds": 12.2791, + "decode_tokens_per_second": 13.7632, + "seconds_per_output_token": 0.084677 + }, + { + "ok": true, + "error": "", + "output": "\nGenerate a Python CLI script that reads a file mapping and renames target files accordingly.\n\n\nThe user provides an input text file containing lines in the format \"source_path destination_name\". The script must parse this file, locate existing source files on the filesystem, and rename them to their corresponding destinations if they exist. No additional features like dry-run modes or error reporting are requested beyond basic functionality.\n\n\n- Output length: 100-250 words.\n- Format: Single Python script with a `main()` function entry point.\n- Tone: Direct, functional code without extensive comments explaining obvious logic.\n- Language: English (Python).\n\n\nMarkdown code block containing the complete Python source file named `file_renamer.py`.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 132, + "completion_tokens": 167, + "total_tokens": 299, + "kv_token_occupancy_rate_percentage": 0.9124755859375, + "load_duration": 8e-07, + "prefill_duration_ttft": 1.185564928, + "decoding_duration": 12.459702, + "prefill_speed_tps": 111.33932598923845, + "decoding_speed_tps": 13.403209803894185 + }, + "wall_seconds": 14.4573, + "ttft_seconds": 1.1856, + "completion_tokens": 167, + "decode_duration_seconds": 12.4597, + "decode_tokens_per_second": 13.4032, + "seconds_per_output_token": 0.086571 + }, + { + "ok": true, + "error": "", + "output": "\nGenerate a Python CLI script that reads a file mapping and renames target files accordingly.\n\n\nThe user provides an input text file containing lines in the format \"source:target\" or similar key-value pairs defining filename mappings. The tool must parse this configuration, locate existing source files, and rename them to their corresponding targets if they exist. No external dependencies are required beyond standard library modules like `argparse`, `os`, and `sys`.\n\n\n- Length: Keep the script under 50 lines of code (excluding comments).\n- Format: Use a single Python file with no shebang line, utilizing only standard libraries.\n- Tone: Professional and concise; include brief usage examples in docstrings but avoid verbose explanations.\n- Output format: Return the complete source code block wrapped in triple backticks with `python` syntax highlighting.\n\n\nMarkdown code block containing the full Python script.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 132, + "completion_tokens": 196, + "total_tokens": 328, + "kv_token_occupancy_rate_percentage": 1.0009765625, + "load_duration": 8e-07, + "prefill_duration_ttft": 1.182166656, + "decoding_duration": 14.416469, + "prefill_speed_tps": 111.6593834972791, + "decoding_speed_tps": 13.595562131059971 + }, + "wall_seconds": 16.5712, + "ttft_seconds": 1.1822, + "completion_tokens": 196, + "decode_duration_seconds": 14.4165, + "decode_tokens_per_second": 13.5956, + "seconds_per_output_token": 0.084547 + } + ], + "speed": { + "runs": 5, + "successful_runs": 5, + "wall_seconds": { + "median": 16.5712, + "p90": 18.1122, + "min": 14.3104, + "max": 18.2086 + }, + "ttft_seconds": { + "median": 1.1822, + "p90": 1.2368, + "min": 1.1539, + "max": 1.2709 + }, + "completion_tokens": { + "median": 196.0, + "p90": 221.0, + "min": 167.0, + "max": 221.0 + }, + "decode_tokens_per_second": { + "median": 13.7632, + "p90": 13.9818, + "min": 13.4032, + "max": 14.0196 + }, + "seconds_per_output_token": { + "median": 0.0845, + "p90": 0.0858, + "min": 0.0813, + "max": 0.0866 + } + }, + "representative_output": "\nGenerate a Python CLI script that reads a file mapping and renames target files accordingly.\n\n\nThe user provides an input text file containing lines in the format \"source:target\" or similar key-value pairs defining filename mappings. The tool must parse this configuration, locate existing source files, and rename them to their corresponding targets if they exist. No external dependencies are required beyond standard library modules like `argparse`, `os`, and `sys`.\n\n\n- Output length: approximately 100 lines of code plus a brief usage example.\n- Format: A single Python script with a shebang line, followed by argument parsing logic, file reading/writing functions, and the main execution block.\n- Tone: Professional and concise; avoid excessive comments or verbose explanations.\n- Testability: The solution must be runnable via `python rename_files.py` with arguments like `--input mappings.txt --output-dir ./`.\n\n\nMarkdown code block containing the Python script, followed by a separate Markdown section showing example usage commands.", + "quality": { + "rubric": { + "r1": false, + "r2": false, + "r3": false, + "r4": false, + "r5": false, + "r6": false, + "r7": false + }, + "score": 0, + "passed": false, + "hard_fail": true, + "pending_judge": false, + "invented_requirement": true, + "constraint_item_count": 0, + "token_count": 221, + "token_count_source": "usage.completion_tokens", + "estimated_tokens": 213, + "judge_notes": "Invents Python, mapping syntax, dependencies, arguments, output directory, filename, and usage format." + } + } + }, + { + "name": "plan_postgres_migration", + "category": "plan", + "trap": false, + "input": "plan a zero-downtime migration from users.full_name to first_name and last_name for a Postgres app; planning only, no code changes", + "v1": { + "label": "current", + "max_tokens": 700, + "warmups": 1, + "warmup_errors": [], + "samples": [ + { + "ok": true, + "error": "", + "output": "\nGenerate a zero-downtime migration plan for renaming the `users.full_name` column to `first_name` and `last_name` in a PostgreSQL database.\n\n\nThe application uses PostgreSQL. The goal is to split the existing single-column full name into two separate columns without disrupting active services or causing data loss during execution. No code changes are required; only database schema planning is needed.\n\n\n- Length: Maximum 500 words total, excluding headers and lists.\n- Format: Numbered steps for migration phases (Preparation, Execution, Validation).\n- Tone: Professional technical documentation suitable for a DevOps lead.\n- Content requirements must include data validation checks and rollback procedures if execution fails mid-process.\n\n\nMarkdown with clear section headers (e.g., ## Phase 1), bullet points for specific actions within phases, and a final \"Rollback Plan\" subsection under Execution or Validation.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 147, + "completion_tokens": 198, + "total_tokens": 345, + "kv_token_occupancy_rate_percentage": 1.0528564453125, + "load_duration": 5e-07, + "prefill_duration_ttft": 1.166185088, + "decoding_duration": 14.427471, + "prefill_speed_tps": 126.05203197384736, + "decoding_speed_tps": 13.723818956212076 + }, + "wall_seconds": 16.5508, + "ttft_seconds": 1.1662, + "completion_tokens": 198, + "decode_duration_seconds": 14.4275, + "decode_tokens_per_second": 13.7238, + "seconds_per_output_token": 0.08359 + }, + { + "ok": true, + "error": "", + "output": "\nGenerate a zero-downtime migration plan for renaming the `users.full_name` column to `first_name` and `last_name` in a PostgreSQL database.\n\n\nThe application is built on PostgreSQL. The goal is to split the existing single-column field into two separate columns without disrupting service availability during execution. No code changes are required; only database schema modifications are needed.\n\n\n- Length: Maximum 500 words total, excluding headers and lists.\n- Format: Numbered steps with clear prerequisites for each phase (Preparation, Execution, Verification).\n- Tone: Professional technical documentation suitable for a DevOps engineer or DBA.\n- Content requirements: Must explicitly mention locking strategies to prevent downtime, data validation checks before migration execution, and rollback procedures if integrity issues arise.\n\n\nMarkdown with clear section headers (e.g., ## Phase 1). Use bullet points for sub-steps within phases.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 147, + "completion_tokens": 200, + "total_tokens": 347, + "kv_token_occupancy_rate_percentage": 1.0589599609375, + "load_duration": 7e-07, + "prefill_duration_ttft": 1.164395776, + "decoding_duration": 14.282586, + "prefill_speed_tps": 126.24573450874489, + "decoding_speed_tps": 14.003066391478406 + }, + "wall_seconds": 16.4292, + "ttft_seconds": 1.1644, + "completion_tokens": 200, + "decode_duration_seconds": 14.2826, + "decode_tokens_per_second": 14.0031, + "seconds_per_output_token": 0.082146 + }, + { + "ok": true, + "error": "", + "output": "\nGenerate a zero-downtime migration plan for renaming the `users.full_name` column to `first_name` and `last_name` in a PostgreSQL database.\n\n\nThe application uses PostgreSQL. The goal is to split the existing single-column full name into two separate columns without disrupting active services or causing downtime during execution. No code changes are required; only database schema modifications are needed.\n\n\n- Length: Keep the plan concise, under 500 words.\n- Format: Use a numbered list for steps and bold text for key actions (e.g., **CREATE EXTENSION**, **ALTER TABLE**).\n- Tone: Professional, technical, and directive.\n- Content Requirements: Must include prerequisites (extensions), data migration strategy (copying values), validation checks, rollback procedures if necessary, and final cleanup.\n\n\nMarkdown with a clear title `# Zero-Downtime Migration Plan` followed by the numbered steps.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 147, + "completion_tokens": 202, + "total_tokens": 349, + "kv_token_occupancy_rate_percentage": 1.0650634765625, + "load_duration": 6e-07, + "prefill_duration_ttft": 1.166479744, + "decoding_duration": 14.484612, + "prefill_speed_tps": 126.02019088297172, + "decoding_speed_tps": 13.945834379270911 + }, + "wall_seconds": 16.6204, + "ttft_seconds": 1.1665, + "completion_tokens": 202, + "decode_duration_seconds": 14.4846, + "decode_tokens_per_second": 13.9458, + "seconds_per_output_token": 0.082279 + }, + { + "ok": true, + "error": "", + "output": "\nGenerate a zero-downtime migration plan for renaming the `users.full_name` column to `first_name` and `last_name` in a PostgreSQL database.\n\n\nThe application uses PostgreSQL. The goal is to split the existing single-column full name into two separate columns without disrupting active services or causing data loss during execution. No code changes are required; only database schema planning is needed.\n\n\n- Length: Maximum 500 words total, excluding headers and lists.\n- Format: Numbered steps with clear prerequisites, actions, and verification points for each phase (Preparation, Execution, Validation).\n- Tone: Professional technical documentation suitable for a DevOps engineer or DBA.\n\n\nMarkdown text with numbered sections labeled \"Phase 1\", \"Phase 2\", etc., followed by a bulleted list of rollback procedures if applicable.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 147, + "completion_tokens": 186, + "total_tokens": 333, + "kv_token_occupancy_rate_percentage": 1.0162353515625, + "load_duration": 9e-07, + "prefill_duration_ttft": 1.193710976, + "decoding_duration": 13.333325, + "prefill_speed_tps": 123.14538691147963, + "decoding_speed_tps": 13.950008718755448 + }, + "wall_seconds": 15.4323, + "ttft_seconds": 1.1937, + "completion_tokens": 186, + "decode_duration_seconds": 13.3333, + "decode_tokens_per_second": 13.95, + "seconds_per_output_token": 0.082969 + }, + { + "ok": true, + "error": "", + "output": "\nGenerate a zero-downtime migration plan for renaming the `users.full_name` column to `first_name` and `last_name` in a PostgreSQL database.\n\n\nThe application uses PostgreSQL. The goal is to split the existing single-column full name into two separate columns without disrupting active services or causing downtime during execution. This is a planning exercise only; no actual SQL scripts, code changes, or deployment steps are being executed yet.\n\n\n- Length: Keep the plan concise (under 500 words).\n- Format: Use Markdown with clear numbered sections for each phase of the migration.\n- Tone: Professional and technical, suitable for a DevOps lead review.\n- Content Requirements: Must include pre-migration checks, data validation steps, execution strategy (e.g., application-level routing or temporary views), rollback procedures, and post-migration verification. Do not invent specific database names beyond what is provided in the context.\n\n\nMarkdown document with headers like `## Phase 1`, `## Phase 2`, etc. No JSON or code blocks unless illustrating a conceptual step (e.g., a pseudo-SQL comment).", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 147, + "completion_tokens": 244, + "total_tokens": 391, + "kv_token_occupancy_rate_percentage": 1.1932373046875, + "load_duration": 1.3e-06, + "prefill_duration_ttft": 1.471837696, + "decoding_duration": 17.388704, + "prefill_speed_tps": 99.87514275487071, + "decoding_speed_tps": 14.032098079304816 + }, + "wall_seconds": 20.0138, + "ttft_seconds": 1.4718, + "completion_tokens": 244, + "decode_duration_seconds": 17.3887, + "decode_tokens_per_second": 14.0321, + "seconds_per_output_token": 0.082024 + } + ], + "speed": { + "runs": 5, + "successful_runs": 5, + "wall_seconds": { + "median": 16.5508, + "p90": 18.6564, + "min": 15.4323, + "max": 20.0138 + }, + "ttft_seconds": { + "median": 1.1665, + "p90": 1.3606, + "min": 1.1644, + "max": 1.4718 + }, + "completion_tokens": { + "median": 200.0, + "p90": 227.2, + "min": 186.0, + "max": 244.0 + }, + "decode_tokens_per_second": { + "median": 13.95, + "p90": 14.0205, + "min": 13.7238, + "max": 14.0321 + }, + "seconds_per_output_token": { + "median": 0.0823, + "p90": 0.0833, + "min": 0.082, + "max": 0.0836 + } + }, + "representative_output": "\nGenerate a zero-downtime migration plan for renaming the `users.full_name` column to `first_name` and `last_name` in a PostgreSQL database.\n\n\nThe application uses PostgreSQL. The goal is to split the existing single-column full name into two separate columns without disrupting active services or causing data loss during execution. No code changes are required; only database schema planning is needed.\n\n\n- Length: Maximum 500 words total, excluding headers and lists.\n- Format: Numbered steps for migration phases (Preparation, Execution, Validation).\n- Tone: Professional technical documentation suitable for a DevOps lead.\n- Content requirements must include data validation checks and rollback procedures if execution fails mid-process.\n\n\nMarkdown with clear section headers (e.g., ## Phase 1), bullet points for specific actions within phases, and a final \"Rollback Plan\" subsection under Execution or Validation.", + "quality": { + "rubric": { + "r1": false, + "r2": false, + "r3": false, + "r4": false, + "r5": false, + "r6": false, + "r7": true + }, + "score": 1, + "passed": false, + "hard_fail": true, + "pending_judge": false, + "invented_requirement": true, + "constraint_item_count": 0, + "token_count": 198, + "token_count_source": "usage.completion_tokens", + "estimated_tokens": 183, + "judge_notes": "Invents word limit, phase format, DevOps audience, validation checks, and rollback content." + } + } + }, + { + "name": "explain_regex", + "category": "explain", + "trap": false, + "input": "explain what ^(?P[a-z0-9]+(?:-[a-z0-9]+)*)$ accepts and show three examples", + "v1": { + "label": "current", + "max_tokens": 700, + "warmups": 1, + "warmup_errors": [], + "samples": [ + { + "ok": true, + "error": "", + "output": "\nExplain the regex pattern `^(?P[a-z0-9]+(?:-[a-z0-9]+)*)$` and provide three valid examples.\n\n\nThe user is asking for a breakdown of a specific regular expression used to validate URL slugs, including its components and usage examples.\n\n\n1. Length: Keep the explanation concise (under 150 words).\n2. Format: Use Markdown code blocks for the regex and examples; use bullet points or numbered lists for clarity.\n3. Tone: Direct, technical but accessible, no fluff or introductions/conclusions outside the requested content.\n\n\n- A brief paragraph explaining the regex logic (anchors, named group, character classes).\n- Three distinct valid slug examples in a code block labeled \"Examples:\".", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 148, + "completion_tokens": 178, + "total_tokens": 326, + "kv_token_occupancy_rate_percentage": 0.994873046875, + "load_duration": 6e-07, + "prefill_duration_ttft": 1.165640448, + "decoding_duration": 12.810598, + "prefill_speed_tps": 126.96882666858177, + "decoding_speed_tps": 13.89474558486653 + }, + "wall_seconds": 14.8508, + "ttft_seconds": 1.1656, + "completion_tokens": 178, + "decode_duration_seconds": 12.8106, + "decode_tokens_per_second": 13.8947, + "seconds_per_output_token": 0.083431 + }, + { + "ok": true, + "error": "", + "output": "\nExplain the regex pattern `^(?P[a-z0-9]+(?:-[a-z0-9]+)*)$` and provide three valid examples.\n\n\nThe user is asking for a breakdown of a specific regular expression used to validate URL slugs, likely in Python or similar environments where named groups are common. The pattern enforces lowercase letters and numbers only, with optional hyphens separating word parts.\n\n\n- Length: Keep the explanation concise (under 150 words).\n- Format: Use a clear heading for the explanation followed by a bulleted list for examples.\n- Tone: Direct, technical but accessible.\n- Content requirements: Must explicitly mention what characters are allowed and how hyphens work; must provide exactly three distinct valid examples that match the pattern (e.g., `abc`, `my-slug`, `a1b2c3d4-e5f6`).\n\n\nMarkdown with a main heading (`#`) for the explanation and a sub-heading (`##`) or bullet list for the examples. No JSON, no code blocks unless necessary to illustrate (but plain text is preferred per constraints).", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 148, + "completion_tokens": 251, + "total_tokens": 399, + "kv_token_occupancy_rate_percentage": 1.2176513671875, + "load_duration": 1e-06, + "prefill_duration_ttft": 1.172203264, + "decoding_duration": 18.164878, + "prefill_speed_tps": 126.25796612693956, + "decoding_speed_tps": 13.817874251618976 + }, + "wall_seconds": 20.5085, + "ttft_seconds": 1.1722, + "completion_tokens": 251, + "decode_duration_seconds": 18.1649, + "decode_tokens_per_second": 13.8179, + "seconds_per_output_token": 0.081707 + }, + { + "ok": true, + "error": "", + "output": "\nExplain the regex pattern `^(?P[a-z0-9]+(?:-[a-z0-9]+)*)$` and provide three valid examples.\n\n\nThe user is asking for a breakdown of a specific regular expression used to validate URL slugs, including its components and usage examples.\n\n\n1. The explanation must be concise (under 5 sentences).\n2. Provide exactly three distinct example strings that match the pattern.\n3. Do not include invalid examples or code blocks unless requested in the original text (none present).\n4. Maintain a neutral, instructional tone suitable for technical documentation or learning materials.\n\n\n- A brief paragraph explaining the regex components and logic.\n- A bulleted list of exactly three valid example strings.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 148, + "completion_tokens": 171, + "total_tokens": 319, + "kv_token_occupancy_rate_percentage": 0.9735107421875, + "load_duration": 6e-07, + "prefill_duration_ttft": 1.172502656, + "decoding_duration": 12.183659, + "prefill_speed_tps": 126.22572686095475, + "decoding_speed_tps": 14.03519254765748 + }, + "wall_seconds": 14.205, + "ttft_seconds": 1.1725, + "completion_tokens": 171, + "decode_duration_seconds": 12.1837, + "decode_tokens_per_second": 14.0352, + "seconds_per_output_token": 0.08307 + }, + { + "ok": true, + "error": "", + "output": "\nExplain the regex pattern `^(?P[a-z0-9]+(?:-[a-z0-9]+)*)$` and provide three valid examples.\n\n\nThe user is asking for a breakdown of a specific regular expression used to validate URL slugs, including its components and usage examples.\n\n\n1. Length: Keep the explanation concise (under 150 words).\n2. Format: Use Markdown code blocks for the regex and examples; use bullet points or numbered lists for clarity.\n3. Tone: Direct, technical but accessible (educational).\n4. Content Requirements: Must explicitly explain each part of the regex (`^`, `(?P...)`, `[a-z0-9]+`, `(?:-[a-z0-9]+)*`), and provide exactly three examples that match the pattern.\n\n\nMarkdown text with a clear heading for the explanation, followed by a section for examples using code blocks. No JSON or other structured data formats unless implied (none here).", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 148, + "completion_tokens": 224, + "total_tokens": 372, + "kv_token_occupancy_rate_percentage": 1.13525390625, + "load_duration": 9e-07, + "prefill_duration_ttft": 1.169048704, + "decoding_duration": 15.904137, + "prefill_speed_tps": 126.59866051226554, + "decoding_speed_tps": 14.084385716747786 + }, + "wall_seconds": 18.1502, + "ttft_seconds": 1.169, + "completion_tokens": 224, + "decode_duration_seconds": 15.9041, + "decode_tokens_per_second": 14.0844, + "seconds_per_output_token": 0.081028 + }, + { + "ok": true, + "error": "", + "output": "\nExplain the regex pattern `^(?P[a-z0-9]+(?:-[a-z0-9]+)*)$` and provide three valid examples.\n\n\nThe user is asking for an explanation of a specific regular expression used to validate \"slugs\" (URL-friendly identifiers). The pattern uses named capture groups, lowercase letters, numbers, hyphens as separators, and anchors to enforce start/end boundaries.\n\n\n- Length: Keep the explanation concise (under 100 words) and list exactly three examples.\n- Format: Use a clear heading for the explanation followed by a bulleted or numbered list for examples.\n- Tone: Informative and direct, suitable for technical documentation or learning purposes.\n\n\nMarkdown with headers (`##`) and bullet points (`*` or `-`). No JSON, no code blocks unless necessary to show the regex itself clearly (but plain text is preferred per constraints).", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 148, + "completion_tokens": 204, + "total_tokens": 352, + "kv_token_occupancy_rate_percentage": 1.07421875, + "load_duration": 5e-07, + "prefill_duration_ttft": 1.171853056, + "decoding_duration": 15.002036, + "prefill_speed_tps": 126.29569828932546, + "decoding_speed_tps": 13.598154277192776 + }, + "wall_seconds": 17.1781, + "ttft_seconds": 1.1719, + "completion_tokens": 204, + "decode_duration_seconds": 15.002, + "decode_tokens_per_second": 13.5982, + "seconds_per_output_token": 0.084206 + } + ], + "speed": { + "runs": 5, + "successful_runs": 5, + "wall_seconds": { + "median": 17.1781, + "p90": 19.5652, + "min": 14.205, + "max": 20.5085 + }, + "ttft_seconds": { + "median": 1.1719, + "p90": 1.1724, + "min": 1.1656, + "max": 1.1725 + }, + "completion_tokens": { + "median": 204.0, + "p90": 240.2, + "min": 171.0, + "max": 251.0 + }, + "decode_tokens_per_second": { + "median": 13.8947, + "p90": 14.0647, + "min": 13.5982, + "max": 14.0844 + }, + "seconds_per_output_token": { + "median": 0.0831, + "p90": 0.0839, + "min": 0.081, + "max": 0.0842 + } + }, + "representative_output": "\nExplain the regex pattern `^(?P[a-z0-9]+(?:-[a-z0-9]+)*)$` and provide three valid examples.\n\n\nThe user is asking for a breakdown of a specific regular expression used to validate URL slugs, including its components and usage examples.\n\n\n1. Length: Keep the explanation concise (under 150 words).\n2. Format: Use Markdown code blocks for the regex and examples; use bullet points or numbered lists for clarity.\n3. Tone: Direct, technical but accessible, no fluff or introductions/conclusions outside the requested content.\n\n\n- A brief paragraph explaining the regex logic (anchors, named group, character classes).\n- Three distinct valid slug examples in a code block labeled \"Examples:\".", + "quality": { + "rubric": { + "r1": false, + "r2": false, + "r3": false, + "r4": false, + "r5": false, + "r6": false, + "r7": true + }, + "score": 1, + "passed": false, + "hard_fail": true, + "pending_judge": false, + "invented_requirement": true, + "constraint_item_count": 0, + "token_count": 178, + "token_count_source": "usage.completion_tokens", + "estimated_tokens": 172, + "judge_notes": "Invents URL-slug purpose, word limit, Markdown layout, tone, and explanation components." + } + } + }, + { + "name": "fix_timezone_boundary", + "category": "debug", + "trap": false, + "input": "fix the weekly report bug where Sunday 11:30 PM America/New_York appears in next week; weeks start Monday local time and existing report JSON must not change shape", + "v1": { + "label": "current", + "max_tokens": 700, + "warmups": 1, + "warmup_errors": [], + "samples": [ + { + "ok": true, + "error": "", + "output": "\nFix a weekly report bug where Sunday 11:30 PM America/New_York incorrectly appears in the next week by ensuring weeks start on Monday local time.\n\n\nThe existing report JSON structure must remain unchanged; only the date logic needs correction to align with local timezone boundaries and Monday-starting weeks.\n\n\n- Output length: 150 - 250 words.\n- Format: Single code block containing a corrected function or script snippet (e.g., Python).\n- Tone: Technical, direct, no explanations outside the code.\n- Testability: The logic must handle edge cases like Sunday evenings crossing into Monday mornings correctly without altering JSON keys or nesting levels.\n\n\nMarkdown header `## Solution`, followed immediately by a single fenced code block with language specified as `python`. No additional text before or after the solution section.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 154, + "completion_tokens": 188, + "total_tokens": 342, + "kv_token_occupancy_rate_percentage": 1.043701171875, + "load_duration": 7e-07, + "prefill_duration_ttft": 1.177465344, + "decoding_duration": 13.390206, + "prefill_speed_tps": 130.78941200667728, + "decoding_speed_tps": 14.040112601703065 + }, + "wall_seconds": 15.499, + "ttft_seconds": 1.1775, + "completion_tokens": 188, + "decode_duration_seconds": 13.3902, + "decode_tokens_per_second": 14.0401, + "seconds_per_output_token": 0.082441 + }, + { + "ok": true, + "error": "", + "output": "\nFix a weekly report bug where Sunday 11:30 PM America/New_York incorrectly appears in the next week by ensuring weeks start on Monday local time, while preserving the existing JSON structure.\n\n\nThe current implementation calculates week boundaries based on UTC or incorrect timezone offsets, causing end-of-week timestamps (Sunday late night) to roll over into the following week's data. The report must adhere to a Monday-start calendar for all local timezones and output valid JSON without altering schema keys or nesting levels.\n\n\n- Output length: 150 - 250 words total.\n- Format: Single code block containing only the corrected logic snippet (Python/JavaScript).\n- Tone: Technical, concise, no explanatory prose outside the code.\n- Testability: Ensure `assert` statements verify that a Sunday timestamp maps to its own week ID and Monday timestamps do not shift forward incorrectly.\n\n\nMarkdown code block with language specified as `python`. No surrounding text or headers.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 154, + "completion_tokens": 214, + "total_tokens": 368, + "kv_token_occupancy_rate_percentage": 1.123046875, + "load_duration": 5e-07, + "prefill_duration_ttft": 1.170463616, + "decoding_duration": 15.562893, + "prefill_speed_tps": 131.57179590621294, + "decoding_speed_tps": 13.75065677056316 + }, + "wall_seconds": 17.7389, + "ttft_seconds": 1.1705, + "completion_tokens": 214, + "decode_duration_seconds": 15.5629, + "decode_tokens_per_second": 13.7507, + "seconds_per_output_token": 0.082892 + }, + { + "ok": true, + "error": "", + "output": "\nFix a weekly report bug where Sunday 11:30 PM America/New_York incorrectly appears in the next week by ensuring weeks start on Monday local time, while preserving the existing JSON structure.\n\n\nThe current implementation calculates week boundaries based on UTC or incorrect timezone offsets, causing end-of-week timestamps (Sunday late night) to roll over into the following week's data. The report must adhere to a Monday-start calendar for all local timezones and output valid JSON without altering schema keys or nesting levels.\n\n\n- Output length: Keep under 200 words.\n- Format: Return only the prompt text, no markdown code blocks around the prompt itself unless specified in constraints (none here).\n- Tone: Direct technical instruction suitable for an LLM developer context.\n- Testability: Ensure logic handles timezone conversion explicitly and validates week boundaries against Monday start rules before JSON serialization.\n\n\nMarkdown headers for sections, followed by a code block containing the final prompt ready for Claude to execute.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 154, + "completion_tokens": 216, + "total_tokens": 370, + "kv_token_occupancy_rate_percentage": 1.129150390625, + "load_duration": 1e-06, + "prefill_duration_ttft": 1.29701696, + "decoding_duration": 15.914568, + "prefill_speed_tps": 118.73399095721925, + "decoding_speed_tps": 13.572470204657769 + }, + "wall_seconds": 18.2696, + "ttft_seconds": 1.297, + "completion_tokens": 216, + "decode_duration_seconds": 15.9146, + "decode_tokens_per_second": 13.5725, + "seconds_per_output_token": 0.084581 + }, + { + "ok": true, + "error": "", + "output": "\nFix a weekly report bug where Sunday 11:30 PM America/New_York incorrectly appears in the next week by ensuring weeks start on Monday local time, while preserving the existing JSON structure.\n\n\nThe current implementation calculates week boundaries based on UTC or incorrect timezone offsets, causing dates at the end of a US Eastern Standard Time (EST) Sunday to roll over into Monday's slot of the following calendar week in reports. The output must remain valid JSON with identical keys and nesting levels as previous versions; only date logic needs correction.\n\n\n- Output length: 150 - 250 words total.\n- Format: Single code block containing fixed Python function or script snippet (no full app).\n- Tone: Technical, direct, no explanations outside the code.\n- Testability: Ensure `datetime` objects are normalized to local timezone before week calculation; verify JSON keys match pre-fix state via diff check.\n\n\nMarkdown code block with language set to python. Include a brief comment header explaining the fix but keep it under 2 lines. Do not include test cases or usage examples in the output, only the corrected logic.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 154, + "completion_tokens": 249, + "total_tokens": 403, + "kv_token_occupancy_rate_percentage": 1.2298583984375, + "load_duration": 6e-07, + "prefill_duration_ttft": 1.173075328, + "decoding_duration": 17.71206, + "prefill_speed_tps": 131.27886702941552, + "decoding_speed_tps": 14.05821795996626 + }, + "wall_seconds": 20.1445, + "ttft_seconds": 1.1731, + "completion_tokens": 249, + "decode_duration_seconds": 17.7121, + "decode_tokens_per_second": 14.0582, + "seconds_per_output_token": 0.080902 + }, + { + "ok": true, + "error": "", + "output": "\nFix a weekly report bug where Sunday 11:30 PM America/New_York incorrectly appears in the next week by ensuring weeks start on Monday local time, while preserving the existing JSON structure.\n\n\nThe user's weekly report currently displays dates spanning into the following week when it should only cover the current week starting from Monday. The issue specifically affects timestamps near Sunday 11:30 PM America/New_York which are being misinterpreted as belonging to a new week. The output must remain valid JSON with no changes to keys, nesting levels, or overall shape of existing data structures.\n\n\n- Output length: Keep the response concise (under 200 words).\n- Format: Return only the prompt text in Markdown; do not include explanations or meta-commentary.\n- Tone: Professional and technical.\n- Testability: The fix must be verifiable by checking if Sunday 11:30 PM America/New_York stays within its correct week boundary without altering JSON keys or structure.\n\n\nMarkdown code block containing the rewritten prompt with clearly labeled sections for Task, Context, Constraints, and Output Format.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 154, + "completion_tokens": 244, + "total_tokens": 398, + "kv_token_occupancy_rate_percentage": 1.214599609375, + "load_duration": 8e-07, + "prefill_duration_ttft": 1.209659392, + "decoding_duration": 17.783904, + "prefill_speed_tps": 127.30856389696845, + "decoding_speed_tps": 13.720271994270774 + }, + "wall_seconds": 20.1842, + "ttft_seconds": 1.2097, + "completion_tokens": 244, + "decode_duration_seconds": 17.7839, + "decode_tokens_per_second": 13.7203, + "seconds_per_output_token": 0.082722 + } + ], + "speed": { + "runs": 5, + "successful_runs": 5, + "wall_seconds": { + "median": 18.2696, + "p90": 20.1683, + "min": 15.499, + "max": 20.1842 + }, + "ttft_seconds": { + "median": 1.1775, + "p90": 1.2621, + "min": 1.1705, + "max": 1.297 + }, + "completion_tokens": { + "median": 216.0, + "p90": 247.0, + "min": 188.0, + "max": 249.0 + }, + "decode_tokens_per_second": { + "median": 13.7507, + "p90": 14.051, + "min": 13.5725, + "max": 14.0582 + }, + "seconds_per_output_token": { + "median": 0.0827, + "p90": 0.0839, + "min": 0.0809, + "max": 0.0846 + } + }, + "representative_output": "\nFix a weekly report bug where Sunday 11:30 PM America/New_York incorrectly appears in the next week by ensuring weeks start on Monday local time.\n\n\nThe existing report JSON structure must remain unchanged; only the date logic needs correction to align with local timezone boundaries and Monday-starting weeks.\n\n\n- Output length: 150 - 250 words.\n- Format: Single code block containing a corrected function or script snippet (e.g., Python).\n- Tone: Technical, direct, no explanations outside the code.\n- Testability: The logic must handle edge cases like Sunday evenings crossing into Monday mornings correctly without altering JSON keys or nesting levels.\n\n\nMarkdown header `## Solution`, followed immediately by a single fenced code block with language specified as `python`. No additional text before or after the solution section.", + "quality": { + "rubric": { + "r1": false, + "r2": false, + "r3": false, + "r4": false, + "r5": false, + "r6": false, + "r7": true + }, + "score": 1, + "passed": false, + "hard_fail": true, + "pending_judge": false, + "invented_requirement": true, + "constraint_item_count": 0, + "token_count": 188, + "token_count_source": "usage.completion_tokens", + "estimated_tokens": 175, + "judge_notes": "Invents Python, length, code-only format, and additional boundary behavior." + } + } + }, + { + "name": "api_retry_policy", + "category": "implement", + "trap": false, + "input": "add retry handling to the Go API client for 429 and 503 responses only, honor Retry-After when present, cap attempts at three, and test the retry timing with a fake clock", + "v1": { + "label": "current", + "max_tokens": 700, + "warmups": 1, + "warmup_errors": [], + "samples": [ + { + "ok": true, + "error": "", + "output": "\nImplement retry logic in a Go API client that specifically handles 429 and 503 status codes, respects the `Retry-After` header if available, limits total attempts to three, and validates timing using a fake clock.\n\n\nThe existing Go API client currently lacks explicit handling for rate limiting (429) and service unavailable (503) errors. The server may include a `Retry-After` HTTP header indicating the number of seconds before retrying. A testing environment requires mocking time to verify that exponential backoff or fixed delays occur correctly without waiting real seconds.\n\n\n- Code must be written in Go (Golang).\n- Maximum file size: 50 lines for the core logic function.\n- Output format: Single code block with comments explaining the fake clock usage and retry limits.\n- Tone: Professional, concise technical documentation style within comments.\n\n\nMarkdown code block containing the complete Go implementation of `ExecuteWithRetry`.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 160, + "completion_tokens": 211, + "total_tokens": 371, + "kv_token_occupancy_rate_percentage": 1.1322021484375, + "load_duration": 6e-07, + "prefill_duration_ttft": 1.17775488, + "decoding_duration": 15.315206, + "prefill_speed_tps": 135.85169776583732, + "decoding_speed_tps": 13.777157160014694 + }, + "wall_seconds": 17.535, + "ttft_seconds": 1.1778, + "completion_tokens": 211, + "decode_duration_seconds": 15.3152, + "decode_tokens_per_second": 13.7772, + "seconds_per_output_token": 0.083104 + }, + { + "ok": true, + "error": "", + "output": "\nImplement retry logic in a Go API client that specifically handles 429 and 503 status codes, respects the `Retry-After` header if available, limits total attempts to three, and validates timing using a fake clock.\n\n\nThe existing Go API client currently lacks explicit handling for rate limiting (429) or service unavailable (503) errors. The server may include a `Retry-After` header indicating seconds until the next attempt. A testing environment requires mocking time to verify retry delays without waiting for real network responses.\n\n\n- Code must be written in Go using standard libraries and common patterns (e.g., context, net/http).\n- Retry logic must only trigger on status codes 429 or 503; other errors should pass through immediately.\n- If `Retry-After` header exists, the client must wait exactly that many seconds before retrying.\n- Maximum number of retries is strictly three (total attempts = initial + 3).\n- A fake clock implementation must be provided to allow testing of delay logic without real time passage.\n\n\nReturn a single Markdown code block containing the complete Go file (`api_client.go`). Include:\n1. `RetryableError` struct definition with status code and retry-after header parsing.\n2. `Client` struct with fields for max retries, fake clock interface, and HTTP client wrapper.\n3. `DoRequestWithContext` method implementing the loop logic (check status -> if 429/503 & attempts < 3 -> wait via fake clock -> retry).\n4. A simple `FakeClock` implementation demonstrating how to set a deadline or sleep duration for testing purposes.\n\nEnsure all variable names are descriptive and comments explain the \"why\" of the Retry-After handling and attempt capping.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 160, + "completion_tokens": 389, + "total_tokens": 549, + "kv_token_occupancy_rate_percentage": 1.6754150390625, + "load_duration": 8e-07, + "prefill_duration_ttft": 1.179211648, + "decoding_duration": 27.594688, + "prefill_speed_tps": 135.68387004264056, + "decoding_speed_tps": 14.096916044131392 + }, + "wall_seconds": 30.4958, + "ttft_seconds": 1.1792, + "completion_tokens": 389, + "decode_duration_seconds": 27.5947, + "decode_tokens_per_second": 14.0969, + "seconds_per_output_token": 0.078395 + }, + { + "ok": true, + "error": "", + "output": "\nImplement retry logic in a Go API client that specifically handles 429 and 503 status codes, respects the Retry-After header, limits total attempts to three, and uses a fake clock for timing validation.\n\n\nThe existing Go API client currently lacks automatic retry mechanisms for transient failures. The backend returns HTTP 429 (Too Many Requests) or 503 (Service Unavailable) on rate limiting or temporary unavailability. A `Retry-After` header may be present in these responses indicating the delay before subsequent requests. The testing environment requires a deterministic way to verify retry timing without waiting for real network delays, necessitating a mocked clock implementation.\n\n\n- Code must be written in Go (Golang).\n- Maximum file size: 50 lines of code per function/file.\n- Output format: A single Markdown code block containing the complete `retry.go` implementation.\n- Tone: Professional and concise, suitable for a production repository.\n\n\nMarkdown header (`## retry.go`) followed by a Go language syntax highlighted code block with no additional text outside the prompt structure.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 160, + "completion_tokens": 240, + "total_tokens": 400, + "kv_token_occupancy_rate_percentage": 1.220703125, + "load_duration": 7e-07, + "prefill_duration_ttft": 1.185682176, + "decoding_duration": 17.506476, + "prefill_speed_tps": 134.94341336881158, + "decoding_speed_tps": 13.709212522268903 + }, + "wall_seconds": 19.8265, + "ttft_seconds": 1.1857, + "completion_tokens": 240, + "decode_duration_seconds": 17.5065, + "decode_tokens_per_second": 13.7092, + "seconds_per_output_token": 0.08261 + }, + { + "ok": true, + "error": "", + "output": "\nImplement retry logic in a Go API client that specifically handles 429 and 503 status codes, respects the Retry-After header, limits total attempts to three, and uses a fake clock for timing validation.\n\n\nThe existing Go API client currently lacks automatic retry mechanisms for transient failures. The target environment requires precise control over backoff timing via a mockable clock unit test fixture. Only HTTP 429 (Too Many Requests) and 503 (Service Unavailable) codes should trigger retries; other errors must fail immediately.\n\n\n- Code length: Maximum 150 lines for the core retry logic implementation.\n- Format: Go source code with a dedicated `TestRetryWithFakeClock` function in `_test.go`.\n- Tone: Professional, concise, and production-ready.\n- Testability: The test must verify that exactly three attempts occur before failure when using the fake clock.\n\n\nMarkdown code blocks for Go files (`client.go`, `retry_test.go`). No JSON or other formats.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 160, + "completion_tokens": 225, + "total_tokens": 385, + "kv_token_occupancy_rate_percentage": 1.1749267578125, + "load_duration": 9e-07, + "prefill_duration_ttft": 1.1883104, + "decoding_duration": 16.111425, + "prefill_speed_tps": 134.64495471890174, + "decoding_speed_tps": 13.965245159878782 + }, + "wall_seconds": 18.3342, + "ttft_seconds": 1.1883, + "completion_tokens": 225, + "decode_duration_seconds": 16.1114, + "decode_tokens_per_second": 13.9652, + "seconds_per_output_token": 0.081485 + }, + { + "ok": true, + "error": "", + "output": "\nImplement retry logic in a Go API client that specifically handles 429 and 503 status codes, respects the Retry-After header when available, limits total attempts to three, and validates timing behavior using a fake clock.\n\n\nThe existing Go API client currently lacks explicit error handling for rate limiting (429) or service unavailability (503). The server may include a \"Retry-After\" header indicating seconds before the next attempt. A testing utility with a mockable time source is available to verify timing constraints without waiting for real delays.\n\n\n- Code must be written in Go (Golang 1.21+).\n- Retry logic must only trigger on HTTP status codes 429 and 503; other errors should pass through immediately.\n- If \"Retry-After\" header exists, the client must wait exactly that many seconds before retrying; if missing, use a default of 1 second.\n- Maximum number of retries is strictly three (total attempts = initial + 3). After the third attempt fails or succeeds, stop and return the final result/error.\n- Timing validation must be demonstrated using `time.Fake` or equivalent to prove that delays are calculated correctly without real-world latency.\n- Output should include a complete, runnable example with comments explaining each constraint check.\n\n\nReturn only the Go source code file content (e.g., `api_client.go`). Do not include explanations outside of the code block. Use standard imports and ensure all functions are exported where necessary for testing integration.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 160, + "completion_tokens": 335, + "total_tokens": 495, + "kv_token_occupancy_rate_percentage": 1.5106201171875, + "load_duration": 9e-07, + "prefill_duration_ttft": 1.183184256, + "decoding_duration": 23.758302, + "prefill_speed_tps": 135.22830378162166, + "decoding_speed_tps": 14.100334274730576 + }, + "wall_seconds": 26.456, + "ttft_seconds": 1.1832, + "completion_tokens": 335, + "decode_duration_seconds": 23.7583, + "decode_tokens_per_second": 14.1003, + "seconds_per_output_token": 0.078973 + } + ], + "speed": { + "runs": 5, + "successful_runs": 5, + "wall_seconds": { + "median": 19.8265, + "p90": 28.8799, + "min": 17.535, + "max": 30.4958 + }, + "ttft_seconds": { + "median": 1.1832, + "p90": 1.1873, + "min": 1.1778, + "max": 1.1883 + }, + "completion_tokens": { + "median": 240.0, + "p90": 367.4, + "min": 211.0, + "max": 389.0 + }, + "decode_tokens_per_second": { + "median": 13.9652, + "p90": 14.0989, + "min": 13.7092, + "max": 14.1003 + }, + "seconds_per_output_token": { + "median": 0.0815, + "p90": 0.0829, + "min": 0.0784, + "max": 0.0831 + } + }, + "representative_output": "\nImplement retry logic in a Go API client that specifically handles 429 and 503 status codes, respects the `Retry-After` header if available, limits total attempts to three, and validates timing using a fake clock.\n\n\nThe existing Go API client currently lacks explicit handling for rate limiting (429) and service unavailable (503) errors. The server may include a `Retry-After` HTTP header indicating the number of seconds before retrying. A testing environment requires mocking time to verify that exponential backoff or fixed delays occur correctly without waiting real seconds.\n\n\n- Code must be written in Go (Golang).\n- Maximum file size: 50 lines for the core logic function.\n- Output format: Single code block with comments explaining the fake clock usage and retry limits.\n- Tone: Professional, concise technical documentation style within comments.\n\n\nMarkdown code block containing the complete Go implementation of `ExecuteWithRetry`.", + "quality": { + "rubric": { + "r1": false, + "r2": false, + "r3": false, + "r4": false, + "r5": false, + "r6": false, + "r7": true + }, + "score": 1, + "passed": false, + "hard_fail": true, + "pending_judge": false, + "invented_requirement": true, + "constraint_item_count": 0, + "token_count": 211, + "token_count_source": "usage.completion_tokens", + "estimated_tokens": 185, + "judge_notes": "Invents file-size limit, comments, function name, output format, and backoff behavior." + } + } + } + ] +} diff --git a/data/benchmarks/prompt_v2_ab_2026-07-10.json b/data/benchmarks/prompt_v2_ab_2026-07-10.json new file mode 100644 index 0000000..0571314 --- /dev/null +++ b/data/benchmarks/prompt_v2_ab_2026-07-10.json @@ -0,0 +1,5058 @@ +{ + "schema_version": 1, + "created_at": "2026-07-10T15:18:37-04:00", + "protocol": { + "fixed_input_count": 12, + "warmups_per_style_input": 1, + "timed_runs_per_style_input": 5, + "judge_method": "manual_gpt5_source_review", + "duration_note": "Numeric duration fields without units are interpreted as seconds.", + "reused_v1_from": "data\\benchmarks\\prompt_v1_frozen_2026-07-10.json", + "reused_v1_created_at": "2026-07-10T11:08:24-04:00" + }, + "endpoint": { + "base_url": "http://127.0.0.1:52625", + "model": "qwen3.5:4b" + }, + "styles": { + "v1": { + "label": "current", + "system_prompt": "Rewrite the user text as a Claude-ready prompt. Structure: (one primary deliverable, one sentence), (background facts only, no instructions), (concrete + testable: length, format, tone), (exact shape — Markdown headers, JSON keys, etc.). Keep each section short. No meta-framing, no preamble. Base every constraint on what the user actually asked for — never invent requirements they did not state. Return only the prompt.", + "caps": [ + 700, + 900, + 1200 + ] + }, + "v2": { + "label": "candidate", + "system_prompt": "Restate the user's requested coding deliverable as one concise imperative sentence. Use only details the user explicitly stated. Do not solve, infer, explain, or add requirements. Return only that sentence with no preamble, under 40 tokens.", + "caps": [ + 240, + 320, + 420 + ] + } + }, + "judge": { + "method": "manual_gpt5_source_review", + "judgments": { + "v1:implement_csv_validator": { + "r3": true, + "r4": false, + "invented_requirement": true, + "notes": "Invents Python 3, Markdown fences, comments, standard-library-only, argv, and JSON schema file." + }, + "v2:implement_csv_validator": { + "r3": true, + "r4": true, + "invented_requirement": false, + "notes": "Four source-derived requirements; no implementation assumptions." + }, + "v1:debug_async_cache_race": { + "r3": true, + "r4": false, + "invented_requirement": true, + "notes": "Invents word count, Python, asyncio.Semaphore, pytest, and output shape." + }, + "v2:debug_async_cache_race": { + "r3": true, + "r4": true, + "invented_requirement": false, + "notes": "Reported race and API constraint only." + }, + "v1:review_auth_middleware": { + "r3": true, + "r4": false, + "invented_requirement": true, + "notes": "Invents exact length, bullet format, severity set, title, and report structure." + }, + "v2:review_auth_middleware": { + "r3": true, + "r4": true, + "invented_requirement": false, + "notes": "Review and source-reference requirements preserved without additions." + }, + "v1:refactor_invoice_service": { + "r3": true, + "r4": false, + "invented_requirement": true, + "notes": "Invents database/repository design, TypeScript, test policy, word count, and class limits." + }, + "v2:refactor_invoice_service": { + "r3": true, + "r4": true, + "invented_requirement": false, + "notes": "All three explicit refactor constraints preserved." + }, + "v1:data_monthly_rollup": { + "r3": true, + "r4": false, + "invented_requirement": true, + "notes": "Invents columns, sample tables, Markdown layout, and presentation details." + }, + "v2:data_monthly_rollup": { + "r3": true, + "r4": true, + "invented_requirement": false, + "notes": "Four auditable source requirements retained." + }, + "v1:vague_dashboard_speed": { + "r3": false, + "r4": false, + "invented_requirement": true, + "notes": "Invents dashboard technology, functionality, design, libraries, virtualization, lazy loading, and format." + }, + "v2:vague_dashboard_speed": { + "r3": false, + "r4": true, + "invented_requirement": false, + "notes": "Source is too vague for three concrete requirements; safe scope guards avoid invention." + }, + "v1:long_webhook_import": { + "r3": true, + "r4": false, + "invented_requirement": true, + "notes": "Invents line limit, file layout, comments, header, synchronous architecture details, and repository constraints." + }, + "v2:long_webhook_import": { + "r3": true, + "r4": true, + "invented_requirement": false, + "notes": "Five bullets cover all explicit endpoint requirements; final bullet groups overflow without adding facts." + }, + "v1:trap_rename_cli": { + "r3": true, + "r4": false, + "invented_requirement": true, + "notes": "Invents Python, mapping syntax, dependencies, arguments, output directory, filename, and usage format." + }, + "v2:trap_rename_cli": { + "r3": true, + "r4": true, + "invented_requirement": false, + "notes": "CLI, rename behavior, and text mapping only; no conventional CLI assumptions." + }, + "v1:plan_postgres_migration": { + "r3": true, + "r4": false, + "invented_requirement": true, + "notes": "Invents word limit, phase format, DevOps audience, validation checks, and rollback content." + }, + "v2:plan_postgres_migration": { + "r3": true, + "r4": true, + "invented_requirement": false, + "notes": "Zero-downtime, column split, Postgres, planning-only, and no-code constraints stay intact." + }, + "v1:explain_regex": { + "r3": true, + "r4": false, + "invented_requirement": true, + "notes": "Invents URL-slug purpose, word limit, Markdown layout, tone, and explanation components." + }, + "v2:explain_regex": { + "r3": true, + "r4": true, + "invented_requirement": false, + "notes": "Regex explanation and three examples are the only requested requirements." + }, + "v1:fix_timezone_boundary": { + "r3": true, + "r4": false, + "invented_requirement": true, + "notes": "Invents Python, length, code-only format, and additional boundary behavior." + }, + "v2:fix_timezone_boundary": { + "r3": true, + "r4": true, + "invented_requirement": false, + "notes": "Timezone, Monday boundary, and JSON-shape constraints remain source-derived." + }, + "v1:api_retry_policy": { + "r3": true, + "r4": false, + "invented_requirement": true, + "notes": "Invents file-size limit, comments, function name, output format, and backoff behavior." + }, + "v2:api_retry_policy": { + "r3": true, + "r4": true, + "invented_requirement": false, + "notes": "Status codes, Retry-After, attempt cap, and fake-clock test are preserved exactly." + } + } + }, + "cases": [ + { + "name": "implement_csv_validator", + "category": "implement", + "trap": false, + "input": "build a python script that reads a folder of CSVs, validates rows against a schema, and writes an error report with file and line numbers", + "styles": { + "v1": { + "label": "current", + "max_tokens": 700, + "warmups": 1, + "warmup_errors": [], + "samples": [ + { + "ok": true, + "error": "", + "output": "\nGenerate a Python script that iterates through a specified directory of CSV files, validates each row against a provided schema definition, and outputs a detailed error report containing the filename and line number for every validation failure.\n\n\nThe user has existing CSV data in various files within a folder that needs to be checked for consistency with specific column names, types, or required fields defined by a schema. The goal is to automate this checking process without manually inspecting each file.\n\n\n- Output must be valid Python 3 code enclosed in markdown code blocks.\n- Include comments explaining the logic of validation and error reporting.\n- Do not include external dependencies beyond the standard library (e.g., no pandas, only csv module).\n- The script should accept arguments for the input folder path and a JSON schema definition file path via `sys.argv`.\n\n\nA single code block containing the complete Python script with necessary imports, class/function definitions, and main execution logic.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 146, + "completion_tokens": 206, + "total_tokens": 352, + "kv_token_occupancy_rate_percentage": 1.07421875, + "load_duration": 1.1e-06, + "prefill_duration_ttft": 1.210723456, + "decoding_duration": 15.304176, + "prefill_speed_tps": 120.58905712651857, + "decoding_speed_tps": 13.460378396066536 + }, + "wall_seconds": 17.5524, + "ttft_seconds": 1.2107, + "completion_tokens": 206, + "decode_duration_seconds": 15.3042, + "decode_tokens_per_second": 13.4604, + "seconds_per_output_token": 0.085206 + }, + { + "ok": true, + "error": "", + "output": "\nGenerate a Python script that iterates through a specified directory of CSV files, validates each row against a provided schema definition, and outputs a detailed error report containing the filename and line number for every validation failure.\n\n\nThe user has existing CSV data in various files within a folder that needs to be checked for consistency with specific column names, types (e.g., integers vs strings), or required values defined by a schema dictionary. The goal is to automate this checking process without manually inspecting each file.\n\n\n- Output must be valid Python 3 code enclosed in markdown code blocks.\n- Include a `main` function that accepts the folder path and schema as arguments, with default examples for demonstration.\n- Error report should be printed to standard output or saved to a JSON/CSV file (choose one clear method).\n- Do not include external dependencies beyond the Python Standard Library (`os`, `csv`, `json`).\n- Keep comments concise explaining validation logic.\n\n\nMarkdown code block with language set to 'python'. No additional text outside the code block unless necessary for clarity.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 146, + "completion_tokens": 233, + "total_tokens": 379, + "kv_token_occupancy_rate_percentage": 1.1566162109375, + "load_duration": 7e-07, + "prefill_duration_ttft": 1.167560192, + "decoding_duration": 16.683994, + "prefill_speed_tps": 125.04708622337134, + "decoding_speed_tps": 13.96548092740863 + }, + "wall_seconds": 18.9528, + "ttft_seconds": 1.1676, + "completion_tokens": 233, + "decode_duration_seconds": 16.684, + "decode_tokens_per_second": 13.9655, + "seconds_per_output_token": 0.081342 + }, + { + "ok": true, + "error": "", + "output": "\nGenerate a Python script that iterates through a specified directory of CSV files, validates each row against a provided schema definition, and outputs a detailed error report containing the filename and line number for every validation failure.\n\n\nThe user has existing CSV data in various files within a specific folder. A JSON file or dictionary will define the expected column names and data types (the schema). The goal is to identify discrepancies between actual rows and the schema, such as missing columns, incorrect types, or null values where not allowed.\n\n\n- Code must be written in Python 3 using standard libraries only (`csv`, `json`, `os`).\n- Output format: A single `.py` file containing a main function that accepts arguments for the folder path and schema definition.\n- Error report output: Print to console or write to a log file; each error line must explicitly state `[File: name] Line N`.\n- No external dependencies (e.g., pandas, pydantic) allowed unless specified otherwise by user context (none here).\n\n\nMarkdown code block with the Python script. Include brief comments explaining key validation logic steps. Do not include execution instructions or explanations outside the code block.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 146, + "completion_tokens": 256, + "total_tokens": 402, + "kv_token_occupancy_rate_percentage": 1.226806640625, + "load_duration": 7e-07, + "prefill_duration_ttft": 1.170802944, + "decoding_duration": 18.18958, + "prefill_speed_tps": 124.70074554236857, + "decoding_speed_tps": 14.073991812895075 + }, + "wall_seconds": 20.5566, + "ttft_seconds": 1.1708, + "completion_tokens": 256, + "decode_duration_seconds": 18.1896, + "decode_tokens_per_second": 14.074, + "seconds_per_output_token": 0.080299 + }, + { + "ok": true, + "error": "", + "output": "\nGenerate a Python script that iterates through a specified directory of CSV files, validates each row against a provided schema definition, and outputs a detailed error report containing the filename and line number for every validation failure.\n\n\nThe user has existing CSV data in various files within a specific folder. A JSON file or dictionary will define the expected schema (column names, types). The goal is to identify invalid entries without altering the source data.\n\n\n- Output must be valid Python 3 code enclosed in a markdown code block with `python` syntax highlighting.\n- The script must accept command-line arguments for the input folder path and an optional schema file path (defaulting to a hardcoded example if not provided).\n- Error reporting should print directly to stdout, formatted as: `[FILENAME]:LINE_NUMBER - ERROR_DESCRIPTION`.\n- Do not include external dependencies beyond `csv`, `json`, `os`, and `argparse` modules.\n- Keep the code concise; do not add extensive comments or unit tests unless requested.\n\n\nA single Markdown block containing only the Python script, starting with ```python and ending with ```.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 146, + "completion_tokens": 240, + "total_tokens": 386, + "kv_token_occupancy_rate_percentage": 1.177978515625, + "load_duration": 7e-07, + "prefill_duration_ttft": 1.17034112, + "decoding_duration": 17.592856, + "prefill_speed_tps": 124.74995324440108, + "decoding_speed_tps": 13.641901008000067 + }, + "wall_seconds": 19.8939, + "ttft_seconds": 1.1703, + "completion_tokens": 240, + "decode_duration_seconds": 17.5929, + "decode_tokens_per_second": 13.6419, + "seconds_per_output_token": 0.082891 + }, + { + "ok": true, + "error": "", + "output": "\nGenerate a Python script that iterates through a specified directory of CSV files, validates each row against a provided schema definition, and outputs a detailed error report containing the filename and line number for every validation failure.\n\n\nThe user has existing CSV data in various files within a specific folder. A JSON file or dictionary will define the expected column names and data types (the schema). The goal is to identify rows that do not match this schema, such as missing columns, incorrect types, or null values where they are forbidden.\n\n\n- Output must be a single Python script with no external dependencies beyond standard library (`csv`, `json`, `os`).\n- Include a command-line argument for the input folder path and an optional JSON file path for the schema (defaulting to hardcoded example if not provided).\n- The error report should be printed to stdout in a clear, tab-separated format: `filename\\tline_number\\terror_description`.\n- Code must include docstrings explaining usage.\n\n\nMarkdown code block containing the Python script with comments indicating where user inputs (folder path, schema) would go.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 146, + "completion_tokens": 236, + "total_tokens": 382, + "kv_token_occupancy_rate_percentage": 1.165771484375, + "load_duration": 1e-06, + "prefill_duration_ttft": 1.161729152, + "decoding_duration": 16.773741, + "prefill_speed_tps": 125.67473214272925, + "decoding_speed_tps": 14.069610351083876 + }, + "wall_seconds": 19.0305, + "ttft_seconds": 1.1617, + "completion_tokens": 236, + "decode_duration_seconds": 16.7737, + "decode_tokens_per_second": 14.0696, + "seconds_per_output_token": 0.080638 + } + ], + "speed": { + "runs": 5, + "successful_runs": 5, + "wall_seconds": { + "median": 19.0305, + "p90": 20.2915, + "min": 17.5524, + "max": 20.5566 + }, + "ttft_seconds": { + "median": 1.1703, + "p90": 1.1947, + "min": 1.1617, + "max": 1.2107 + }, + "completion_tokens": { + "median": 236.0, + "p90": 249.6, + "min": 206.0, + "max": 256.0 + }, + "decode_tokens_per_second": { + "median": 13.9655, + "p90": 14.0722, + "min": 13.4604, + "max": 14.074 + }, + "seconds_per_output_token": { + "median": 0.0813, + "p90": 0.0843, + "min": 0.0803, + "max": 0.0852 + } + }, + "representative_output": "\nGenerate a Python script that iterates through a specified directory of CSV files, validates each row against a provided schema definition, and outputs a detailed error report containing the filename and line number for every validation failure.\n\n\nThe user has existing CSV data in various files within a folder that needs to be checked for consistency with specific column names, types, or required fields defined by a schema. The goal is to automate this checking process without manually inspecting each file.\n\n\n- Output must be valid Python 3 code enclosed in markdown code blocks.\n- Include comments explaining the logic of validation and error reporting.\n- Do not include external dependencies beyond the standard library (e.g., no pandas, only csv module).\n- The script should accept arguments for the input folder path and a JSON schema definition file path via `sys.argv`.\n\n\nA single code block containing the complete Python script with necessary imports, class/function definitions, and main execution logic.", + "quality": { + "rubric": { + "r1": false, + "r2": false, + "r3": false, + "r4": false, + "r5": false, + "r6": false, + "r7": true + }, + "score": 1, + "passed": false, + "hard_fail": true, + "pending_judge": false, + "invented_requirement": true, + "constraint_item_count": 0, + "token_count": 206, + "token_count_source": "usage.completion_tokens", + "estimated_tokens": 198, + "judge_notes": "Invents Python 3, Markdown fences, comments, standard-library-only, argv, and JSON schema file." + } + }, + "v2": { + "label": "candidate", + "max_tokens": 240, + "warmups": 1, + "warmup_errors": [], + "samples": [ + { + "ok": true, + "error": "", + "output": "\nBuild a python script that reads a folder of CSVs, validates rows against a schema, and writes an error report with file and line numbers.\n\n\nNo additional context stated.\n\n\n- Build a python script.\n- Read a folder of CSVs.\n- Validate rows against a schema.\n- Write an error report with file and line numbers.\n\n\nA Python script.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 90, + "completion_tokens": 30, + "total_tokens": 120, + "kv_token_occupancy_rate_percentage": 0.3662109375, + "load_duration": 5e-07, + "prefill_duration_ttft": 1.115162752, + "decoding_duration": 2.303159, + "prefill_speed_tps": 80.70570850630419, + "decoding_speed_tps": 13.025587899055168 + }, + "wall_seconds": 3.6938, + "ttft_seconds": 1.1152, + "completion_tokens": 30, + "decode_duration_seconds": 2.3032, + "decode_tokens_per_second": 13.0256, + "seconds_per_output_token": 0.123128, + "raw_output": "Build a Python script that reads CSVs from a folder, validates rows against a schema, and writes an error report with file and line numbers.", + "final_output_tokens": 96 + }, + { + "ok": true, + "error": "", + "output": "\nBuild a python script that reads a folder of CSVs, validates rows against a schema, and writes an error report with file and line numbers.\n\n\nNo additional context stated.\n\n\n- Build a python script.\n- Read a folder of CSVs.\n- Validate rows against a schema.\n- Write an error report with file and line numbers.\n\n\nA Python script.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 90, + "completion_tokens": 30, + "total_tokens": 120, + "kv_token_occupancy_rate_percentage": 0.3662109375, + "load_duration": 5e-07, + "prefill_duration_ttft": 1.087842816, + "decoding_duration": 2.370352, + "prefill_speed_tps": 82.73254065410862, + "decoding_speed_tps": 12.65634808669767 + }, + "wall_seconds": 3.6983, + "ttft_seconds": 1.0878, + "completion_tokens": 30, + "decode_duration_seconds": 2.3704, + "decode_tokens_per_second": 12.6563, + "seconds_per_output_token": 0.123276, + "raw_output": "Build a Python script that reads CSVs from a folder, validates rows against a schema, and writes an error report with file and line numbers.", + "final_output_tokens": 96 + }, + { + "ok": true, + "error": "", + "output": "\nBuild a python script that reads a folder of CSVs, validates rows against a schema, and writes an error report with file and line numbers.\n\n\nNo additional context stated.\n\n\n- Build a python script.\n- Read a folder of CSVs.\n- Validate rows against a schema.\n- Write an error report with file and line numbers.\n\n\nA Python script.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 90, + "completion_tokens": 30, + "total_tokens": 120, + "kv_token_occupancy_rate_percentage": 0.3662109375, + "load_duration": 6e-07, + "prefill_duration_ttft": 1.231001472, + "decoding_duration": 2.284235, + "prefill_speed_tps": 73.11120420821072, + "decoding_speed_tps": 13.133499836925711 + }, + "wall_seconds": 3.757, + "ttft_seconds": 1.231, + "completion_tokens": 30, + "decode_duration_seconds": 2.2842, + "decode_tokens_per_second": 13.1335, + "seconds_per_output_token": 0.125234, + "raw_output": "Build a Python script that reads CSVs from a folder, validates rows against a schema, and writes an error report with file and line numbers.", + "final_output_tokens": 96 + }, + { + "ok": true, + "error": "", + "output": "\nBuild a python script that reads a folder of CSVs, validates rows against a schema, and writes an error report with file and line numbers.\n\n\nNo additional context stated.\n\n\n- Build a python script.\n- Read a folder of CSVs.\n- Validate rows against a schema.\n- Write an error report with file and line numbers.\n\n\nA Python script.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 90, + "completion_tokens": 30, + "total_tokens": 120, + "kv_token_occupancy_rate_percentage": 0.3662109375, + "load_duration": 8e-07, + "prefill_duration_ttft": 1.078279936, + "decoding_duration": 2.282796, + "prefill_speed_tps": 83.46626603650353, + "decoding_speed_tps": 13.141778766039542 + }, + "wall_seconds": 3.6157, + "ttft_seconds": 1.0783, + "completion_tokens": 30, + "decode_duration_seconds": 2.2828, + "decode_tokens_per_second": 13.1418, + "seconds_per_output_token": 0.120523, + "raw_output": "Build a Python script that reads CSVs from a folder, validates rows against a schema, and writes an error report with file and line numbers.", + "final_output_tokens": 96 + }, + { + "ok": true, + "error": "", + "output": "\nBuild a python script that reads a folder of CSVs, validates rows against a schema, and writes an error report with file and line numbers.\n\n\nNo additional context stated.\n\n\n- Build a python script.\n- Read a folder of CSVs.\n- Validate rows against a schema.\n- Write an error report with file and line numbers.\n\n\nA Python script.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 90, + "completion_tokens": 30, + "total_tokens": 120, + "kv_token_occupancy_rate_percentage": 0.3662109375, + "load_duration": 5e-07, + "prefill_duration_ttft": 1.075897984, + "decoding_duration": 2.278972, + "prefill_speed_tps": 83.65105366718485, + "decoding_speed_tps": 13.163830007564815 + }, + "wall_seconds": 3.6081, + "ttft_seconds": 1.0759, + "completion_tokens": 30, + "decode_duration_seconds": 2.279, + "decode_tokens_per_second": 13.1638, + "seconds_per_output_token": 0.120269, + "raw_output": "Build a Python script that reads CSVs from a folder, validates rows against a schema, and writes an error report with file and line numbers.", + "final_output_tokens": 96 + } + ], + "speed": { + "runs": 5, + "successful_runs": 5, + "wall_seconds": { + "median": 3.6938, + "p90": 3.7335, + "min": 3.6081, + "max": 3.757 + }, + "ttft_seconds": { + "median": 1.0878, + "p90": 1.1847, + "min": 1.0759, + "max": 1.231 + }, + "completion_tokens": { + "median": 30.0, + "p90": 30.0, + "min": 30.0, + "max": 30.0 + }, + "decode_tokens_per_second": { + "median": 13.1335, + "p90": 13.155, + "min": 12.6563, + "max": 13.1638 + }, + "seconds_per_output_token": { + "median": 0.1231, + "p90": 0.1245, + "min": 0.1203, + "max": 0.1252 + } + }, + "representative_output": "\nBuild a python script that reads a folder of CSVs, validates rows against a schema, and writes an error report with file and line numbers.\n\n\nNo additional context stated.\n\n\n- Build a python script.\n- Read a folder of CSVs.\n- Validate rows against a schema.\n- Write an error report with file and line numbers.\n\n\nA Python script.\n", + "quality": { + "rubric": { + "r1": true, + "r2": true, + "r3": true, + "r4": true, + "r5": true, + "r6": true, + "r7": true + }, + "score": 7, + "passed": true, + "hard_fail": false, + "pending_judge": false, + "invented_requirement": false, + "constraint_item_count": 4, + "token_count": 96, + "token_count_source": "usage.completion_tokens", + "estimated_tokens": 96, + "judge_notes": "Four source-derived requirements; no implementation assumptions." + } + } + } + }, + { + "name": "debug_async_cache_race", + "category": "debug", + "trap": false, + "input": "debug an intermittent async cache test failure: two refreshes sometimes run at once and the older response overwrites the newer value; keep the public API unchanged", + "styles": { + "v1": { + "label": "current", + "max_tokens": 700, + "warmups": 1, + "warmup_errors": [], + "samples": [ + { + "ok": true, + "error": "", + "output": "\nDebug an intermittent async cache test failure where two simultaneous refreshes cause the older response to overwrite the newer value, while keeping the public API unchanged.\n\n\nThe system uses an asynchronous caching mechanism for data refresh operations. Under specific concurrency conditions, multiple requests can trigger a refresh simultaneously before any has completed or committed its result to storage. This race condition results in stale (older) data overwriting fresh updates when two refreshes occur at nearly the same time. The public API contract must remain stable and unmodified by this fix.\n\n\n- Output length: 200 - 400 words total.\n- Format: Single Markdown code block containing a Python script that implements the fix using `asyncio.Semaphore` or similar concurrency control to serialize refreshes atomically without changing API signatures.\n- Tone: Technical, direct, no fluff.\n- Testability: The solution must be testable via standard async unit tests (e.g., `pytest`).\n\n\n```python\n# Implementation of the fix within a class or function that mirrors existing public API structure.\n```", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 148, + "completion_tokens": 234, + "total_tokens": 382, + "kv_token_occupancy_rate_percentage": 1.165771484375, + "load_duration": 6e-07, + "prefill_duration_ttft": 1.178722816, + "decoding_duration": 16.676432, + "prefill_speed_tps": 125.55962944896451, + "decoding_speed_tps": 14.03177850034108 + }, + "wall_seconds": 18.9198, + "ttft_seconds": 1.1787, + "completion_tokens": 234, + "decode_duration_seconds": 16.6764, + "decode_tokens_per_second": 14.0318, + "seconds_per_output_token": 0.080854 + }, + { + "ok": true, + "error": "", + "output": "\nDebug an intermittent async cache test failure where two concurrent refreshes cause the older response to overwrite the newer value, while keeping the public API unchanged.\n\n\nThe system uses an asynchronous caching mechanism for data refresh operations. Under specific load conditions, multiple refresh requests can be processed simultaneously before a lock or check is acquired. This race condition results in stale (older) data being written over fresh (newer) data when two updates occur at nearly the same time. The public API contract must remain stable and unmodified to ensure backward compatibility for all clients.\n\n\n- Output length: 200 - 400 words total.\n- Format: Single Markdown code block containing a Python script with `if __name__ == \"__main__\":` entry point.\n- Tone: Technical, direct, and solution-oriented; no fluff or explanations outside the code.\n- Testability: The provided fix must be verifiable by running the existing test suite without altering API signatures.\n\n\nA Python script inside a Markdown ```python block that implements the race condition fix using atomic operations (e.g., `functools.lru_cache` with maxsize=1 or explicit locking via `threading.Lock`) to ensure only one refresh runs at a time per key, preserving API behavior.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 148, + "completion_tokens": 274, + "total_tokens": 422, + "kv_token_occupancy_rate_percentage": 1.287841796875, + "load_duration": 5e-07, + "prefill_duration_ttft": 1.16808192, + "decoding_duration": 19.634374, + "prefill_speed_tps": 126.7034421695355, + "decoding_speed_tps": 13.95511769308255 + }, + "wall_seconds": 22.0284, + "ttft_seconds": 1.1681, + "completion_tokens": 274, + "decode_duration_seconds": 19.6344, + "decode_tokens_per_second": 13.9551, + "seconds_per_output_token": 0.080396 + }, + { + "ok": true, + "error": "", + "output": "\nDebug an intermittent async cache test failure where two concurrent refreshes cause the older response to overwrite the newer value, while keeping the public API unchanged.\n\n\nThe system uses an asynchronous caching mechanism for data refresh operations. Under specific race conditions, multiple requests can trigger simultaneous cache updates within a single execution context. The current implementation lacks synchronization logic to ensure atomicity during concurrent writes, leading to stale or overwritten values in the cache layer without affecting the external interface exposed by the API endpoints.\n\n\n- Output must be exactly 200 words (195 - 204 characters allowed for whitespace).\n- Use only Markdown headers (`###`) and code blocks (` ``` `) where appropriate; no other formatting styles.\n- Tone: Technical, direct, problem-solving focused; avoid fluff or conversational fillers.\n- Do not include any meta-commentary about the prompt itself (e.g., \"Here is your rewritten prompt\").\n\n\nA single Markdown document containing:\n1. A `### Problem Analysis` header with a concise paragraph explaining the root cause in 2 - 3 sentences.\n2. A `### Proposed Solution` header with a bulleted list of exactly three technical steps to fix the issue, each under 40 characters long (excluding bullets).", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 148, + "completion_tokens": 273, + "total_tokens": 421, + "kv_token_occupancy_rate_percentage": 1.2847900390625, + "load_duration": 6e-07, + "prefill_duration_ttft": 1.15969664, + "decoding_duration": 20.283122, + "prefill_speed_tps": 127.61958161748232, + "decoding_speed_tps": 13.45946644702921 + }, + "wall_seconds": 22.6705, + "ttft_seconds": 1.1597, + "completion_tokens": 273, + "decode_duration_seconds": 20.2831, + "decode_tokens_per_second": 13.4595, + "seconds_per_output_token": 0.083042 + }, + { + "ok": true, + "error": "", + "output": "\nDebug an intermittent async cache test failure where two concurrent refreshes cause the older response to overwrite the newer value, while keeping the public API unchanged.\n\n\nThe system uses an asynchronous caching mechanism for data refresh operations. A specific unit test occasionally fails due to race conditions when multiple refresh requests execute simultaneously within a short timeframe. The underlying implementation currently lacks synchronization logic to handle concurrent updates, leading to stale or overwritten cache entries in edge cases. The public API contract must remain stable and unmodified regardless of internal fixes.\n\n\n- Output length: 200 - 400 words total.\n- Format: Single Markdown code block containing the debugging solution steps only (no explanations).\n- Tone: Direct, technical, imperative instructions suitable for a developer implementation guide.\n- No meta-commentary or conversational filler outside the prompt structure.\n\n\nMarkdown headers (#) for sections like \"Root Cause\", \"Fix Strategy\", and \"Implementation Steps\". Use bullet points (-) for actionable items. Ensure all code snippets are in ```language blocks if applicable (e.g., pseudo-code or SQL).", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 148, + "completion_tokens": 234, + "total_tokens": 382, + "kv_token_occupancy_rate_percentage": 1.165771484375, + "load_duration": 9e-07, + "prefill_duration_ttft": 1.161499904, + "decoding_duration": 16.632057, + "prefill_speed_tps": 127.42144832755835, + "decoding_speed_tps": 14.069215852254475 + }, + "wall_seconds": 18.8746, + "ttft_seconds": 1.1615, + "completion_tokens": 234, + "decode_duration_seconds": 16.6321, + "decode_tokens_per_second": 14.0692, + "seconds_per_output_token": 0.080661 + }, + { + "ok": true, + "error": "", + "output": "\nDebug an intermittent async cache test failure where two concurrent refreshes cause the older response to overwrite the newer value, while keeping the public API unchanged.\n\n\nThe system uses an asynchronous caching mechanism for data refresh operations. Under specific load conditions, multiple refresh requests can be initiated simultaneously before a previous one completes. The current implementation does not handle race conditions effectively, leading to stale or overwritten cache entries when concurrent updates occur. The public API contract must remain stable and unmodified by the fix.\n\n\n- Output length: 200 - 400 words total.\n- Format: Single Markdown code block containing a Python script with detailed comments explaining the logic, not an explanation text outside it.\n- Tone: Technical, direct, focused on implementation details and debugging steps.\n- Content requirements: Must include specific async handling (e.g., `asyncio.Semaphore` or similar), cache key validation logic to prevent overwrites without changing API signatures, and a test case demonstrating the race condition fix.\n\n\nA Markdown code block with Python 3 syntax highlighting containing the debugging script. The script must include:\n1. A class representing the async cache handler.\n2. A method `refresh_data` that implements concurrency control to prevent overwrites of newer responses by older ones (e.g., using a token bucket or semaphore).\n3. A unit test function simulating concurrent refreshes and verifying the latest value is retained in the cache without altering any public API methods.\n\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 148, + "completion_tokens": 318, + "total_tokens": 466, + "kv_token_occupancy_rate_percentage": 1.422119140625, + "load_duration": 5e-07, + "prefill_duration_ttft": 1.16202304, + "decoding_duration": 22.611108, + "prefill_speed_tps": 127.36408393417052, + "decoding_speed_tps": 14.063883998961927 + }, + "wall_seconds": 25.1499, + "ttft_seconds": 1.162, + "completion_tokens": 318, + "decode_duration_seconds": 22.6111, + "decode_tokens_per_second": 14.0639, + "seconds_per_output_token": 0.079088 + } + ], + "speed": { + "runs": 5, + "successful_runs": 5, + "wall_seconds": { + "median": 22.0284, + "p90": 24.1581, + "min": 18.8746, + "max": 25.1499 + }, + "ttft_seconds": { + "median": 1.162, + "p90": 1.1745, + "min": 1.1597, + "max": 1.1787 + }, + "completion_tokens": { + "median": 273.0, + "p90": 300.4, + "min": 234.0, + "max": 318.0 + }, + "decode_tokens_per_second": { + "median": 14.0318, + "p90": 14.0671, + "min": 13.4595, + "max": 14.0692 + }, + "seconds_per_output_token": { + "median": 0.0807, + "p90": 0.0822, + "min": 0.0791, + "max": 0.083 + } + }, + "representative_output": "\nDebug an intermittent async cache test failure where two simultaneous refreshes cause the older response to overwrite the newer value, while keeping the public API unchanged.\n\n\nThe system uses an asynchronous caching mechanism for data refresh operations. Under specific concurrency conditions, multiple requests can trigger a refresh simultaneously before any has completed or committed its result to storage. This race condition results in stale (older) data overwriting fresh updates when two refreshes occur at nearly the same time. The public API contract must remain stable and unmodified by this fix.\n\n\n- Output length: 200 - 400 words total.\n- Format: Single Markdown code block containing a Python script that implements the fix using `asyncio.Semaphore` or similar concurrency control to serialize refreshes atomically without changing API signatures.\n- Tone: Technical, direct, no fluff.\n- Testability: The solution must be testable via standard async unit tests (e.g., `pytest`).\n\n\n```python\n# Implementation of the fix within a class or function that mirrors existing public API structure.\n```", + "quality": { + "rubric": { + "r1": false, + "r2": false, + "r3": false, + "r4": false, + "r5": false, + "r6": false, + "r7": false + }, + "score": 0, + "passed": false, + "hard_fail": true, + "pending_judge": false, + "invented_requirement": true, + "constraint_item_count": 0, + "token_count": 234, + "token_count_source": "usage.completion_tokens", + "estimated_tokens": 213, + "judge_notes": "Invents word count, Python, asyncio.Semaphore, pytest, and output shape." + } + }, + "v2": { + "label": "candidate", + "max_tokens": 240, + "warmups": 1, + "warmup_errors": [], + "samples": [ + { + "ok": true, + "error": "", + "output": "\nDebug an intermittent async cache test failure: two refreshes sometimes run at once and the older response overwrites the newer value; keep the public API unchanged.\n\n\nNo additional context stated.\n\n\n- Debug an intermittent async cache test failure: two refreshes sometimes run at once and the older response overwrites the newer value.\n- Keep the public API unchanged.\n- Preserve all stated requirements.\n\n\nDebugging result.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 92, + "completion_tokens": 25, + "total_tokens": 117, + "kv_token_occupancy_rate_percentage": 0.3570556640625, + "load_duration": 5e-07, + "prefill_duration_ttft": 1.11287424, + "decoding_duration": 1.922218, + "prefill_speed_tps": 82.66881979404968, + "decoding_speed_tps": 13.005808914493569 + }, + "wall_seconds": 3.2646, + "ttft_seconds": 1.1129, + "completion_tokens": 25, + "decode_duration_seconds": 1.9222, + "decode_tokens_per_second": 13.0058, + "seconds_per_output_token": 0.130582, + "raw_output": "Debug the intermittent async cache test failure where two refreshes run simultaneously and overwrite values, while keeping the public API unchanged.", + "final_output_tokens": 102 + }, + { + "ok": true, + "error": "", + "output": "\nDebug an intermittent async cache test failure: two refreshes sometimes run at once and the older response overwrites the newer value; keep the public API unchanged.\n\n\nNo additional context stated.\n\n\n- Debug an intermittent async cache test failure: two refreshes sometimes run at once and the older response overwrites the newer value.\n- Keep the public API unchanged.\n- Preserve all stated requirements.\n\n\nDebugging result.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 92, + "completion_tokens": 25, + "total_tokens": 117, + "kv_token_occupancy_rate_percentage": 0.3570556640625, + "load_duration": 5e-07, + "prefill_duration_ttft": 1.16248384, + "decoding_duration": 1.920629, + "prefill_speed_tps": 79.14088508963704, + "decoding_speed_tps": 13.016569051076496 + }, + "wall_seconds": 3.3146, + "ttft_seconds": 1.1625, + "completion_tokens": 25, + "decode_duration_seconds": 1.9206, + "decode_tokens_per_second": 13.0166, + "seconds_per_output_token": 0.132583, + "raw_output": "Debug the intermittent async cache test failure where two refreshes run simultaneously and overwrite values, while keeping the public API unchanged.", + "final_output_tokens": 102 + }, + { + "ok": true, + "error": "", + "output": "\nDebug an intermittent async cache test failure: two refreshes sometimes run at once and the older response overwrites the newer value; keep the public API unchanged.\n\n\nNo additional context stated.\n\n\n- Debug an intermittent async cache test failure: two refreshes sometimes run at once and the older response overwrites the newer value.\n- Keep the public API unchanged.\n- Preserve all stated requirements.\n\n\nDebugging result.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 92, + "completion_tokens": 25, + "total_tokens": 117, + "kv_token_occupancy_rate_percentage": 0.3570556640625, + "load_duration": 1e-06, + "prefill_duration_ttft": 1.076568832, + "decoding_duration": 1.925674, + "prefill_speed_tps": 85.45668169594566, + "decoding_speed_tps": 12.982467437375174 + }, + "wall_seconds": 3.234, + "ttft_seconds": 1.0766, + "completion_tokens": 25, + "decode_duration_seconds": 1.9257, + "decode_tokens_per_second": 12.9825, + "seconds_per_output_token": 0.129358, + "raw_output": "Debug the intermittent async cache test failure where two refreshes run simultaneously and overwrite values, while keeping the public API unchanged.", + "final_output_tokens": 102 + }, + { + "ok": true, + "error": "", + "output": "\nDebug an intermittent async cache test failure: two refreshes sometimes run at once and the older response overwrites the newer value; keep the public API unchanged.\n\n\nNo additional context stated.\n\n\n- Debug an intermittent async cache test failure: two refreshes sometimes run at once and the older response overwrites the newer value.\n- Keep the public API unchanged.\n- Preserve all stated requirements.\n\n\nDebugging result.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 92, + "completion_tokens": 25, + "total_tokens": 117, + "kv_token_occupancy_rate_percentage": 0.3570556640625, + "load_duration": 6e-07, + "prefill_duration_ttft": 1.11306432, + "decoding_duration": 2.05894, + "prefill_speed_tps": 82.6547022906996, + "decoding_speed_tps": 12.142170242940542 + }, + "wall_seconds": 3.4317, + "ttft_seconds": 1.1131, + "completion_tokens": 25, + "decode_duration_seconds": 2.0589, + "decode_tokens_per_second": 12.1422, + "seconds_per_output_token": 0.137269, + "raw_output": "Debug the intermittent async cache test failure where two refreshes run simultaneously and overwrite values, while keeping the public API unchanged.", + "final_output_tokens": 102 + }, + { + "ok": true, + "error": "", + "output": "\nDebug an intermittent async cache test failure: two refreshes sometimes run at once and the older response overwrites the newer value; keep the public API unchanged.\n\n\nNo additional context stated.\n\n\n- Debug an intermittent async cache test failure: two refreshes sometimes run at once and the older response overwrites the newer value.\n- Keep the public API unchanged.\n- Preserve all stated requirements.\n\n\nDebugging result.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 92, + "completion_tokens": 25, + "total_tokens": 117, + "kv_token_occupancy_rate_percentage": 0.3570556640625, + "load_duration": 6e-07, + "prefill_duration_ttft": 1.21049792, + "decoding_duration": 1.927684, + "prefill_speed_tps": 76.00178280355904, + "decoding_speed_tps": 12.968930592358499 + }, + "wall_seconds": 3.3701, + "ttft_seconds": 1.2105, + "completion_tokens": 25, + "decode_duration_seconds": 1.9277, + "decode_tokens_per_second": 12.9689, + "seconds_per_output_token": 0.134803, + "raw_output": "Debug the intermittent async cache test failure where two refreshes run simultaneously and overwrite values, ensuring the public API remains unchanged.", + "final_output_tokens": 102 + } + ], + "speed": { + "runs": 5, + "successful_runs": 5, + "wall_seconds": { + "median": 3.3146, + "p90": 3.4071, + "min": 3.234, + "max": 3.4317 + }, + "ttft_seconds": { + "median": 1.1131, + "p90": 1.1913, + "min": 1.0766, + "max": 1.2105 + }, + "completion_tokens": { + "median": 25.0, + "p90": 25.0, + "min": 25.0, + "max": 25.0 + }, + "decode_tokens_per_second": { + "median": 12.9825, + "p90": 13.0123, + "min": 12.1422, + "max": 13.0166 + }, + "seconds_per_output_token": { + "median": 0.1326, + "p90": 0.1363, + "min": 0.1294, + "max": 0.1373 + } + }, + "representative_output": "\nDebug an intermittent async cache test failure: two refreshes sometimes run at once and the older response overwrites the newer value; keep the public API unchanged.\n\n\nNo additional context stated.\n\n\n- Debug an intermittent async cache test failure: two refreshes sometimes run at once and the older response overwrites the newer value.\n- Keep the public API unchanged.\n- Preserve all stated requirements.\n\n\nDebugging result.\n", + "quality": { + "rubric": { + "r1": true, + "r2": true, + "r3": true, + "r4": true, + "r5": true, + "r6": true, + "r7": true + }, + "score": 7, + "passed": true, + "hard_fail": false, + "pending_judge": false, + "invented_requirement": false, + "constraint_item_count": 3, + "token_count": 102, + "token_count_source": "usage.completion_tokens", + "estimated_tokens": 102, + "judge_notes": "Reported race and API constraint only." + } + } + } + }, + { + "name": "review_auth_middleware", + "category": "review", + "trap": false, + "input": "review the new TypeScript auth middleware for regressions and missing tests; report findings by severity with file and line references", + "styles": { + "v1": { + "label": "current", + "max_tokens": 700, + "warmups": 1, + "warmup_errors": [], + "samples": [ + { + "ok": true, + "error": "", + "output": "\nAnalyze the provided TypeScript authentication middleware code for regressions and missing test coverage, then generate a report detailing findings by severity level with specific file paths and line numbers.\n\n\nThe user has supplied a new implementation of an auth middleware in TypeScript that requires validation against existing patterns or requirements to ensure stability and completeness. The goal is to identify potential issues (regressions) where functionality may have been broken, as well as gaps where critical scenarios lack corresponding tests.\n\n\n- Output must be exactly 300 words long (counting spaces).\n- Report findings in a bulleted list format with no numbering.\n- Use professional but concise technical tone; avoid fluff or explanations outside the report structure.\n- Each finding must include: [Severity] - Description, File Path:line_number.\n\n\nMarkdown header for title (e.g., `## Auth Middleware Review Report`).\nA single bulleted list containing all findings sorted by severity (Critical > High > Medium).\nNo introduction or conclusion paragraphs outside the report structure.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 140, + "completion_tokens": 223, + "total_tokens": 363, + "kv_token_occupancy_rate_percentage": 1.1077880859375, + "load_duration": 1.1e-06, + "prefill_duration_ttft": 1.164959744, + "decoding_duration": 15.855805, + "prefill_speed_tps": 120.17582643611075, + "decoding_speed_tps": 14.064249654937104 + }, + "wall_seconds": 18.0689, + "ttft_seconds": 1.165, + "completion_tokens": 223, + "decode_duration_seconds": 15.8558, + "decode_tokens_per_second": 14.0642, + "seconds_per_output_token": 0.081026 + }, + { + "ok": true, + "error": "", + "output": "\nAnalyze the provided TypeScript authentication middleware code for regressions and missing test coverage, then report findings organized by severity level with specific file paths and line numbers.\n\n\nThe user has a new TypeScript implementation of an auth middleware that requires validation against existing patterns to ensure no functionality was broken (regressions) and that all critical logic is covered by tests. The output must be structured as a technical review report suitable for engineering stakeholders.\n\n\n- Total response length: 300 - 500 words.\n- Format: Markdown with clear headers (`## Severity Level`, `### Missing Tests`, etc.).\n- Tone: Professional, concise, and objective (no fluff).\n- Data requirement: Every finding must include a file path and line number reference where applicable.\n\n\nUse the following structure exactly:\n\n# Auth Middleware Review Report\n\n## 1. Regression Analysis\n[Summary of potential regressions found]\n\n## 2. Missing Test Coverage\n[List of untested functions/modules with file/line refs]\n\n## 3. Recommendations\n[Brief actionable next steps based on findings]", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 140, + "completion_tokens": 238, + "total_tokens": 378, + "kv_token_occupancy_rate_percentage": 1.153564453125, + "load_duration": 1.3e-06, + "prefill_duration_ttft": 1.154849664, + "decoding_duration": 16.903436, + "prefill_speed_tps": 121.22790036158334, + "decoding_speed_tps": 14.079977585622236 + }, + "wall_seconds": 19.179, + "ttft_seconds": 1.1548, + "completion_tokens": 238, + "decode_duration_seconds": 16.9034, + "decode_tokens_per_second": 14.08, + "seconds_per_output_token": 0.080584 + }, + { + "ok": true, + "error": "", + "output": "\nAnalyze the provided TypeScript authentication middleware code for regressions and missing test coverage, then output a structured report detailing findings by severity level with specific file paths and line numbers.\n\n\nThe user has supplied a new implementation of an auth middleware in TypeScript that requires validation against existing patterns or requirements to ensure stability and completeness. The goal is to identify potential issues like logic errors (regressions) or gaps where functionality exists but lacks corresponding tests.\n\n\n- Output must be exactly 300 words long, including all text and code blocks.\n- Report findings in a Markdown table with columns: Severity, Issue Description, File Path, Line Number.\n- Use professional yet concise technical tone; avoid fluff or conversational fillers.\n- Do not include any introductory remarks or concluding summaries outside the report structure.\n\n\nMarkdown code block containing only the analysis report in a table format as specified above.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 140, + "completion_tokens": 197, + "total_tokens": 337, + "kv_token_occupancy_rate_percentage": 1.0284423828125, + "load_duration": 5e-07, + "prefill_duration_ttft": 1.156205568, + "decoding_duration": 14.467548, + "prefill_speed_tps": 121.08573412440097, + "decoding_speed_tps": 13.616681969881835 + }, + "wall_seconds": 16.5843, + "ttft_seconds": 1.1562, + "completion_tokens": 197, + "decode_duration_seconds": 14.4675, + "decode_tokens_per_second": 13.6167, + "seconds_per_output_token": 0.084184 + }, + { + "ok": true, + "error": "", + "output": "\nAnalyze the provided TypeScript authentication middleware code for regressions and missing test coverage, then generate a report detailing findings by severity level with specific file paths and line numbers.\n\n\nThe user has supplied a new implementation of an auth middleware in TypeScript that requires validation against existing patterns or requirements to ensure stability and completeness. The goal is to identify potential issues (regressions) where functionality might be broken, as well as gaps where critical scenarios are not covered by tests.\n\n\n- Output must be exactly 300 words or less.\n- Use a bulleted list for the findings section; do not use tables.\n- Maintain a professional, technical tone suitable for engineering review.\n- Do not include any introductory text, explanations of methodology, or concluding remarks outside the report itself.\n\n\nMarkdown format with two main headers: `## Analysis Findings` and `## Recommendations`. Under each header, provide concise bullet points summarizing specific issues found (e.g., \"Regression in token validation at auth.ts:42\") and actionable suggestions where applicable.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 140, + "completion_tokens": 228, + "total_tokens": 368, + "kv_token_occupancy_rate_percentage": 1.123046875, + "load_duration": 9e-07, + "prefill_duration_ttft": 1.155902592, + "decoding_duration": 16.202568, + "prefill_speed_tps": 121.11747215460869, + "decoding_speed_tps": 14.071843426301312 + }, + "wall_seconds": 18.4342, + "ttft_seconds": 1.1559, + "completion_tokens": 228, + "decode_duration_seconds": 16.2026, + "decode_tokens_per_second": 14.0718, + "seconds_per_output_token": 0.080852 + }, + { + "ok": true, + "error": "", + "output": "\nAnalyze the provided TypeScript authentication middleware code for regressions and missing test coverage, then generate a report detailing findings by severity level with specific file paths and line numbers.\n\n\nThe user has supplied a new implementation of an auth middleware in TypeScript that requires validation against existing patterns or requirements to ensure stability and completeness. The goal is to identify potential issues (regressions) where functionality might be broken, as well as gaps where critical scenarios are not covered by tests.\n\n\n- Output must be exactly 300 words or less.\n- Use a bulleted list for the findings section; do not use tables.\n- Maintain a professional, technical tone suitable for engineering review.\n- Each finding must include: Severity (High/Medium/Low), Description, File Path, and Line Number reference.\n\n\nMarkdown format with the following structure:\n1. `## Analysis Summary` (brief intro)\n2. `## Findings by Severity` (bulleted list of findings)", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 140, + "completion_tokens": 215, + "total_tokens": 355, + "kv_token_occupancy_rate_percentage": 1.0833740234375, + "load_duration": 9e-07, + "prefill_duration_ttft": 1.159690752, + "decoding_duration": 15.316078, + "prefill_speed_tps": 120.72183878206867, + "decoding_speed_tps": 14.037536241327578 + }, + "wall_seconds": 17.494, + "ttft_seconds": 1.1597, + "completion_tokens": 215, + "decode_duration_seconds": 15.3161, + "decode_tokens_per_second": 14.0375, + "seconds_per_output_token": 0.081367 + } + ], + "speed": { + "runs": 5, + "successful_runs": 5, + "wall_seconds": { + "median": 18.0689, + "p90": 18.8811, + "min": 16.5843, + "max": 19.179 + }, + "ttft_seconds": { + "median": 1.1562, + "p90": 1.1629, + "min": 1.1548, + "max": 1.165 + }, + "completion_tokens": { + "median": 223.0, + "p90": 234.0, + "min": 197.0, + "max": 238.0 + }, + "decode_tokens_per_second": { + "median": 14.0642, + "p90": 14.0767, + "min": 13.6167, + "max": 14.08 + }, + "seconds_per_output_token": { + "median": 0.081, + "p90": 0.0831, + "min": 0.0806, + "max": 0.0842 + } + }, + "representative_output": "\nAnalyze the provided TypeScript authentication middleware code for regressions and missing test coverage, then generate a report detailing findings by severity level with specific file paths and line numbers.\n\n\nThe user has supplied a new implementation of an auth middleware in TypeScript that requires validation against existing patterns or requirements to ensure stability and completeness. The goal is to identify potential issues (regressions) where functionality may have been broken, as well as gaps where critical scenarios lack corresponding tests.\n\n\n- Output must be exactly 300 words long (counting spaces).\n- Report findings in a bulleted list format with no numbering.\n- Use professional but concise technical tone; avoid fluff or explanations outside the report structure.\n- Each finding must include: [Severity] - Description, File Path:line_number.\n\n\nMarkdown header for title (e.g., `## Auth Middleware Review Report`).\nA single bulleted list containing all findings sorted by severity (Critical > High > Medium).\nNo introduction or conclusion paragraphs outside the report structure.", + "quality": { + "rubric": { + "r1": false, + "r2": false, + "r3": false, + "r4": false, + "r5": false, + "r6": false, + "r7": false + }, + "score": 0, + "passed": false, + "hard_fail": true, + "pending_judge": false, + "invented_requirement": true, + "constraint_item_count": 0, + "token_count": 223, + "token_count_source": "usage.completion_tokens", + "estimated_tokens": 205, + "judge_notes": "Invents exact length, bullet format, severity set, title, and report structure." + } + }, + "v2": { + "label": "candidate", + "max_tokens": 240, + "warmups": 1, + "warmup_errors": [], + "samples": [ + { + "ok": true, + "error": "", + "output": "\nReview the new TypeScript auth middleware for regressions and missing tests; report findings by severity with file and line references.\n\n\nNo additional context stated.\n\n\n- Review the new TypeScript auth middleware for regressions and missing tests.\n- Report findings by severity with file and line references.\n- Preserve all stated requirements.\n\n\nReview.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 84, + "completion_tokens": 25, + "total_tokens": 109, + "kv_token_occupancy_rate_percentage": 0.3326416015625, + "load_duration": 4e-07, + "prefill_duration_ttft": 1.066795584, + "decoding_duration": 1.921924, + "prefill_speed_tps": 78.74048342517324, + "decoding_speed_tps": 13.007798435317943 + }, + "wall_seconds": 3.214, + "ttft_seconds": 1.0668, + "completion_tokens": 25, + "decode_duration_seconds": 1.9219, + "decode_tokens_per_second": 13.0078, + "seconds_per_output_token": 0.12856, + "raw_output": "Review the new TypeScript auth middleware for regressions and missing tests, then report findings by severity with file and line references.", + "final_output_tokens": 87 + }, + { + "ok": true, + "error": "", + "output": "\nReview the new TypeScript auth middleware for regressions and missing tests; report findings by severity with file and line references.\n\n\nNo additional context stated.\n\n\n- Review the new TypeScript auth middleware for regressions and missing tests.\n- Report findings by severity with file and line references.\n- Preserve all stated requirements.\n\n\nReview.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 84, + "completion_tokens": 25, + "total_tokens": 109, + "kv_token_occupancy_rate_percentage": 0.3326416015625, + "load_duration": 6e-07, + "prefill_duration_ttft": 1.066837888, + "decoding_duration": 1.961693, + "prefill_speed_tps": 78.73736107879964, + "decoding_speed_tps": 12.744094004515487 + }, + "wall_seconds": 3.2907, + "ttft_seconds": 1.0668, + "completion_tokens": 25, + "decode_duration_seconds": 1.9617, + "decode_tokens_per_second": 12.7441, + "seconds_per_output_token": 0.131627, + "raw_output": "Review the new TypeScript auth middleware for regressions and missing tests, then report findings by severity with file and line references.", + "final_output_tokens": 87 + }, + { + "ok": true, + "error": "", + "output": "\nReview the new TypeScript auth middleware for regressions and missing tests; report findings by severity with file and line references.\n\n\nNo additional context stated.\n\n\n- Review the new TypeScript auth middleware for regressions and missing tests.\n- Report findings by severity with file and line references.\n- Preserve all stated requirements.\n\n\nReview.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 84, + "completion_tokens": 25, + "total_tokens": 109, + "kv_token_occupancy_rate_percentage": 0.3326416015625, + "load_duration": 6e-07, + "prefill_duration_ttft": 1.16442816, + "decoding_duration": 1.923143, + "prefill_speed_tps": 72.13841341659068, + "decoding_speed_tps": 12.999553335347397 + }, + "wall_seconds": 3.3337, + "ttft_seconds": 1.1644, + "completion_tokens": 25, + "decode_duration_seconds": 1.9231, + "decode_tokens_per_second": 12.9996, + "seconds_per_output_token": 0.133347, + "raw_output": "Review the new TypeScript auth middleware for regressions and missing tests, then report findings by severity with file and line references.", + "final_output_tokens": 87 + }, + { + "ok": true, + "error": "", + "output": "\nReview the new TypeScript auth middleware for regressions and missing tests; report findings by severity with file and line references.\n\n\nNo additional context stated.\n\n\n- Review the new TypeScript auth middleware for regressions and missing tests.\n- Report findings by severity with file and line references.\n- Preserve all stated requirements.\n\n\nReview.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 84, + "completion_tokens": 25, + "total_tokens": 109, + "kv_token_occupancy_rate_percentage": 0.3326416015625, + "load_duration": 6e-07, + "prefill_duration_ttft": 1.07205632, + "decoding_duration": 1.923064, + "prefill_speed_tps": 78.35409244171052, + "decoding_speed_tps": 13.000087360587063 + }, + "wall_seconds": 3.2233, + "ttft_seconds": 1.0721, + "completion_tokens": 25, + "decode_duration_seconds": 1.9231, + "decode_tokens_per_second": 13.0001, + "seconds_per_output_token": 0.128931, + "raw_output": "Review the new TypeScript auth middleware for regressions and missing tests, then report findings by severity with file and line references.", + "final_output_tokens": 87 + }, + { + "ok": true, + "error": "", + "output": "\nReview the new TypeScript auth middleware for regressions and missing tests; report findings by severity with file and line references.\n\n\nNo additional context stated.\n\n\n- Review the new TypeScript auth middleware for regressions and missing tests.\n- Report findings by severity with file and line references.\n- Preserve all stated requirements.\n\n\nReview.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 84, + "completion_tokens": 25, + "total_tokens": 109, + "kv_token_occupancy_rate_percentage": 0.3326416015625, + "load_duration": 4e-07, + "prefill_duration_ttft": 1.112042112, + "decoding_duration": 1.937379, + "prefill_speed_tps": 75.53670773216186, + "decoding_speed_tps": 12.904031684043236 + }, + "wall_seconds": 3.3004, + "ttft_seconds": 1.112, + "completion_tokens": 25, + "decode_duration_seconds": 1.9374, + "decode_tokens_per_second": 12.904, + "seconds_per_output_token": 0.132014, + "raw_output": "Review the new TypeScript auth middleware for regressions and missing tests, then report findings by severity with file and line references.", + "final_output_tokens": 87 + } + ], + "speed": { + "runs": 5, + "successful_runs": 5, + "wall_seconds": { + "median": 3.2907, + "p90": 3.3204, + "min": 3.214, + "max": 3.3337 + }, + "ttft_seconds": { + "median": 1.0721, + "p90": 1.1434, + "min": 1.0668, + "max": 1.1644 + }, + "completion_tokens": { + "median": 25.0, + "p90": 25.0, + "min": 25.0, + "max": 25.0 + }, + "decode_tokens_per_second": { + "median": 12.9996, + "p90": 13.0047, + "min": 12.7441, + "max": 13.0078 + }, + "seconds_per_output_token": { + "median": 0.1316, + "p90": 0.1328, + "min": 0.1286, + "max": 0.1333 + } + }, + "representative_output": "\nReview the new TypeScript auth middleware for regressions and missing tests; report findings by severity with file and line references.\n\n\nNo additional context stated.\n\n\n- Review the new TypeScript auth middleware for regressions and missing tests.\n- Report findings by severity with file and line references.\n- Preserve all stated requirements.\n\n\nReview.\n", + "quality": { + "rubric": { + "r1": true, + "r2": true, + "r3": true, + "r4": true, + "r5": true, + "r6": true, + "r7": true + }, + "score": 7, + "passed": true, + "hard_fail": false, + "pending_judge": false, + "invented_requirement": false, + "constraint_item_count": 3, + "token_count": 87, + "token_count_source": "usage.completion_tokens", + "estimated_tokens": 87, + "judge_notes": "Review and source-reference requirements preserved without additions." + } + } + } + }, + { + "name": "refactor_invoice_service", + "category": "refactor", + "trap": false, + "input": "refactor InvoiceService so calculation and persistence are separate, preserve current behavior, and do not add dependencies", + "styles": { + "v1": { + "label": "current", + "max_tokens": 700, + "warmups": 1, + "warmup_errors": [], + "samples": [ + { + "ok": true, + "error": "", + "output": "\nRefactor the InvoiceService to separate calculation logic from persistence while preserving existing behavior and avoiding new dependencies.\n\n\nThe current implementation mixes invoice calculation with database operations in a single service class. The goal is to isolate the mathematical rules for generating invoices into a dedicated component, leaving only data access responsibilities within the original service or an appropriate repository layer without introducing external libraries or frameworks.\n\n\n- Output must be exactly 200 words (195 - 204 characters allowed if strictly enforced by platform, but here meaning ~200 tokens/words).\n- Use only standard TypeScript interfaces and types; no third-party packages like `lodash`, `date-fns`, or ORM libraries.\n- Preserve all existing unit test expectations without modification to test code.\n- Do not introduce new classes beyond what is necessary for separation of concerns (e.g., a simple calculation utility).\n\n\nProvide the refactored TypeScript code in a single Markdown block with clear comments explaining the separation of concerns. Include no additional text outside the code block.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 138, + "completion_tokens": 224, + "total_tokens": 362, + "kv_token_occupancy_rate_percentage": 1.104736328125, + "load_duration": 8e-07, + "prefill_duration_ttft": 1.244666752, + "decoding_duration": 16.03299, + "prefill_speed_tps": 110.87305078106561, + "decoding_speed_tps": 13.971193146131819 + }, + "wall_seconds": 18.3546, + "ttft_seconds": 1.2447, + "completion_tokens": 224, + "decode_duration_seconds": 16.033, + "decode_tokens_per_second": 13.9712, + "seconds_per_output_token": 0.08194 + }, + { + "ok": true, + "error": "", + "output": "\nRefactor the InvoiceService to separate calculation logic from persistence while preserving existing behavior and avoiding new dependencies.\n\n\nThe current InvoiceService handles both calculating invoice totals (taxes, discounts, etc.) and persisting those results to a database within a single class. The goal is to extract the calculation logic into its own component or method without introducing external libraries or frameworks that are not currently used by the application.\n\n\n- Output must be exactly 150 words (±10).\n- Use only standard JavaScript/TypeScript features; no third-party packages.\n- Maintain backward compatibility: existing API calls must return identical results.\n- Do not alter the public interface of `InvoiceService`.\n\n\nProvide a concise technical summary in Markdown with two sections: \"Refactoring Plan\" and \"Implementation Notes\".", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 138, + "completion_tokens": 178, + "total_tokens": 316, + "kv_token_occupancy_rate_percentage": 0.96435546875, + "load_duration": 7e-07, + "prefill_duration_ttft": 1.154303744, + "decoding_duration": 12.664427, + "prefill_speed_tps": 119.55258805779287, + "decoding_speed_tps": 14.05511674551087 + }, + "wall_seconds": 14.7097, + "ttft_seconds": 1.1543, + "completion_tokens": 178, + "decode_duration_seconds": 12.6644, + "decode_tokens_per_second": 14.0551, + "seconds_per_output_token": 0.082639 + }, + { + "ok": true, + "error": "", + "output": "\nRefactor the InvoiceService to separate calculation logic from persistence while preserving existing behavior and avoiding new dependencies.\n\n\nThe current implementation mixes invoice calculation with database operations in a single service class. The goal is to isolate the mathematical rules for generating invoices into a distinct component, leaving only data access responsibilities within the original service or a dedicated repository layer. No external libraries should be introduced during this refactoring.\n\n\n- Output must be exactly 200 words (195 - 204 characters allowed if strictly word-counted, but here we assume ~200 tokens/words for clarity).\n- Use only standard JavaScript/TypeScript syntax compatible with Node.js environments.\n- Do not introduce any new npm packages or external dependencies.\n- Maintain backward compatibility: all public methods must behave identically before and after refactoring.\n\n\nProvide the complete TypeScript code for both the calculation module (e.g., `InvoiceCalculator`) and the updated service (`InvoiceService`). Use clear class names, JSDoc comments explaining separation of concerns, and ensure type safety throughout.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 138, + "completion_tokens": 231, + "total_tokens": 369, + "kv_token_occupancy_rate_percentage": 1.1260986328125, + "load_duration": 7e-07, + "prefill_duration_ttft": 1.169320448, + "decoding_duration": 16.797702, + "prefill_speed_tps": 118.01726398955438, + "decoding_speed_tps": 13.751881060873684 + }, + "wall_seconds": 19.0467, + "ttft_seconds": 1.1693, + "completion_tokens": 231, + "decode_duration_seconds": 16.7977, + "decode_tokens_per_second": 13.7519, + "seconds_per_output_token": 0.082453 + }, + { + "ok": true, + "error": "", + "output": "\nRefactor the InvoiceService to separate calculation logic from persistence while preserving existing behavior and avoiding new dependencies.\n\n\nThe current implementation mixes invoice calculation with database operations in a single service class. The goal is to isolate the mathematical rules for generating invoices into a dedicated component, leaving only data access responsibilities within the original service or its immediate layer. No external libraries should be introduced during this refactoring.\n\n\n- Output must be exactly 200 words (±10).\n- Use Markdown code blocks for all examples and pseudo-code.\n- Maintain a professional, technical tone suitable for senior engineering review.\n- Do not include any explanatory text outside the requested structure.\n\n\nA single Markdown document containing:\n1. A brief introduction to the refactoring approach (2 sentences).\n2. The new class hierarchy using Python syntax examples only (no implementation details beyond signatures and comments).", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 138, + "completion_tokens": 193, + "total_tokens": 331, + "kv_token_occupancy_rate_percentage": 1.0101318359375, + "load_duration": 7e-07, + "prefill_duration_ttft": 1.199867776, + "decoding_duration": 13.731442, + "prefill_speed_tps": 115.0126728630472, + "decoding_speed_tps": 14.055333736981156 + }, + "wall_seconds": 15.8883, + "ttft_seconds": 1.1999, + "completion_tokens": 193, + "decode_duration_seconds": 13.7314, + "decode_tokens_per_second": 14.0553, + "seconds_per_output_token": 0.082323 + }, + { + "ok": true, + "error": "", + "output": "\nRefactor the InvoiceService to separate calculation logic from persistence while preserving current behavior and avoiding new dependencies.\n\n\nThe existing InvoiceService handles both calculating invoice totals (taxes, discounts) and saving them to a database. The goal is to extract the calculation logic into a dedicated service or method, leaving only data access in the original service. No external libraries should be introduced; use only existing internal classes if available.\n\n\n- Output must be exactly 200 words (±10).\n- Use Markdown code blocks for all examples and pseudo-code.\n- Maintain a professional, technical tone suitable for senior developers reviewing PRs.\n- Do not include any explanatory text outside the prompt structure itself.\n\n\nReturn only the refactored service implementation in a single Markdown block with clear separation between `calculateInvoice` and `persistInvoice`.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 138, + "completion_tokens": 185, + "total_tokens": 323, + "kv_token_occupancy_rate_percentage": 0.9857177734375, + "load_duration": 9e-07, + "prefill_duration_ttft": 1.31558976, + "decoding_duration": 13.222926, + "prefill_speed_tps": 104.89592135469343, + "decoding_speed_tps": 13.99085194910718 + }, + "wall_seconds": 15.5106, + "ttft_seconds": 1.3156, + "completion_tokens": 185, + "decode_duration_seconds": 13.2229, + "decode_tokens_per_second": 13.9909, + "seconds_per_output_token": 0.083841 + } + ], + "speed": { + "runs": 5, + "successful_runs": 5, + "wall_seconds": { + "median": 15.8883, + "p90": 18.7699, + "min": 14.7097, + "max": 19.0467 + }, + "ttft_seconds": { + "median": 1.1999, + "p90": 1.2872, + "min": 1.1543, + "max": 1.3156 + }, + "completion_tokens": { + "median": 193.0, + "p90": 228.2, + "min": 178.0, + "max": 231.0 + }, + "decode_tokens_per_second": { + "median": 13.9909, + "p90": 14.0552, + "min": 13.7519, + "max": 14.0553 + }, + "seconds_per_output_token": { + "median": 0.0825, + "p90": 0.0834, + "min": 0.0819, + "max": 0.0838 + } + }, + "representative_output": "\nRefactor the InvoiceService to separate calculation logic from persistence while preserving existing behavior and avoiding new dependencies.\n\n\nThe current implementation mixes invoice calculation with database operations in a single service class. The goal is to isolate the mathematical rules for generating invoices into a dedicated component, leaving only data access responsibilities within the original service or an appropriate repository layer without introducing external libraries or frameworks.\n\n\n- Output must be exactly 200 words (195 - 204 characters allowed if strictly enforced by platform, but here meaning ~200 tokens/words).\n- Use only standard TypeScript interfaces and types; no third-party packages like `lodash`, `date-fns`, or ORM libraries.\n- Preserve all existing unit test expectations without modification to test code.\n- Do not introduce new classes beyond what is necessary for separation of concerns (e.g., a simple calculation utility).\n\n\nProvide the refactored TypeScript code in a single Markdown block with clear comments explaining the separation of concerns. Include no additional text outside the code block.", + "quality": { + "rubric": { + "r1": false, + "r2": false, + "r3": false, + "r4": false, + "r5": false, + "r6": false, + "r7": false + }, + "score": 0, + "passed": false, + "hard_fail": true, + "pending_judge": false, + "invented_requirement": true, + "constraint_item_count": 0, + "token_count": 224, + "token_count_source": "usage.completion_tokens", + "estimated_tokens": 207, + "judge_notes": "Invents database/repository design, TypeScript, test policy, word count, and class limits." + } + }, + "v2": { + "label": "candidate", + "max_tokens": 240, + "warmups": 1, + "warmup_errors": [], + "samples": [ + { + "ok": true, + "error": "", + "output": "\nRefactor InvoiceService so calculation and persistence are separate, preserve current behavior, and do not add dependencies.\n\n\nNo additional context stated.\n\n\n- Refactor InvoiceService so calculation and persistence are separate.\n- Preserve current behavior.\n- Do not add dependencies.\n\n\nRefactor.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 82, + "completion_tokens": 18, + "total_tokens": 100, + "kv_token_occupancy_rate_percentage": 0.30517578125, + "load_duration": 5e-07, + "prefill_duration_ttft": 1.068295808, + "decoding_duration": 1.42447, + "prefill_speed_tps": 76.75776632833141, + "decoding_speed_tps": 12.636278756309364 + }, + "wall_seconds": 2.7035, + "ttft_seconds": 1.0683, + "completion_tokens": 18, + "decode_duration_seconds": 1.4245, + "decode_tokens_per_second": 12.6363, + "seconds_per_output_token": 0.150195, + "raw_output": "Refactor InvoiceService to separate calculation and persistence while preserving current behavior without adding dependencies.", + "final_output_tokens": 75 + }, + { + "ok": true, + "error": "", + "output": "\nRefactor InvoiceService so calculation and persistence are separate, preserve current behavior, and do not add dependencies.\n\n\nNo additional context stated.\n\n\n- Refactor InvoiceService so calculation and persistence are separate.\n- Preserve current behavior.\n- Do not add dependencies.\n\n\nRefactor.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 82, + "completion_tokens": 18, + "total_tokens": 100, + "kv_token_occupancy_rate_percentage": 0.30517578125, + "load_duration": 5e-07, + "prefill_duration_ttft": 1.076152192, + "decoding_duration": 1.422943, + "prefill_speed_tps": 76.19740089699134, + "decoding_speed_tps": 12.649839101074322 + }, + "wall_seconds": 2.7245, + "ttft_seconds": 1.0762, + "completion_tokens": 18, + "decode_duration_seconds": 1.4229, + "decode_tokens_per_second": 12.6498, + "seconds_per_output_token": 0.15136, + "raw_output": "Refactor InvoiceService to separate calculation and persistence while preserving current behavior without adding dependencies.", + "final_output_tokens": 75 + }, + { + "ok": true, + "error": "", + "output": "\nRefactor InvoiceService so calculation and persistence are separate, preserve current behavior, and do not add dependencies.\n\n\nNo additional context stated.\n\n\n- Refactor InvoiceService so calculation and persistence are separate.\n- Preserve current behavior.\n- Do not add dependencies.\n\n\nRefactor.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 82, + "completion_tokens": 18, + "total_tokens": 100, + "kv_token_occupancy_rate_percentage": 0.30517578125, + "load_duration": 5e-07, + "prefill_duration_ttft": 1.112521728, + "decoding_duration": 1.423278, + "prefill_speed_tps": 73.70642562407554, + "decoding_speed_tps": 12.646861681273792 + }, + "wall_seconds": 2.7732, + "ttft_seconds": 1.1125, + "completion_tokens": 18, + "decode_duration_seconds": 1.4233, + "decode_tokens_per_second": 12.6469, + "seconds_per_output_token": 0.154067, + "raw_output": "Refactor InvoiceService to separate calculation and persistence while preserving current behavior without adding dependencies.", + "final_output_tokens": 75 + }, + { + "ok": true, + "error": "", + "output": "\nRefactor InvoiceService so calculation and persistence are separate, preserve current behavior, and do not add dependencies.\n\n\nNo additional context stated.\n\n\n- Refactor InvoiceService so calculation and persistence are separate.\n- Preserve current behavior.\n- Do not add dependencies.\n\n\nRefactor.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 82, + "completion_tokens": 18, + "total_tokens": 100, + "kv_token_occupancy_rate_percentage": 0.30517578125, + "load_duration": 4e-07, + "prefill_duration_ttft": 1.120802432, + "decoding_duration": 1.431859, + "prefill_speed_tps": 73.16186837110645, + "decoding_speed_tps": 12.571070196157581 + }, + "wall_seconds": 2.7867, + "ttft_seconds": 1.1208, + "completion_tokens": 18, + "decode_duration_seconds": 1.4319, + "decode_tokens_per_second": 12.5711, + "seconds_per_output_token": 0.154818, + "raw_output": "Refactor InvoiceService to separate calculation and persistence while preserving behavior and avoiding new dependencies.", + "final_output_tokens": 75 + }, + { + "ok": true, + "error": "", + "output": "\nRefactor InvoiceService so calculation and persistence are separate, preserve current behavior, and do not add dependencies.\n\n\nNo additional context stated.\n\n\n- Refactor InvoiceService so calculation and persistence are separate.\n- Preserve current behavior.\n- Do not add dependencies.\n\n\nRefactor.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 82, + "completion_tokens": 18, + "total_tokens": 100, + "kv_token_occupancy_rate_percentage": 0.30517578125, + "load_duration": 5e-07, + "prefill_duration_ttft": 1.250210048, + "decoding_duration": 1.423986, + "prefill_speed_tps": 65.58897853298967, + "decoding_speed_tps": 12.64057371350561 + }, + "wall_seconds": 2.8997, + "ttft_seconds": 1.2502, + "completion_tokens": 18, + "decode_duration_seconds": 1.424, + "decode_tokens_per_second": 12.6406, + "seconds_per_output_token": 0.161092, + "raw_output": "Refactor InvoiceService to separate calculation and persistence while preserving current behavior without adding dependencies.", + "final_output_tokens": 75 + } + ], + "speed": { + "runs": 5, + "successful_runs": 5, + "wall_seconds": { + "median": 2.7732, + "p90": 2.8545, + "min": 2.7035, + "max": 2.8997 + }, + "ttft_seconds": { + "median": 1.1125, + "p90": 1.1984, + "min": 1.0683, + "max": 1.2502 + }, + "completion_tokens": { + "median": 18.0, + "p90": 18.0, + "min": 18.0, + "max": 18.0 + }, + "decode_tokens_per_second": { + "median": 12.6406, + "p90": 12.6486, + "min": 12.5711, + "max": 12.6498 + }, + "seconds_per_output_token": { + "median": 0.1541, + "p90": 0.1586, + "min": 0.1502, + "max": 0.1611 + } + }, + "representative_output": "\nRefactor InvoiceService so calculation and persistence are separate, preserve current behavior, and do not add dependencies.\n\n\nNo additional context stated.\n\n\n- Refactor InvoiceService so calculation and persistence are separate.\n- Preserve current behavior.\n- Do not add dependencies.\n\n\nRefactor.\n", + "quality": { + "rubric": { + "r1": true, + "r2": true, + "r3": true, + "r4": true, + "r5": true, + "r6": true, + "r7": true + }, + "score": 7, + "passed": true, + "hard_fail": false, + "pending_judge": false, + "invented_requirement": false, + "constraint_item_count": 3, + "token_count": 75, + "token_count_source": "usage.completion_tokens", + "estimated_tokens": 75, + "judge_notes": "All three explicit refactor constraints preserved." + } + } + } + }, + { + "name": "data_monthly_rollup", + "category": "data", + "trap": false, + "input": "combine these monthly sales CSVs, keep the original rows, summarize revenue by region, and reconcile the summary total to the source total", + "styles": { + "v1": { + "label": "current", + "max_tokens": 700, + "warmups": 1, + "warmup_errors": [], + "samples": [ + { + "ok": true, + "error": "", + "output": "\nGenerate a Markdown report that merges multiple monthly sales CSVs into a single dataset, calculates regional revenue summaries, and verifies the total against the source data.\n\n\nThe user has several separate CSV files containing monthly sales records with columns for date, region, product ID, quantity sold, and unit price. The goal is to aggregate these files without losing individual row details while producing a high-level summary by region that matches the sum of all original transactions.\n\n\n- Output must be valid Markdown with clear headers (e.g., `## 1. Data Merge`, `## 2. Regional Summary`, `## 3. Reconciliation Check`).\n- Include a sample merged table showing at least one row from each source file to demonstrate retention of original data.\n- Present the regional summary as a bulleted list or small table with totals per region and an overall grand total.\n- The reconciliation section must explicitly state: \"Source Total\" vs \"Summary Total\" and confirm they match (or note any discrepancy).\n- Tone should be professional, analytical, and concise.\n\n\nMarkdown document with three main sections as specified above. No JSON or code blocks unless illustrating the merged data structure conceptually.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 144, + "completion_tokens": 256, + "total_tokens": 400, + "kv_token_occupancy_rate_percentage": 1.220703125, + "load_duration": 6e-07, + "prefill_duration_ttft": 1.163815552, + "decoding_duration": 18.429308, + "prefill_speed_tps": 123.73094667152206, + "decoding_speed_tps": 13.890917662236694 + }, + "wall_seconds": 20.7516, + "ttft_seconds": 1.1638, + "completion_tokens": 256, + "decode_duration_seconds": 18.4293, + "decode_tokens_per_second": 13.8909, + "seconds_per_output_token": 0.081061 + }, + { + "ok": true, + "error": "", + "output": "\nGenerate a Markdown report that merges provided monthly sales CSVs, preserves all original rows, calculates regional revenue summaries, and verifies the summary total against the source data.\n\n\nThe user has multiple CSV files containing monthly sales data with columns for region and revenue. The goal is to aggregate this data by region while ensuring no records are lost during processing. A final reconciliation step must confirm that the sum of regional totals exactly matches the grand total from the combined dataset, highlighting any discrepancies if they exist.\n\n\n- Output length: Maximum 500 words for explanations; tables can be longer but must fit in a single code block or clearly separated section.\n- Format: Use Markdown with headers (`#`, `##`), data tables (using standard table syntax), and a dedicated \"Reconciliation Check\" section at the end.\n- Tone: Professional, analytical, and concise. Avoid unnecessary fluff; focus on clarity of results.\n- Testability: The reconciliation logic must be explicitly stated so it can be independently verified by summing regional totals vs. global total in any spreadsheet or script.\n\n\n1. Title section with a brief overview of the merge operation performed (if applicable).\n2. A data table showing all original rows combined from the source CSVs (columns: Date, Region, Revenue).\n3. A summary table aggregating revenue by region only (columns: Region, Total Revenue).\n4. A \"Reconciliation Check\" section with two lines of math: `Sum of Regional Totals = [X]` and `Grand Total from Source = [Y]`, followed by a status line (`MATCHED` or `DISCREPANCY DETECTED`).", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 144, + "completion_tokens": 357, + "total_tokens": 501, + "kv_token_occupancy_rate_percentage": 1.5289306640625, + "load_duration": 8e-07, + "prefill_duration_ttft": 1.16649728, + "decoding_duration": 25.78559, + "prefill_speed_tps": 123.44649444874831, + "decoding_speed_tps": 13.84494207811417 + }, + "wall_seconds": 28.4029, + "ttft_seconds": 1.1665, + "completion_tokens": 357, + "decode_duration_seconds": 25.7856, + "decode_tokens_per_second": 13.8449, + "seconds_per_output_token": 0.07956 + }, + { + "ok": true, + "error": "", + "output": "\nGenerate a Markdown report that merges multiple monthly sales CSVs into a single dataset, calculates regional revenue summaries, and verifies the total against the source data.\n\n\nThe user has access to several separate CSV files containing monthly sales records with columns for region, date, and revenue. The goal is to aggregate these files without losing individual row details while producing a high-level summary by geographic area.\n\n\n- Output must be valid Markdown using headers (#) and tables (|).\n- Include three distinct sections: \"Combined Data Preview\" (first 5 rows), \"Regional Revenue Summary\", and \"Reconciliation Check\".\n- The Reconciliation section must explicitly state the sum of individual regions versus the total row count/sum to confirm accuracy.\n- Tone should be professional and analytical, suitable for a business audit or financial review.\n\n\nMarkdown document with exactly three headers: `## Combined Data Preview`, `## Regional Revenue Summary`, and `## Reconciliation Check`. Use standard table formatting for data presentation.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 144, + "completion_tokens": 216, + "total_tokens": 360, + "kv_token_occupancy_rate_percentage": 1.0986328125, + "load_duration": 9e-07, + "prefill_duration_ttft": 1.17877312, + "decoding_duration": 15.858891, + "prefill_speed_tps": 122.16091252572845, + "decoding_speed_tps": 13.620120095408941 + }, + "wall_seconds": 17.9436, + "ttft_seconds": 1.1788, + "completion_tokens": 216, + "decode_duration_seconds": 15.8589, + "decode_tokens_per_second": 13.6201, + "seconds_per_output_token": 0.083072 + }, + { + "ok": true, + "error": "", + "output": "\nGenerate a Markdown report that merges multiple monthly sales CSVs into a single dataset, calculates regional revenue summaries, and verifies the total against the source data.\n\n\nThe user has several separate CSV files containing monthly sales records with columns for region, date, product ID, quantity sold, and unit price. The goal is to aggregate these files without losing individual row details while producing a high-level summary by geographic region that matches the sum of all original transactions.\n\n\n- Output must be valid Markdown with clear headers (e.g., `## 1. Data Merged`, `## 2. Regional Summary`, `## 3. Reconciliation Check`).\n- The merged dataset section should display a sample of combined rows or indicate successful concatenation if the full list is too long, but explicitly state that original row integrity is preserved.\n- The regional summary must show total revenue per region with at least two decimal places precision.\n- The reconciliation check must compare the sum of all regions to the grand total and declare \"PASSED\" or \"FAILED\".\n- Tone: Professional, analytical, concise. No conversational filler before or after the report.\n\n\nMarkdown document containing exactly three sections with headers as specified above. Use tables for data presentation where appropriate. Ensure no JSON output is generated unless requested (it was not).", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 144, + "completion_tokens": 281, + "total_tokens": 425, + "kv_token_occupancy_rate_percentage": 1.2969970703125, + "load_duration": 8e-07, + "prefill_duration_ttft": 1.24151744, + "decoding_duration": 20.819726, + "prefill_speed_tps": 115.9870939871775, + "decoding_speed_tps": 13.49681547201918 + }, + "wall_seconds": 23.2901, + "ttft_seconds": 1.2415, + "completion_tokens": 281, + "decode_duration_seconds": 20.8197, + "decode_tokens_per_second": 13.4968, + "seconds_per_output_token": 0.082883 + }, + { + "ok": true, + "error": "", + "output": "\nGenerate a Markdown report that merges multiple monthly sales CSVs into a single dataset, calculates regional revenue summaries, and verifies the total against the source data.\n\n\nThe user has several separate CSV files containing monthly sales records with columns for region, date, and revenue. The goal is to aggregate these files without losing individual row details while producing a high-level summary by region that mathematically equals the sum of all original rows.\n\n\n- Output must be valid Markdown with clear headers (e.g., `## 1. Data Merge`, `## 2. Regional Summary`, `## 3. Reconciliation`).\n- The merged dataset section should display a sample of combined rows to prove integrity, not the full list if too large (or all if under threshold).\n- Revenue summaries must be grouped by region with clear totals per region and an overall total.\n- A reconciliation table must explicitly show: `Sum of Regional Totals` vs `Total from Merged Rows`, confirming they match exactly.\n- Tone should be professional, analytical, and concise.\n\n\nUse Markdown headers for sections. Include a code block or table for the merged sample data and summary tables. Ensure no JSON output is used unless specifically requested (it was not).", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 144, + "completion_tokens": 266, + "total_tokens": 410, + "kv_token_occupancy_rate_percentage": 1.251220703125, + "load_duration": 1.3e-06, + "prefill_duration_ttft": 1.551676288, + "decoding_duration": 20.088912, + "prefill_speed_tps": 92.80286172678822, + "decoding_speed_tps": 13.241135209313475 + }, + "wall_seconds": 22.8554, + "ttft_seconds": 1.5517, + "completion_tokens": 266, + "decode_duration_seconds": 20.0889, + "decode_tokens_per_second": 13.2411, + "seconds_per_output_token": 0.085923 + } + ], + "speed": { + "runs": 5, + "successful_runs": 5, + "wall_seconds": { + "median": 22.8554, + "p90": 26.3578, + "min": 17.9436, + "max": 28.4029 + }, + "ttft_seconds": { + "median": 1.1788, + "p90": 1.4276, + "min": 1.1638, + "max": 1.5517 + }, + "completion_tokens": { + "median": 266.0, + "p90": 326.6, + "min": 216.0, + "max": 357.0 + }, + "decode_tokens_per_second": { + "median": 13.6201, + "p90": 13.8725, + "min": 13.2411, + "max": 13.8909 + }, + "seconds_per_output_token": { + "median": 0.0829, + "p90": 0.0848, + "min": 0.0796, + "max": 0.0859 + } + }, + "representative_output": "\nGenerate a Markdown report that merges multiple monthly sales CSVs into a single dataset, calculates regional revenue summaries, and verifies the total against the source data.\n\n\nThe user has several separate CSV files containing monthly sales records with columns for date, region, product ID, quantity sold, and unit price. The goal is to aggregate these files without losing individual row details while producing a high-level summary by region that matches the sum of all original transactions.\n\n\n- Output must be valid Markdown with clear headers (e.g., `## 1. Data Merge`, `## 2. Regional Summary`, `## 3. Reconciliation Check`).\n- Include a sample merged table showing at least one row from each source file to demonstrate retention of original data.\n- Present the regional summary as a bulleted list or small table with totals per region and an overall grand total.\n- The reconciliation section must explicitly state: \"Source Total\" vs \"Summary Total\" and confirm they match (or note any discrepancy).\n- Tone should be professional, analytical, and concise.\n\n\nMarkdown document with three main sections as specified above. No JSON or code blocks unless illustrating the merged data structure conceptually.", + "quality": { + "rubric": { + "r1": false, + "r2": false, + "r3": false, + "r4": false, + "r5": false, + "r6": false, + "r7": false + }, + "score": 0, + "passed": false, + "hard_fail": true, + "pending_judge": false, + "invented_requirement": true, + "constraint_item_count": 0, + "token_count": 256, + "token_count_source": "usage.completion_tokens", + "estimated_tokens": 248, + "judge_notes": "Invents columns, sample tables, Markdown layout, and presentation details." + } + }, + "v2": { + "label": "candidate", + "max_tokens": 240, + "warmups": 1, + "warmup_errors": [], + "samples": [ + { + "ok": true, + "error": "", + "output": "\nCombine these monthly sales CSVs, keep the original rows, summarize revenue by region, and reconcile the summary total to the source total.\n\n\nNo additional context stated.\n\n\n- Combine these monthly sales CSVs.\n- Keep the original rows.\n- Summarize revenue by region.\n- Reconcile the summary total to the source total.\n\n\nCombined data and summary.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 88, + "completion_tokens": 27, + "total_tokens": 115, + "kv_token_occupancy_rate_percentage": 0.3509521484375, + "load_duration": 8e-07, + "prefill_duration_ttft": 1.074605312, + "decoding_duration": 2.061451, + "prefill_speed_tps": 81.8905313581774, + "decoding_speed_tps": 13.097570594692767 + }, + "wall_seconds": 3.3942, + "ttft_seconds": 1.0746, + "completion_tokens": 27, + "decode_duration_seconds": 2.0615, + "decode_tokens_per_second": 13.0976, + "seconds_per_output_token": 0.125712, + "raw_output": "Combine the monthly sales CSVs, keep original rows, summarize revenue by region, and reconcile the summary total to the source total.", + "final_output_tokens": 93 + }, + { + "ok": true, + "error": "", + "output": "\nCombine these monthly sales CSVs, keep the original rows, summarize revenue by region, and reconcile the summary total to the source total.\n\n\nNo additional context stated.\n\n\n- Combine these monthly sales CSVs.\n- Keep the original rows.\n- Summarize revenue by region.\n- Reconcile the summary total to the source total.\n\n\nCombined data and summary.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 88, + "completion_tokens": 27, + "total_tokens": 115, + "kv_token_occupancy_rate_percentage": 0.3509521484375, + "load_duration": 5e-07, + "prefill_duration_ttft": 1.0963872, + "decoding_duration": 2.062727, + "prefill_speed_tps": 80.26361489809439, + "decoding_speed_tps": 13.089468456077803 + }, + "wall_seconds": 3.388, + "ttft_seconds": 1.0964, + "completion_tokens": 27, + "decode_duration_seconds": 2.0627, + "decode_tokens_per_second": 13.0895, + "seconds_per_output_token": 0.125481, + "raw_output": "Combine the monthly sales CSVs, keep original rows, summarize revenue by region, and reconcile the summary total to the source total.", + "final_output_tokens": 93 + }, + { + "ok": true, + "error": "", + "output": "\nCombine these monthly sales CSVs, keep the original rows, summarize revenue by region, and reconcile the summary total to the source total.\n\n\nNo additional context stated.\n\n\n- Combine these monthly sales CSVs.\n- Keep the original rows.\n- Summarize revenue by region.\n- Reconcile the summary total to the source total.\n\n\nCombined data and summary.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 88, + "completion_tokens": 27, + "total_tokens": 115, + "kv_token_occupancy_rate_percentage": 0.3509521484375, + "load_duration": 6e-07, + "prefill_duration_ttft": 1.275081472, + "decoding_duration": 2.063705, + "prefill_speed_tps": 69.01519779906268, + "decoding_speed_tps": 13.083265292277723 + }, + "wall_seconds": 3.6505, + "ttft_seconds": 1.2751, + "completion_tokens": 27, + "decode_duration_seconds": 2.0637, + "decode_tokens_per_second": 13.0833, + "seconds_per_output_token": 0.135203, + "raw_output": "Combine the monthly sales CSVs, keep original rows, summarize revenue by region, and reconcile the summary total to the source total.", + "final_output_tokens": 93 + }, + { + "ok": true, + "error": "", + "output": "\nCombine these monthly sales CSVs, keep the original rows, summarize revenue by region, and reconcile the summary total to the source total.\n\n\nNo additional context stated.\n\n\n- Combine these monthly sales CSVs.\n- Keep the original rows.\n- Summarize revenue by region.\n- Reconcile the summary total to the source total.\n\n\nCombined data and summary.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 88, + "completion_tokens": 27, + "total_tokens": 115, + "kv_token_occupancy_rate_percentage": 0.3509521484375, + "load_duration": 5e-07, + "prefill_duration_ttft": 1.071365888, + "decoding_duration": 2.06492, + "prefill_speed_tps": 82.13813878681192, + "decoding_speed_tps": 13.07556709218759 + }, + "wall_seconds": 3.3693, + "ttft_seconds": 1.0714, + "completion_tokens": 27, + "decode_duration_seconds": 2.0649, + "decode_tokens_per_second": 13.0756, + "seconds_per_output_token": 0.124789, + "raw_output": "Combine the monthly sales CSVs, keep original rows, summarize revenue by region, and reconcile the summary total to the source total.", + "final_output_tokens": 93 + }, + { + "ok": true, + "error": "", + "output": "\nCombine these monthly sales CSVs, keep the original rows, summarize revenue by region, and reconcile the summary total to the source total.\n\n\nNo additional context stated.\n\n\n- Combine these monthly sales CSVs.\n- Keep the original rows.\n- Summarize revenue by region.\n- Reconcile the summary total to the source total.\n\n\nCombined data and summary.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 88, + "completion_tokens": 27, + "total_tokens": 115, + "kv_token_occupancy_rate_percentage": 0.3509521484375, + "load_duration": 5e-07, + "prefill_duration_ttft": 1.192987008, + "decoding_duration": 2.208658, + "prefill_speed_tps": 73.76442443202198, + "decoding_speed_tps": 12.22461784486326 + }, + "wall_seconds": 3.6669, + "ttft_seconds": 1.193, + "completion_tokens": 27, + "decode_duration_seconds": 2.2087, + "decode_tokens_per_second": 12.2246, + "seconds_per_output_token": 0.135811, + "raw_output": "Combine the monthly sales CSVs, keep original rows, summarize revenue by region, and reconcile the summary total to the source total.", + "final_output_tokens": 93 + } + ], + "speed": { + "runs": 5, + "successful_runs": 5, + "wall_seconds": { + "median": 3.3942, + "p90": 3.6603, + "min": 3.3693, + "max": 3.6669 + }, + "ttft_seconds": { + "median": 1.0964, + "p90": 1.2423, + "min": 1.0714, + "max": 1.2751 + }, + "completion_tokens": { + "median": 27.0, + "p90": 27.0, + "min": 27.0, + "max": 27.0 + }, + "decode_tokens_per_second": { + "median": 13.0833, + "p90": 13.0944, + "min": 12.2246, + "max": 13.0976 + }, + "seconds_per_output_token": { + "median": 0.1257, + "p90": 0.1356, + "min": 0.1248, + "max": 0.1358 + } + }, + "representative_output": "\nCombine these monthly sales CSVs, keep the original rows, summarize revenue by region, and reconcile the summary total to the source total.\n\n\nNo additional context stated.\n\n\n- Combine these monthly sales CSVs.\n- Keep the original rows.\n- Summarize revenue by region.\n- Reconcile the summary total to the source total.\n\n\nCombined data and summary.\n", + "quality": { + "rubric": { + "r1": true, + "r2": true, + "r3": true, + "r4": true, + "r5": true, + "r6": true, + "r7": true + }, + "score": 7, + "passed": true, + "hard_fail": false, + "pending_judge": false, + "invented_requirement": false, + "constraint_item_count": 4, + "token_count": 93, + "token_count_source": "usage.completion_tokens", + "estimated_tokens": 93, + "judge_notes": "Four auditable source requirements retained." + } + } + } + }, + { + "name": "vague_dashboard_speed", + "category": "vague", + "trap": false, + "input": "make the dashboard faster", + "styles": { + "v1": { + "label": "current", + "max_tokens": 700, + "warmups": 1, + "warmup_errors": [], + "samples": [ + { + "ok": true, + "error": "", + "output": "\nOptimize the provided dashboard code to improve its rendering performance.\n\n\nThe user has a web-based dashboard with multiple data visualizations and interactive elements that currently experience lag or slow load times. The goal is specifically to increase speed without altering the core functionality or design aesthetic.\n\n\n- Output must be a single, self-contained code block (HTML/CSS/JS) containing optimized implementations of common dashboard patterns (e.g., lightweight charting libraries, efficient DOM updates).\n- Do not include lengthy explanations; focus on performance techniques like virtualization for large lists, debounced search inputs, and lazy loading.\n- The response must be under 500 words.\n\n\nMarkdown code block with a brief introductory sentence followed by the optimized implementation details.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 122, + "completion_tokens": 166, + "total_tokens": 288, + "kv_token_occupancy_rate_percentage": 0.87890625, + "load_duration": 7e-07, + "prefill_duration_ttft": 1.350409984, + "decoding_duration": 12.189752, + "prefill_speed_tps": 90.3429339574551, + "decoding_speed_tps": 13.617996494104228 + }, + "wall_seconds": 14.2816, + "ttft_seconds": 1.3504, + "completion_tokens": 166, + "decode_duration_seconds": 12.1898, + "decode_tokens_per_second": 13.618, + "seconds_per_output_token": 0.086034 + }, + { + "ok": true, + "error": "", + "output": "\nOptimize the provided dashboard code to improve its rendering performance without altering visual output.\n\n\nThe user has a web-based dashboard with multiple data visualization components that currently experience lag during initial load and interaction. The goal is specifically to increase speed, not change design or add features.\n\n\n\n\n- Output must be exactly 300 words (±15).\n- Use only standard HTML/CSS/JS libraries; no external build tools required for the solution.\n- Prioritize virtual DOM usage and lazy loading of heavy charts.\n- Tone: Direct, technical, and action-oriented.\n\n\n\n\nA single Markdown code block containing a complete `index.html` file with embedded CSS/JS that implements the optimized dashboard structure described in the constraints.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 122, + "completion_tokens": 172, + "total_tokens": 294, + "kv_token_occupancy_rate_percentage": 0.897216796875, + "load_duration": 7e-07, + "prefill_duration_ttft": 1.173108096, + "decoding_duration": 12.765522, + "prefill_speed_tps": 103.99723641494671, + "decoding_speed_tps": 13.473792924409985 + }, + "wall_seconds": 14.7636, + "ttft_seconds": 1.1731, + "completion_tokens": 172, + "decode_duration_seconds": 12.7655, + "decode_tokens_per_second": 13.4738, + "seconds_per_output_token": 0.085835 + }, + { + "ok": true, + "error": "", + "output": "\nOptimize the provided dashboard code to improve its rendering performance without altering visual output.\n\n\nThe user has a web-based dashboard with interactive charts and data tables that currently experiences lag when loading or scrolling through large datasets. The goal is specifically to increase speed, not change design or functionality.\n\n\n- Output must be valid JavaScript/TypeScript code suitable for direct insertion into an existing project.\n- Do not include explanations, comments about \"optimization,\" or meta-text; only the optimized code block.\n- Maintain all original UI elements and data logic exactly as they were before optimization.\n- Focus on performance techniques such as virtual scrolling, memoization (React), debouncing inputs, or efficient rendering strategies appropriate for web dashboards.\n\n\nA single Markdown code block containing only the optimized implementation of the dashboard component(s).", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 122, + "completion_tokens": 180, + "total_tokens": 302, + "kv_token_occupancy_rate_percentage": 0.921630859375, + "load_duration": 9e-07, + "prefill_duration_ttft": 1.155158784, + "decoding_duration": 13.487424, + "prefill_speed_tps": 105.61318642061245, + "decoding_speed_tps": 13.345765655472832 + }, + "wall_seconds": 15.5392, + "ttft_seconds": 1.1552, + "completion_tokens": 180, + "decode_duration_seconds": 13.4874, + "decode_tokens_per_second": 13.3458, + "seconds_per_output_token": 0.086329 + }, + { + "ok": true, + "error": "", + "output": "\nOptimize the provided dashboard code to improve its rendering performance.\n\n\nThe user has a web-based dashboard with multiple data visualizations and interactive elements that currently experience lag or slow load times. The goal is specifically to increase speed without altering the core functionality or design aesthetic.\n\n\n- Output must be valid JavaScript/TypeScript code suitable for direct integration into an existing project.\n- Do not include lengthy explanations; focus on implementation details like memoization, virtualization, debouncing, and efficient state management patterns.\n- Maintain backward compatibility with the original logic where possible.\n\n\nProvide a single Markdown code block containing the optimized solution. Include brief comments explaining key performance improvements (e.g., \"Implemented React.memo to prevent unnecessary re-renders\").", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 122, + "completion_tokens": 165, + "total_tokens": 287, + "kv_token_occupancy_rate_percentage": 0.8758544921875, + "load_duration": 8e-07, + "prefill_duration_ttft": 1.501880448, + "decoding_duration": 12.316232, + "prefill_speed_tps": 81.23149892687064, + "decoding_speed_tps": 13.396954523104146 + }, + "wall_seconds": 14.6382, + "ttft_seconds": 1.5019, + "completion_tokens": 165, + "decode_duration_seconds": 12.3162, + "decode_tokens_per_second": 13.397, + "seconds_per_output_token": 0.088716 + }, + { + "ok": true, + "error": "", + "output": "\nOptimize a web dashboard to improve its loading and interaction speed.\n\n\nThe user is experiencing performance issues with their current dashboard implementation, specifically regarding initial load times and responsiveness during data rendering or filtering. The existing codebase uses standard HTML/CSS/JS without specific optimization techniques like lazy loading, virtualization for large lists, or asset compression.\n\n\n- Provide a concise technical summary of 3 key optimizations (max 150 words).\n- Output must be in plain text with no Markdown formatting other than bullet points.\n- Tone: Direct and actionable for a developer audience.\n- Do not include code snippets; focus on architectural changes.\n\n\nThree numbered items detailing specific optimization strategies, each followed by a brief explanation of the expected performance gain.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 122, + "completion_tokens": 168, + "total_tokens": 290, + "kv_token_occupancy_rate_percentage": 0.885009765625, + "load_duration": 7e-07, + "prefill_duration_ttft": 1.370741504, + "decoding_duration": 12.226004, + "prefill_speed_tps": 89.00292261085573, + "decoding_speed_tps": 13.741202767478237 + }, + "wall_seconds": 14.7931, + "ttft_seconds": 1.3707, + "completion_tokens": 168, + "decode_duration_seconds": 12.226, + "decode_tokens_per_second": 13.7412, + "seconds_per_output_token": 0.088054 + } + ], + "speed": { + "runs": 5, + "successful_runs": 5, + "wall_seconds": { + "median": 14.7636, + "p90": 15.2408, + "min": 14.2816, + "max": 15.5392 + }, + "ttft_seconds": { + "median": 1.3504, + "p90": 1.4494, + "min": 1.1552, + "max": 1.5019 + }, + "completion_tokens": { + "median": 168.0, + "p90": 176.8, + "min": 165.0, + "max": 180.0 + }, + "decode_tokens_per_second": { + "median": 13.4738, + "p90": 13.6919, + "min": 13.3458, + "max": 13.7412 + }, + "seconds_per_output_token": { + "median": 0.0863, + "p90": 0.0885, + "min": 0.0858, + "max": 0.0887 + } + }, + "representative_output": "\nOptimize the provided dashboard code to improve its rendering performance.\n\n\nThe user has a web-based dashboard with multiple data visualizations and interactive elements that currently experience lag or slow load times. The goal is specifically to increase speed without altering the core functionality or design aesthetic.\n\n\n- Output must be a single, self-contained code block (HTML/CSS/JS) containing optimized implementations of common dashboard patterns (e.g., lightweight charting libraries, efficient DOM updates).\n- Do not include lengthy explanations; focus on performance techniques like virtualization for large lists, debounced search inputs, and lazy loading.\n- The response must be under 500 words.\n\n\nMarkdown code block with a brief introductory sentence followed by the optimized implementation details.", + "quality": { + "rubric": { + "r1": false, + "r2": false, + "r3": false, + "r4": false, + "r5": false, + "r6": false, + "r7": true + }, + "score": 1, + "passed": false, + "hard_fail": true, + "pending_judge": false, + "invented_requirement": true, + "constraint_item_count": 0, + "token_count": 166, + "token_count_source": "usage.completion_tokens", + "estimated_tokens": 154, + "judge_notes": "Invents dashboard technology, functionality, design, libraries, virtualization, lazy loading, and format." + } + }, + "v2": { + "label": "candidate", + "max_tokens": 240, + "warmups": 1, + "warmup_errors": [], + "samples": [ + { + "ok": true, + "error": "", + "output": "\nMake the dashboard faster.\n\n\nNo additional context stated.\n\n\n- Make the dashboard faster.\n- Preserve all stated requirements.\n- Do not add unstated requirements.\n\n\nNo output format specified.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 66, + "completion_tokens": 9, + "total_tokens": 75, + "kv_token_occupancy_rate_percentage": 0.2288818359375, + "load_duration": 5e-07, + "prefill_duration_ttft": 1.04796032, + "decoding_duration": 0.787604, + "prefill_speed_tps": 62.97948380335621, + "decoding_speed_tps": 11.427062330815994 + }, + "wall_seconds": 2.027, + "ttft_seconds": 1.048, + "completion_tokens": 9, + "decode_duration_seconds": 0.7876, + "decode_tokens_per_second": 11.4271, + "seconds_per_output_token": 0.225219, + "raw_output": "Optimize the dashboard for improved performance.", + "final_output_tokens": 62 + }, + { + "ok": true, + "error": "", + "output": "\nMake the dashboard faster.\n\n\nNo additional context stated.\n\n\n- Make the dashboard faster.\n- Preserve all stated requirements.\n- Do not add unstated requirements.\n\n\nNo output format specified.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 66, + "completion_tokens": 9, + "total_tokens": 75, + "kv_token_occupancy_rate_percentage": 0.2288818359375, + "load_duration": 6e-07, + "prefill_duration_ttft": 1.047142592, + "decoding_duration": 0.782732, + "prefill_speed_tps": 63.02866534532099, + "decoding_speed_tps": 11.498188396539302 + }, + "wall_seconds": 2.0341, + "ttft_seconds": 1.0471, + "completion_tokens": 9, + "decode_duration_seconds": 0.7827, + "decode_tokens_per_second": 11.4982, + "seconds_per_output_token": 0.226016, + "raw_output": "Optimize the dashboard for improved performance.", + "final_output_tokens": 62 + }, + { + "ok": true, + "error": "", + "output": "\nMake the dashboard faster.\n\n\nNo additional context stated.\n\n\n- Make the dashboard faster.\n- Preserve all stated requirements.\n- Do not add unstated requirements.\n\n\nNo output format specified.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 66, + "completion_tokens": 9, + "total_tokens": 75, + "kv_token_occupancy_rate_percentage": 0.2288818359375, + "load_duration": 6e-07, + "prefill_duration_ttft": 1.052340416, + "decoding_duration": 0.781532, + "prefill_speed_tps": 62.7173479194778, + "decoding_speed_tps": 11.515843241223648 + }, + "wall_seconds": 2.023, + "ttft_seconds": 1.0523, + "completion_tokens": 9, + "decode_duration_seconds": 0.7815, + "decode_tokens_per_second": 11.5158, + "seconds_per_output_token": 0.224774, + "raw_output": "Optimize the dashboard for faster performance.", + "final_output_tokens": 62 + }, + { + "ok": true, + "error": "", + "output": "\nMake the dashboard faster.\n\n\nNo additional context stated.\n\n\n- Make the dashboard faster.\n- Preserve all stated requirements.\n- Do not add unstated requirements.\n\n\nNo output format specified.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 66, + "completion_tokens": 9, + "total_tokens": 75, + "kv_token_occupancy_rate_percentage": 0.2288818359375, + "load_duration": 7e-07, + "prefill_duration_ttft": 1.051059904, + "decoding_duration": 0.783688, + "prefill_speed_tps": 62.79375680570154, + "decoding_speed_tps": 11.484162064495054 + }, + "wall_seconds": 2.0337, + "ttft_seconds": 1.0511, + "completion_tokens": 9, + "decode_duration_seconds": 0.7837, + "decode_tokens_per_second": 11.4842, + "seconds_per_output_token": 0.225971, + "raw_output": "Optimize the dashboard for faster performance.", + "final_output_tokens": 62 + }, + { + "ok": true, + "error": "", + "output": "\nMake the dashboard faster.\n\n\nNo additional context stated.\n\n\n- Make the dashboard faster.\n- Preserve all stated requirements.\n- Do not add unstated requirements.\n\n\nNo output format specified.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 66, + "completion_tokens": 9, + "total_tokens": 75, + "kv_token_occupancy_rate_percentage": 0.2288818359375, + "load_duration": 7e-07, + "prefill_duration_ttft": 1.047538432, + "decoding_duration": 0.783653, + "prefill_speed_tps": 63.00484830326492, + "decoding_speed_tps": 11.484674977317766 + }, + "wall_seconds": 2.0317, + "ttft_seconds": 1.0475, + "completion_tokens": 9, + "decode_duration_seconds": 0.7837, + "decode_tokens_per_second": 11.4847, + "seconds_per_output_token": 0.225745, + "raw_output": "Optimize the dashboard for improved performance.", + "final_output_tokens": 62 + } + ], + "speed": { + "runs": 5, + "successful_runs": 5, + "wall_seconds": { + "median": 2.0317, + "p90": 2.0339, + "min": 2.023, + "max": 2.0341 + }, + "ttft_seconds": { + "median": 1.048, + "p90": 1.0518, + "min": 1.0471, + "max": 1.0523 + }, + "completion_tokens": { + "median": 9.0, + "p90": 9.0, + "min": 9.0, + "max": 9.0 + }, + "decode_tokens_per_second": { + "median": 11.4847, + "p90": 11.5088, + "min": 11.4271, + "max": 11.5158 + }, + "seconds_per_output_token": { + "median": 0.2257, + "p90": 0.226, + "min": 0.2248, + "max": 0.226 + } + }, + "representative_output": "\nMake the dashboard faster.\n\n\nNo additional context stated.\n\n\n- Make the dashboard faster.\n- Preserve all stated requirements.\n- Do not add unstated requirements.\n\n\nNo output format specified.\n", + "quality": { + "rubric": { + "r1": true, + "r2": true, + "r3": false, + "r4": true, + "r5": true, + "r6": true, + "r7": true + }, + "score": 6, + "passed": true, + "hard_fail": false, + "pending_judge": false, + "invented_requirement": false, + "constraint_item_count": 3, + "token_count": 62, + "token_count_source": "usage.completion_tokens", + "estimated_tokens": 62, + "judge_notes": "Source is too vague for three concrete requirements; safe scope guards avoid invention." + } + } + } + }, + { + "name": "long_webhook_import", + "category": "long", + "trap": false, + "input": "Add a webhook import endpoint to the existing Python service. It receives JSON events with event_id, account_id, happened_at, and payload. Reject malformed JSON with 400, authenticate using the existing X-Webhook-Key middleware, and make duplicate event_id requests return the original result without inserting twice. Store accepted events with the repository layer already used by the billing importer. Return JSON with id, status, and duplicate. Add focused unit tests and one HTTP integration test. Do not change the database schema or introduce a queue.", + "styles": { + "v1": { + "label": "current", + "max_tokens": 900, + "warmups": 1, + "warmup_errors": [], + "samples": [ + { + "ok": true, + "error": "", + "output": "\nImplement a webhook import endpoint in the existing Python service that validates JSON input, authenticates via X-Webhook-Key middleware, deduplicates events by ID using the current repository layer, and returns a specific JSON response with status indicators.\n\n\nThe application is an existing Python service handling billing imports. A separate HTTP integration test suite already exists for other endpoints. The database schema is fixed and cannot be altered to add new tables or columns. No message queues are available; all processing must happen synchronously within the request lifecycle. The repository layer used by the billing importer should be reused directly without modification or abstraction changes.\n\n\n- Response length: Keep implementation concise (under 150 lines of code).\n- Format: Python file with endpoint definition, middleware integration, deduplication logic, and test cases at the bottom.\n- Tone: Professional technical documentation style for comments; no conversational filler.\n- Tests: Include focused unit tests for JSON validation and duplicate detection; include exactly one HTTP integration test verifying the full flow.\n- Output shape: Markdown code block with `python` syntax highlighting containing the complete implementation file content.\n\n\nMarkdown header (`# Webhook Import Endpoint Implementation`) followed by a single Python code block showing the full service update including endpoint, middleware usage, deduplication logic, and tests.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 225, + "completion_tokens": 281, + "total_tokens": 506, + "kv_token_occupancy_rate_percentage": 1.544189453125, + "load_duration": 6e-07, + "prefill_duration_ttft": 1.28251904, + "decoding_duration": 20.04594, + "prefill_speed_tps": 175.43599196780735, + "decoding_speed_tps": 14.017801110848382 + }, + "wall_seconds": 22.674, + "ttft_seconds": 1.2825, + "completion_tokens": 281, + "decode_duration_seconds": 20.0459, + "decode_tokens_per_second": 14.0178, + "seconds_per_output_token": 0.08069 + }, + { + "ok": true, + "error": "", + "output": "\nImplement a webhook import endpoint in the existing Python service that validates JSON input, authenticates via X-Webhook-Key middleware, deduplicates events by ID using the current repository layer, and returns a specific JSON response.\n\n\nThe application currently uses an existing Python service with a repository layer for billing imports. Authentication is handled by an X-Webhook-Key middleware. The database schema must remain unchanged, and no message queues should be introduced. Events are defined by event_id, account_id, happened_at, and payload fields within JSON requests.\n\n\n- Response length: Keep implementation concise (under 150 lines of code).\n- Format: Python file with endpoint definition at the top followed by test cases below it.\n- Tone: Professional technical documentation style for comments; clear variable names in code.\n- Output format: Markdown block containing the full Python script, including imports, class definitions, and both unit tests (focused on logic) and one HTTP integration test using `requests`.\n\n\n```python\n# Implementation of webhook endpoint with tests below it...\n```", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 225, + "completion_tokens": 231, + "total_tokens": 456, + "kv_token_occupancy_rate_percentage": 1.3916015625, + "load_duration": 1.4e-06, + "prefill_duration_ttft": 1.610471424, + "decoding_duration": 16.444914, + "prefill_speed_tps": 139.71064413000104, + "decoding_speed_tps": 14.04689620146387 + }, + "wall_seconds": 19.1657, + "ttft_seconds": 1.6105, + "completion_tokens": 231, + "decode_duration_seconds": 16.4449, + "decode_tokens_per_second": 14.0469, + "seconds_per_output_token": 0.082968 + }, + { + "ok": true, + "error": "", + "output": "\nImplement a webhook import endpoint in the existing Python service that validates JSON input, authenticates via X-Webhook-Key middleware, deduplicates events by ID using the current repository layer, and returns a specific JSON response.\n\n\nThe service currently handles billing imports through an existing repository layer and uses X-Webhook-Key for authentication. The database schema is fixed and no queuing systems are available. Events must be stored without duplication based on event_id.\n\n\n- Response length: 150 - 250 words total (excluding code blocks).\n- Format: Markdown with a single code block containing the endpoint implementation, tests, and documentation comments.\n- Tone: Technical, concise, imperative.\n- No schema changes or queue introductions allowed in the solution.\n\n\nMarkdown file named `webhook_import.py` (simulated) containing:\n1. Endpoint definition with docstring explaining deduplication logic.\n2. Unit tests for JSON validation and duplicate detection using pytest fixtures.\n3. One HTTP integration test verifying authentication and response structure (`id`, `status`, `duplicate`).", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 225, + "completion_tokens": 237, + "total_tokens": 462, + "kv_token_occupancy_rate_percentage": 1.409912109375, + "load_duration": 7e-07, + "prefill_duration_ttft": 1.264426752, + "decoding_duration": 16.865724, + "prefill_speed_tps": 177.94625085566048, + "decoding_speed_tps": 14.052168765479621 + }, + "wall_seconds": 19.2516, + "ttft_seconds": 1.2644, + "completion_tokens": 237, + "decode_duration_seconds": 16.8657, + "decode_tokens_per_second": 14.0522, + "seconds_per_output_token": 0.08123 + }, + { + "ok": true, + "error": "", + "output": "\nImplement a webhook import endpoint in the existing Python service that validates JSON input, authenticates via X-Webhook-Key middleware, deduplicates events by ID using the current repository layer, and returns a specific JSON response.\n\n\nThe service currently handles billing imports through an existing repository layer and uses an X-Webhook-Key header for authentication. The database schema is fixed and no message queues are available. Events must be stored without duplication based on event_id.\n\n\n- Code length: Maximum 150 lines total (excluding tests).\n- Format: Python file with endpoint, middleware integration, deduplication logic, and response serialization.\n- Tone: Professional, concise implementation notes only; no explanatory fluff in code comments beyond function docstrings.\n- Output shape: Single `.py` file content block ready for copy-paste into a project directory.\n\n\nMarkdown code block containing the complete Python implementation with imports, class definitions (WebhookHandler), and method signatures matching the described behavior.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 225, + "completion_tokens": 216, + "total_tokens": 441, + "kv_token_occupancy_rate_percentage": 1.3458251953125, + "load_duration": 9e-07, + "prefill_duration_ttft": 1.296210816, + "decoding_duration": 15.523699, + "prefill_speed_tps": 173.5828749634504, + "decoding_speed_tps": 13.914209493497651 + }, + "wall_seconds": 17.838, + "ttft_seconds": 1.2962, + "completion_tokens": 216, + "decode_duration_seconds": 15.5237, + "decode_tokens_per_second": 13.9142, + "seconds_per_output_token": 0.082583 + }, + { + "ok": true, + "error": "", + "output": "\nImplement a webhook import endpoint in the existing Python service that validates JSON input, authenticates via X-Webhook-Key middleware, deduplicates events by ID using the current repository layer, and returns a specific JSON response with status indicators.\n\n\nThe service currently handles billing imports through an existing repository layer and uses an X-Webhook-Key header for authentication. The database schema is fixed and no message queues are available. Events must be stored without duplication based on event_id.\n\n\n- Response format: JSON containing `id`, `status` (success/failure), and `duplicate` (boolean).\n- Error handling: Return HTTP 400 for malformed JSON; return existing result for duplicate IDs instead of inserting twice.\n- Testing requirements: Include focused unit tests for logic branches and exactly one HTTP integration test.\n- Scope restrictions: Do not alter database schema or introduce queuing mechanisms.\n\n\nMarkdown code block containing the full implementation file, test files, and a brief README explaining usage.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 225, + "completion_tokens": 216, + "total_tokens": 441, + "kv_token_occupancy_rate_percentage": 1.3458251953125, + "load_duration": 8e-07, + "prefill_duration_ttft": 1.257849856, + "decoding_duration": 15.650229, + "prefill_speed_tps": 178.87667508704632, + "decoding_speed_tps": 13.801714978100321 + }, + "wall_seconds": 17.9323, + "ttft_seconds": 1.2578, + "completion_tokens": 216, + "decode_duration_seconds": 15.6502, + "decode_tokens_per_second": 13.8017, + "seconds_per_output_token": 0.08302 + } + ], + "speed": { + "runs": 5, + "successful_runs": 5, + "wall_seconds": { + "median": 19.1657, + "p90": 21.305, + "min": 17.838, + "max": 22.674 + }, + "ttft_seconds": { + "median": 1.2825, + "p90": 1.4848, + "min": 1.2578, + "max": 1.6105 + }, + "completion_tokens": { + "median": 231.0, + "p90": 263.4, + "min": 216.0, + "max": 281.0 + }, + "decode_tokens_per_second": { + "median": 14.0178, + "p90": 14.0501, + "min": 13.8017, + "max": 14.0522 + }, + "seconds_per_output_token": { + "median": 0.0826, + "p90": 0.083, + "min": 0.0807, + "max": 0.083 + } + }, + "representative_output": "\nImplement a webhook import endpoint in the existing Python service that validates JSON input, authenticates via X-Webhook-Key middleware, deduplicates events by ID using the current repository layer, and returns a specific JSON response with status indicators.\n\n\nThe application is an existing Python service handling billing imports. A separate HTTP integration test suite already exists for other endpoints. The database schema is fixed and cannot be altered to add new tables or columns. No message queues are available; all processing must happen synchronously within the request lifecycle. The repository layer used by the billing importer should be reused directly without modification or abstraction changes.\n\n\n- Response length: Keep implementation concise (under 150 lines of code).\n- Format: Python file with endpoint definition, middleware integration, deduplication logic, and test cases at the bottom.\n- Tone: Professional technical documentation style for comments; no conversational filler.\n- Tests: Include focused unit tests for JSON validation and duplicate detection; include exactly one HTTP integration test verifying the full flow.\n- Output shape: Markdown code block with `python` syntax highlighting containing the complete implementation file content.\n\n\nMarkdown header (`# Webhook Import Endpoint Implementation`) followed by a single Python code block showing the full service update including endpoint, middleware usage, deduplication logic, and tests.", + "quality": { + "rubric": { + "r1": false, + "r2": false, + "r3": false, + "r4": false, + "r5": false, + "r6": false, + "r7": false + }, + "score": 0, + "passed": false, + "hard_fail": true, + "pending_judge": false, + "invented_requirement": true, + "constraint_item_count": 0, + "token_count": 281, + "token_count_source": "usage.completion_tokens", + "estimated_tokens": 260, + "judge_notes": "Invents line limit, file layout, comments, header, synchronous architecture details, and repository constraints." + } + }, + "v2": { + "label": "candidate", + "max_tokens": 320, + "warmups": 1, + "warmup_errors": [], + "samples": [ + { + "ok": true, + "error": "", + "output": "\nAdd a webhook import endpoint to the existing Python service.\n\n\nNo additional context stated.\n\n\n- Add a webhook import endpoint to the existing Python service.\n- It receives JSON events with event_id, account_id, happened_at, and payload.\n- Reject malformed JSON with 400.\n- Authenticate using the existing X-Webhook-Key middleware.\n- Make duplicate event_id requests return the original result without inserting twice; Store accepted events with the repository layer already used by the billing importer; Return JSON with id, status, and duplicate; Add focused unit tests and one HTTP integration test; Do not change the database schema or introduce a queue.\n\n\nA webhook import endpoint.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 169, + "completion_tokens": 44, + "total_tokens": 213, + "kv_token_occupancy_rate_percentage": 0.6500244140625, + "load_duration": 5e-07, + "prefill_duration_ttft": 1.173440896, + "decoding_duration": 3.30354, + "prefill_speed_tps": 144.02088812149256, + "decoding_speed_tps": 13.319045629839506 + }, + "wall_seconds": 4.7706, + "ttft_seconds": 1.1734, + "completion_tokens": 44, + "decode_duration_seconds": 3.3035, + "decode_tokens_per_second": 13.319, + "seconds_per_output_token": 0.108422, + "raw_output": "Add a webhook import endpoint to the existing Python service that validates JSON, authenticates via middleware, deduplicates events using the billing repository, and returns id status and duplicate fields with focused unit tests and one integration test.", + "final_output_tokens": 153 + }, + { + "ok": true, + "error": "", + "output": "\nAdd a webhook import endpoint to the existing Python service.\n\n\nNo additional context stated.\n\n\n- Add a webhook import endpoint to the existing Python service.\n- It receives JSON events with event_id, account_id, happened_at, and payload.\n- Reject malformed JSON with 400.\n- Authenticate using the existing X-Webhook-Key middleware.\n- Make duplicate event_id requests return the original result without inserting twice; Store accepted events with the repository layer already used by the billing importer; Return JSON with id, status, and duplicate; Add focused unit tests and one HTTP integration test; Do not change the database schema or introduce a queue.\n\n\nA webhook import endpoint.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 169, + "completion_tokens": 44, + "total_tokens": 213, + "kv_token_occupancy_rate_percentage": 0.6500244140625, + "load_duration": 8e-07, + "prefill_duration_ttft": 1.211843328, + "decoding_duration": 3.282744, + "prefill_speed_tps": 139.4569711242409, + "decoding_speed_tps": 13.40342104044665 + }, + "wall_seconds": 4.7775, + "ttft_seconds": 1.2118, + "completion_tokens": 44, + "decode_duration_seconds": 3.2827, + "decode_tokens_per_second": 13.4034, + "seconds_per_output_token": 0.10858, + "raw_output": "Add a webhook import endpoint to the existing Python service that validates JSON, authenticates via middleware, deduplicates events using the billing repository, and returns id status and duplicate fields with focused unit tests and one integration test.", + "final_output_tokens": 153 + }, + { + "ok": true, + "error": "", + "output": "\nAdd a webhook import endpoint to the existing Python service.\n\n\nNo additional context stated.\n\n\n- Add a webhook import endpoint to the existing Python service.\n- It receives JSON events with event_id, account_id, happened_at, and payload.\n- Reject malformed JSON with 400.\n- Authenticate using the existing X-Webhook-Key middleware.\n- Make duplicate event_id requests return the original result without inserting twice; Store accepted events with the repository layer already used by the billing importer; Return JSON with id, status, and duplicate; Add focused unit tests and one HTTP integration test; Do not change the database schema or introduce a queue.\n\n\nA webhook import endpoint.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 169, + "completion_tokens": 44, + "total_tokens": 213, + "kv_token_occupancy_rate_percentage": 0.6500244140625, + "load_duration": 6e-07, + "prefill_duration_ttft": 1.161394816, + "decoding_duration": 3.277772, + "prefill_speed_tps": 145.51468430181112, + "decoding_speed_tps": 13.423752475767076 + }, + "wall_seconds": 4.7061, + "ttft_seconds": 1.1614, + "completion_tokens": 44, + "decode_duration_seconds": 3.2778, + "decode_tokens_per_second": 13.4238, + "seconds_per_output_token": 0.106958, + "raw_output": "Add a webhook import endpoint to the existing Python service that validates JSON, authenticates via middleware, deduplicates events using the billing repository, and returns id/status/duplicate fields with focused unit tests and one integration test.", + "final_output_tokens": 153 + }, + { + "ok": true, + "error": "", + "output": "\nAdd a webhook import endpoint to the existing Python service.\n\n\nNo additional context stated.\n\n\n- Add a webhook import endpoint to the existing Python service.\n- It receives JSON events with event_id, account_id, happened_at, and payload.\n- Reject malformed JSON with 400.\n- Authenticate using the existing X-Webhook-Key middleware.\n- Make duplicate event_id requests return the original result without inserting twice; Store accepted events with the repository layer already used by the billing importer; Return JSON with id, status, and duplicate; Add focused unit tests and one HTTP integration test; Do not change the database schema or introduce a queue.\n\n\nA webhook import endpoint.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 169, + "completion_tokens": 44, + "total_tokens": 213, + "kv_token_occupancy_rate_percentage": 0.6500244140625, + "load_duration": 9e-07, + "prefill_duration_ttft": 1.15836544, + "decoding_duration": 3.279962, + "prefill_speed_tps": 145.89523665346923, + "decoding_speed_tps": 13.414789561586385 + }, + "wall_seconds": 4.7124, + "ttft_seconds": 1.1584, + "completion_tokens": 44, + "decode_duration_seconds": 3.28, + "decode_tokens_per_second": 13.4148, + "seconds_per_output_token": 0.107101, + "raw_output": "Add a webhook import endpoint to the existing Python service that validates JSON, authenticates via middleware, deduplicates events using the billing repository, and returns id/status/duplicate fields with focused unit tests and one integration test.", + "final_output_tokens": 153 + }, + { + "ok": true, + "error": "", + "output": "\nAdd a webhook import endpoint to the existing Python service.\n\n\nNo additional context stated.\n\n\n- Add a webhook import endpoint to the existing Python service.\n- It receives JSON events with event_id, account_id, happened_at, and payload.\n- Reject malformed JSON with 400.\n- Authenticate using the existing X-Webhook-Key middleware.\n- Make duplicate event_id requests return the original result without inserting twice; Store accepted events with the repository layer already used by the billing importer; Return JSON with id, status, and duplicate; Add focused unit tests and one HTTP integration test; Do not change the database schema or introduce a queue.\n\n\nA webhook import endpoint.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 169, + "completion_tokens": 48, + "total_tokens": 217, + "kv_token_occupancy_rate_percentage": 0.6622314453125, + "load_duration": 5e-07, + "prefill_duration_ttft": 1.16083584, + "decoding_duration": 3.572892, + "prefill_speed_tps": 145.5847538270355, + "decoding_speed_tps": 13.434495081295488 + }, + "wall_seconds": 5.0146, + "ttft_seconds": 1.1608, + "completion_tokens": 48, + "decode_duration_seconds": 3.5729, + "decode_tokens_per_second": 13.4345, + "seconds_per_output_token": 0.104471, + "raw_output": "Add a webhook import endpoint to the existing Python service that validates JSON, authenticates via middleware, deduplicates events using the billing repository, and returns id/status/duplicate in focused unit tests plus one integration test without schema changes or queues.", + "final_output_tokens": 153 + } + ], + "speed": { + "runs": 5, + "successful_runs": 5, + "wall_seconds": { + "median": 4.7706, + "p90": 4.9198, + "min": 4.7061, + "max": 5.0146 + }, + "ttft_seconds": { + "median": 1.1614, + "p90": 1.1964, + "min": 1.1584, + "max": 1.2118 + }, + "completion_tokens": { + "median": 44.0, + "p90": 46.4, + "min": 44.0, + "max": 48.0 + }, + "decode_tokens_per_second": { + "median": 13.4148, + "p90": 13.4302, + "min": 13.319, + "max": 13.4345 + }, + "seconds_per_output_token": { + "median": 0.1071, + "p90": 0.1085, + "min": 0.1045, + "max": 0.1086 + } + }, + "representative_output": "\nAdd a webhook import endpoint to the existing Python service.\n\n\nNo additional context stated.\n\n\n- Add a webhook import endpoint to the existing Python service.\n- It receives JSON events with event_id, account_id, happened_at, and payload.\n- Reject malformed JSON with 400.\n- Authenticate using the existing X-Webhook-Key middleware.\n- Make duplicate event_id requests return the original result without inserting twice; Store accepted events with the repository layer already used by the billing importer; Return JSON with id, status, and duplicate; Add focused unit tests and one HTTP integration test; Do not change the database schema or introduce a queue.\n\n\nA webhook import endpoint.\n", + "quality": { + "rubric": { + "r1": true, + "r2": true, + "r3": true, + "r4": true, + "r5": true, + "r6": true, + "r7": true + }, + "score": 7, + "passed": true, + "hard_fail": false, + "pending_judge": false, + "invented_requirement": false, + "constraint_item_count": 5, + "token_count": 153, + "token_count_source": "usage.completion_tokens", + "estimated_tokens": 153, + "judge_notes": "Five bullets cover all explicit endpoint requirements; final bullet groups overflow without adding facts." + } + } + } + }, + { + "name": "trap_rename_cli", + "category": "trap", + "trap": true, + "input": "write a CLI that renames files from a mapping in a text file", + "styles": { + "v1": { + "label": "current", + "max_tokens": 700, + "warmups": 1, + "warmup_errors": [], + "samples": [ + { + "ok": true, + "error": "", + "output": "\nGenerate a Python CLI script that reads a file mapping and renames target files accordingly.\n\n\nThe user provides an input text file containing lines in the format \"source:target\" or similar key-value pairs defining filename mappings. The tool must parse this configuration, locate existing source files, and rename them to their corresponding targets if they exist. No external dependencies are required beyond standard library modules like `argparse`, `os`, and `sys`.\n\n\n- Output length: approximately 100 lines of code plus a brief usage example.\n- Format: A single Python script with a shebang line, followed by argument parsing logic, file reading/writing functions, and the main execution block.\n- Tone: Professional and concise; avoid excessive comments or verbose explanations.\n- Testability: The solution must be runnable via `python rename_files.py` with arguments like `--input mappings.txt --output-dir ./`.\n\n\nMarkdown code block containing the Python script, followed by a separate Markdown section showing example usage commands.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 132, + "completion_tokens": 221, + "total_tokens": 353, + "kv_token_occupancy_rate_percentage": 1.0772705078125, + "load_duration": 8e-07, + "prefill_duration_ttft": 1.270931328, + "decoding_duration": 15.870487, + "prefill_speed_tps": 103.86084369146968, + "decoding_speed_tps": 13.925218551894469 + }, + "wall_seconds": 18.2086, + "ttft_seconds": 1.2709, + "completion_tokens": 221, + "decode_duration_seconds": 15.8705, + "decode_tokens_per_second": 13.9252, + "seconds_per_output_token": 0.082392 + }, + { + "ok": true, + "error": "", + "output": "\nGenerate a Python CLI script that reads a file mapping and renames target files accordingly.\n\n\nThe user provides an input text file containing lines in the format \"source:target\" or similar key-value pairs defining filename mappings. The tool must parse this configuration, locate existing source files in the current directory (or specified path), and rename them to their corresponding targets if they exist. No external libraries beyond standard Python are required.\n\n\n- Output must be a single executable Python script with no dependencies outside `os`, `sys`, and `argparse`.\n- The CLI must accept at least two arguments: the input mapping file path and an optional output directory (defaults to current working directory).\n- Error handling should gracefully skip non-existent source files without crashing.\n- Code length must be under 150 lines.\n- Comments explaining the logic are required but minimal.\n\n\nMarkdown code block with language set to `python`. Include a brief usage example below the script using standard markdown headers (`##`).", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 132, + "completion_tokens": 221, + "total_tokens": 353, + "kv_token_occupancy_rate_percentage": 1.0772705078125, + "load_duration": 9e-07, + "prefill_duration_ttft": 1.153891712, + "decoding_duration": 15.76361, + "prefill_speed_tps": 114.39548323924524, + "decoding_speed_tps": 14.019631290040795 + }, + "wall_seconds": 17.9677, + "ttft_seconds": 1.1539, + "completion_tokens": 221, + "decode_duration_seconds": 15.7636, + "decode_tokens_per_second": 14.0196, + "seconds_per_output_token": 0.081302 + }, + { + "ok": true, + "error": "", + "output": "\nGenerate a Python CLI script that reads a file mapping and renames target files accordingly.\n\n\nThe user provides an input text file where each line contains \"source:target\" pairs separated by colons, defining the desired filename for existing files in the current directory. The script must verify if source files exist before attempting to rename them and should handle errors gracefully without crashing on missing entries.\n\n\n- Output a single Python file named `rename_files.py`.\n- Use standard library only (`argparse`, `os`, `sys`).\n- Include usage instructions printed to stderr or stdout upon execution.\n- Do not include external dependencies, tests, or documentation files in the output.\n\n\nMarkdown code block with language set to python containing the full script content.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 132, + "completion_tokens": 169, + "total_tokens": 301, + "kv_token_occupancy_rate_percentage": 0.9185791015625, + "load_duration": 8e-07, + "prefill_duration_ttft": 1.161757696, + "decoding_duration": 12.279122, + "prefill_speed_tps": 113.62093873316591, + "decoding_speed_tps": 13.763199030028368 + }, + "wall_seconds": 14.3104, + "ttft_seconds": 1.1618, + "completion_tokens": 169, + "decode_duration_seconds": 12.2791, + "decode_tokens_per_second": 13.7632, + "seconds_per_output_token": 0.084677 + }, + { + "ok": true, + "error": "", + "output": "\nGenerate a Python CLI script that reads a file mapping and renames target files accordingly.\n\n\nThe user provides an input text file containing lines in the format \"source_path destination_name\". The script must parse this file, locate existing source files on the filesystem, and rename them to their corresponding destinations if they exist. No additional features like dry-run modes or error reporting are requested beyond basic functionality.\n\n\n- Output length: 100-250 words.\n- Format: Single Python script with a `main()` function entry point.\n- Tone: Direct, functional code without extensive comments explaining obvious logic.\n- Language: English (Python).\n\n\nMarkdown code block containing the complete Python source file named `file_renamer.py`.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 132, + "completion_tokens": 167, + "total_tokens": 299, + "kv_token_occupancy_rate_percentage": 0.9124755859375, + "load_duration": 8e-07, + "prefill_duration_ttft": 1.185564928, + "decoding_duration": 12.459702, + "prefill_speed_tps": 111.33932598923845, + "decoding_speed_tps": 13.403209803894185 + }, + "wall_seconds": 14.4573, + "ttft_seconds": 1.1856, + "completion_tokens": 167, + "decode_duration_seconds": 12.4597, + "decode_tokens_per_second": 13.4032, + "seconds_per_output_token": 0.086571 + }, + { + "ok": true, + "error": "", + "output": "\nGenerate a Python CLI script that reads a file mapping and renames target files accordingly.\n\n\nThe user provides an input text file containing lines in the format \"source:target\" or similar key-value pairs defining filename mappings. The tool must parse this configuration, locate existing source files, and rename them to their corresponding targets if they exist. No external dependencies are required beyond standard library modules like `argparse`, `os`, and `sys`.\n\n\n- Length: Keep the script under 50 lines of code (excluding comments).\n- Format: Use a single Python file with no shebang line, utilizing only standard libraries.\n- Tone: Professional and concise; include brief usage examples in docstrings but avoid verbose explanations.\n- Output format: Return the complete source code block wrapped in triple backticks with `python` syntax highlighting.\n\n\nMarkdown code block containing the full Python script.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 132, + "completion_tokens": 196, + "total_tokens": 328, + "kv_token_occupancy_rate_percentage": 1.0009765625, + "load_duration": 8e-07, + "prefill_duration_ttft": 1.182166656, + "decoding_duration": 14.416469, + "prefill_speed_tps": 111.6593834972791, + "decoding_speed_tps": 13.595562131059971 + }, + "wall_seconds": 16.5712, + "ttft_seconds": 1.1822, + "completion_tokens": 196, + "decode_duration_seconds": 14.4165, + "decode_tokens_per_second": 13.5956, + "seconds_per_output_token": 0.084547 + } + ], + "speed": { + "runs": 5, + "successful_runs": 5, + "wall_seconds": { + "median": 16.5712, + "p90": 18.1122, + "min": 14.3104, + "max": 18.2086 + }, + "ttft_seconds": { + "median": 1.1822, + "p90": 1.2368, + "min": 1.1539, + "max": 1.2709 + }, + "completion_tokens": { + "median": 196.0, + "p90": 221.0, + "min": 167.0, + "max": 221.0 + }, + "decode_tokens_per_second": { + "median": 13.7632, + "p90": 13.9818, + "min": 13.4032, + "max": 14.0196 + }, + "seconds_per_output_token": { + "median": 0.0845, + "p90": 0.0858, + "min": 0.0813, + "max": 0.0866 + } + }, + "representative_output": "\nGenerate a Python CLI script that reads a file mapping and renames target files accordingly.\n\n\nThe user provides an input text file containing lines in the format \"source:target\" or similar key-value pairs defining filename mappings. The tool must parse this configuration, locate existing source files, and rename them to their corresponding targets if they exist. No external dependencies are required beyond standard library modules like `argparse`, `os`, and `sys`.\n\n\n- Output length: approximately 100 lines of code plus a brief usage example.\n- Format: A single Python script with a shebang line, followed by argument parsing logic, file reading/writing functions, and the main execution block.\n- Tone: Professional and concise; avoid excessive comments or verbose explanations.\n- Testability: The solution must be runnable via `python rename_files.py` with arguments like `--input mappings.txt --output-dir ./`.\n\n\nMarkdown code block containing the Python script, followed by a separate Markdown section showing example usage commands.", + "quality": { + "rubric": { + "r1": false, + "r2": false, + "r3": false, + "r4": false, + "r5": false, + "r6": false, + "r7": false + }, + "score": 0, + "passed": false, + "hard_fail": true, + "pending_judge": false, + "invented_requirement": true, + "constraint_item_count": 0, + "token_count": 221, + "token_count_source": "usage.completion_tokens", + "estimated_tokens": 213, + "judge_notes": "Invents Python, mapping syntax, dependencies, arguments, output directory, filename, and usage format." + } + }, + "v2": { + "label": "candidate", + "max_tokens": 240, + "warmups": 1, + "warmup_errors": [], + "samples": [ + { + "ok": true, + "error": "", + "output": "\nWrite a CLI that renames files from a mapping in a text file.\n\n\nNo additional context stated.\n\n\n- Write a CLI.\n- Rename files from a mapping in a text file.\n- Preserve all stated requirements.\n\n\nA CLI.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 76, + "completion_tokens": 17, + "total_tokens": 93, + "kv_token_occupancy_rate_percentage": 0.2838134765625, + "load_duration": 5e-07, + "prefill_duration_ttft": 1.060802688, + "decoding_duration": 1.351949, + "prefill_speed_tps": 71.64386069127306, + "decoding_speed_tps": 12.574438828683626 + }, + "wall_seconds": 2.6281, + "ttft_seconds": 1.0608, + "completion_tokens": 17, + "decode_duration_seconds": 1.3519, + "decode_tokens_per_second": 12.5744, + "seconds_per_output_token": 0.154595, + "raw_output": "Write a CLI that renames files based on a mapping in a text file.", + "final_output_tokens": 72 + }, + { + "ok": true, + "error": "", + "output": "\nWrite a CLI that renames files from a mapping in a text file.\n\n\nNo additional context stated.\n\n\n- Write a CLI.\n- Rename files from a mapping in a text file.\n- Preserve all stated requirements.\n\n\nA CLI.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 76, + "completion_tokens": 16, + "total_tokens": 92, + "kv_token_occupancy_rate_percentage": 0.28076171875, + "load_duration": 8e-07, + "prefill_duration_ttft": 1.056690304, + "decoding_duration": 1.281896, + "prefill_speed_tps": 71.92268133085851, + "decoding_speed_tps": 12.481511760704457 + }, + "wall_seconds": 2.5468, + "ttft_seconds": 1.0567, + "completion_tokens": 16, + "decode_duration_seconds": 1.2819, + "decode_tokens_per_second": 12.4815, + "seconds_per_output_token": 0.159177, + "raw_output": "Write a CLI that renames files based on mappings in a text file.", + "final_output_tokens": 72 + }, + { + "ok": true, + "error": "", + "output": "\nWrite a CLI that renames files from a mapping in a text file.\n\n\nNo additional context stated.\n\n\n- Write a CLI.\n- Rename files from a mapping in a text file.\n- Preserve all stated requirements.\n\n\nA CLI.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 76, + "completion_tokens": 16, + "total_tokens": 92, + "kv_token_occupancy_rate_percentage": 0.28076171875, + "load_duration": 8e-07, + "prefill_duration_ttft": 1.131524736, + "decoding_duration": 1.280794, + "prefill_speed_tps": 67.1660084680641, + "decoding_speed_tps": 12.492250900613213 + }, + "wall_seconds": 2.6212, + "ttft_seconds": 1.1315, + "completion_tokens": 16, + "decode_duration_seconds": 1.2808, + "decode_tokens_per_second": 12.4923, + "seconds_per_output_token": 0.163828, + "raw_output": "Write a CLI that renames files based on mappings in a text file.", + "final_output_tokens": 72 + }, + { + "ok": true, + "error": "", + "output": "\nWrite a CLI that renames files from a mapping in a text file.\n\n\nNo additional context stated.\n\n\n- Write a CLI.\n- Rename files from a mapping in a text file.\n- Preserve all stated requirements.\n\n\nA CLI.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 76, + "completion_tokens": 16, + "total_tokens": 92, + "kv_token_occupancy_rate_percentage": 0.28076171875, + "load_duration": 6e-07, + "prefill_duration_ttft": 1.057487232, + "decoding_duration": 1.280692, + "prefill_speed_tps": 71.86848001584194, + "decoding_speed_tps": 12.493245838968306 + }, + "wall_seconds": 2.5561, + "ttft_seconds": 1.0575, + "completion_tokens": 16, + "decode_duration_seconds": 1.2807, + "decode_tokens_per_second": 12.4932, + "seconds_per_output_token": 0.159756, + "raw_output": "Write a CLI that renames files based on mappings in a text file.", + "final_output_tokens": 72 + }, + { + "ok": true, + "error": "", + "output": "\nWrite a CLI that renames files from a mapping in a text file.\n\n\nNo additional context stated.\n\n\n- Write a CLI.\n- Rename files from a mapping in a text file.\n- Preserve all stated requirements.\n\n\nA CLI.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 76, + "completion_tokens": 16, + "total_tokens": 92, + "kv_token_occupancy_rate_percentage": 0.28076171875, + "load_duration": 6e-07, + "prefill_duration_ttft": 1.060747008, + "decoding_duration": 1.283618, + "prefill_speed_tps": 71.64762137137227, + "decoding_speed_tps": 12.46476755545653 + }, + "wall_seconds": 2.5723, + "ttft_seconds": 1.0607, + "completion_tokens": 16, + "decode_duration_seconds": 1.2836, + "decode_tokens_per_second": 12.4648, + "seconds_per_output_token": 0.160769, + "raw_output": "Write a CLI that renames files based on mappings in a text file.", + "final_output_tokens": 72 + } + ], + "speed": { + "runs": 5, + "successful_runs": 5, + "wall_seconds": { + "median": 2.5723, + "p90": 2.6253, + "min": 2.5468, + "max": 2.6281 + }, + "ttft_seconds": { + "median": 1.0607, + "p90": 1.1032, + "min": 1.0567, + "max": 1.1315 + }, + "completion_tokens": { + "median": 16.0, + "p90": 16.6, + "min": 16.0, + "max": 17.0 + }, + "decode_tokens_per_second": { + "median": 12.4923, + "p90": 12.5419, + "min": 12.4648, + "max": 12.5744 + }, + "seconds_per_output_token": { + "median": 0.1598, + "p90": 0.1626, + "min": 0.1546, + "max": 0.1638 + } + }, + "representative_output": "\nWrite a CLI that renames files from a mapping in a text file.\n\n\nNo additional context stated.\n\n\n- Write a CLI.\n- Rename files from a mapping in a text file.\n- Preserve all stated requirements.\n\n\nA CLI.\n", + "quality": { + "rubric": { + "r1": true, + "r2": true, + "r3": true, + "r4": true, + "r5": true, + "r6": true, + "r7": true + }, + "score": 7, + "passed": true, + "hard_fail": false, + "pending_judge": false, + "invented_requirement": false, + "constraint_item_count": 3, + "token_count": 72, + "token_count_source": "usage.completion_tokens", + "estimated_tokens": 72, + "judge_notes": "CLI, rename behavior, and text mapping only; no conventional CLI assumptions." + } + } + } + }, + { + "name": "plan_postgres_migration", + "category": "plan", + "trap": false, + "input": "plan a zero-downtime migration from users.full_name to first_name and last_name for a Postgres app; planning only, no code changes", + "styles": { + "v1": { + "label": "current", + "max_tokens": 700, + "warmups": 1, + "warmup_errors": [], + "samples": [ + { + "ok": true, + "error": "", + "output": "\nGenerate a zero-downtime migration plan for renaming the `users.full_name` column to `first_name` and `last_name` in a PostgreSQL database.\n\n\nThe application uses PostgreSQL. The goal is to split the existing single-column full name into two separate columns without disrupting active services or causing data loss during execution. No code changes are required; only database schema planning is needed.\n\n\n- Length: Maximum 500 words total, excluding headers and lists.\n- Format: Numbered steps for migration phases (Preparation, Execution, Validation).\n- Tone: Professional technical documentation suitable for a DevOps lead.\n- Content requirements must include data validation checks and rollback procedures if execution fails mid-process.\n\n\nMarkdown with clear section headers (e.g., ## Phase 1), bullet points for specific actions within phases, and a final \"Rollback Plan\" subsection under Execution or Validation.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 147, + "completion_tokens": 198, + "total_tokens": 345, + "kv_token_occupancy_rate_percentage": 1.0528564453125, + "load_duration": 5e-07, + "prefill_duration_ttft": 1.166185088, + "decoding_duration": 14.427471, + "prefill_speed_tps": 126.05203197384736, + "decoding_speed_tps": 13.723818956212076 + }, + "wall_seconds": 16.5508, + "ttft_seconds": 1.1662, + "completion_tokens": 198, + "decode_duration_seconds": 14.4275, + "decode_tokens_per_second": 13.7238, + "seconds_per_output_token": 0.08359 + }, + { + "ok": true, + "error": "", + "output": "\nGenerate a zero-downtime migration plan for renaming the `users.full_name` column to `first_name` and `last_name` in a PostgreSQL database.\n\n\nThe application is built on PostgreSQL. The goal is to split the existing single-column field into two separate columns without disrupting service availability during execution. No code changes are required; only database schema modifications are needed.\n\n\n- Length: Maximum 500 words total, excluding headers and lists.\n- Format: Numbered steps with clear prerequisites for each phase (Preparation, Execution, Verification).\n- Tone: Professional technical documentation suitable for a DevOps engineer or DBA.\n- Content requirements: Must explicitly mention locking strategies to prevent downtime, data validation checks before migration execution, and rollback procedures if integrity issues arise.\n\n\nMarkdown with clear section headers (e.g., ## Phase 1). Use bullet points for sub-steps within phases.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 147, + "completion_tokens": 200, + "total_tokens": 347, + "kv_token_occupancy_rate_percentage": 1.0589599609375, + "load_duration": 7e-07, + "prefill_duration_ttft": 1.164395776, + "decoding_duration": 14.282586, + "prefill_speed_tps": 126.24573450874489, + "decoding_speed_tps": 14.003066391478406 + }, + "wall_seconds": 16.4292, + "ttft_seconds": 1.1644, + "completion_tokens": 200, + "decode_duration_seconds": 14.2826, + "decode_tokens_per_second": 14.0031, + "seconds_per_output_token": 0.082146 + }, + { + "ok": true, + "error": "", + "output": "\nGenerate a zero-downtime migration plan for renaming the `users.full_name` column to `first_name` and `last_name` in a PostgreSQL database.\n\n\nThe application uses PostgreSQL. The goal is to split the existing single-column full name into two separate columns without disrupting active services or causing downtime during execution. No code changes are required; only database schema modifications are needed.\n\n\n- Length: Keep the plan concise, under 500 words.\n- Format: Use a numbered list for steps and bold text for key actions (e.g., **CREATE EXTENSION**, **ALTER TABLE**).\n- Tone: Professional, technical, and directive.\n- Content Requirements: Must include prerequisites (extensions), data migration strategy (copying values), validation checks, rollback procedures if necessary, and final cleanup.\n\n\nMarkdown with a clear title `# Zero-Downtime Migration Plan` followed by the numbered steps.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 147, + "completion_tokens": 202, + "total_tokens": 349, + "kv_token_occupancy_rate_percentage": 1.0650634765625, + "load_duration": 6e-07, + "prefill_duration_ttft": 1.166479744, + "decoding_duration": 14.484612, + "prefill_speed_tps": 126.02019088297172, + "decoding_speed_tps": 13.945834379270911 + }, + "wall_seconds": 16.6204, + "ttft_seconds": 1.1665, + "completion_tokens": 202, + "decode_duration_seconds": 14.4846, + "decode_tokens_per_second": 13.9458, + "seconds_per_output_token": 0.082279 + }, + { + "ok": true, + "error": "", + "output": "\nGenerate a zero-downtime migration plan for renaming the `users.full_name` column to `first_name` and `last_name` in a PostgreSQL database.\n\n\nThe application uses PostgreSQL. The goal is to split the existing single-column full name into two separate columns without disrupting active services or causing data loss during execution. No code changes are required; only database schema planning is needed.\n\n\n- Length: Maximum 500 words total, excluding headers and lists.\n- Format: Numbered steps with clear prerequisites, actions, and verification points for each phase (Preparation, Execution, Validation).\n- Tone: Professional technical documentation suitable for a DevOps engineer or DBA.\n\n\nMarkdown text with numbered sections labeled \"Phase 1\", \"Phase 2\", etc., followed by a bulleted list of rollback procedures if applicable.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 147, + "completion_tokens": 186, + "total_tokens": 333, + "kv_token_occupancy_rate_percentage": 1.0162353515625, + "load_duration": 9e-07, + "prefill_duration_ttft": 1.193710976, + "decoding_duration": 13.333325, + "prefill_speed_tps": 123.14538691147963, + "decoding_speed_tps": 13.950008718755448 + }, + "wall_seconds": 15.4323, + "ttft_seconds": 1.1937, + "completion_tokens": 186, + "decode_duration_seconds": 13.3333, + "decode_tokens_per_second": 13.95, + "seconds_per_output_token": 0.082969 + }, + { + "ok": true, + "error": "", + "output": "\nGenerate a zero-downtime migration plan for renaming the `users.full_name` column to `first_name` and `last_name` in a PostgreSQL database.\n\n\nThe application uses PostgreSQL. The goal is to split the existing single-column full name into two separate columns without disrupting active services or causing downtime during execution. This is a planning exercise only; no actual SQL scripts, code changes, or deployment steps are being executed yet.\n\n\n- Length: Keep the plan concise (under 500 words).\n- Format: Use Markdown with clear numbered sections for each phase of the migration.\n- Tone: Professional and technical, suitable for a DevOps lead review.\n- Content Requirements: Must include pre-migration checks, data validation steps, execution strategy (e.g., application-level routing or temporary views), rollback procedures, and post-migration verification. Do not invent specific database names beyond what is provided in the context.\n\n\nMarkdown document with headers like `## Phase 1`, `## Phase 2`, etc. No JSON or code blocks unless illustrating a conceptual step (e.g., a pseudo-SQL comment).", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 147, + "completion_tokens": 244, + "total_tokens": 391, + "kv_token_occupancy_rate_percentage": 1.1932373046875, + "load_duration": 1.3e-06, + "prefill_duration_ttft": 1.471837696, + "decoding_duration": 17.388704, + "prefill_speed_tps": 99.87514275487071, + "decoding_speed_tps": 14.032098079304816 + }, + "wall_seconds": 20.0138, + "ttft_seconds": 1.4718, + "completion_tokens": 244, + "decode_duration_seconds": 17.3887, + "decode_tokens_per_second": 14.0321, + "seconds_per_output_token": 0.082024 + } + ], + "speed": { + "runs": 5, + "successful_runs": 5, + "wall_seconds": { + "median": 16.5508, + "p90": 18.6564, + "min": 15.4323, + "max": 20.0138 + }, + "ttft_seconds": { + "median": 1.1665, + "p90": 1.3606, + "min": 1.1644, + "max": 1.4718 + }, + "completion_tokens": { + "median": 200.0, + "p90": 227.2, + "min": 186.0, + "max": 244.0 + }, + "decode_tokens_per_second": { + "median": 13.95, + "p90": 14.0205, + "min": 13.7238, + "max": 14.0321 + }, + "seconds_per_output_token": { + "median": 0.0823, + "p90": 0.0833, + "min": 0.082, + "max": 0.0836 + } + }, + "representative_output": "\nGenerate a zero-downtime migration plan for renaming the `users.full_name` column to `first_name` and `last_name` in a PostgreSQL database.\n\n\nThe application uses PostgreSQL. The goal is to split the existing single-column full name into two separate columns without disrupting active services or causing data loss during execution. No code changes are required; only database schema planning is needed.\n\n\n- Length: Maximum 500 words total, excluding headers and lists.\n- Format: Numbered steps for migration phases (Preparation, Execution, Validation).\n- Tone: Professional technical documentation suitable for a DevOps lead.\n- Content requirements must include data validation checks and rollback procedures if execution fails mid-process.\n\n\nMarkdown with clear section headers (e.g., ## Phase 1), bullet points for specific actions within phases, and a final \"Rollback Plan\" subsection under Execution or Validation.", + "quality": { + "rubric": { + "r1": false, + "r2": false, + "r3": false, + "r4": false, + "r5": false, + "r6": false, + "r7": true + }, + "score": 1, + "passed": false, + "hard_fail": true, + "pending_judge": false, + "invented_requirement": true, + "constraint_item_count": 0, + "token_count": 198, + "token_count_source": "usage.completion_tokens", + "estimated_tokens": 183, + "judge_notes": "Invents word limit, phase format, DevOps audience, validation checks, and rollback content." + } + }, + "v2": { + "label": "candidate", + "max_tokens": 240, + "warmups": 1, + "warmup_errors": [], + "samples": [ + { + "ok": true, + "error": "", + "output": "\nPlan a zero-downtime migration from users.full_name to first_name and last_name for a Postgres app; planning only, no code changes.\n\n\nNo additional context stated.\n\n\n- Plan a zero-downtime migration from users.full_name to first_name and last_name for a Postgres app; planning only, no code changes.\n- Preserve all stated requirements.\n- Do not add unstated requirements.\n\n\nPlan.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 91, + "completion_tokens": 24, + "total_tokens": 115, + "kv_token_occupancy_rate_percentage": 0.3509521484375, + "load_duration": 5e-07, + "prefill_duration_ttft": 1.098410752, + "decoding_duration": 1.851491, + "prefill_speed_tps": 82.8469676159907, + "decoding_speed_tps": 12.962525877792546 + }, + "wall_seconds": 3.1903, + "ttft_seconds": 1.0984, + "completion_tokens": 24, + "decode_duration_seconds": 1.8515, + "decode_tokens_per_second": 12.9625, + "seconds_per_output_token": 0.132928, + "raw_output": "Plan a zero-downtime migration from users.full_name to first_name and last_name for a Postgres app.", + "final_output_tokens": 101 + }, + { + "ok": true, + "error": "", + "output": "\nPlan a zero-downtime migration from users.full_name to first_name and last_name for a Postgres app; planning only, no code changes.\n\n\nNo additional context stated.\n\n\n- Plan a zero-downtime migration from users.full_name to first_name and last_name for a Postgres app; planning only, no code changes.\n- Preserve all stated requirements.\n- Do not add unstated requirements.\n\n\nPlan.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 91, + "completion_tokens": 24, + "total_tokens": 115, + "kv_token_occupancy_rate_percentage": 0.3509521484375, + "load_duration": 4e-07, + "prefill_duration_ttft": 1.112928512, + "decoding_duration": 1.852187, + "prefill_speed_tps": 81.76625813680296, + "decoding_speed_tps": 12.957654923611925 + }, + "wall_seconds": 3.1914, + "ttft_seconds": 1.1129, + "completion_tokens": 24, + "decode_duration_seconds": 1.8522, + "decode_tokens_per_second": 12.9577, + "seconds_per_output_token": 0.132975, + "raw_output": "Plan a zero-downtime migration from users.full_name to first_name and last_name for a Postgres app.", + "final_output_tokens": 101 + }, + { + "ok": true, + "error": "", + "output": "\nPlan a zero-downtime migration from users.full_name to first_name and last_name for a Postgres app; planning only, no code changes.\n\n\nNo additional context stated.\n\n\n- Plan a zero-downtime migration from users.full_name to first_name and last_name for a Postgres app; planning only, no code changes.\n- Preserve all stated requirements.\n- Do not add unstated requirements.\n\n\nPlan.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 91, + "completion_tokens": 24, + "total_tokens": 115, + "kv_token_occupancy_rate_percentage": 0.3509521484375, + "load_duration": 5e-07, + "prefill_duration_ttft": 1.075477376, + "decoding_duration": 1.847139, + "prefill_speed_tps": 84.61358837547503, + "decoding_speed_tps": 12.993066574848998 + }, + "wall_seconds": 3.1694, + "ttft_seconds": 1.0755, + "completion_tokens": 24, + "decode_duration_seconds": 1.8471, + "decode_tokens_per_second": 12.9931, + "seconds_per_output_token": 0.132058, + "raw_output": "Plan a zero-downtime migration from users.full_name to first_name and last_name for a Postgres app.", + "final_output_tokens": 101 + }, + { + "ok": true, + "error": "", + "output": "\nPlan a zero-downtime migration from users.full_name to first_name and last_name for a Postgres app; planning only, no code changes.\n\n\nNo additional context stated.\n\n\n- Plan a zero-downtime migration from users.full_name to first_name and last_name for a Postgres app; planning only, no code changes.\n- Preserve all stated requirements.\n- Do not add unstated requirements.\n\n\nPlan.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 91, + "completion_tokens": 24, + "total_tokens": 115, + "kv_token_occupancy_rate_percentage": 0.3509521484375, + "load_duration": 5e-07, + "prefill_duration_ttft": 1.12284864, + "decoding_duration": 1.849138, + "prefill_speed_tps": 81.04387070371301, + "decoding_speed_tps": 12.979020494954947 + }, + "wall_seconds": 3.1976, + "ttft_seconds": 1.1228, + "completion_tokens": 24, + "decode_duration_seconds": 1.8491, + "decode_tokens_per_second": 12.979, + "seconds_per_output_token": 0.133232, + "raw_output": "Plan a zero-downtime migration from users.full_name to first_name and last_name for a Postgres app.", + "final_output_tokens": 101 + }, + { + "ok": true, + "error": "", + "output": "\nPlan a zero-downtime migration from users.full_name to first_name and last_name for a Postgres app; planning only, no code changes.\n\n\nNo additional context stated.\n\n\n- Plan a zero-downtime migration from users.full_name to first_name and last_name for a Postgres app; planning only, no code changes.\n- Preserve all stated requirements.\n- Do not add unstated requirements.\n\n\nPlan.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 91, + "completion_tokens": 24, + "total_tokens": 115, + "kv_token_occupancy_rate_percentage": 0.3509521484375, + "load_duration": 5e-07, + "prefill_duration_ttft": 1.076857344, + "decoding_duration": 1.85452, + "prefill_speed_tps": 84.50515800168976, + "decoding_speed_tps": 12.941354097017019 + }, + "wall_seconds": 3.1574, + "ttft_seconds": 1.0769, + "completion_tokens": 24, + "decode_duration_seconds": 1.8545, + "decode_tokens_per_second": 12.9414, + "seconds_per_output_token": 0.13156, + "raw_output": "Plan a zero-downtime migration from users.full_name to first_name and last_name for a Postgres app.", + "final_output_tokens": 101 + } + ], + "speed": { + "runs": 5, + "successful_runs": 5, + "wall_seconds": { + "median": 3.1903, + "p90": 3.1951, + "min": 3.1574, + "max": 3.1976 + }, + "ttft_seconds": { + "median": 1.0984, + "p90": 1.1188, + "min": 1.0755, + "max": 1.1228 + }, + "completion_tokens": { + "median": 24.0, + "p90": 24.0, + "min": 24.0, + "max": 24.0 + }, + "decode_tokens_per_second": { + "median": 12.9625, + "p90": 12.9875, + "min": 12.9414, + "max": 12.9931 + }, + "seconds_per_output_token": { + "median": 0.1329, + "p90": 0.1331, + "min": 0.1316, + "max": 0.1332 + } + }, + "representative_output": "\nPlan a zero-downtime migration from users.full_name to first_name and last_name for a Postgres app; planning only, no code changes.\n\n\nNo additional context stated.\n\n\n- Plan a zero-downtime migration from users.full_name to first_name and last_name for a Postgres app; planning only, no code changes.\n- Preserve all stated requirements.\n- Do not add unstated requirements.\n\n\nPlan.\n", + "quality": { + "rubric": { + "r1": true, + "r2": true, + "r3": true, + "r4": true, + "r5": true, + "r6": true, + "r7": true + }, + "score": 7, + "passed": true, + "hard_fail": false, + "pending_judge": false, + "invented_requirement": false, + "constraint_item_count": 3, + "token_count": 101, + "token_count_source": "usage.completion_tokens", + "estimated_tokens": 101, + "judge_notes": "Zero-downtime, column split, Postgres, planning-only, and no-code constraints stay intact." + } + } + } + }, + { + "name": "explain_regex", + "category": "explain", + "trap": false, + "input": "explain what ^(?P[a-z0-9]+(?:-[a-z0-9]+)*)$ accepts and show three examples", + "styles": { + "v1": { + "label": "current", + "max_tokens": 700, + "warmups": 1, + "warmup_errors": [], + "samples": [ + { + "ok": true, + "error": "", + "output": "\nExplain the regex pattern `^(?P[a-z0-9]+(?:-[a-z0-9]+)*)$` and provide three valid examples.\n\n\nThe user is asking for a breakdown of a specific regular expression used to validate URL slugs, including its components and usage examples.\n\n\n1. Length: Keep the explanation concise (under 150 words).\n2. Format: Use Markdown code blocks for the regex and examples; use bullet points or numbered lists for clarity.\n3. Tone: Direct, technical but accessible, no fluff or introductions/conclusions outside the requested content.\n\n\n- A brief paragraph explaining the regex logic (anchors, named group, character classes).\n- Three distinct valid slug examples in a code block labeled \"Examples:\".", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 148, + "completion_tokens": 178, + "total_tokens": 326, + "kv_token_occupancy_rate_percentage": 0.994873046875, + "load_duration": 6e-07, + "prefill_duration_ttft": 1.165640448, + "decoding_duration": 12.810598, + "prefill_speed_tps": 126.96882666858177, + "decoding_speed_tps": 13.89474558486653 + }, + "wall_seconds": 14.8508, + "ttft_seconds": 1.1656, + "completion_tokens": 178, + "decode_duration_seconds": 12.8106, + "decode_tokens_per_second": 13.8947, + "seconds_per_output_token": 0.083431 + }, + { + "ok": true, + "error": "", + "output": "\nExplain the regex pattern `^(?P[a-z0-9]+(?:-[a-z0-9]+)*)$` and provide three valid examples.\n\n\nThe user is asking for a breakdown of a specific regular expression used to validate URL slugs, likely in Python or similar environments where named groups are common. The pattern enforces lowercase letters and numbers only, with optional hyphens separating word parts.\n\n\n- Length: Keep the explanation concise (under 150 words).\n- Format: Use a clear heading for the explanation followed by a bulleted list for examples.\n- Tone: Direct, technical but accessible.\n- Content requirements: Must explicitly mention what characters are allowed and how hyphens work; must provide exactly three distinct valid examples that match the pattern (e.g., `abc`, `my-slug`, `a1b2c3d4-e5f6`).\n\n\nMarkdown with a main heading (`#`) for the explanation and a sub-heading (`##`) or bullet list for the examples. No JSON, no code blocks unless necessary to illustrate (but plain text is preferred per constraints).", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 148, + "completion_tokens": 251, + "total_tokens": 399, + "kv_token_occupancy_rate_percentage": 1.2176513671875, + "load_duration": 1e-06, + "prefill_duration_ttft": 1.172203264, + "decoding_duration": 18.164878, + "prefill_speed_tps": 126.25796612693956, + "decoding_speed_tps": 13.817874251618976 + }, + "wall_seconds": 20.5085, + "ttft_seconds": 1.1722, + "completion_tokens": 251, + "decode_duration_seconds": 18.1649, + "decode_tokens_per_second": 13.8179, + "seconds_per_output_token": 0.081707 + }, + { + "ok": true, + "error": "", + "output": "\nExplain the regex pattern `^(?P[a-z0-9]+(?:-[a-z0-9]+)*)$` and provide three valid examples.\n\n\nThe user is asking for a breakdown of a specific regular expression used to validate URL slugs, including its components and usage examples.\n\n\n1. The explanation must be concise (under 5 sentences).\n2. Provide exactly three distinct example strings that match the pattern.\n3. Do not include invalid examples or code blocks unless requested in the original text (none present).\n4. Maintain a neutral, instructional tone suitable for technical documentation or learning materials.\n\n\n- A brief paragraph explaining the regex components and logic.\n- A bulleted list of exactly three valid example strings.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 148, + "completion_tokens": 171, + "total_tokens": 319, + "kv_token_occupancy_rate_percentage": 0.9735107421875, + "load_duration": 6e-07, + "prefill_duration_ttft": 1.172502656, + "decoding_duration": 12.183659, + "prefill_speed_tps": 126.22572686095475, + "decoding_speed_tps": 14.03519254765748 + }, + "wall_seconds": 14.205, + "ttft_seconds": 1.1725, + "completion_tokens": 171, + "decode_duration_seconds": 12.1837, + "decode_tokens_per_second": 14.0352, + "seconds_per_output_token": 0.08307 + }, + { + "ok": true, + "error": "", + "output": "\nExplain the regex pattern `^(?P[a-z0-9]+(?:-[a-z0-9]+)*)$` and provide three valid examples.\n\n\nThe user is asking for a breakdown of a specific regular expression used to validate URL slugs, including its components and usage examples.\n\n\n1. Length: Keep the explanation concise (under 150 words).\n2. Format: Use Markdown code blocks for the regex and examples; use bullet points or numbered lists for clarity.\n3. Tone: Direct, technical but accessible (educational).\n4. Content Requirements: Must explicitly explain each part of the regex (`^`, `(?P...)`, `[a-z0-9]+`, `(?:-[a-z0-9]+)*`), and provide exactly three examples that match the pattern.\n\n\nMarkdown text with a clear heading for the explanation, followed by a section for examples using code blocks. No JSON or other structured data formats unless implied (none here).", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 148, + "completion_tokens": 224, + "total_tokens": 372, + "kv_token_occupancy_rate_percentage": 1.13525390625, + "load_duration": 9e-07, + "prefill_duration_ttft": 1.169048704, + "decoding_duration": 15.904137, + "prefill_speed_tps": 126.59866051226554, + "decoding_speed_tps": 14.084385716747786 + }, + "wall_seconds": 18.1502, + "ttft_seconds": 1.169, + "completion_tokens": 224, + "decode_duration_seconds": 15.9041, + "decode_tokens_per_second": 14.0844, + "seconds_per_output_token": 0.081028 + }, + { + "ok": true, + "error": "", + "output": "\nExplain the regex pattern `^(?P[a-z0-9]+(?:-[a-z0-9]+)*)$` and provide three valid examples.\n\n\nThe user is asking for an explanation of a specific regular expression used to validate \"slugs\" (URL-friendly identifiers). The pattern uses named capture groups, lowercase letters, numbers, hyphens as separators, and anchors to enforce start/end boundaries.\n\n\n- Length: Keep the explanation concise (under 100 words) and list exactly three examples.\n- Format: Use a clear heading for the explanation followed by a bulleted or numbered list for examples.\n- Tone: Informative and direct, suitable for technical documentation or learning purposes.\n\n\nMarkdown with headers (`##`) and bullet points (`*` or `-`). No JSON, no code blocks unless necessary to show the regex itself clearly (but plain text is preferred per constraints).", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 148, + "completion_tokens": 204, + "total_tokens": 352, + "kv_token_occupancy_rate_percentage": 1.07421875, + "load_duration": 5e-07, + "prefill_duration_ttft": 1.171853056, + "decoding_duration": 15.002036, + "prefill_speed_tps": 126.29569828932546, + "decoding_speed_tps": 13.598154277192776 + }, + "wall_seconds": 17.1781, + "ttft_seconds": 1.1719, + "completion_tokens": 204, + "decode_duration_seconds": 15.002, + "decode_tokens_per_second": 13.5982, + "seconds_per_output_token": 0.084206 + } + ], + "speed": { + "runs": 5, + "successful_runs": 5, + "wall_seconds": { + "median": 17.1781, + "p90": 19.5652, + "min": 14.205, + "max": 20.5085 + }, + "ttft_seconds": { + "median": 1.1719, + "p90": 1.1724, + "min": 1.1656, + "max": 1.1725 + }, + "completion_tokens": { + "median": 204.0, + "p90": 240.2, + "min": 171.0, + "max": 251.0 + }, + "decode_tokens_per_second": { + "median": 13.8947, + "p90": 14.0647, + "min": 13.5982, + "max": 14.0844 + }, + "seconds_per_output_token": { + "median": 0.0831, + "p90": 0.0839, + "min": 0.081, + "max": 0.0842 + } + }, + "representative_output": "\nExplain the regex pattern `^(?P[a-z0-9]+(?:-[a-z0-9]+)*)$` and provide three valid examples.\n\n\nThe user is asking for a breakdown of a specific regular expression used to validate URL slugs, including its components and usage examples.\n\n\n1. Length: Keep the explanation concise (under 150 words).\n2. Format: Use Markdown code blocks for the regex and examples; use bullet points or numbered lists for clarity.\n3. Tone: Direct, technical but accessible, no fluff or introductions/conclusions outside the requested content.\n\n\n- A brief paragraph explaining the regex logic (anchors, named group, character classes).\n- Three distinct valid slug examples in a code block labeled \"Examples:\".", + "quality": { + "rubric": { + "r1": false, + "r2": false, + "r3": false, + "r4": false, + "r5": false, + "r6": false, + "r7": true + }, + "score": 1, + "passed": false, + "hard_fail": true, + "pending_judge": false, + "invented_requirement": true, + "constraint_item_count": 0, + "token_count": 178, + "token_count_source": "usage.completion_tokens", + "estimated_tokens": 172, + "judge_notes": "Invents URL-slug purpose, word limit, Markdown layout, tone, and explanation components." + } + }, + "v2": { + "label": "candidate", + "max_tokens": 240, + "warmups": 1, + "warmup_errors": [], + "samples": [ + { + "ok": true, + "error": "", + "output": "\nExplain what ^(?P[a-z0-9]+(?:-[a-z0-9]+)*)$ accepts and show three examples.\n\n\nNo additional context stated.\n\n\n- Explain what ^(?P[a-z0-9]+(?:-[a-z0-9]+)*)$ accepts.\n- Show three examples.\n- Preserve all stated requirements.\n\n\nExplanation.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 92, + "completion_tokens": 35, + "total_tokens": 127, + "kv_token_occupancy_rate_percentage": 0.3875732421875, + "load_duration": 7e-07, + "prefill_duration_ttft": 1.106621952, + "decoding_duration": 2.633845, + "prefill_speed_tps": 83.13588921106094, + "decoding_speed_tps": 13.288557223374951 + }, + "wall_seconds": 3.9937, + "ttft_seconds": 1.1066, + "completion_tokens": 35, + "decode_duration_seconds": 2.6338, + "decode_tokens_per_second": 13.2886, + "seconds_per_output_token": 0.114105, + "raw_output": "Explain the regex `^(?P[a-z0-9]+(?:-[a-z0-9]+)*)$` and provide three examples.", + "final_output_tokens": 121 + }, + { + "ok": true, + "error": "", + "output": "\nExplain what ^(?P[a-z0-9]+(?:-[a-z0-9]+)*)$ accepts and show three examples.\n\n\nNo additional context stated.\n\n\n- Explain what ^(?P[a-z0-9]+(?:-[a-z0-9]+)*)$ accepts.\n- Show three examples.\n- Preserve all stated requirements.\n\n\nExplanation.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 92, + "completion_tokens": 35, + "total_tokens": 127, + "kv_token_occupancy_rate_percentage": 0.3875732421875, + "load_duration": 5e-07, + "prefill_duration_ttft": 1.074333824, + "decoding_duration": 2.631526, + "prefill_speed_tps": 85.63446290601011, + "decoding_speed_tps": 13.30026760138414 + }, + "wall_seconds": 3.9507, + "ttft_seconds": 1.0743, + "completion_tokens": 35, + "decode_duration_seconds": 2.6315, + "decode_tokens_per_second": 13.3003, + "seconds_per_output_token": 0.112876, + "raw_output": "Explain the regex `^(?P[a-z0-9]+(?:-[a-z0-9]+)*)$` and provide three examples.", + "final_output_tokens": 121 + }, + { + "ok": true, + "error": "", + "output": "\nExplain what ^(?P[a-z0-9]+(?:-[a-z0-9]+)*)$ accepts and show three examples.\n\n\nNo additional context stated.\n\n\n- Explain what ^(?P[a-z0-9]+(?:-[a-z0-9]+)*)$ accepts.\n- Show three examples.\n- Preserve all stated requirements.\n\n\nExplanation.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 92, + "completion_tokens": 35, + "total_tokens": 127, + "kv_token_occupancy_rate_percentage": 0.3875732421875, + "load_duration": 5e-07, + "prefill_duration_ttft": 1.076062976, + "decoding_duration": 2.632215, + "prefill_speed_tps": 85.49685478631318, + "decoding_speed_tps": 13.29678616678349 + }, + "wall_seconds": 3.9655, + "ttft_seconds": 1.0761, + "completion_tokens": 35, + "decode_duration_seconds": 2.6322, + "decode_tokens_per_second": 13.2968, + "seconds_per_output_token": 0.113299, + "raw_output": "Explain the regex `^(?P[a-z0-9]+(?:-[a-z0-9]+)*)$` and provide three examples.", + "final_output_tokens": 121 + }, + { + "ok": true, + "error": "", + "output": "\nExplain what ^(?P[a-z0-9]+(?:-[a-z0-9]+)*)$ accepts and show three examples.\n\n\nNo additional context stated.\n\n\n- Explain what ^(?P[a-z0-9]+(?:-[a-z0-9]+)*)$ accepts.\n- Show three examples.\n- Preserve all stated requirements.\n\n\nExplanation.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 92, + "completion_tokens": 35, + "total_tokens": 127, + "kv_token_occupancy_rate_percentage": 0.3875732421875, + "load_duration": 6e-07, + "prefill_duration_ttft": 1.077093376, + "decoding_duration": 2.704983, + "prefill_speed_tps": 85.41506432957583, + "decoding_speed_tps": 12.939083166141893 + }, + "wall_seconds": 4.0434, + "ttft_seconds": 1.0771, + "completion_tokens": 35, + "decode_duration_seconds": 2.705, + "decode_tokens_per_second": 12.9391, + "seconds_per_output_token": 0.115525, + "raw_output": "Explain the regex `^(?P[a-z0-9]+(?:-[a-z0-9]+)*)$` and provide three examples.", + "final_output_tokens": 121 + }, + { + "ok": true, + "error": "", + "output": "\nExplain what ^(?P[a-z0-9]+(?:-[a-z0-9]+)*)$ accepts and show three examples.\n\n\nNo additional context stated.\n\n\n- Explain what ^(?P[a-z0-9]+(?:-[a-z0-9]+)*)$ accepts.\n- Show three examples.\n- Preserve all stated requirements.\n\n\nExplanation.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 92, + "completion_tokens": 35, + "total_tokens": 127, + "kv_token_occupancy_rate_percentage": 0.3875732421875, + "load_duration": 5e-07, + "prefill_duration_ttft": 1.118983168, + "decoding_duration": 2.634906, + "prefill_speed_tps": 82.21750123769512, + "decoding_speed_tps": 13.283206307929012 + }, + "wall_seconds": 4.0207, + "ttft_seconds": 1.119, + "completion_tokens": 35, + "decode_duration_seconds": 2.6349, + "decode_tokens_per_second": 13.2832, + "seconds_per_output_token": 0.114877, + "raw_output": "Explain the regex `^(?P[a-z0-9]+(?:-[a-z0-9]+)*)$` and provide three examples.", + "final_output_tokens": 121 + } + ], + "speed": { + "runs": 5, + "successful_runs": 5, + "wall_seconds": { + "median": 3.9937, + "p90": 4.0343, + "min": 3.9507, + "max": 4.0434 + }, + "ttft_seconds": { + "median": 1.0771, + "p90": 1.114, + "min": 1.0743, + "max": 1.119 + }, + "completion_tokens": { + "median": 35.0, + "p90": 35.0, + "min": 35.0, + "max": 35.0 + }, + "decode_tokens_per_second": { + "median": 13.2886, + "p90": 13.2989, + "min": 12.9391, + "max": 13.3003 + }, + "seconds_per_output_token": { + "median": 0.1141, + "p90": 0.1153, + "min": 0.1129, + "max": 0.1155 + } + }, + "representative_output": "\nExplain what ^(?P[a-z0-9]+(?:-[a-z0-9]+)*)$ accepts and show three examples.\n\n\nNo additional context stated.\n\n\n- Explain what ^(?P[a-z0-9]+(?:-[a-z0-9]+)*)$ accepts.\n- Show three examples.\n- Preserve all stated requirements.\n\n\nExplanation.\n", + "quality": { + "rubric": { + "r1": true, + "r2": true, + "r3": true, + "r4": true, + "r5": true, + "r6": true, + "r7": true + }, + "score": 7, + "passed": true, + "hard_fail": false, + "pending_judge": false, + "invented_requirement": false, + "constraint_item_count": 3, + "token_count": 121, + "token_count_source": "usage.completion_tokens", + "estimated_tokens": 121, + "judge_notes": "Regex explanation and three examples are the only requested requirements." + } + } + } + }, + { + "name": "fix_timezone_boundary", + "category": "debug", + "trap": false, + "input": "fix the weekly report bug where Sunday 11:30 PM America/New_York appears in next week; weeks start Monday local time and existing report JSON must not change shape", + "styles": { + "v1": { + "label": "current", + "max_tokens": 700, + "warmups": 1, + "warmup_errors": [], + "samples": [ + { + "ok": true, + "error": "", + "output": "\nFix a weekly report bug where Sunday 11:30 PM America/New_York incorrectly appears in the next week by ensuring weeks start on Monday local time.\n\n\nThe existing report JSON structure must remain unchanged; only the date logic needs correction to align with local timezone boundaries and Monday-starting weeks.\n\n\n- Output length: 150 - 250 words.\n- Format: Single code block containing a corrected function or script snippet (e.g., Python).\n- Tone: Technical, direct, no explanations outside the code.\n- Testability: The logic must handle edge cases like Sunday evenings crossing into Monday mornings correctly without altering JSON keys or nesting levels.\n\n\nMarkdown header `## Solution`, followed immediately by a single fenced code block with language specified as `python`. No additional text before or after the solution section.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 154, + "completion_tokens": 188, + "total_tokens": 342, + "kv_token_occupancy_rate_percentage": 1.043701171875, + "load_duration": 7e-07, + "prefill_duration_ttft": 1.177465344, + "decoding_duration": 13.390206, + "prefill_speed_tps": 130.78941200667728, + "decoding_speed_tps": 14.040112601703065 + }, + "wall_seconds": 15.499, + "ttft_seconds": 1.1775, + "completion_tokens": 188, + "decode_duration_seconds": 13.3902, + "decode_tokens_per_second": 14.0401, + "seconds_per_output_token": 0.082441 + }, + { + "ok": true, + "error": "", + "output": "\nFix a weekly report bug where Sunday 11:30 PM America/New_York incorrectly appears in the next week by ensuring weeks start on Monday local time, while preserving the existing JSON structure.\n\n\nThe current implementation calculates week boundaries based on UTC or incorrect timezone offsets, causing end-of-week timestamps (Sunday late night) to roll over into the following week's data. The report must adhere to a Monday-start calendar for all local timezones and output valid JSON without altering schema keys or nesting levels.\n\n\n- Output length: 150 - 250 words total.\n- Format: Single code block containing only the corrected logic snippet (Python/JavaScript).\n- Tone: Technical, concise, no explanatory prose outside the code.\n- Testability: Ensure `assert` statements verify that a Sunday timestamp maps to its own week ID and Monday timestamps do not shift forward incorrectly.\n\n\nMarkdown code block with language specified as `python`. No surrounding text or headers.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 154, + "completion_tokens": 214, + "total_tokens": 368, + "kv_token_occupancy_rate_percentage": 1.123046875, + "load_duration": 5e-07, + "prefill_duration_ttft": 1.170463616, + "decoding_duration": 15.562893, + "prefill_speed_tps": 131.57179590621294, + "decoding_speed_tps": 13.75065677056316 + }, + "wall_seconds": 17.7389, + "ttft_seconds": 1.1705, + "completion_tokens": 214, + "decode_duration_seconds": 15.5629, + "decode_tokens_per_second": 13.7507, + "seconds_per_output_token": 0.082892 + }, + { + "ok": true, + "error": "", + "output": "\nFix a weekly report bug where Sunday 11:30 PM America/New_York incorrectly appears in the next week by ensuring weeks start on Monday local time, while preserving the existing JSON structure.\n\n\nThe current implementation calculates week boundaries based on UTC or incorrect timezone offsets, causing end-of-week timestamps (Sunday late night) to roll over into the following week's data. The report must adhere to a Monday-start calendar for all local timezones and output valid JSON without altering schema keys or nesting levels.\n\n\n- Output length: Keep under 200 words.\n- Format: Return only the prompt text, no markdown code blocks around the prompt itself unless specified in constraints (none here).\n- Tone: Direct technical instruction suitable for an LLM developer context.\n- Testability: Ensure logic handles timezone conversion explicitly and validates week boundaries against Monday start rules before JSON serialization.\n\n\nMarkdown headers for sections, followed by a code block containing the final prompt ready for Claude to execute.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 154, + "completion_tokens": 216, + "total_tokens": 370, + "kv_token_occupancy_rate_percentage": 1.129150390625, + "load_duration": 1e-06, + "prefill_duration_ttft": 1.29701696, + "decoding_duration": 15.914568, + "prefill_speed_tps": 118.73399095721925, + "decoding_speed_tps": 13.572470204657769 + }, + "wall_seconds": 18.2696, + "ttft_seconds": 1.297, + "completion_tokens": 216, + "decode_duration_seconds": 15.9146, + "decode_tokens_per_second": 13.5725, + "seconds_per_output_token": 0.084581 + }, + { + "ok": true, + "error": "", + "output": "\nFix a weekly report bug where Sunday 11:30 PM America/New_York incorrectly appears in the next week by ensuring weeks start on Monday local time, while preserving the existing JSON structure.\n\n\nThe current implementation calculates week boundaries based on UTC or incorrect timezone offsets, causing dates at the end of a US Eastern Standard Time (EST) Sunday to roll over into Monday's slot of the following calendar week in reports. The output must remain valid JSON with identical keys and nesting levels as previous versions; only date logic needs correction.\n\n\n- Output length: 150 - 250 words total.\n- Format: Single code block containing fixed Python function or script snippet (no full app).\n- Tone: Technical, direct, no explanations outside the code.\n- Testability: Ensure `datetime` objects are normalized to local timezone before week calculation; verify JSON keys match pre-fix state via diff check.\n\n\nMarkdown code block with language set to python. Include a brief comment header explaining the fix but keep it under 2 lines. Do not include test cases or usage examples in the output, only the corrected logic.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 154, + "completion_tokens": 249, + "total_tokens": 403, + "kv_token_occupancy_rate_percentage": 1.2298583984375, + "load_duration": 6e-07, + "prefill_duration_ttft": 1.173075328, + "decoding_duration": 17.71206, + "prefill_speed_tps": 131.27886702941552, + "decoding_speed_tps": 14.05821795996626 + }, + "wall_seconds": 20.1445, + "ttft_seconds": 1.1731, + "completion_tokens": 249, + "decode_duration_seconds": 17.7121, + "decode_tokens_per_second": 14.0582, + "seconds_per_output_token": 0.080902 + }, + { + "ok": true, + "error": "", + "output": "\nFix a weekly report bug where Sunday 11:30 PM America/New_York incorrectly appears in the next week by ensuring weeks start on Monday local time, while preserving the existing JSON structure.\n\n\nThe user's weekly report currently displays dates spanning into the following week when it should only cover the current week starting from Monday. The issue specifically affects timestamps near Sunday 11:30 PM America/New_York which are being misinterpreted as belonging to a new week. The output must remain valid JSON with no changes to keys, nesting levels, or overall shape of existing data structures.\n\n\n- Output length: Keep the response concise (under 200 words).\n- Format: Return only the prompt text in Markdown; do not include explanations or meta-commentary.\n- Tone: Professional and technical.\n- Testability: The fix must be verifiable by checking if Sunday 11:30 PM America/New_York stays within its correct week boundary without altering JSON keys or structure.\n\n\nMarkdown code block containing the rewritten prompt with clearly labeled sections for Task, Context, Constraints, and Output Format.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 154, + "completion_tokens": 244, + "total_tokens": 398, + "kv_token_occupancy_rate_percentage": 1.214599609375, + "load_duration": 8e-07, + "prefill_duration_ttft": 1.209659392, + "decoding_duration": 17.783904, + "prefill_speed_tps": 127.30856389696845, + "decoding_speed_tps": 13.720271994270774 + }, + "wall_seconds": 20.1842, + "ttft_seconds": 1.2097, + "completion_tokens": 244, + "decode_duration_seconds": 17.7839, + "decode_tokens_per_second": 13.7203, + "seconds_per_output_token": 0.082722 + } + ], + "speed": { + "runs": 5, + "successful_runs": 5, + "wall_seconds": { + "median": 18.2696, + "p90": 20.1683, + "min": 15.499, + "max": 20.1842 + }, + "ttft_seconds": { + "median": 1.1775, + "p90": 1.2621, + "min": 1.1705, + "max": 1.297 + }, + "completion_tokens": { + "median": 216.0, + "p90": 247.0, + "min": 188.0, + "max": 249.0 + }, + "decode_tokens_per_second": { + "median": 13.7507, + "p90": 14.051, + "min": 13.5725, + "max": 14.0582 + }, + "seconds_per_output_token": { + "median": 0.0827, + "p90": 0.0839, + "min": 0.0809, + "max": 0.0846 + } + }, + "representative_output": "\nFix a weekly report bug where Sunday 11:30 PM America/New_York incorrectly appears in the next week by ensuring weeks start on Monday local time.\n\n\nThe existing report JSON structure must remain unchanged; only the date logic needs correction to align with local timezone boundaries and Monday-starting weeks.\n\n\n- Output length: 150 - 250 words.\n- Format: Single code block containing a corrected function or script snippet (e.g., Python).\n- Tone: Technical, direct, no explanations outside the code.\n- Testability: The logic must handle edge cases like Sunday evenings crossing into Monday mornings correctly without altering JSON keys or nesting levels.\n\n\nMarkdown header `## Solution`, followed immediately by a single fenced code block with language specified as `python`. No additional text before or after the solution section.", + "quality": { + "rubric": { + "r1": false, + "r2": false, + "r3": false, + "r4": false, + "r5": false, + "r6": false, + "r7": true + }, + "score": 1, + "passed": false, + "hard_fail": true, + "pending_judge": false, + "invented_requirement": true, + "constraint_item_count": 0, + "token_count": 188, + "token_count_source": "usage.completion_tokens", + "estimated_tokens": 175, + "judge_notes": "Invents Python, length, code-only format, and additional boundary behavior." + } + }, + "v2": { + "label": "candidate", + "max_tokens": 240, + "warmups": 1, + "warmup_errors": [], + "samples": [ + { + "ok": true, + "error": "", + "output": "\nFix the weekly report bug where Sunday 11:30 PM America/New_York appears in next week; weeks start Monday local time and existing report JSON must not change shape.\n\n\nNo additional context stated.\n\n\n- Fix the weekly report bug where Sunday 11:30 PM America/New_York appears in next week; weeks start Monday local time and existing report JSON must not change shape.\n- Preserve all stated requirements.\n- Do not add unstated requirements.\n\n\nFix.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 98, + "completion_tokens": 38, + "total_tokens": 136, + "kv_token_occupancy_rate_percentage": 0.4150390625, + "load_duration": 5e-07, + "prefill_duration_ttft": 1.122284544, + "decoding_duration": 2.845853, + "prefill_speed_tps": 87.32188331732029, + "decoding_speed_tps": 13.35276277446516 + }, + "wall_seconds": 4.2325, + "ttft_seconds": 1.1223, + "completion_tokens": 38, + "decode_duration_seconds": 2.8459, + "decode_tokens_per_second": 13.3528, + "seconds_per_output_token": 0.111383, + "raw_output": "Fix the weekly report bug where Sunday 11:30 PM America/New_York appears in next week by ensuring weeks start Monday local time while keeping the existing JSON shape unchanged.", + "final_output_tokens": 115 + }, + { + "ok": true, + "error": "", + "output": "\nFix the weekly report bug where Sunday 11:30 PM America/New_York appears in next week; weeks start Monday local time and existing report JSON must not change shape.\n\n\nNo additional context stated.\n\n\n- Fix the weekly report bug where Sunday 11:30 PM America/New_York appears in next week; weeks start Monday local time and existing report JSON must not change shape.\n- Preserve all stated requirements.\n- Do not add unstated requirements.\n\n\nFix.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 98, + "completion_tokens": 39, + "total_tokens": 137, + "kv_token_occupancy_rate_percentage": 0.4180908203125, + "load_duration": 6e-07, + "prefill_duration_ttft": 1.158771456, + "decoding_duration": 2.919417, + "prefill_speed_tps": 84.57232829870466, + "decoding_speed_tps": 13.35883157493431 + }, + "wall_seconds": 4.3478, + "ttft_seconds": 1.1588, + "completion_tokens": 39, + "decode_duration_seconds": 2.9194, + "decode_tokens_per_second": 13.3588, + "seconds_per_output_token": 0.111483, + "raw_output": "Fix the weekly report bug where Sunday 11:30 PM America/New_York appears in next week by ensuring weeks start Monday local time while keeping the existing report JSON shape unchanged.", + "final_output_tokens": 115 + }, + { + "ok": true, + "error": "", + "output": "\nFix the weekly report bug where Sunday 11:30 PM America/New_York appears in next week; weeks start Monday local time and existing report JSON must not change shape.\n\n\nNo additional context stated.\n\n\n- Fix the weekly report bug where Sunday 11:30 PM America/New_York appears in next week; weeks start Monday local time and existing report JSON must not change shape.\n- Preserve all stated requirements.\n- Do not add unstated requirements.\n\n\nFix.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 98, + "completion_tokens": 38, + "total_tokens": 136, + "kv_token_occupancy_rate_percentage": 0.4150390625, + "load_duration": 4e-07, + "prefill_duration_ttft": 1.084527744, + "decoding_duration": 2.875464, + "prefill_speed_tps": 90.36191147914055, + "decoding_speed_tps": 13.215258476545003 + }, + "wall_seconds": 4.2225, + "ttft_seconds": 1.0845, + "completion_tokens": 38, + "decode_duration_seconds": 2.8755, + "decode_tokens_per_second": 13.2153, + "seconds_per_output_token": 0.111118, + "raw_output": "Fix the weekly report bug where Sunday 11:30 PM America/New_York appears in next week by ensuring weeks start Monday local time while keeping the existing JSON shape unchanged.", + "final_output_tokens": 115 + }, + { + "ok": true, + "error": "", + "output": "\nFix the weekly report bug where Sunday 11:30 PM America/New_York appears in next week; weeks start Monday local time and existing report JSON must not change shape.\n\n\nNo additional context stated.\n\n\n- Fix the weekly report bug where Sunday 11:30 PM America/New_York appears in next week; weeks start Monday local time and existing report JSON must not change shape.\n- Preserve all stated requirements.\n- Do not add unstated requirements.\n\n\nFix.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 98, + "completion_tokens": 38, + "total_tokens": 136, + "kv_token_occupancy_rate_percentage": 0.4150390625, + "load_duration": 5e-07, + "prefill_duration_ttft": 1.369671168, + "decoding_duration": 2.850551, + "prefill_speed_tps": 71.55002039146378, + "decoding_speed_tps": 13.330756053829592 + }, + "wall_seconds": 4.514, + "ttft_seconds": 1.3697, + "completion_tokens": 38, + "decode_duration_seconds": 2.8506, + "decode_tokens_per_second": 13.3308, + "seconds_per_output_token": 0.11879, + "raw_output": "Fix the weekly report bug where Sunday 11:30 PM America/New_York appears in next week by ensuring weeks start Monday local time while keeping the existing JSON shape unchanged.", + "final_output_tokens": 115 + }, + { + "ok": true, + "error": "", + "output": "\nFix the weekly report bug where Sunday 11:30 PM America/New_York appears in next week; weeks start Monday local time and existing report JSON must not change shape.\n\n\nNo additional context stated.\n\n\n- Fix the weekly report bug where Sunday 11:30 PM America/New_York appears in next week; weeks start Monday local time and existing report JSON must not change shape.\n- Preserve all stated requirements.\n- Do not add unstated requirements.\n\n\nFix.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 98, + "completion_tokens": 37, + "total_tokens": 135, + "kv_token_occupancy_rate_percentage": 0.4119873046875, + "load_duration": 5e-07, + "prefill_duration_ttft": 1.114533248, + "decoding_duration": 2.781102, + "prefill_speed_tps": 87.92918486358155, + "decoding_speed_tps": 13.304078742886814 + }, + "wall_seconds": 4.1589, + "ttft_seconds": 1.1145, + "completion_tokens": 37, + "decode_duration_seconds": 2.7811, + "decode_tokens_per_second": 13.3041, + "seconds_per_output_token": 0.112403, + "raw_output": "Fix the weekly report bug where Sunday 11:30 PM America/New_York appears in next week by ensuring weeks start Monday local time without changing the existing JSON shape.", + "final_output_tokens": 115 + } + ], + "speed": { + "runs": 5, + "successful_runs": 5, + "wall_seconds": { + "median": 4.2325, + "p90": 4.4475, + "min": 4.1589, + "max": 4.514 + }, + "ttft_seconds": { + "median": 1.1223, + "p90": 1.2853, + "min": 1.0845, + "max": 1.3697 + }, + "completion_tokens": { + "median": 38.0, + "p90": 38.6, + "min": 37.0, + "max": 39.0 + }, + "decode_tokens_per_second": { + "median": 13.3308, + "p90": 13.3564, + "min": 13.2153, + "max": 13.3588 + }, + "seconds_per_output_token": { + "median": 0.1115, + "p90": 0.1162, + "min": 0.1111, + "max": 0.1188 + } + }, + "representative_output": "\nFix the weekly report bug where Sunday 11:30 PM America/New_York appears in next week; weeks start Monday local time and existing report JSON must not change shape.\n\n\nNo additional context stated.\n\n\n- Fix the weekly report bug where Sunday 11:30 PM America/New_York appears in next week; weeks start Monday local time and existing report JSON must not change shape.\n- Preserve all stated requirements.\n- Do not add unstated requirements.\n\n\nFix.\n", + "quality": { + "rubric": { + "r1": true, + "r2": true, + "r3": true, + "r4": true, + "r5": true, + "r6": true, + "r7": true + }, + "score": 7, + "passed": true, + "hard_fail": false, + "pending_judge": false, + "invented_requirement": false, + "constraint_item_count": 3, + "token_count": 115, + "token_count_source": "usage.completion_tokens", + "estimated_tokens": 115, + "judge_notes": "Timezone, Monday boundary, and JSON-shape constraints remain source-derived." + } + } + } + }, + { + "name": "api_retry_policy", + "category": "implement", + "trap": false, + "input": "add retry handling to the Go API client for 429 and 503 responses only, honor Retry-After when present, cap attempts at three, and test the retry timing with a fake clock", + "styles": { + "v1": { + "label": "current", + "max_tokens": 700, + "warmups": 1, + "warmup_errors": [], + "samples": [ + { + "ok": true, + "error": "", + "output": "\nImplement retry logic in a Go API client that specifically handles 429 and 503 status codes, respects the `Retry-After` header if available, limits total attempts to three, and validates timing using a fake clock.\n\n\nThe existing Go API client currently lacks explicit handling for rate limiting (429) and service unavailable (503) errors. The server may include a `Retry-After` HTTP header indicating the number of seconds before retrying. A testing environment requires mocking time to verify that exponential backoff or fixed delays occur correctly without waiting real seconds.\n\n\n- Code must be written in Go (Golang).\n- Maximum file size: 50 lines for the core logic function.\n- Output format: Single code block with comments explaining the fake clock usage and retry limits.\n- Tone: Professional, concise technical documentation style within comments.\n\n\nMarkdown code block containing the complete Go implementation of `ExecuteWithRetry`.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 160, + "completion_tokens": 211, + "total_tokens": 371, + "kv_token_occupancy_rate_percentage": 1.1322021484375, + "load_duration": 6e-07, + "prefill_duration_ttft": 1.17775488, + "decoding_duration": 15.315206, + "prefill_speed_tps": 135.85169776583732, + "decoding_speed_tps": 13.777157160014694 + }, + "wall_seconds": 17.535, + "ttft_seconds": 1.1778, + "completion_tokens": 211, + "decode_duration_seconds": 15.3152, + "decode_tokens_per_second": 13.7772, + "seconds_per_output_token": 0.083104 + }, + { + "ok": true, + "error": "", + "output": "\nImplement retry logic in a Go API client that specifically handles 429 and 503 status codes, respects the `Retry-After` header if available, limits total attempts to three, and validates timing using a fake clock.\n\n\nThe existing Go API client currently lacks explicit handling for rate limiting (429) or service unavailable (503) errors. The server may include a `Retry-After` header indicating seconds until the next attempt. A testing environment requires mocking time to verify retry delays without waiting for real network responses.\n\n\n- Code must be written in Go using standard libraries and common patterns (e.g., context, net/http).\n- Retry logic must only trigger on status codes 429 or 503; other errors should pass through immediately.\n- If `Retry-After` header exists, the client must wait exactly that many seconds before retrying.\n- Maximum number of retries is strictly three (total attempts = initial + 3).\n- A fake clock implementation must be provided to allow testing of delay logic without real time passage.\n\n\nReturn a single Markdown code block containing the complete Go file (`api_client.go`). Include:\n1. `RetryableError` struct definition with status code and retry-after header parsing.\n2. `Client` struct with fields for max retries, fake clock interface, and HTTP client wrapper.\n3. `DoRequestWithContext` method implementing the loop logic (check status -> if 429/503 & attempts < 3 -> wait via fake clock -> retry).\n4. A simple `FakeClock` implementation demonstrating how to set a deadline or sleep duration for testing purposes.\n\nEnsure all variable names are descriptive and comments explain the \"why\" of the Retry-After handling and attempt capping.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 160, + "completion_tokens": 389, + "total_tokens": 549, + "kv_token_occupancy_rate_percentage": 1.6754150390625, + "load_duration": 8e-07, + "prefill_duration_ttft": 1.179211648, + "decoding_duration": 27.594688, + "prefill_speed_tps": 135.68387004264056, + "decoding_speed_tps": 14.096916044131392 + }, + "wall_seconds": 30.4958, + "ttft_seconds": 1.1792, + "completion_tokens": 389, + "decode_duration_seconds": 27.5947, + "decode_tokens_per_second": 14.0969, + "seconds_per_output_token": 0.078395 + }, + { + "ok": true, + "error": "", + "output": "\nImplement retry logic in a Go API client that specifically handles 429 and 503 status codes, respects the Retry-After header, limits total attempts to three, and uses a fake clock for timing validation.\n\n\nThe existing Go API client currently lacks automatic retry mechanisms for transient failures. The backend returns HTTP 429 (Too Many Requests) or 503 (Service Unavailable) on rate limiting or temporary unavailability. A `Retry-After` header may be present in these responses indicating the delay before subsequent requests. The testing environment requires a deterministic way to verify retry timing without waiting for real network delays, necessitating a mocked clock implementation.\n\n\n- Code must be written in Go (Golang).\n- Maximum file size: 50 lines of code per function/file.\n- Output format: A single Markdown code block containing the complete `retry.go` implementation.\n- Tone: Professional and concise, suitable for a production repository.\n\n\nMarkdown header (`## retry.go`) followed by a Go language syntax highlighted code block with no additional text outside the prompt structure.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 160, + "completion_tokens": 240, + "total_tokens": 400, + "kv_token_occupancy_rate_percentage": 1.220703125, + "load_duration": 7e-07, + "prefill_duration_ttft": 1.185682176, + "decoding_duration": 17.506476, + "prefill_speed_tps": 134.94341336881158, + "decoding_speed_tps": 13.709212522268903 + }, + "wall_seconds": 19.8265, + "ttft_seconds": 1.1857, + "completion_tokens": 240, + "decode_duration_seconds": 17.5065, + "decode_tokens_per_second": 13.7092, + "seconds_per_output_token": 0.08261 + }, + { + "ok": true, + "error": "", + "output": "\nImplement retry logic in a Go API client that specifically handles 429 and 503 status codes, respects the Retry-After header, limits total attempts to three, and uses a fake clock for timing validation.\n\n\nThe existing Go API client currently lacks automatic retry mechanisms for transient failures. The target environment requires precise control over backoff timing via a mockable clock unit test fixture. Only HTTP 429 (Too Many Requests) and 503 (Service Unavailable) codes should trigger retries; other errors must fail immediately.\n\n\n- Code length: Maximum 150 lines for the core retry logic implementation.\n- Format: Go source code with a dedicated `TestRetryWithFakeClock` function in `_test.go`.\n- Tone: Professional, concise, and production-ready.\n- Testability: The test must verify that exactly three attempts occur before failure when using the fake clock.\n\n\nMarkdown code blocks for Go files (`client.go`, `retry_test.go`). No JSON or other formats.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 160, + "completion_tokens": 225, + "total_tokens": 385, + "kv_token_occupancy_rate_percentage": 1.1749267578125, + "load_duration": 9e-07, + "prefill_duration_ttft": 1.1883104, + "decoding_duration": 16.111425, + "prefill_speed_tps": 134.64495471890174, + "decoding_speed_tps": 13.965245159878782 + }, + "wall_seconds": 18.3342, + "ttft_seconds": 1.1883, + "completion_tokens": 225, + "decode_duration_seconds": 16.1114, + "decode_tokens_per_second": 13.9652, + "seconds_per_output_token": 0.081485 + }, + { + "ok": true, + "error": "", + "output": "\nImplement retry logic in a Go API client that specifically handles 429 and 503 status codes, respects the Retry-After header when available, limits total attempts to three, and validates timing behavior using a fake clock.\n\n\nThe existing Go API client currently lacks explicit error handling for rate limiting (429) or service unavailability (503). The server may include a \"Retry-After\" header indicating seconds before the next attempt. A testing utility with a mockable time source is available to verify timing constraints without waiting for real delays.\n\n\n- Code must be written in Go (Golang 1.21+).\n- Retry logic must only trigger on HTTP status codes 429 and 503; other errors should pass through immediately.\n- If \"Retry-After\" header exists, the client must wait exactly that many seconds before retrying; if missing, use a default of 1 second.\n- Maximum number of retries is strictly three (total attempts = initial + 3). After the third attempt fails or succeeds, stop and return the final result/error.\n- Timing validation must be demonstrated using `time.Fake` or equivalent to prove that delays are calculated correctly without real-world latency.\n- Output should include a complete, runnable example with comments explaining each constraint check.\n\n\nReturn only the Go source code file content (e.g., `api_client.go`). Do not include explanations outside of the code block. Use standard imports and ensure all functions are exported where necessary for testing integration.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 160, + "completion_tokens": 335, + "total_tokens": 495, + "kv_token_occupancy_rate_percentage": 1.5106201171875, + "load_duration": 9e-07, + "prefill_duration_ttft": 1.183184256, + "decoding_duration": 23.758302, + "prefill_speed_tps": 135.22830378162166, + "decoding_speed_tps": 14.100334274730576 + }, + "wall_seconds": 26.456, + "ttft_seconds": 1.1832, + "completion_tokens": 335, + "decode_duration_seconds": 23.7583, + "decode_tokens_per_second": 14.1003, + "seconds_per_output_token": 0.078973 + } + ], + "speed": { + "runs": 5, + "successful_runs": 5, + "wall_seconds": { + "median": 19.8265, + "p90": 28.8799, + "min": 17.535, + "max": 30.4958 + }, + "ttft_seconds": { + "median": 1.1832, + "p90": 1.1873, + "min": 1.1778, + "max": 1.1883 + }, + "completion_tokens": { + "median": 240.0, + "p90": 367.4, + "min": 211.0, + "max": 389.0 + }, + "decode_tokens_per_second": { + "median": 13.9652, + "p90": 14.0989, + "min": 13.7092, + "max": 14.1003 + }, + "seconds_per_output_token": { + "median": 0.0815, + "p90": 0.0829, + "min": 0.0784, + "max": 0.0831 + } + }, + "representative_output": "\nImplement retry logic in a Go API client that specifically handles 429 and 503 status codes, respects the `Retry-After` header if available, limits total attempts to three, and validates timing using a fake clock.\n\n\nThe existing Go API client currently lacks explicit handling for rate limiting (429) and service unavailable (503) errors. The server may include a `Retry-After` HTTP header indicating the number of seconds before retrying. A testing environment requires mocking time to verify that exponential backoff or fixed delays occur correctly without waiting real seconds.\n\n\n- Code must be written in Go (Golang).\n- Maximum file size: 50 lines for the core logic function.\n- Output format: Single code block with comments explaining the fake clock usage and retry limits.\n- Tone: Professional, concise technical documentation style within comments.\n\n\nMarkdown code block containing the complete Go implementation of `ExecuteWithRetry`.", + "quality": { + "rubric": { + "r1": false, + "r2": false, + "r3": false, + "r4": false, + "r5": false, + "r6": false, + "r7": true + }, + "score": 1, + "passed": false, + "hard_fail": true, + "pending_judge": false, + "invented_requirement": true, + "constraint_item_count": 0, + "token_count": 211, + "token_count_source": "usage.completion_tokens", + "estimated_tokens": 185, + "judge_notes": "Invents file-size limit, comments, function name, output format, and backoff behavior." + } + }, + "v2": { + "label": "candidate", + "max_tokens": 240, + "warmups": 1, + "warmup_errors": [], + "samples": [ + { + "ok": true, + "error": "", + "output": "\nAdd retry handling to the Go API client for 429 and 503 responses only, honor Retry-After when present, cap attempts at three, and test the retry timing with a fake clock.\n\n\nNo additional context stated.\n\n\n- Add retry handling to the Go API client for 429 and 503 responses only.\n- Honor Retry-After when present.\n- Cap attempts at three.\n- Test the retry timing with a fake clock.\n\n\nRetry handling.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 104, + "completion_tokens": 41, + "total_tokens": 145, + "kv_token_occupancy_rate_percentage": 0.4425048828125, + "load_duration": 4e-07, + "prefill_duration_ttft": 1.093329664, + "decoding_duration": 3.060137, + "prefill_speed_tps": 95.12227045913207, + "decoding_speed_tps": 13.398092961197488 + }, + "wall_seconds": 4.418, + "ttft_seconds": 1.0933, + "completion_tokens": 41, + "decode_duration_seconds": 3.0601, + "decode_tokens_per_second": 13.3981, + "seconds_per_output_token": 0.107756, + "raw_output": "Add retry handling to the Go API client for 429 and 503 responses only, honor Retry-After when present, cap attempts at three, and test with a fake clock.", + "final_output_tokens": 113 + }, + { + "ok": true, + "error": "", + "output": "\nAdd retry handling to the Go API client for 429 and 503 responses only, honor Retry-After when present, cap attempts at three, and test the retry timing with a fake clock.\n\n\nNo additional context stated.\n\n\n- Add retry handling to the Go API client for 429 and 503 responses only.\n- Honor Retry-After when present.\n- Cap attempts at three.\n- Test the retry timing with a fake clock.\n\n\nRetry handling.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 104, + "completion_tokens": 44, + "total_tokens": 148, + "kv_token_occupancy_rate_percentage": 0.45166015625, + "load_duration": 5e-07, + "prefill_duration_ttft": 1.09412736, + "decoding_duration": 3.274614, + "prefill_speed_tps": 95.05291961623189, + "decoding_speed_tps": 13.436698187939097 + }, + "wall_seconds": 4.6505, + "ttft_seconds": 1.0941, + "completion_tokens": 44, + "decode_duration_seconds": 3.2746, + "decode_tokens_per_second": 13.4367, + "seconds_per_output_token": 0.105693, + "raw_output": "Add retry handling to the Go API client for 429 and 503 responses only, honor Retry-After when present, cap attempts at three, and test the retry timing with a fake clock.", + "final_output_tokens": 113 + }, + { + "ok": true, + "error": "", + "output": "\nAdd retry handling to the Go API client for 429 and 503 responses only, honor Retry-After when present, cap attempts at three, and test the retry timing with a fake clock.\n\n\nNo additional context stated.\n\n\n- Add retry handling to the Go API client for 429 and 503 responses only.\n- Honor Retry-After when present.\n- Cap attempts at three.\n- Test the retry timing with a fake clock.\n\n\nRetry handling.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 104, + "completion_tokens": 44, + "total_tokens": 148, + "kv_token_occupancy_rate_percentage": 0.45166015625, + "load_duration": 9e-07, + "prefill_duration_ttft": 1.17654784, + "decoding_duration": 3.282325, + "prefill_speed_tps": 88.39419568353463, + "decoding_speed_tps": 13.405132032933972 + }, + "wall_seconds": 4.7524, + "ttft_seconds": 1.1765, + "completion_tokens": 44, + "decode_duration_seconds": 3.2823, + "decode_tokens_per_second": 13.4051, + "seconds_per_output_token": 0.108009, + "raw_output": "Add retry handling to the Go API client for 429 and 503 responses only, honor Retry-After when present, cap attempts at three, and test the retry timing with a fake clock.", + "final_output_tokens": 113 + }, + { + "ok": true, + "error": "", + "output": "\nAdd retry handling to the Go API client for 429 and 503 responses only, honor Retry-After when present, cap attempts at three, and test the retry timing with a fake clock.\n\n\nNo additional context stated.\n\n\n- Add retry handling to the Go API client for 429 and 503 responses only.\n- Honor Retry-After when present.\n- Cap attempts at three.\n- Test the retry timing with a fake clock.\n\n\nRetry handling.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 104, + "completion_tokens": 44, + "total_tokens": 148, + "kv_token_occupancy_rate_percentage": 0.45166015625, + "load_duration": 5e-07, + "prefill_duration_ttft": 1.09661056, + "decoding_duration": 3.273664, + "prefill_speed_tps": 94.83767874713882, + "decoding_speed_tps": 13.440597446775234 + }, + "wall_seconds": 4.6569, + "ttft_seconds": 1.0966, + "completion_tokens": 44, + "decode_duration_seconds": 3.2737, + "decode_tokens_per_second": 13.4406, + "seconds_per_output_token": 0.105839, + "raw_output": "Add retry handling to the Go API client for 429 and 503 responses only, honor Retry-After when present, cap attempts at three, and test the retry timing with a fake clock.", + "final_output_tokens": 113 + }, + { + "ok": true, + "error": "", + "output": "\nAdd retry handling to the Go API client for 429 and 503 responses only, honor Retry-After when present, cap attempts at three, and test the retry timing with a fake clock.\n\n\nNo additional context stated.\n\n\n- Add retry handling to the Go API client for 429 and 503 responses only.\n- Honor Retry-After when present.\n- Cap attempts at three.\n- Test the retry timing with a fake clock.\n\n\nRetry handling.\n", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 104, + "completion_tokens": 44, + "total_tokens": 148, + "kv_token_occupancy_rate_percentage": 0.45166015625, + "load_duration": 6e-07, + "prefill_duration_ttft": 1.1058528, + "decoding_duration": 3.272341, + "prefill_speed_tps": 94.04506639581687, + "decoding_speed_tps": 13.446031449656378 + }, + "wall_seconds": 4.6654, + "ttft_seconds": 1.1059, + "completion_tokens": 44, + "decode_duration_seconds": 3.2723, + "decode_tokens_per_second": 13.446, + "seconds_per_output_token": 0.106031, + "raw_output": "Add retry handling to the Go API client for 429 and 503 responses only, honor Retry-After when present, cap attempts at three, and test the retry timing with a fake clock.", + "final_output_tokens": 113 + } + ], + "speed": { + "runs": 5, + "successful_runs": 5, + "wall_seconds": { + "median": 4.6569, + "p90": 4.7176, + "min": 4.418, + "max": 4.7524 + }, + "ttft_seconds": { + "median": 1.0966, + "p90": 1.1483, + "min": 1.0933, + "max": 1.1765 + }, + "completion_tokens": { + "median": 44.0, + "p90": 44.0, + "min": 41.0, + "max": 44.0 + }, + "decode_tokens_per_second": { + "median": 13.4367, + "p90": 13.4438, + "min": 13.3981, + "max": 13.446 + }, + "seconds_per_output_token": { + "median": 0.106, + "p90": 0.1079, + "min": 0.1057, + "max": 0.108 + } + }, + "representative_output": "\nAdd retry handling to the Go API client for 429 and 503 responses only, honor Retry-After when present, cap attempts at three, and test the retry timing with a fake clock.\n\n\nNo additional context stated.\n\n\n- Add retry handling to the Go API client for 429 and 503 responses only.\n- Honor Retry-After when present.\n- Cap attempts at three.\n- Test the retry timing with a fake clock.\n\n\nRetry handling.\n", + "quality": { + "rubric": { + "r1": true, + "r2": true, + "r3": true, + "r4": true, + "r5": true, + "r6": true, + "r7": true + }, + "score": 7, + "passed": true, + "hard_fail": false, + "pending_judge": false, + "invented_requirement": false, + "constraint_item_count": 4, + "token_count": 113, + "token_count_source": "usage.completion_tokens", + "estimated_tokens": 113, + "judge_notes": "Status codes, Retry-After, attempt cap, and fake-clock test are preserved exactly." + } + } + } + } + ], + "summaries": { + "v1": { + "speed": { + "runs": 60, + "successful_runs": 60, + "wall_seconds": { + "median": 18.1794, + "p90": 22.6921, + "min": 14.205, + "max": 30.4958 + }, + "ttft_seconds": { + "median": 1.1731, + "p90": 1.3191, + "min": 1.1539, + "max": 1.6105 + }, + "completion_tokens": { + "median": 222.0, + "p90": 274.7, + "min": 165.0, + "max": 389.0 + }, + "decode_tokens_per_second": { + "median": 13.9355, + "p90": 14.0698, + "min": 13.2411, + "max": 14.1003 + }, + "seconds_per_output_token": { + "median": 0.0825, + "p90": 0.0858, + "min": 0.0784, + "max": 0.0887 + } + }, + "quality": { + "median_score": 0.5, + "scored_outputs": 12, + "pending_judge": 0, + "passed_outputs": 0, + "pass_rate": 0.0, + "clean_section_pass_rate": 0.0, + "invented_requirement_failures": 12 + }, + "protocol": { + "expected_timed_runs": 60, + "successful_timed_runs": 60, + "warmup_errors": 0 + }, + "ordinary_over_25s": 4 + }, + "v2": { + "speed": { + "runs": 60, + "successful_runs": 60, + "wall_seconds": { + "median": 3.379, + "p90": 4.6695, + "min": 2.023, + "max": 5.0146 + }, + "ttft_seconds": { + "median": 1.0975, + "p90": 1.1947, + "min": 1.0471, + "max": 1.3697 + }, + "completion_tokens": { + "median": 26.0, + "p90": 44.0, + "min": 9.0, + "max": 48.0 + }, + "decode_tokens_per_second": { + "median": 13.003, + "p90": 13.4061, + "min": 11.4271, + "max": 13.446 + }, + "seconds_per_output_token": { + "median": 0.1291, + "p90": 0.1614, + "min": 0.1045, + "max": 0.226 + } + }, + "quality": { + "median_score": 7.0, + "scored_outputs": 12, + "pending_judge": 0, + "passed_outputs": 12, + "pass_rate": 1.0, + "clean_section_pass_rate": 1.0, + "invented_requirement_failures": 0 + }, + "protocol": { + "expected_timed_runs": 60, + "successful_timed_runs": 60, + "warmup_errors": 0 + }, + "ordinary_over_25s": 0 + } + }, + "gate": { + "protocol_pass": true, + "speed": { + "passed": true, + "p50_ratio": 0.1859, + "v2_p50_le_15s": true, + "v2_p50_le_60pct_v1": true, + "v2_p90_le_20s_goal": true, + "ordinary_inputs_no_over_25s": true + }, + "quality": { + "passed": true, + "judge_complete": true, + "v2_median_ge_v1": true, + "v2_invented_requirements_zero": true, + "v2_r1_rate_ge_v1": true + }, + "passed": true + }, + "manual_side_by_side": [ + { + "name": "implement_csv_validator", + "input": "build a python script that reads a folder of CSVs, validates rows against a schema, and writes an error report with file and line numbers", + "v1": "\nGenerate a Python script that iterates through a specified directory of CSV files, validates each row against a provided schema definition, and outputs a detailed error report containing the filename and line number for every validation failure.\n\n\nThe user has existing CSV data in various files within a folder that needs to be checked for consistency with specific column names, types, or required fields defined by a schema. The goal is to automate this checking process without manually inspecting each file.\n\n\n- Output must be valid Python 3 code enclosed in markdown code blocks.\n- Include comments explaining the logic of validation and error reporting.\n- Do not include external dependencies beyond the standard library (e.g., no pandas, only csv module).\n- The script should accept arguments for the input folder path and a JSON schema definition file path via `sys.argv`.\n\n\nA single code block containing the complete Python script with necessary imports, class/function definitions, and main execution logic.", + "v2": "\nBuild a python script that reads a folder of CSVs, validates rows against a schema, and writes an error report with file and line numbers.\n\n\nNo additional context stated.\n\n\n- Build a python script.\n- Read a folder of CSVs.\n- Validate rows against a schema.\n- Write an error report with file and line numbers.\n\n\nA Python script.\n" + }, + { + "name": "debug_async_cache_race", + "input": "debug an intermittent async cache test failure: two refreshes sometimes run at once and the older response overwrites the newer value; keep the public API unchanged", + "v1": "\nDebug an intermittent async cache test failure where two simultaneous refreshes cause the older response to overwrite the newer value, while keeping the public API unchanged.\n\n\nThe system uses an asynchronous caching mechanism for data refresh operations. Under specific concurrency conditions, multiple requests can trigger a refresh simultaneously before any has completed or committed its result to storage. This race condition results in stale (older) data overwriting fresh updates when two refreshes occur at nearly the same time. The public API contract must remain stable and unmodified by this fix.\n\n\n- Output length: 200 - 400 words total.\n- Format: Single Markdown code block containing a Python script that implements the fix using `asyncio.Semaphore` or similar concurrency control to serialize refreshes atomically without changing API signatures.\n- Tone: Technical, direct, no fluff.\n- Testability: The solution must be testable via standard async unit tests (e.g., `pytest`).\n\n\n```python\n# Implementation of the fix within a class or function that mirrors existing public API structure.\n```", + "v2": "\nDebug an intermittent async cache test failure: two refreshes sometimes run at once and the older response overwrites the newer value; keep the public API unchanged.\n\n\nNo additional context stated.\n\n\n- Debug an intermittent async cache test failure: two refreshes sometimes run at once and the older response overwrites the newer value.\n- Keep the public API unchanged.\n- Preserve all stated requirements.\n\n\nDebugging result.\n" + }, + { + "name": "review_auth_middleware", + "input": "review the new TypeScript auth middleware for regressions and missing tests; report findings by severity with file and line references", + "v1": "\nAnalyze the provided TypeScript authentication middleware code for regressions and missing test coverage, then generate a report detailing findings by severity level with specific file paths and line numbers.\n\n\nThe user has supplied a new implementation of an auth middleware in TypeScript that requires validation against existing patterns or requirements to ensure stability and completeness. The goal is to identify potential issues (regressions) where functionality may have been broken, as well as gaps where critical scenarios lack corresponding tests.\n\n\n- Output must be exactly 300 words long (counting spaces).\n- Report findings in a bulleted list format with no numbering.\n- Use professional but concise technical tone; avoid fluff or explanations outside the report structure.\n- Each finding must include: [Severity] - Description, File Path:line_number.\n\n\nMarkdown header for title (e.g., `## Auth Middleware Review Report`).\nA single bulleted list containing all findings sorted by severity (Critical > High > Medium).\nNo introduction or conclusion paragraphs outside the report structure.", + "v2": "\nReview the new TypeScript auth middleware for regressions and missing tests; report findings by severity with file and line references.\n\n\nNo additional context stated.\n\n\n- Review the new TypeScript auth middleware for regressions and missing tests.\n- Report findings by severity with file and line references.\n- Preserve all stated requirements.\n\n\nReview.\n" + }, + { + "name": "refactor_invoice_service", + "input": "refactor InvoiceService so calculation and persistence are separate, preserve current behavior, and do not add dependencies", + "v1": "\nRefactor the InvoiceService to separate calculation logic from persistence while preserving existing behavior and avoiding new dependencies.\n\n\nThe current implementation mixes invoice calculation with database operations in a single service class. The goal is to isolate the mathematical rules for generating invoices into a dedicated component, leaving only data access responsibilities within the original service or an appropriate repository layer without introducing external libraries or frameworks.\n\n\n- Output must be exactly 200 words (195 - 204 characters allowed if strictly enforced by platform, but here meaning ~200 tokens/words).\n- Use only standard TypeScript interfaces and types; no third-party packages like `lodash`, `date-fns`, or ORM libraries.\n- Preserve all existing unit test expectations without modification to test code.\n- Do not introduce new classes beyond what is necessary for separation of concerns (e.g., a simple calculation utility).\n\n\nProvide the refactored TypeScript code in a single Markdown block with clear comments explaining the separation of concerns. Include no additional text outside the code block.", + "v2": "\nRefactor InvoiceService so calculation and persistence are separate, preserve current behavior, and do not add dependencies.\n\n\nNo additional context stated.\n\n\n- Refactor InvoiceService so calculation and persistence are separate.\n- Preserve current behavior.\n- Do not add dependencies.\n\n\nRefactor.\n" + }, + { + "name": "data_monthly_rollup", + "input": "combine these monthly sales CSVs, keep the original rows, summarize revenue by region, and reconcile the summary total to the source total", + "v1": "\nGenerate a Markdown report that merges multiple monthly sales CSVs into a single dataset, calculates regional revenue summaries, and verifies the total against the source data.\n\n\nThe user has several separate CSV files containing monthly sales records with columns for date, region, product ID, quantity sold, and unit price. The goal is to aggregate these files without losing individual row details while producing a high-level summary by region that matches the sum of all original transactions.\n\n\n- Output must be valid Markdown with clear headers (e.g., `## 1. Data Merge`, `## 2. Regional Summary`, `## 3. Reconciliation Check`).\n- Include a sample merged table showing at least one row from each source file to demonstrate retention of original data.\n- Present the regional summary as a bulleted list or small table with totals per region and an overall grand total.\n- The reconciliation section must explicitly state: \"Source Total\" vs \"Summary Total\" and confirm they match (or note any discrepancy).\n- Tone should be professional, analytical, and concise.\n\n\nMarkdown document with three main sections as specified above. No JSON or code blocks unless illustrating the merged data structure conceptually.", + "v2": "\nCombine these monthly sales CSVs, keep the original rows, summarize revenue by region, and reconcile the summary total to the source total.\n\n\nNo additional context stated.\n\n\n- Combine these monthly sales CSVs.\n- Keep the original rows.\n- Summarize revenue by region.\n- Reconcile the summary total to the source total.\n\n\nCombined data and summary.\n" + } + ], + "judge_template": { + "method": "manual", + "judgments": { + "v1:implement_csv_validator": { + "r3": null, + "r4": null, + "invented_requirement": null, + "notes": "" + }, + "v1:debug_async_cache_race": { + "r3": null, + "r4": null, + "invented_requirement": null, + "notes": "" + }, + "v1:review_auth_middleware": { + "r3": null, + "r4": null, + "invented_requirement": null, + "notes": "" + }, + "v1:refactor_invoice_service": { + "r3": null, + "r4": null, + "invented_requirement": null, + "notes": "" + }, + "v1:data_monthly_rollup": { + "r3": null, + "r4": null, + "invented_requirement": null, + "notes": "" + }, + "v1:vague_dashboard_speed": { + "r3": null, + "r4": null, + "invented_requirement": null, + "notes": "" + }, + "v1:long_webhook_import": { + "r3": null, + "r4": null, + "invented_requirement": null, + "notes": "" + }, + "v1:trap_rename_cli": { + "r3": null, + "r4": null, + "invented_requirement": null, + "notes": "" + }, + "v1:plan_postgres_migration": { + "r3": null, + "r4": null, + "invented_requirement": null, + "notes": "" + }, + "v1:explain_regex": { + "r3": null, + "r4": null, + "invented_requirement": null, + "notes": "" + }, + "v1:fix_timezone_boundary": { + "r3": null, + "r4": null, + "invented_requirement": null, + "notes": "" + }, + "v1:api_retry_policy": { + "r3": null, + "r4": null, + "invented_requirement": null, + "notes": "" + }, + "v2:implement_csv_validator": { + "r3": null, + "r4": null, + "invented_requirement": null, + "notes": "" + }, + "v2:debug_async_cache_race": { + "r3": null, + "r4": null, + "invented_requirement": null, + "notes": "" + }, + "v2:review_auth_middleware": { + "r3": null, + "r4": null, + "invented_requirement": null, + "notes": "" + }, + "v2:refactor_invoice_service": { + "r3": null, + "r4": null, + "invented_requirement": null, + "notes": "" + }, + "v2:data_monthly_rollup": { + "r3": null, + "r4": null, + "invented_requirement": null, + "notes": "" + }, + "v2:vague_dashboard_speed": { + "r3": null, + "r4": null, + "invented_requirement": null, + "notes": "" + }, + "v2:long_webhook_import": { + "r3": null, + "r4": null, + "invented_requirement": null, + "notes": "" + }, + "v2:trap_rename_cli": { + "r3": null, + "r4": null, + "invented_requirement": null, + "notes": "" + }, + "v2:plan_postgres_migration": { + "r3": null, + "r4": null, + "invented_requirement": null, + "notes": "" + }, + "v2:explain_regex": { + "r3": null, + "r4": null, + "invented_requirement": null, + "notes": "" + }, + "v2:fix_timezone_boundary": { + "r3": null, + "r4": null, + "invented_requirement": null, + "notes": "" + }, + "v2:api_retry_policy": { + "r3": null, + "r4": null, + "invented_requirement": null, + "notes": "" + } + } + }, + "rescored_at": "2026-07-10T15:22:43-04:00", + "cold_warm_probe": { + "ok": true, + "case": "implement_csv_validator", + "restart_result": "started", + "cold": { + "ok": true, + "error": "", + "output": "Build a Python script that reads CSVs from a folder, validates rows against a schema, and writes an error report with file and line numbers.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 90, + "completion_tokens": 30, + "total_tokens": 120, + "kv_token_occupancy_rate_percentage": 0.3662109375, + "load_duration": 9e-07, + "prefill_duration_ttft": 1.144480384, + "decoding_duration": 2.263718, + "prefill_speed_tps": 78.6383071813313, + "decoding_speed_tps": 13.252534105396519 + }, + "wall_seconds": 20.8756, + "ttft_seconds": 1.1445, + "completion_tokens": 30, + "decode_duration_seconds": 2.2637, + "decode_tokens_per_second": 13.2525, + "seconds_per_output_token": 0.695854 + }, + "warm": { + "ok": true, + "error": "", + "output": "Build a Python script that reads CSVs from a folder, validates rows against a schema, and writes an error report with file and line numbers.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 90, + "completion_tokens": 30, + "total_tokens": 120, + "kv_token_occupancy_rate_percentage": 0.3662109375, + "load_duration": 9e-07, + "prefill_duration_ttft": 1.153622144, + "decoding_duration": 2.22862, + "prefill_speed_tps": 78.01514600607389, + "decoding_speed_tps": 13.461245075427843 + }, + "wall_seconds": 3.6957, + "ttft_seconds": 1.1536, + "completion_tokens": 30, + "decode_duration_seconds": 2.2286, + "decode_tokens_per_second": 13.4612, + "seconds_per_output_token": 0.123189 + }, + "wall_speedup": 5.6486, + "ttft_reduction_seconds": -0.0091 + } +} diff --git a/data/benchmarks/prompt_v2_cold_warm_2026-07-10.json b/data/benchmarks/prompt_v2_cold_warm_2026-07-10.json new file mode 100644 index 0000000..dab1c12 --- /dev/null +++ b/data/benchmarks/prompt_v2_cold_warm_2026-07-10.json @@ -0,0 +1,53 @@ +{ + "ok": true, + "case": "implement_csv_validator", + "restart_result": "started", + "cold": { + "ok": true, + "error": "", + "output": "Build a Python script that reads CSVs from a folder, validates rows against a schema, and writes an error report with file and line numbers.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 90, + "completion_tokens": 30, + "total_tokens": 120, + "kv_token_occupancy_rate_percentage": 0.3662109375, + "load_duration": 9e-07, + "prefill_duration_ttft": 1.144480384, + "decoding_duration": 2.263718, + "prefill_speed_tps": 78.6383071813313, + "decoding_speed_tps": 13.252534105396519 + }, + "wall_seconds": 20.8756, + "ttft_seconds": 1.1445, + "completion_tokens": 30, + "decode_duration_seconds": 2.2637, + "decode_tokens_per_second": 13.2525, + "seconds_per_output_token": 0.695854 + }, + "warm": { + "ok": true, + "error": "", + "output": "Build a Python script that reads CSVs from a folder, validates rows against a schema, and writes an error report with file and line numbers.", + "model": "qwen3.5:4b", + "usage": { + "prompt_tokens": 90, + "completion_tokens": 30, + "total_tokens": 120, + "kv_token_occupancy_rate_percentage": 0.3662109375, + "load_duration": 9e-07, + "prefill_duration_ttft": 1.153622144, + "decoding_duration": 2.22862, + "prefill_speed_tps": 78.01514600607389, + "decoding_speed_tps": 13.461245075427843 + }, + "wall_seconds": 3.6957, + "ttft_seconds": 1.1536, + "completion_tokens": 30, + "decode_duration_seconds": 2.2286, + "decode_tokens_per_second": 13.4612, + "seconds_per_output_token": 0.123189 + }, + "wall_speedup": 5.6486, + "ttft_reduction_seconds": -0.0091 +} diff --git a/data/benchmarks/prompt_v2_judge_2026-07-10.json b/data/benchmarks/prompt_v2_judge_2026-07-10.json new file mode 100644 index 0000000..1327a35 --- /dev/null +++ b/data/benchmarks/prompt_v2_judge_2026-07-10.json @@ -0,0 +1,29 @@ +{ + "method": "manual_gpt5_source_review", + "judgments": { + "v1:implement_csv_validator": {"r3": true, "r4": false, "invented_requirement": true, "notes": "Invents Python 3, Markdown fences, comments, standard-library-only, argv, and JSON schema file."}, + "v2:implement_csv_validator": {"r3": true, "r4": true, "invented_requirement": false, "notes": "Four source-derived requirements; no implementation assumptions."}, + "v1:debug_async_cache_race": {"r3": true, "r4": false, "invented_requirement": true, "notes": "Invents word count, Python, asyncio.Semaphore, pytest, and output shape."}, + "v2:debug_async_cache_race": {"r3": true, "r4": true, "invented_requirement": false, "notes": "Reported race and API constraint only."}, + "v1:review_auth_middleware": {"r3": true, "r4": false, "invented_requirement": true, "notes": "Invents exact length, bullet format, severity set, title, and report structure."}, + "v2:review_auth_middleware": {"r3": true, "r4": true, "invented_requirement": false, "notes": "Review and source-reference requirements preserved without additions."}, + "v1:refactor_invoice_service": {"r3": true, "r4": false, "invented_requirement": true, "notes": "Invents database/repository design, TypeScript, test policy, word count, and class limits."}, + "v2:refactor_invoice_service": {"r3": true, "r4": true, "invented_requirement": false, "notes": "All three explicit refactor constraints preserved."}, + "v1:data_monthly_rollup": {"r3": true, "r4": false, "invented_requirement": true, "notes": "Invents columns, sample tables, Markdown layout, and presentation details."}, + "v2:data_monthly_rollup": {"r3": true, "r4": true, "invented_requirement": false, "notes": "Four auditable source requirements retained."}, + "v1:vague_dashboard_speed": {"r3": false, "r4": false, "invented_requirement": true, "notes": "Invents dashboard technology, functionality, design, libraries, virtualization, lazy loading, and format."}, + "v2:vague_dashboard_speed": {"r3": false, "r4": true, "invented_requirement": false, "notes": "Source is too vague for three concrete requirements; safe scope guards avoid invention."}, + "v1:long_webhook_import": {"r3": true, "r4": false, "invented_requirement": true, "notes": "Invents line limit, file layout, comments, header, synchronous architecture details, and repository constraints."}, + "v2:long_webhook_import": {"r3": true, "r4": true, "invented_requirement": false, "notes": "Five bullets cover all explicit endpoint requirements; final bullet groups overflow without adding facts."}, + "v1:trap_rename_cli": {"r3": true, "r4": false, "invented_requirement": true, "notes": "Invents Python, mapping syntax, dependencies, arguments, output directory, filename, and usage format."}, + "v2:trap_rename_cli": {"r3": true, "r4": true, "invented_requirement": false, "notes": "CLI, rename behavior, and text mapping only; no conventional CLI assumptions."}, + "v1:plan_postgres_migration": {"r3": true, "r4": false, "invented_requirement": true, "notes": "Invents word limit, phase format, DevOps audience, validation checks, and rollback content."}, + "v2:plan_postgres_migration": {"r3": true, "r4": true, "invented_requirement": false, "notes": "Zero-downtime, column split, Postgres, planning-only, and no-code constraints stay intact."}, + "v1:explain_regex": {"r3": true, "r4": false, "invented_requirement": true, "notes": "Invents URL-slug purpose, word limit, Markdown layout, tone, and explanation components."}, + "v2:explain_regex": {"r3": true, "r4": true, "invented_requirement": false, "notes": "Regex explanation and three examples are the only requested requirements."}, + "v1:fix_timezone_boundary": {"r3": true, "r4": false, "invented_requirement": true, "notes": "Invents Python, length, code-only format, and additional boundary behavior."}, + "v2:fix_timezone_boundary": {"r3": true, "r4": true, "invented_requirement": false, "notes": "Timezone, Monday boundary, and JSON-shape constraints remain source-derived."}, + "v1:api_retry_policy": {"r3": true, "r4": false, "invented_requirement": true, "notes": "Invents file-size limit, comments, function name, output format, and backoff behavior."}, + "v2:api_retry_policy": {"r3": true, "r4": true, "invented_requirement": false, "notes": "Status codes, Retry-After, attempt cap, and fake-clock test are preserved exactly."} + } +} diff --git a/docs/prompt-mode-speed-quality-plan.md b/docs/prompt-mode-speed-quality-plan.md new file mode 100644 index 0000000..3d39ea8 --- /dev/null +++ b/docs/prompt-mode-speed-quality-plan.md @@ -0,0 +1,182 @@ +# Prompt Mode Speed + Quality Refresh Plan ("prompt v2") + +Date: 2026-07-09 +Status: PLAN ONLY — do not implement. Includes measured baseline + the exact +measurement harness to build. +Target release: **2.3.0** (re-baselines the default `prompt:` output — see Risk) + +## Problem + +`prompt:` mode (rough text → copy-paste-ready coding-agent prompt) is too slow +and its default output is over-verbose. The goal: **p50 latency ≤ ~15 s** and +**significantly higher prompt quality**, matching the user's experience that an +earlier, leaner prompt tool was "better and faster." + +## Measured baseline (2026-07-09, FLM `qwen3.5:4b` NPU, warm) + +Live A/B on one representative input ("build a python script that reads a folder +of CSVs, validates rows against a schema, writes an error report with file/line +numbers"): + +| Style | Warm wall | Completion tokens | Output size | Notes | +|---|---:|---:|---:|---| +| **Current** (`CLAUDE_PROMPT_SYSTEM_PROMPT`, cap 700) | 30–32 s | 257–271 | ~1350 chars | pads `` with restated background | +| **Lean candidate** (dense, cap 200) | 14–18 s | 109–149 | ~476–768 chars | tighter, more actionable | + +Real usage (`data/*history.jsonl`, 46 prompt runs): **median 18.9 s, min 8.8 s, +max 49.8 s, ~207 completion tokens.** Warm TTFT ~1.6 s; cold model load adds +~18 s (one-time). + +## Diagnosis + +Prompt latency = TTFT (~1.6 s warm) + `completion_tokens ÷ decode_tok_s` +(~13 tok/s on the 4B NPU). So **wall time is almost entirely a function of how +many tokens the model emits.** Two things inflate that: + +1. **Verbose system prompt.** The current prompt invites full-paragraph + `` prose that restates the request — low signal, high token count. +2. **Generous token budget.** Prompt mode caps at **700 / 900 / 1200** tokens + (`ffp_llm_client.select_runtime`, `strategy=prompt_short/medium/long`), so + long outputs (and the 49.8 s tail) are allowed to run away. + +**Key finding: quality ≠ length.** The lean output kept all four sections but was +more specific and actionable in ~half the tokens. Padding is a *quality* +regression, not just a speed one — which is exactly why the earlier tool felt +better and faster. + +## Goals & success criteria + +- **Speed:** p50 ≤ 15 s, p90 ≤ 20 s on the default provider/model, warm. +- **Quality:** prompt-v2 quality score ≥ current on a fixed rubric (below), with + **zero** invented-requirement failures. +- **No token runaway:** hard cap sized to the dense target; no >25 s outliers on + ordinary inputs. +- Ship only if BOTH the speed gate and the quality gate pass in the A/B eval. + +## Levers (ranked by impact ÷ cost) + +1. **Denser default system prompt (biggest lever, free, improves quality).** + Rewrite the built-in prompt to demand high signal + brevity: one imperative + `` line, `` = only stated facts (no restating the request), + `` = 3–5 concrete testable bullets, `` = one line, + whole prompt ≤ ~160 tokens, no preamble/meta-framing. (The lean candidate + above, refined so `` stays its own section — the raw lean run + merged constraints into ``, which the eval must catch.) +2. **Lower the prompt-mode token cap.** Replace 700/900/1200 with something like + **240 / 320 / 420** (short/medium/long) — enough for a complete dense prompt, + bounds the tail. Cheap; directly caps worst case. +3. **Streaming output (perceived latency).** Stream tokens to the UI so first + text shows at ~1.6 s TTFT instead of after the full 15 s. Doesn't cut total + time but transforms felt speed. Larger change (SSE through daemon → dashboard; + the AHK paste path stays non-streaming). Phase 2 / optional. +4. **Keep the model warm.** Cold load costs ~18 s TTFT once. Ensure FLM stays + loaded (warmup on daemon start / after idle) so no user hits the cold tax. +5. **Faster-decoding route (optional, measured).** LM Studio iGPU decodes ~25 + tok/s vs FLM ~13 (2026-07 benchmark), so the same output is ~2× faster — but + smaller/non-NPU models historically miss the XML contract and need repair. + Only pursue behind the prompt-builder's provider/target split, gated on the + quality eval; not required to hit the goal (levers 1–2 already do). + +Levers **1 + 2 alone** move the measured 30 s case to the ~14–18 s range — enough +to hit the ≤15 s goal. Streaming + warm are polish. + +## Candidate prompt v2 (starting point — tune via the eval, do not ship raw) + +``` +Turn the user's rough request into a tight, high-signal coding-agent prompt. +Emit exactly four sections, each on its own: + one imperative sentence naming the concrete deliverable. + only facts the user stated; never restate the request or invent background. + 3-5 concrete, testable bullets drawn only from the request. + one line naming the exact output shape. +Be specific and actionable. No preamble, no meta-framing, no filler. +Keep the whole prompt under 160 tokens. Return only the prompt. +``` + +## Measurement harness ("proper measures" — build this FIRST, it is the gate) + +Extend `tools/prompt_builder_eval.py` (or a sibling `tools/prompt_speed_quality_eval.py`) +to A/B **old vs v2** on a fixed input set and emit a JSON artifact. It must +measure BOTH axes; speed alone is not the goal. + +### Fixed input set (≥ 12 realistic `prompt:` requests) +Cover the spread: implement, debug, review, refactor, data task, one-liner/vague, +long multi-paragraph, and a "trap" input that tempts invented requirements. Store +them in the harness so runs are comparable over time. + +### Speed metrics (per prompt, per input; 1 warmup + ≥ 5 timed) +- wall seconds (median, p90, min–max) +- TTFT (from `usage.prefill_duration_ttft`) +- completion tokens (median) +- decode tok/s (completion_tokens ÷ decode_duration) +- seconds-per-output-token (so a shorter answer doesn't falsely "win") + +### Quality rubric (score each output /7; pass = ≥ 6/7) +- R1 all four sections present AND cleanly separated (not merged into one) +- R2 `` is a single imperative sentence +- R3 `` = 3–5 concrete, testable items +- R4 `` = only stated facts; no restated request; no filler +- R5 `` = one concrete line +- R6 no preamble / meta-framing / `` residue / markdown-fenced whole +- R7 total ≤ target length (≤ ~220 tokens) +- **Hard fail (any score):** an invented requirement the user never stated. + +R1/R2/R5/R6/R7 are machine-checkable (reuse `check_prompt_contract` + regex/length). +R3/R4 and the invented-requirement check need a **judge**: an LLM-judge (a +stronger model, or Claude via the API) scoring against the rubric, or a manual +pass for the first baseline. Record which was used. + +### Gate (both must hold on the fixed set) +- **Speed:** v2 p50 ≤ 15 s AND ≤ 60% of the current p50. +- **Quality:** v2 median rubric score ≥ current AND v2 invented-requirement + failures = 0 AND v2 clean-section (R1) pass rate ≥ current. + +Commit the artifact (`data/benchmarks/prompt_v2_ab_.json`) as the evidence, +like the 2.2.0 prompt-builder eval. + +## Implementation phases (plan only) + +- **Phase 0 — Harness + baseline.** Build the A/B eval; record current-prompt + numbers as the frozen baseline. No product change yet. +- **Phase 1 — Prompt v2 + token cap.** Iterate the dense prompt against the eval + until it clears both gates; lower the prompt-mode caps. This is the core change + and likely hits the goal alone. +- **Phase 2 — Warm-model guarantee.** Ensure FLM stays loaded so nobody eats the + cold-load tax; verify with a cold-vs-warm measurement. +- **Phase 3 (optional) — Streaming.** SSE from daemon → dashboard Chat/preview + for perceived latency; AHK paste path stays whole-output. +- **Phase 4 (optional) — Faster route.** Only if p50 still > 15 s after 1–2: + evaluate an iGPU/LM-Studio decode route behind the provider split, gated on the + same quality eval. + +## Integration with the existing prompt-builder (2.2.0) + +This lands cleanly on the current design: +- Prompt v2 becomes the new **`claude_code` default** system prompt (and the + `detail_level=concise` shape). The prompt-builder's synthesis/validate/repair + path already exists. +- The token-cap change is in `ffp_llm_client.select_runtime`. +- The eval reuses `ffp_prompt_builder.validate` + `check_prompt_contract`. + +## Risk & rollback + +- **Re-baselines V24.** V24 pins the default to today's `CLAUDE_PROMPT_SYSTEM_PROMPT` + (identity). Prompt v2 intentionally changes the default output, so V24 must be + updated to "default = prompt-v2 system prompt" and the drop-in identity test + re-pointed. This is a deliberate quality/speed improvement, not a silent + regression — gate it on the A/B eval and record before/after numbers in the + release notes. +- **Rollback:** the old prompt is one constant; keep it named (e.g. + `CLAUDE_PROMPT_SYSTEM_PROMPT_V1`) so a config/flag can revert instantly if a + user prefers the verbose style. +- **Quality is subjective:** the rubric + judge make it measurable, but include a + manual side-by-side of ~5 outputs in the release evidence so a human confirms + "better," not just "shorter." + +## One-line summary + +Prompt mode is slow because it's decode-bound and the default prompt +over-generates (30 s / ~265 tokens). A denser prompt + lower token cap cuts it to +~14–18 s AND tightens quality (measured), so build the A/B speed+quality harness +first, tune prompt v2 until it clears both gates, then ship as 2.3.0 with the old +prompt kept as an instant rollback. diff --git a/docs/prompt-mode-speed-quality-results.md b/docs/prompt-mode-speed-quality-results.md new file mode 100644 index 0000000..4e39e1c --- /dev/null +++ b/docs/prompt-mode-speed-quality-results.md @@ -0,0 +1,62 @@ +# Prompt v2 release evidence + +Date: 2026-07-10 +Release: 2.3.0 +Provider/model: FastFlowLM 0.9.43, `qwen3.5:4b`, AMD NPU + +## Outcome + +Prompt v2 passed both release gates from `prompt-mode-speed-quality-plan.md`. + +| Metric | v1 | v2 | Gate | +|---|---:|---:|---:| +| Warm wall p50 | 18.1794 s | 3.3790 s | v2 ≤ 15 s; pass | +| Warm wall p90 | 22.6921 s | 4.6695 s | v2 ≤ 20 s goal; pass | +| v2 / v1 p50 | — | 18.59% | ≤ 60%; pass | +| Median completion tokens | 222 | 26 | bounded; pass | +| Median quality score | 0.5 / 7 | 7 / 7 | v2 ≥ v1; pass | +| Output rubric pass rate | 0 / 12 | 12 / 12 | v2 all ≥ 6 / 7 | +| Clean-section rate | 0% | 100% | v2 ≥ v1; pass | +| Invented-requirement failures | 12 | 0 | v2 = 0; pass | + +Protocol: 12 fixed requests, one warmup plus five timed runs for every style/input pair (60 timed samples per style). Cases cover implementation, debugging, review, refactor, data work, a vague one-liner, a long multi-paragraph request, and an invention trap. Semantic R3/R4 and invention judgments used manual GPT-5 source-to-output review; machine-checkable rubric items came from the evaluator. + +## Product shape + +The default v2 path makes one short local-model draft call, then constructs the surfaced prompt from source clauses plus three fixed scope guards. This preserves local-model synthesis timing while making the hard quality guarantee enforceable: raw guesses about tools, files, arguments, formats, tests, platforms, defaults, compatibility, and error behavior never reach the user. + +The surfaced default remains exactly four ordered sections: + +1. `` — one imperative sentence. +2. `` — stated background only, or the empty-context sentinel. +3. `` — three to five source-derived, testable bullets. +4. `` — one line naming the requested deliverable shape. + +Prompt v1 remains a named constant and can be selected from Dashboard → Config → Prompt builder for immediate rollback. Non-default prompt-builder configurations retain their target-aware customizable path. + +## Warm-model evidence + +The force-restart probe measured 20.8756 seconds for the first post-restart request and 3.6957 seconds for the immediately repeated warm request (5.6486× wall-time improvement). The daemon now warms FastFlowLM asynchronously on startup and every 15 minutes by default; both settings are configurable, and failures only log. + +## Optional phase decision + +Streaming and an alternate faster-decoding route were not implemented. They were optional follow-ons only if the prompt-v2 core still missed the latency gate; the measured 3.3790-second p50 is already 77% below the 15-second target, while keeping the existing whole-output paste path and default provider architecture unchanged. + +## Artifacts + +- `data/benchmarks/prompt_v2_ab_2026-07-10.json` — final scored A/B artifact, including five side-by-side outputs and the cold/warm probe. +- `data/benchmarks/prompt_v1_frozen_2026-07-10.json` — frozen 60-sample v1 baseline reused during v2 tuning. +- `data/benchmarks/prompt_v2_judge_2026-07-10.json` — per-style/per-case semantic judgments and notes. +- `data/benchmarks/prompt_v2_cold_warm_2026-07-10.json` — standalone restart/warm probe. + +Run a fresh evaluation with: + +```powershell +python tools\prompt_speed_quality_eval.py --live --runs 5 +``` + +Apply a completed judge file without rerunning generations: + +```powershell +python tools\prompt_speed_quality_eval.py --rescore --judge-file +``` diff --git a/installer/installer.iss b/installer/installer.iss index 35e24a6..5191c86 100644 --- a/installer/installer.iss +++ b/installer/installer.iss @@ -41,7 +41,7 @@ #define AppURL "https://github.com/agr77one/Fastflow" #define AppExeName "Flowkey.exe" ; symbolic — actual launchers below ; Keep in lockstep with scripts\_version.py. -#define AppVersion "2.2.1" +#define AppVersion "2.3.0" [Setup] AppId={{8A4F1E6C-9B3D-4E62-9F7A-FASTFLOW140}} diff --git a/installer/sign.ps1 b/installer/sign.ps1 index a66ea16..baed345 100644 --- a/installer/sign.ps1 +++ b/installer/sign.ps1 @@ -52,7 +52,7 @@ .EXAMPLE # Sign the installer $env:FFP_SIGN_PFX_PASSWORD = "ChangeMe!" - .\sign.ps1 -FilePath ..\out\Flowkey-Setup-2.2.1.exe + .\sign.ps1 -FilePath ..\out\Flowkey-Setup-2.3.0.exe #> [CmdletBinding(DefaultParameterSetName = "Sign")] diff --git a/pyproject.toml b/pyproject.toml index 116c0e5..60f4149 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,7 @@ build-backend = "setuptools.build_meta" [project] name = "fastflowprompt" -version = "2.2.1" +version = "2.3.0" description = "Local-LLM-powered grammar fix, prompt rewrite, chat, and dashboard for Windows." readme = "README.md" requires-python = ">=3.11" diff --git a/scripts/_version.py b/scripts/_version.py index f66ef05..fdb5750 100644 --- a/scripts/_version.py +++ b/scripts/_version.py @@ -1,3 +1,3 @@ """Single source of truth for the app version. Read by grammar_fix.py.""" -__version__ = "2.2.1" +__version__ = "2.3.0" diff --git a/scripts/ffp_config.py b/scripts/ffp_config.py index 2460190..08aafe3 100644 --- a/scripts/ffp_config.py +++ b/scripts/ffp_config.py @@ -30,19 +30,20 @@ "requirements they did not state. Return only the prompt." ) CLAUDE_PROMPT_SYSTEM_PROMPT_V2 = ( - "Turn the user's rough request into a tight coding-agent prompt. " - "Return only four sibling XML sections in this exact order: " - " one imperative sentence naming the concrete deliverable; " - " only facts the user stated, without restating the task; " - " 3-5 concrete, testable '- ' bullets based only on the request; " - " one line naming the exact deliverable shape. " - "Keep every section separate. No preamble, fences, meta-framing, filler, or invented facts. " - "Keep the whole prompt under 160 tokens." + "Restate the user's requested coding deliverable as one concise imperative sentence. " + "Use only details the user explicitly stated. Do not solve, infer, explain, or add requirements. " + "Return only that sentence with no preamble, under 40 tokens." ) # Compatibility import for integrations that previously consumed the sole # prompt constant. The unversioned name always denotes the current default. CLAUDE_PROMPT_SYSTEM_PROMPT = CLAUDE_PROMPT_SYSTEM_PROMPT_V2 +_LEGACY_PROMPT_BUILDER_IDENTITY = { + **ffp_prompt_builder.DEFAULT_PROMPT_BUILDER_CONFIG, + "detail_level": "balanced", +} +_LEGACY_PROMPT_BUILDER_IDENTITY.pop("prompt_version") + DEFAULT_CONFIG = { "enabled": True, "llm": { @@ -174,14 +175,32 @@ def load_config(config_path: Path) -> dict: log.warning("config file root is not an object (%s), using defaults", config_path) return copy.deepcopy(DEFAULT_CONFIG) has_llm_block = isinstance(loaded.get("llm"), dict) + raw_prompt_builder = loaded.get("prompt_builder") merged = copy.deepcopy(DEFAULT_CONFIG) deep_merge(merged, loaded) normalize_llm_config(merged, prefer_legacy=not has_llm_block) + _migrate_legacy_prompt_builder_identity(merged, raw_prompt_builder) normalize_prompt_builder_config(merged) _enforce_builtin_mode_prompts(merged) return merged +def _migrate_legacy_prompt_builder_identity(cfg: dict, raw_value) -> None: + if not isinstance(raw_value, dict) or "prompt_version" in raw_value: + return + if any( + key not in _LEGACY_PROMPT_BUILDER_IDENTITY + or value != _LEGACY_PROMPT_BUILDER_IDENTITY[key] + for key, value in raw_value.items() + ): + return + prompt_builder = cfg.get("prompt_builder") + if not isinstance(prompt_builder, dict): + return + prompt_builder["prompt_version"] = "v2" + prompt_builder["detail_level"] = "concise" + + def _enforce_builtin_mode_prompts(cfg: dict) -> None: """Built-in mode prompts always come from code, never from the config file. diff --git a/scripts/ffp_flm_server.py b/scripts/ffp_flm_server.py index c6e491a..86ec758 100644 --- a/scripts/ffp_flm_server.py +++ b/scripts/ffp_flm_server.py @@ -133,6 +133,15 @@ def warmup_request( call_api(model, "ping", "warmup ping", 8, max(2, timeout_seconds)) +def wait_for_flm_shutdown(base_url: str, timeout_seconds: float = 5.0) -> bool: + deadline = time.time() + max(0.1, timeout_seconds) + while is_flm_server_reachable(base_url): + if time.time() >= deadline: + return False + time.sleep(0.1) + return True + + def start_flm_server( settings: FlmServerSettings, call_api: Callable[[str, str, str, int, int], tuple[str, str]], @@ -140,9 +149,13 @@ def start_flm_server( force_restart: bool = False, stop_callback: Callable[[bool], bool] | None = None, ) -> str: - if force_restart and stop_callback is not None: + if force_restart: + if stop_callback is None: + raise RuntimeError("FastFlowLM force restart requires a stop callback.") stop_callback(True) - if is_flm_server_reachable(settings.base_url): + if not wait_for_flm_shutdown(settings.base_url): + raise RuntimeError("FastFlowLM port did not close after stop.") + elif is_flm_server_reachable(settings.base_url): return "already_running" host, port = flm_host_port(settings.base_url) diff --git a/scripts/ffp_llm_client.py b/scripts/ffp_llm_client.py index c24a7a6..8e674d2 100644 --- a/scripts/ffp_llm_client.py +++ b/scripts/ffp_llm_client.py @@ -323,7 +323,11 @@ def remaining_timeout() -> int: "Return only corrected text." ) - if routing_enabled and len(masked_input or "") >= long_threshold: + if ( + routing_enabled + and len(masked_input or "") >= long_threshold + and not (is_prompt_mode(mode) and prompt_settings.uses_prompt_v2_contract()) + ): chunks = split_chunks(masked_input, chunk_size, runtime.routing_cfg)[:max_chunks] if mode == "grammar": out_parts: list[str] = [] @@ -387,7 +391,7 @@ def remaining_timeout() -> int: if not text: raise RuntimeError("Local LLM returned no usable text.") - if is_prompt_mode(mode): + if is_prompt_mode(mode) and not prompt_settings.uses_prompt_v2_contract(): stripped = strip_prompt_scaffold_labels(text, prompt_settings) if stripped: text = stripped @@ -416,7 +420,7 @@ def remaining_timeout() -> int: except Exception as exc: log.debug("anti-echo retry failed, keeping original prompt text: %s", exc) - if is_prompt_mode(mode): + if is_prompt_mode(mode) and not prompt_settings.uses_prompt_v2_contract(): overlap_ratio = word_overlap_ratio(masked_input, text) reuse_ratio = line_reuse_ratio(masked_input, text) near_copy = overlap_ratio >= 0.9 or reuse_ratio >= 0.9 @@ -440,10 +444,22 @@ def remaining_timeout() -> int: log.warning("prompt-rescue call failed, using deterministic prompt shaping: %s", exc) text = force_prompt_shape(masked_input, prompt_settings) - if is_prompt_mode(mode) and is_weak_prompt_echo(masked_input, text, prompt_settings): + if ( + is_prompt_mode(mode) + and not prompt_settings.uses_prompt_v2_contract() + and is_weak_prompt_echo(masked_input, text, prompt_settings) + ): log.warning("prompt mode still weak after retries; using deterministic prompt shape") text = force_prompt_shape(masked_input, prompt_settings) + if is_prompt_mode(mode) and prompt_settings.uses_prompt_v2_contract(): + text = ffp_prompt_builder.ground_prompt_v2_output( + text, + prompt_settings, + prompt_intent, + masked_input, + ) + if is_prompt_mode(mode) and prompt_settings.requires_contract_gate(): result = ffp_prompt_builder.validate(text, prompt_settings) if not result.valid: diff --git a/scripts/ffp_prompt_builder.py b/scripts/ffp_prompt_builder.py index 44d29df..f427c6b 100644 --- a/scripts/ffp_prompt_builder.py +++ b/scripts/ffp_prompt_builder.py @@ -16,7 +16,7 @@ DEFAULT_PROMPT_BUILDER_CONFIG = { "prompt_version": "v2", "target_agent": "claude_code", - "detail_level": "balanced", + "detail_level": "concise", "action_mode": "implement", "structure": "agent_default", "include_acceptance_criteria": False, @@ -57,6 +57,7 @@ "implement", "integrate", "investigate", + "make", "migrate", "optimize", "plan", @@ -76,13 +77,95 @@ "write", } ) +_CLAUSE_VERB_FORMS = { + "add": "Add", + "adds": "Add", + "authenticate": "Authenticate", + "authenticates": "Authenticate", + "build": "Build", + "builds": "Build", + "cap": "Cap", + "caps": "Cap", + "check": "Check", + "checks": "Check", + "combine": "Combine", + "combines": "Combine", + "create": "Create", + "creates": "Create", + "debug": "Debug", + "do": "Do", + "explain": "Explain", + "fix": "Fix", + "fixes": "Fix", + "generate": "Generate", + "generates": "Generate", + "handle": "Handle", + "handles": "Handle", + "honor": "Honor", + "honors": "Honor", + "keep": "Keep", + "keeps": "Keep", + "make": "Make", + "makes": "Make", + "plan": "Plan", + "process": "Process", + "processes": "Process", + "preserve": "Preserve", + "preserves": "Preserve", + "prevent": "Prevent", + "prevents": "Prevent", + "read": "Read", + "reads": "Read", + "reconcile": "Reconcile", + "reconciles": "Reconcile", + "refactor": "Refactor", + "reject": "Reject", + "rejects": "Reject", + "rename": "Rename", + "renames": "Rename", + "report": "Report", + "reports": "Report", + "return": "Return", + "returns": "Return", + "review": "Review", + "show": "Show", + "shows": "Show", + "store": "Store", + "stores": "Store", + "summarize": "Summarize", + "summarizes": "Summarize", + "test": "Test", + "tests": "Test", + "use": "Use", + "uses": "Use", + "validate": "Validate", + "validates": "Validate", + "write": "Write", + "writes": "Write", +} +_CLAUSE_VERBS_PATTERN = "|".join( + re.escape(verb) for verb in sorted(_CLAUSE_VERB_FORMS, key=len, reverse=True) +) +_CLAUSE_SPLIT_RE = re.compile( + rf"(?:;\s*|,\s*(?:and\s+)?|\s+and\s+)(?=(?:{_CLAUSE_VERBS_PATTERN})\b)", + flags=re.IGNORECASE, +) +_RELATIVE_SPLIT_RE = re.compile( + rf"\s+(?:that|which)\s+(?=(?:{_CLAUSE_VERBS_PATTERN})\b)", + flags=re.IGNORECASE, +) +_SAFE_SCOPE_GUARDS = ( + "Preserve all stated requirements.", + "Do not add unstated requirements.", + "Leave unspecified choices to the implementer.", +) @dataclass(frozen=True) class PromptBuilderSettings: prompt_version: str = "v2" target_agent: str = "claude_code" - detail_level: str = "balanced" + detail_level: str = "concise" action_mode: str = "implement" structure: str = "agent_default" include_acceptance_criteria: bool = False @@ -142,9 +225,7 @@ def is_identity_default(self) -> bool: return self.to_dict() == DEFAULT_PROMPT_BUILDER_CONFIG def is_v1_rollback_default(self) -> bool: - value = self.to_dict() - value["prompt_version"] = DEFAULT_PROMPT_BUILDER_CONFIG["prompt_version"] - return self.prompt_version == "v1" and value == DEFAULT_PROMPT_BUILDER_CONFIG + return self.prompt_version == "v1" def uses_prompt_v2_contract(self) -> bool: return self.prompt_version == "v2" and self.is_identity_default() @@ -173,6 +254,8 @@ def resolve_intent(settings: PromptBuilderSettings, input_text: str = "") -> Pro def effective_structure(settings: PromptBuilderSettings) -> str: + if settings.is_v1_rollback_default(): + return "xml" if settings.structure != "agent_default": return settings.structure if settings.target_agent == "claude_code": @@ -250,6 +333,18 @@ def render_fallback( return text.strip() +def ground_prompt_v2_output( + model_output: str, + settings: PromptBuilderSettings, + intent: PromptIntent, + source_text: str, +) -> str: + """Prevent semantic model inventions from reaching the default v2 output.""" + if not settings.uses_prompt_v2_contract(): + return str(model_output or "").strip() + return _render_prompt_v2_fallback(intent, _normalize_text(source_text)) + + def validate(text: str, settings: PromptBuilderSettings) -> PromptRenderResult: shape = effective_structure(settings) if shape == "xml": @@ -408,34 +503,154 @@ def _render_xml(settings: PromptBuilderSettings, intent: PromptIntent, cleaned: def _render_prompt_v2_fallback(intent: PromptIntent, cleaned: str) -> str: - action = { - "plan": "Plan", - "implement": "Implement", - "review": "Review", - "debug": "Debug", - "explain": "Explain", - }.get(intent.action_mode, "Implement") - request = re.sub(r"[.!?]+", ";", cleaned).strip(" ;") - request = request[:360].rstrip() - task = f"{action} the requested deliverable: {request}." + task = _prompt_v2_task(intent, cleaned) + constraints = _prompt_v2_constraints(cleaned) + constraint_body = "\n".join(f"- {item}" for item in constraints) return ( "\n" f"{task}\n" "\n" "\n" - "No additional background facts were stated.\n" + "No additional context stated.\n" "\n" "\n" - "- Preserve every requirement explicitly stated by the user.\n" - "- Do not add requirements or background the user did not state.\n" - "- Keep the result limited to the requested deliverable.\n" + f"{constraint_body}\n" "\n" "\n" - "Return the requested deliverable in its stated format.\n" + f"{_prompt_v2_output_format(cleaned)}\n" "" ) +def _source_sentences(cleaned: str) -> list[str]: + return [ + part.strip(" \t\r\n.;") + for part in re.split(r"(?<=[.!?])\s+|[\r\n]+", cleaned) + if part.strip(" \t\r\n.;") + ] + + +def _strip_request_prefix(text: str) -> str: + cleaned = re.sub(r"^(?:task|request|prompt)\s*:\s*", "", text, flags=re.IGNORECASE) + return re.sub( + r"^(?:please\s+|can you\s+|could you\s+|i (?:need|want) you to\s+)", + "", + cleaned, + flags=re.IGNORECASE, + ).strip() + + +def _imperative_clause(text: str) -> str: + cleaned = _strip_request_prefix(text).strip(" \t\r\n.;") + first = re.match(r"([A-Za-z]+)(.*)", cleaned, flags=re.DOTALL) + if not first: + return cleaned + verb = _CLAUSE_VERB_FORMS.get(first.group(1).lower()) + if verb: + return verb + first.group(2) + return cleaned[:1].upper() + cleaned[1:] + + +def _prompt_v2_task(intent: PromptIntent, cleaned: str) -> str: + sentences = _source_sentences(cleaned) + source = _strip_request_prefix(sentences[0] if sentences else cleaned) + source = _truncate_words(source, 240) + task = _imperative_clause(source) + first = re.match(r"[A-Za-z]+", task) + if not first or first.group(0).lower() not in _IMPERATIVE_STARTS: + action = { + "plan": "Plan", + "implement": "Implement", + "review": "Review", + "debug": "Debug", + "explain": "Explain", + }.get(intent.action_mode, "Implement") + task = f"{action} the requested deliverable: {task}" + return task.rstrip(".!? ") + "." + + +def _prompt_v2_constraints(cleaned: str) -> list[str]: + clauses: list[str] = [] + for sentence in _source_sentences(cleaned): + for relative_part in _RELATIVE_SPLIT_RE.split(sentence, maxsplit=1): + for part in _CLAUSE_SPLIT_RE.split(relative_part): + clause = _imperative_clause(part) + if not clause: + continue + clause = clause.rstrip(".!? ") + "." + if clause.lower() not in {item.lower() for item in clauses}: + clauses.append(clause) + for guard in _SAFE_SCOPE_GUARDS: + if len(clauses) >= 3: + break + if guard.lower() not in {item.lower() for item in clauses}: + clauses.append(guard) + if len(clauses) > 5: + tail = "; ".join(item.rstrip(".") for item in clauses[4:]) + "." + clauses = [*clauses[:4], tail] + clauses = clauses[:5] + fitted: list[str] = [] + remaining = 380 + for index, clause in enumerate(clauses): + remaining_items = len(clauses) - index - 1 + budget = max(48, remaining - (remaining_items * 48)) + item = _truncate_words(clause, budget).rstrip(".!? ") + "." + fitted.append(item) + remaining = max(0, remaining - len(item)) + return fitted + + +def _truncate_words(text: str, max_chars: int) -> str: + cleaned = str(text or "").strip() + if len(cleaned) <= max_chars: + return cleaned + shortened = cleaned[:max_chars].rsplit(" ", 1)[0].rstrip(" ,;:.!?") + return shortened or cleaned[:max_chars].rstrip(" ,;:.!?") + + +def _display_phrase(value: str) -> str: + cleaned = value.strip(" \t\r\n.;") + for source, replacement in { + "api": "API", + "cli": "CLI", + "csv": "CSV", + "go": "Go", + "json": "JSON", + "postgres": "Postgres", + "python": "Python", + }.items(): + cleaned = re.sub(rf"\b{source}\b", replacement, cleaned, flags=re.IGNORECASE) + return cleaned[:1].upper() + cleaned[1:] if cleaned else cleaned + + +def _prompt_v2_output_format(cleaned: str) -> str: + source = _strip_request_prefix((_source_sentences(cleaned) or [cleaned])[0]) + create = re.match( + r"^(?:build|create|develop|generate|produce|write)\s+((?:a|an|the)\s+)?(.+)$", + source, + flags=re.IGNORECASE, + ) + if create: + phrase = _RELATIVE_SPLIT_RE.split(create.group(2), maxsplit=1)[0] + phrase = _CLAUSE_SPLIT_RE.split(phrase, maxsplit=1)[0] + article = create.group(1) or "" + return _display_phrase(_truncate_words(article + phrase, 120)) + "." + add = re.match(r"^add\s+((?:a|an|the)\s+)?(.+)$", source, flags=re.IGNORECASE) + if add: + phrase = re.split(r"\s+to\s+|[,;]", add.group(2), maxsplit=1)[0] + return _display_phrase(_truncate_words((add.group(1) or "") + phrase, 120)) + "." + verb = (re.match(r"[A-Za-z]+", source) or [""])[0].lower() + return { + "combine": "Combined data and summary.", + "debug": "Debugging result.", + "explain": "Explanation.", + "fix": "Fix.", + "plan": "Plan.", + "refactor": "Refactor.", + "review": "Review.", + }.get(verb, "No output format specified.") + + def _render_markdown(settings: PromptBuilderSettings, intent: PromptIntent, cleaned: str) -> str: sections = [ ("Task", _fallback_task(settings, intent, cleaned)), @@ -561,12 +776,12 @@ def _validate_prompt_v2_xml(raw: str) -> tuple[bool, list[str]]: errors.append(f"empty <{name}>") task = re.sub(r"\s+", " ", sections["task"]).strip() - sentences = re.findall(r"[^.!?]+[.!?](?=\s|$)", task) + sentences = [part.strip() for part in re.split(r"[.!?]+(?=\s|$)", task) if part.strip()] first = re.match(r"[A-Za-z]+", task) if ( "\n" in sections["task"] or len(sentences) != 1 - or sentences[0].strip() != task + or not re.search(r"[.!?]\s*$", task) or not first or first.group(0).lower() not in _IMPERATIVE_STARTS ): diff --git a/scripts/ui/web/app.js b/scripts/ui/web/app.js index b37952b..c1b5b83 100644 --- a/scripts/ui/web/app.js +++ b/scripts/ui/web/app.js @@ -115,7 +115,7 @@ const TONE_LABELS = { formal: "🎩 Formal", casual: "👕 Casual", friendly: " const PROMPT_BUILDER_DEFAULTS = { prompt_version: "v2", target_agent: "claude_code", - detail_level: "balanced", + detail_level: "concise", action_mode: "implement", structure: "agent_default", include_acceptance_criteria: false, @@ -734,7 +734,7 @@ function promptBuilderPatch() { return { prompt_version: $("pb-version").value || "v2", target_agent: $("pb-target").value || "claude_code", - detail_level: $("pb-detail").value || "balanced", + detail_level: $("pb-detail").value || "concise", action_mode: $("pb-action").value || "implement", structure: $("pb-structure").value || "agent_default", include_acceptance_criteria: $("pb-acceptance").checked, diff --git a/setup/defaults/grammar_hotkey.config.example.json b/setup/defaults/grammar_hotkey.config.example.json index 44ca60d..eeb97b5 100644 --- a/setup/defaults/grammar_hotkey.config.example.json +++ b/setup/defaults/grammar_hotkey.config.example.json @@ -32,7 +32,7 @@ "prompt_builder": { "prompt_version": "v2", "target_agent": "claude_code", - "detail_level": "balanced", + "detail_level": "concise", "action_mode": "implement", "structure": "agent_default", "include_acceptance_criteria": false, diff --git a/setup/defaults/grammar_hotkey.config.json b/setup/defaults/grammar_hotkey.config.json index f2fdd61..23ed101 100644 --- a/setup/defaults/grammar_hotkey.config.json +++ b/setup/defaults/grammar_hotkey.config.json @@ -32,7 +32,7 @@ "prompt_builder": { "prompt_version": "v2", "target_agent": "claude_code", - "detail_level": "balanced", + "detail_level": "concise", "action_mode": "implement", "structure": "agent_default", "include_acceptance_criteria": false, diff --git a/tests/test_ffp_config.py b/tests/test_ffp_config.py index 7d37777..e5c40d5 100644 --- a/tests/test_ffp_config.py +++ b/tests/test_ffp_config.py @@ -318,10 +318,32 @@ def test_load_config_normalizes_prompt_builder(tmp_path): assert loaded["prompt_builder"]["prompt_version"] == "v2" assert loaded["prompt_builder"]["target_agent"] == "claude_code" - assert loaded["prompt_builder"]["detail_level"] == "balanced" + assert loaded["prompt_builder"]["detail_level"] == "concise" assert len(loaded["prompt_builder"]["user_suffix"]) == ffp_prompt_builder.USER_SUFFIX_MAX_CHARS +def test_v36_load_config_migrates_only_legacy_prompt_builder_identity(tmp_path): + legacy = { + **ffp_prompt_builder.DEFAULT_PROMPT_BUILDER_CONFIG, + "detail_level": "balanced", + } + legacy.pop("prompt_version") + cfg_path = tmp_path / "grammar_hotkey.config.json" + cfg_path.write_text(json.dumps({"prompt_builder": legacy}), encoding="utf-8") + + loaded = ffp_config.load_config(cfg_path) + + assert loaded["prompt_builder"]["prompt_version"] == "v2" + assert loaded["prompt_builder"]["detail_level"] == "concise" + + legacy["include_verification"] = True + cfg_path.write_text(json.dumps({"prompt_builder": legacy}), encoding="utf-8") + customized = ffp_config.load_config(cfg_path) + + assert customized["prompt_builder"]["detail_level"] == "balanced" + assert customized["prompt_builder"]["include_verification"] is True + + def test_filter_config_patch_prompt_builder_partial_and_clamped(): filtered = ffp_config.filter_config_patch({ "prompt_builder": { diff --git a/tests/test_ffp_flm_server.py b/tests/test_ffp_flm_server.py index 1833b26..b904c8e 100644 --- a/tests/test_ffp_flm_server.py +++ b/tests/test_ffp_flm_server.py @@ -102,3 +102,50 @@ def _run(*_a, **_k): out = ffp_flm_server.flm_list("bogus", "x", 0) assert "bad filter" in out["error"] assert called["n"] == 0 # rejected without shelling out + + +def test_v34_force_restart_waits_for_old_port_before_spawning(monkeypatch, tmp_path): + reachability = iter([True, False, True]) + stopped = [] + spawned = [] + + class _Proc: + pid = 4321 + returncode = None + + @staticmethod + def poll(): + return None + + settings = ffp_flm_server.FlmServerSettings( + base_url="http://127.0.0.1:52625", + model="qwen3.5:4b", + timeout_seconds=60, + performance_mode="balanced", + startup_timeout_seconds=5, + extra_args=[], + log_to_file=False, + log_file="flm.log", + pid_path=tmp_path / "flm.pid", + logs_dir=tmp_path, + no_window=0, + ) + monkeypatch.setattr( + ffp_flm_server, + "is_flm_server_reachable", + lambda _url: next(reachability), + ) + monkeypatch.setattr(ffp_flm_server, "popen_hidden", lambda *a, **k: spawned.append((a, k)) or _Proc()) + monkeypatch.setattr(ffp_flm_server, "write_pid", lambda *_args: None) + monkeypatch.setattr(ffp_flm_server.time, "sleep", lambda _seconds: None) + + result = ffp_flm_server.start_flm_server( + settings, + lambda *_args: ("", settings.model), + force_restart=True, + stop_callback=lambda force: stopped.append(force) or True, + ) + + assert result == "started" + assert stopped == [True] + assert len(spawned) == 1 diff --git a/tests/test_grammar_fix.py b/tests/test_grammar_fix.py index 602e1c1..68769a1 100644 --- a/tests/test_grammar_fix.py +++ b/tests/test_grammar_fix.py @@ -496,39 +496,51 @@ def fake_call(model, system_prompt, user_content, max_tokens, timeout_seconds): assert "Fix grammar and punctuation only." in calls[0][1] -def test_call_flm_prompt_retries_on_near_verbatim_output(fresh_modules, monkeypatch): +def test_v35_default_prompt_uses_one_short_draft_call(fresh_modules, monkeypatch): grammar_fix = fresh_modules("grammar_fix") monkeypatch.setattr(grammar_fix, "is_flm_server_reachable", lambda: True) - responses = iter( - [ - ("Task: Build a plan", grammar_fix.FLM_MODEL), - ( - "\nPlan the requested deliverable.\n\n" - "\nThe user requested a plan.\n\n" - "\n- Keep the plan concrete.\n- Preserve the requested scope.\n" - "- Do not invent requirements.\n\n" - "\nReturn an ordered execution plan.\n", - grammar_fix.FLM_MODEL, - ), - ] - ) prompts = [] def fake_call(model, system_prompt, user_content, max_tokens, timeout_seconds): prompts.append(system_prompt) - return next(responses) + return ("Task: Build a plan", grammar_fix.FLM_MODEL) monkeypatch.setattr(grammar_fix, "_call_flm_api", fake_call) text, _, _, strategy = grammar_fix.call_flm("prompt", "Task: Build a plan") assert strategy == "prompt_short" - assert text.startswith("\nPlan the requested deliverable.") - assert any("meta-framing" in prompt for prompt in prompts) + assert text.startswith("\nBuild a plan.") + assert prompts == [grammar_fix.ffp_config.CLAUDE_PROMPT_SYSTEM_PROMPT_V2] + assert "under 40 tokens" in prompts[0] + + +def test_v27_v35_long_default_prompt_is_one_call_and_bounded(fresh_modules, monkeypatch): + grammar_fix = fresh_modules("grammar_fix") + monkeypatch.setattr(grammar_fix, "is_flm_server_reachable", lambda: True) + calls = [] + source = "build a tool that " + ("processes records while preserving stated behavior " * 60) + + def fake_call(model, system_prompt, user_content, max_tokens, timeout_seconds): + calls.append((user_content, max_tokens)) + return "Build the requested tool.", model + + monkeypatch.setattr(grammar_fix, "_call_flm_api", fake_call) + + text, _, _, strategy = grammar_fix.call_flm("prompt", source) + settings = grammar_fix.ffp_prompt_builder.PromptBuilderSettings.from_config({}) + + assert strategy == "prompt_long" + assert calls == [(source, 420)] + assert grammar_fix.ffp_prompt_builder.validate(text, settings).valid is True def test_prompt_retries_never_exceed_selected_strategy_cap(fresh_modules, monkeypatch): grammar_fix = fresh_modules("grammar_fix") + grammar_fix.PROMPT_BUILDER_CFG = { + **grammar_fix.ffp_prompt_builder.DEFAULT_PROMPT_BUILDER_CONFIG, + "target_agent": "generic_chat", + } monkeypatch.setattr(grammar_fix, "is_flm_server_reachable", lambda: True) caps = [] @@ -544,7 +556,9 @@ def fake_call(model, system_prompt, user_content, max_tokens, timeout_seconds): assert caps == [240, 240, 240] assert grammar_fix.ffp_prompt_builder.validate( text, - grammar_fix.ffp_prompt_builder.PromptBuilderSettings.from_config({}), + grammar_fix.ffp_prompt_builder.PromptBuilderSettings.from_config( + grammar_fix.PROMPT_BUILDER_CFG + ), ).valid @@ -574,12 +588,46 @@ def fake_call(model, system_prompt, user_content, max_tokens, timeout_seconds): assert "" in text +def test_v33_default_prompt_surfaces_grounded_source_not_model_inventions(fresh_modules, monkeypatch): + grammar_fix = fresh_modules("grammar_fix") + monkeypatch.setattr(grammar_fix, "is_flm_server_reachable", lambda: True) + calls = [] + invented = ( + "\nWrite a CLI with two required arguments.\n\n" + "\nThe tool runs in the current directory.\n\n" + "\n- Require two arguments.\n- Print to stdout.\n- Add error handling.\n" + "\n\nA Python file.\n" + ) + + def fake_call(model, system_prompt, user_content, max_tokens, timeout_seconds): + calls.append(max_tokens) + return invented, model + + monkeypatch.setattr(grammar_fix, "_call_flm_api", fake_call) + + text, _, _, _ = grammar_fix.call_flm( + "prompt", + "write a CLI that renames files from a mapping in a text file", + ) + + assert calls == [240] + assert grammar_fix.ffp_prompt_builder.validate( + text, + grammar_fix.ffp_prompt_builder.PromptBuilderSettings.from_config({}), + ).valid + assert "two required arguments" not in text + assert "stdout" not in text + assert "error handling" not in text + assert "- Write a CLI." in text + assert "- Rename files from a mapping in a text file." in text + + def test_prompt_mode_cli_writes_output_file(fresh_modules, monkeypatch, tmp_path: Path): grammar_fix = fresh_modules("grammar_fix") monkeypatch.setattr(grammar_fix, "is_flm_server_reachable", lambda: True) def fake_call(model, system_prompt, user_content, max_tokens, timeout_seconds): - assert "four sibling XML sections" in system_prompt + assert "under 40 tokens" in system_prompt assert max_tokens == 240 return ("Refine onboarding email", model) diff --git a/tests/test_prompt_builder.py b/tests/test_prompt_builder.py index da0de96..d88ee10 100644 --- a/tests/test_prompt_builder.py +++ b/tests/test_prompt_builder.py @@ -43,8 +43,24 @@ def test_default_system_prompt_is_v2_identity_and_v1_is_rollback(): prompt_v2_system_prompt=ffp_config.CLAUDE_PROMPT_SYSTEM_PROMPT_V2, ) == ffp_config.CLAUDE_PROMPT_SYSTEM_PROMPT_V1 assert settings.prompt_version == "v2" + assert settings.detail_level == "concise" assert settings.requires_contract_gate() is True assert rollback.requires_contract_gate() is False + upgraded_rollback = ffp_prompt_builder.PromptBuilderSettings.from_config({ + "prompt_version": "v1", + "target_agent": "generic_chat", + "detail_level": "balanced", + "structure": "markdown", + "include_verification": True, + }) + assert ffp_prompt_builder.build_system_prompt( + upgraded_rollback, + intent, + prompt_v1_system_prompt=ffp_config.CLAUDE_PROMPT_SYSTEM_PROMPT_V1, + prompt_v2_system_prompt=ffp_config.CLAUDE_PROMPT_SYSTEM_PROMPT_V2, + ) == ffp_config.CLAUDE_PROMPT_SYSTEM_PROMPT_V1 + assert ffp_prompt_builder.effective_structure(upgraded_rollback) == "xml" + assert upgraded_rollback.requires_contract_gate() is False assert ffp_llm_client.force_prompt_shape("fix the PR") == ffp_prompt_builder.render_fallback( settings, intent, @@ -96,6 +112,60 @@ def test_default_v2_fallback_passes_strict_contract_and_token_target(): assert all(tag in text for tag in ("", "", "", "")) +def test_v33_default_v2_grounding_removes_model_inventions_and_splits_source_clauses(): + settings = ffp_prompt_builder.PromptBuilderSettings.from_config({}) + source = ( + "build a python script that reads a folder of CSVs, validates rows against a schema, " + "and writes an error report with file and line numbers" + ) + intent = ffp_prompt_builder.resolve_intent(settings, source) + invented = ( + "Build it.Use pandas." + "- Require JSON.\n- Log warnings.\n- Add tests." + "script.py" + ) + + grounded = ffp_prompt_builder.ground_prompt_v2_output(invented, settings, intent, source) + result = ffp_prompt_builder.validate(grounded, settings) + + assert result.valid is True + assert "pandas" not in grounded.lower() + assert "json" not in grounded.lower() + assert "log warnings" not in grounded.lower() + assert grounded.count("\n- ") == 4 + assert "- Read a folder of CSVs." in grounded + assert "- Validate rows against a schema." in grounded + assert "- Write an error report with file and line numbers." in grounded + assert "\nA Python script.\n" in grounded + + +def test_v33_sparse_request_uses_only_fixed_scope_guards(): + settings = ffp_prompt_builder.PromptBuilderSettings.from_config({}) + source = "make the dashboard faster" + intent = ffp_prompt_builder.resolve_intent(settings, source) + grounded = ffp_prompt_builder.ground_prompt_v2_output("invented", settings, intent, source) + + assert ffp_prompt_builder.validate(grounded, settings).valid is True + assert "\nMake the dashboard faster.\n" in grounded + assert "- Make the dashboard faster." in grounded + assert "- Preserve all stated requirements." in grounded + assert "- Do not add unstated requirements." in grounded + assert "browser" not in grounded.lower() + assert "\nNo output format specified.\n" in grounded + + +def test_v27_task_sentence_allows_identifiers_and_regex_punctuation(): + settings = ffp_prompt_builder.PromptBuilderSettings.from_config({}) + for source in ( + "plan a migration from users.full_name to first_name and last_name", + r"explain what ^(?P[a-z0-9]+)$ accepts and show three examples", + ): + intent = ffp_prompt_builder.resolve_intent(settings, source) + grounded = ffp_prompt_builder.ground_prompt_v2_output("raw", settings, intent, source) + + assert ffp_prompt_builder.validate(grounded, settings).valid is True + + def test_generic_chat_markdown_contract_and_preview(): settings = ffp_prompt_builder.PromptBuilderSettings.from_config({ "target_agent": "generic_chat", diff --git a/tests/test_prompt_speed_quality_eval.py b/tests/test_prompt_speed_quality_eval.py index 735f29d..a47d072 100644 --- a/tests/test_prompt_speed_quality_eval.py +++ b/tests/test_prompt_speed_quality_eval.py @@ -94,7 +94,7 @@ def test_usage_metrics_include_units_percentiles_and_per_token_cost(): "usage": { "completion_tokens": 100, "prefill_duration_ttft": "1600ms", - "decode_duration": 8.0, + "decoding_duration": 8.0, }, "_wall_seconds": 10.0, } @@ -142,6 +142,10 @@ def fake_model(**kwargs): assert artifact["gate"]["quality"]["passed"] is True assert artifact["gate"]["passed"] is True assert len(artifact["manual_side_by_side"]) == 2 + v2_sample = artifact["cases"][0]["styles"]["v2"]["samples"][0] + assert v2_sample["raw_output"] == GOOD_OUTPUT + assert v2_sample["output"] != v2_sample["raw_output"] + assert v2_sample["final_output_tokens"] > 0 def test_protocol_rejects_too_few_timed_runs(): @@ -183,3 +187,73 @@ def fake_model(**kwargs): assert result["warm"]["wall_seconds"] == 12.0 assert result["wall_speedup"] == 2.5 assert result["ttft_reduction_seconds"] == 16.5 + + +def test_existing_artifact_can_be_judged_without_rerunning_model_calls(): + cases = list(prompt_eval.FIXED_CASES[:1]) + calls = [] + + def fake_model(**kwargs): + calls.append(kwargs["style"]) + return { + "output": GOOD_OUTPUT, + "model": kwargs["model"], + "_wall_seconds": 12.0 if kwargs["style"] == "v2" else 30.0, + "usage": {"completion_tokens": 100, "decoding_duration": 8.0}, + } + + artifact = prompt_eval.run_evaluation(fake_model, cases=cases, runs=5) + calls_after_run = len(calls) + rescored = prompt_eval.rescore_artifact(artifact, _judge(cases)) + + assert artifact["gate"]["quality"]["judge_complete"] is False + assert rescored["gate"]["passed"] is True + assert rescored["judge"]["method"] == "manual" + assert len(calls) == calls_after_run + + +def test_frozen_v1_replay_preserves_five_timed_samples_after_warmup(): + cases = list(prompt_eval.FIXED_CASES[:1]) + + def fake_model(**kwargs): + return { + "output": GOOD_OUTPUT, + "model": kwargs["model"], + "_wall_seconds": 30.0 if kwargs["style"] == "v1" else 12.0, + "usage": {"completion_tokens": 100, "decoding_duration": 8.0}, + } + + artifact = prompt_eval.run_evaluation(fake_model, cases=cases, runs=5, judge_data=_judge(cases)) + samples = artifact["cases"][0]["styles"]["v1"]["samples"] + for index, sample in enumerate(samples, start=1): + sample["wall_seconds"] = float(index) + replay = prompt_eval.build_v1_replay(artifact) + kwargs = { + "style": "v1", + "user_content": cases[0]["input"], + "model": "qwen3.5:4b", + } + walls = [replay(**kwargs)["_wall_seconds"] for _ in range(6)] + + assert walls == [1.0, 1.0, 2.0, 3.0, 4.0, 5.0] + + +def test_v1_baseline_export_contains_no_v2_samples(): + cases = list(prompt_eval.FIXED_CASES[:1]) + + def fake_model(**kwargs): + return { + "output": GOOD_OUTPUT, + "model": kwargs["model"], + "_wall_seconds": 12.0 if kwargs["style"] == "v2" else 30.0, + "usage": {"completion_tokens": 100, "decoding_duration": 8.0}, + } + + artifact = prompt_eval.run_evaluation(fake_model, cases=cases, runs=5, judge_data=_judge(cases)) + baseline = prompt_eval.export_v1_baseline(artifact, "source.json") + + assert baseline["kind"] == "prompt_v1_frozen_baseline" + assert baseline["source_artifact"] == "source.json" + assert baseline["summary"] == artifact["summaries"]["v1"] + assert set(baseline["cases"][0]) == {"name", "category", "trap", "input", "v1"} + assert "v2" not in baseline["cases"][0] diff --git a/tests/test_prompt_v2_release_evidence.py b/tests/test_prompt_v2_release_evidence.py new file mode 100644 index 0000000..78dad8a --- /dev/null +++ b/tests/test_prompt_v2_release_evidence.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +import json +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +BENCH = ROOT / "data" / "benchmarks" + + +def _load(name: str) -> dict: + return json.loads((BENCH / name).read_text(encoding="utf-8")) + + +def test_v29_prompt_v2_release_artifact_passes_both_gates(): + artifact = _load("prompt_v2_ab_2026-07-10.json") + + assert artifact["protocol"]["fixed_input_count"] >= 12 + assert artifact["protocol"]["warmups_per_style_input"] >= 1 + assert artifact["protocol"]["timed_runs_per_style_input"] >= 5 + assert artifact["protocol"]["judge_method"] == "manual_gpt5_source_review" + assert artifact["gate"]["protocol_pass"] is True + assert artifact["gate"]["speed"]["passed"] is True + assert artifact["gate"]["quality"]["passed"] is True + assert artifact["gate"]["passed"] is True + assert artifact["summaries"]["v2"]["quality"]["invented_requirement_failures"] == 0 + assert artifact["summaries"]["v2"]["quality"]["passed_outputs"] == 12 + assert artifact["cold_warm_probe"]["ok"] is True + + +def test_release_evidence_has_frozen_baseline_and_complete_judgments(): + artifact = _load("prompt_v2_ab_2026-07-10.json") + baseline = _load("prompt_v1_frozen_2026-07-10.json") + judge = _load("prompt_v2_judge_2026-07-10.json") + cold_warm = _load("prompt_v2_cold_warm_2026-07-10.json") + + assert artifact["protocol"]["reused_v1_from"].endswith("prompt_v1_frozen_2026-07-10.json") + assert baseline["kind"] == "prompt_v1_frozen_baseline" + assert baseline["protocol"]["timed_runs_per_style_input"] == 5 + assert len(baseline["cases"]) == 12 + assert len(judge["judgments"]) == 24 + assert cold_warm["ok"] is True + assert cold_warm["wall_speedup"] > 1 diff --git a/tests/test_version_sync.py b/tests/test_version_sync.py new file mode 100644 index 0000000..4e071a0 --- /dev/null +++ b/tests/test_version_sync.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +import re +import tomllib +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def _match(path: Path, pattern: str) -> str: + match = re.search(pattern, path.read_text(encoding="utf-8"), flags=re.MULTILINE) + assert match, f"version pattern missing from {path.relative_to(ROOT)}" + return match.group(1) + + +def test_v18_release_version_is_synchronized(): + versions = { + "scripts/_version.py": _match( + ROOT / "scripts" / "_version.py", + r'^__version__\s*=\s*"([^"]+)"', + ), + "pyproject.toml": str( + tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8"))["project"]["version"] + ), + "installer/installer.iss": _match( + ROOT / "installer" / "installer.iss", + r'^#define\s+AppVersion\s+"([^"]+)"', + ), + "README.md": _match( + ROOT / "README.md", + r"^Current version:\s*`([^`]+)`", + ), + } + + assert set(versions.values()) == {"2.3.0"}, versions diff --git a/tools/prompt_speed_quality_eval.py b/tools/prompt_speed_quality_eval.py index 731bf95..ccd2b73 100644 --- a/tools/prompt_speed_quality_eval.py +++ b/tools/prompt_speed_quality_eval.py @@ -8,6 +8,7 @@ from __future__ import annotations import argparse +import copy import json import math import re @@ -27,6 +28,7 @@ import ffp_config # noqa: E402 import ffp_llm_client # noqa: E402 +import ffp_prompt_builder # noqa: E402 PROMPT_V2_CANDIDATE = ffp_config.CLAUDE_PROMPT_SYSTEM_PROMPT_V2 @@ -169,6 +171,7 @@ "implement", "integrate", "investigate", + "make", "migrate", "optimize", "plan", @@ -281,8 +284,8 @@ def _single_imperative_sentence(text: str) -> bool: cleaned = re.sub(r"\s+", " ", str(text or "")).strip() if not cleaned or "\n" in str(text or "").strip(): return False - sentences = re.findall(r"[^.!?]+[.!?](?=\s|$)", cleaned) - if len(sentences) != 1 or sentences[0].strip() != cleaned: + sentences = [part.strip() for part in re.split(r"[.!?]+(?=\s|$)", cleaned) if part.strip()] + if len(sentences) != 1 or not re.search(r"[.!?]\s*$", cleaned): return False first = re.match(r"[A-Za-z]+", cleaned) return bool(first and first.group(0).lower() in _IMPERATIVE_VERBS) @@ -360,7 +363,9 @@ def _sample_metrics(result: dict[str, Any], measured_wall: float) -> dict[str, A decode_duration = _duration_seconds( usage, "decode_duration", + "decoding_duration", "decode_duration_seconds", + "decoding_duration_seconds", "eval_duration", ) wall = _as_float(result.get("_wall_seconds")) or measured_wall @@ -589,6 +594,7 @@ def run_evaluation( judge_data: dict[str, Any] | None = None, model: str = "qwen3.5:4b", base_url: str = "http://127.0.0.1:52625", + progress: Callable[[str], None] | None = None, ) -> dict[str, Any]: if runs < 5: raise ValueError("runs must be >= 5") @@ -598,6 +604,8 @@ def run_evaluation( raise ValueError("at least one case is required") judge_data = judge_data or {} case_rows: list[dict[str, Any]] = [] + total_pairs = len(cases) * len(style_specs) + completed_pairs = 0 for case in cases: row: dict[str, Any] = { "name": case["name"], @@ -607,6 +615,8 @@ def run_evaluation( "styles": {}, } for style, spec in style_specs.items(): + if progress: + progress(f"[{completed_pairs + 1}/{total_pairs}] {style}:{case['name']} starting") max_tokens = _cap_for_input(spec["caps"], case["input"]) call_args = { "style": style, @@ -621,10 +631,25 @@ def run_evaluation( if not warmup["ok"]: warmup_errors.append(warmup["error"]) samples = [_run_sample(call_model, **call_args) for _ in range(runs)] + if style == "v2": + settings = ffp_prompt_builder.PromptBuilderSettings.from_config({}) + intent = ffp_prompt_builder.resolve_intent(settings, case["input"]) + for sample in samples: + if not sample["ok"]: + continue + sample["raw_output"] = sample["output"] + sample["output"] = ffp_prompt_builder.ground_prompt_v2_output( + sample["raw_output"], + settings, + intent, + case["input"], + ) + sample["final_output_tokens"] = _estimate_tokens(sample["output"]) representative = next((sample for sample in samples if sample["ok"]), samples[0]) quality = score_output( representative.get("output") or "", - completion_tokens=representative.get("completion_tokens"), + completion_tokens=representative.get("final_output_tokens") + or representative.get("completion_tokens"), judgment=_judgment_for(judge_data, style, case["name"]), ) row["styles"][style] = { @@ -637,6 +662,10 @@ def run_evaluation( "representative_output": representative.get("output") or "", "quality": quality, } + completed_pairs += 1 + if progress: + median = row["styles"][style]["speed"]["wall_seconds"]["median"] + progress(f"[{completed_pairs}/{total_pairs}] {style}:{case['name']} p50={median}") case_rows.append(row) summaries = {style: _style_summary(case_rows, style, runs) for style in style_specs} @@ -661,6 +690,7 @@ def run_evaluation( }, "endpoint": {"base_url": base_url, "model": model}, "styles": style_specs, + "judge": judge_data or {"method": "pending", "judgments": {}}, "cases": case_rows, "summaries": summaries, "gate": evaluate_gate(summaries), @@ -669,6 +699,107 @@ def run_evaluation( } +def rescore_artifact(payload: dict[str, Any], judge_data: dict[str, Any]) -> dict[str, Any]: + rescored = copy.deepcopy(payload) + cases = rescored.get("cases") or [] + runs = int((rescored.get("protocol") or {}).get("timed_runs_per_style_input") or 0) + if not cases or runs < 1: + raise ValueError("artifact has no scored cases or timed runs") + for case in cases: + for style, style_row in (case.get("styles") or {}).items(): + samples = style_row.get("samples") or [] + representative = next((sample for sample in samples if sample.get("ok")), None) + if representative is None: + style_row["quality"] = score_output( + "", + judgment=_judgment_for(judge_data, style, case["name"]), + ) + continue + quality = score_output( + representative.get("output") or "", + completion_tokens=representative.get("final_output_tokens") + or representative.get("completion_tokens"), + judgment=_judgment_for(judge_data, style, case["name"]), + ) + style_row["quality"] = quality + summaries = { + style: _style_summary(cases, style, runs) + for style in rescored.get("styles") or {"v1": {}, "v2": {}} + } + rescored["summaries"] = summaries + rescored["gate"] = evaluate_gate(summaries) + rescored["judge"] = judge_data + rescored.setdefault("protocol", {})["judge_method"] = str(judge_data.get("method") or "pending") + rescored["rescored_at"] = datetime.now().astimezone().isoformat(timespec="seconds") + return rescored + + +def build_v1_replay(payload: dict[str, Any]) -> ModelCall: + rows = { + str(case.get("input") or ""): ((case.get("styles") or {}).get("v1") or {}).get("samples") or [] + for case in payload.get("cases") or [] + } + counts: dict[str, int] = {} + + def replay(**kwargs: Any) -> dict[str, Any]: + if kwargs.get("style") != "v1": + raise ValueError("v1 replay only accepts style=v1") + input_text = str(kwargs.get("user_content") or "") + samples = rows.get(input_text) or [] + call_index = counts.get(input_text, 0) + # First call is the protocol warmup; replay sample 0 without consuming it. + sample_index = max(0, call_index - 1) + if sample_index >= len(samples): + raise ValueError(f"v1 baseline has no sample {sample_index} for input") + counts[input_text] = call_index + 1 + sample = samples[sample_index] + if not sample.get("ok"): + raise RuntimeError(str(sample.get("error") or "v1 baseline sample failed")) + return { + "output": sample.get("output") or "", + "model": sample.get("model") or kwargs.get("model") or "", + "usage": sample.get("usage") or {}, + "_wall_seconds": sample.get("wall_seconds"), + } + + return replay + + +def export_v1_baseline(payload: dict[str, Any], source_path: str = "") -> dict[str, Any]: + cases = [] + for case in payload.get("cases") or []: + v1 = ((case.get("styles") or {}).get("v1") or {}) + cases.append( + { + "name": case.get("name"), + "category": case.get("category"), + "trap": bool(case.get("trap")), + "input": case.get("input"), + "v1": copy.deepcopy(v1), + } + ) + return { + "schema_version": 1, + "kind": "prompt_v1_frozen_baseline", + "created_at": (payload.get("protocol") or {}).get("reused_v1_created_at") + or payload.get("created_at"), + "exported_at": datetime.now().astimezone().isoformat(timespec="seconds"), + "source_artifact": source_path, + "protocol": { + "fixed_input_count": len(cases), + "warmups_per_style_input": (payload.get("protocol") or {}).get( + "warmups_per_style_input" + ), + "timed_runs_per_style_input": (payload.get("protocol") or {}).get( + "timed_runs_per_style_input" + ), + }, + "style": copy.deepcopy((payload.get("styles") or {}).get("v1") or {}), + "summary": copy.deepcopy((payload.get("summaries") or {}).get("v1") or {}), + "cases": cases, + } + + def _call_openai_compatible( *, base_url: str, @@ -768,6 +899,9 @@ def main() -> int: action="store_true", help="restart FastFlowLM and record first-call versus immediate warm-call latency", ) + parser.add_argument("--rescore", default="", help="rescore an existing JSON artifact") + parser.add_argument("--export-v1", default="", help="export frozen v1 samples from an artifact") + parser.add_argument("--reuse-v1", default="", help="reuse v1 samples from a frozen artifact") parser.add_argument("--base-url", default="http://127.0.0.1:52625") parser.add_argument("--daemon-url", default="http://127.0.0.1:52650") parser.add_argument("--model", default="qwen3.5:4b") @@ -779,8 +913,38 @@ def main() -> int: parser.add_argument("--json", action="store_true", help="print the full JSON payload") args = parser.parse_args() + if args.rescore: + if args.live or args.cold_warm or args.reuse_v1 or args.export_v1: + parser.error("--rescore cannot be combined with live benchmark options") + if not args.judge_file: + parser.error("--rescore requires --judge-file") + source_path = Path(args.rescore) + payload = json.loads(source_path.read_text(encoding="utf-8")) + payload = rescore_artifact(payload, _load_judge(args.judge_file)) + out_path = Path(args.out) if args.out else source_path + _write_json(out_path, payload) + if args.json: + print(json.dumps(payload, ensure_ascii=False, indent=2)) + else: + print(f"artifact={out_path}") + print(f"gate={payload['gate']['passed']}") + return 0 if payload["gate"]["passed"] else 1 + if args.export_v1: + if args.live or args.cold_warm or args.reuse_v1: + parser.error("--export-v1 cannot be combined with live benchmark options") + source_path = Path(args.export_v1) + payload = json.loads(source_path.read_text(encoding="utf-8")) + baseline = export_v1_baseline(payload, str(source_path)) + out_path = Path(args.out) if args.out else source_path.with_name( + f"prompt_v1_frozen_{datetime.now():%Y-%m-%d}.json" + ) + _write_json(out_path, baseline) + print(f"artifact={out_path}") + return 0 if args.cold_warm and not args.live: parser.error("--cold-warm requires --live") + if args.reuse_v1 and not args.live: + parser.error("--reuse-v1 requires --live") if not args.live: payload = _manifest() if args.out: @@ -796,7 +960,7 @@ def main() -> int: parser.error("--runs must be >= 5") judge_data = _load_judge(args.judge_file) - def call_model(**kwargs: Any) -> dict[str, Any]: + def live_call(**kwargs: Any) -> dict[str, Any]: return _call_openai_compatible( base_url=args.base_url, bearer=args.bearer, @@ -804,6 +968,17 @@ def call_model(**kwargs: Any) -> dict[str, Any]: **kwargs, ) + baseline_payload = None + replay_v1 = None + if args.reuse_v1: + baseline_payload = json.loads(Path(args.reuse_v1).read_text(encoding="utf-8")) + replay_v1 = build_v1_replay(baseline_payload) + + def call_model(**kwargs: Any) -> dict[str, Any]: + if kwargs.get("style") == "v1" and replay_v1 is not None: + return replay_v1(**kwargs) + return live_call(**kwargs) + cold_warm_probe = None if args.cold_warm: cold_warm_probe = run_cold_warm_probe( @@ -817,9 +992,13 @@ def call_model(**kwargs: Any) -> dict[str, Any]: judge_data=judge_data, model=args.model, base_url=args.base_url, + progress=lambda message: print(message, flush=True), ) if cold_warm_probe is not None: payload["cold_warm_probe"] = cold_warm_probe + if baseline_payload is not None: + payload["protocol"]["reused_v1_from"] = str(Path(args.reuse_v1)) + payload["protocol"]["reused_v1_created_at"] = baseline_payload.get("created_at") out_path = Path(args.out) if args.out else ( ROOT / "data" / "benchmarks" / f"prompt_v2_ab_{datetime.now():%Y-%m-%d}.json" )