From cfea93b5c2570e01af7eff76423048338aa3204c Mon Sep 17 00:00:00 2001 From: agrechenkov Date: Tue, 28 Jul 2026 09:14:50 -0400 Subject: [PATCH] 2.4.3: clarify shape for vague prompt requests + R8 echo gate Reported: `prompt:` on "develop a app that allows to perfrom a full schedule for my meeting a proper PM would" returned that sentence as the task, again as its only real constraint, and again as the output format, padded with two scope guards, typos intact. Root cause: default prompt v2 is fully deterministic - ground_prompt_v2_output discards the model draft and renders from the source clauses. A request with four splittable clauses (the CSV case) decomposes well; one unsplittable clause has nothing to decompose, so the renderer emitted it everywhere and padded to reach the 3-constraint minimum. The existing echo detector cannot catch this: is_weak_prompt_echo returns False once output has target structure (V32), and the v2 finalizer always emits structure. - Clarify shape. is_underspecified() (fewer than 2 real source clauses) switches to a render that names the open questions - platform, scope, data source, success criteria - and tells the agent to settle them before building. Adds no requirements: every line asserts only that something was not stated. V33 anti-invention still holds (verified: no agenda/calendar/database/framework). - Language normalization. Surfaced text gets article agreement and a fixed, audited typo map, so "a app ... perfrom" no longer repeats in every section. Exceptions preserved ("a user", "a unique", "a one-off", "a utility"). - R8 rubric item. Fails any output whose constraints hold nothing beyond a restatement of the task plus boilerplate, or whose output_format re-quotes the request. Disqualifying on its own, so a valid-looking echo cannot pass at any score. Machine-checkable; no judge needed. Rubric is now 8 items, pass >= 7. - The reported request is a permanent fixed-set case. Two self-caught metric errors on the way (SPEC B39/B40): coverage-of-candidate flagged good CSV decomposition, and per-item coverage flagged long requests whose task derives from sentence 1. Final metric is coverage-of-task, judged set-wise on constraints. Validated across all 14 fixed cases: ZERO false verdicts. Live on qwen3.5:4b: the reported input now yields the clarify shape (5.87s), the CSV case is byte-identical to before (8.89s, 4 real constraints). Updated test_v33_sparse_request to the new shape while keeping the anti-invention assertions it existed to protect. ruff clean, 485 pytest (+26). SPEC V44/V45/V46, T30, B38-B40. Co-Authored-By: Claude Opus 4.8 --- CHANGELOG.md | 14 ++ README.md | 3 +- SPEC.md | 7 + ...mar_hotkey.config.json.bak-20260728-083028 | 189 +++++++++++++++ installer/installer.iss | 2 +- installer/sign.ps1 | 2 +- pyproject.toml | 2 +- scripts/_version.py | 2 +- scripts/ffp_prompt_builder.py | 118 +++++++++- tests/test_prompt_builder.py | 25 +- tests/test_prompt_clarify_shape.py | 222 ++++++++++++++++++ tests/test_prompt_speed_quality_eval.py | 9 +- tests/test_version_sync.py | 2 +- tools/prompt_speed_quality_eval.py | 85 ++++++- 14 files changed, 663 insertions(+), 19 deletions(-) create mode 100644 config/grammar_hotkey.config.json.bak-20260728-083028 create mode 100644 tests/test_prompt_clarify_shape.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 48fb52a..3eae6b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,20 @@ ## Unreleased +## 2.4.3 + +**`prompt:` handles vague requests properly.** A request too thin to break into requirements now produces a prompt that names what is missing, instead of handing the request back reworded. + +### Fixed + +- **A vague request no longer returns itself.** Given *"develop a app that allows to perfrom a full schedule for my meeting a proper PM would"*, `prompt:` emitted that same sentence as the task, again as its only real constraint, and again as the output format, padded with two generic scope lines. It was structurally correct and invented nothing, so every automated check passed — but it told the reader nothing they had not just typed. When a request cannot be decomposed into distinct requirements, the output now names the open questions (platform, scope, data source, success criteria) and instructs the agent to settle them before building. It still adds no requirements: each line asserts only that something was *not* specified. +- **Typos and article agreement are corrected in the output.** Because the prompt is built from your own wording, "a app" and "perfrom" were repeated in every section. Surfaced text now fixes article agreement and a fixed list of common misspellings, without changing meaning. Well-known exceptions ("a user", "a unique", "a one-off") are preserved. + +### Measurement + +- **New rubric item R8 — no section may restate the task.** The eight-item rubric now fails any output whose constraints contain nothing beyond a restatement of the task plus boilerplate, or whose output format re-quotes the request. R8 is disqualifying on its own, so a structurally valid echo can no longer pass the release gate at any score. It is machine-checkable — no judge required. +- The reported request is now a permanent case in the fixed evaluation set, and R8 was validated against all 14 cases with **zero false verdicts** (it fails the old echo and clears every legitimate output, including multi-clause requests whose first bullet naturally restates the opening sentence). + ## 2.4.2 **Clear failures instead of cryptic ones.** A benchmark that cannot fit in memory is now refused with an explanation, keep-warm no longer fights an in-progress benchmark, and errors the local server reports are passed through instead of being replaced by a generic message. diff --git a/README.md b/README.md index 572ac32..fff3b0e 100644 --- a/README.md +++ b/README.md @@ -4,10 +4,11 @@ Flowkey is a Windows desktop assistant that adds local-LLM hotkeys for grammar f Everything runs locally through [FastFlowLM](https://fastflowlm.com) (AMD Ryzen AI NPU) or, on machines without the NPU, through [Ollama](https://ollama.com) (CPU/GPU) as a secondary provider. No cloud service, analytics, or telemetry is used by the app. -Current version: `2.4.2` +Current version: `2.4.3` ## What's new in 2.4 +- **`prompt:` handles vague requests properly.** If a request is too thin to break into concrete requirements, the generated prompt now names what's missing — platform, scope, data source, success criteria — and tells the coding agent to settle those before building, rather than handing your sentence back to you reworded. It still never invents requirements you didn't state. Typos and article agreement in your wording are cleaned up on the way out. - **Benchmarks fail clearly, not cryptically.** Starting a benchmark on a model too large for your machine used to end in a raw graphics-driver error code. Flowkey now checks the model's memory footprint (plus room for the context sweep) first and tells you what it needs versus what's available. Background model warmups also pause while a benchmark runs, instead of competing with it for the same memory, and errors reported by the local server are passed through verbatim rather than replaced with a generic message. - **The model picker shows every model, sized honestly.** Config → **Models** is now one card, styled like the rest of the dashboard (the old picker used a native browser dropdown that ignored the app's theme). Nothing is hidden from it: models too big for your machine are marked rather than silently dropped, and Mixture-of-Experts models are sized by the parameters actually active per token plus FastFlowLM's own measured memory footprint — so a 35B-total / 3B-active model is no longer mistaken for a 35B one. If a FastFlowLM upgrade invalidates your active model, the card now says so instead of letting the next hotkey fail with an opaque error. - **Chat streams as it thinks.** The dashboard **Chat** tab now fills the reply in token-by-token over a live stream instead of waiting on a blank "Thinking…" until the whole answer is ready — the first words typically appear in ~1.6 s (warm) rather than after the full response. It works on both FastFlowLM and Ollama, and if anything interrupts the stream the partial reply is kept and the error is shown. Grammar, `prompt:`, and the other hotkeys are unchanged (they still paste the finished result). diff --git a/SPEC.md b/SPEC.md index e99f664..378a0d3 100644 --- a/SPEC.md +++ b/SPEC.md @@ -94,6 +94,9 @@ Caveman-encoded (compression, not amputation). Paths / ids / action names / numb - V41: `flm bench` start → refuse iff catalog `footprint_gb` + 32k-context headroom > usable mem, w/ plain-language reason; unknown footprint ∨ lookup failure → allow (⊥ block on a guess) - V42: benchmark running → keep-warm tick ⊥ warms (`skipped_benchmark_running`); ∵ bench 10-20min > 15min keepalive ∴ collision is the norm - V43: provider 200-response carrying `error` ∧ ⊥ `choices` → raise that message verbatim (⊥ generic "no usable text"); `error` + `choices` → ignore, return completion +- V44: source ⊥ decomposable (real clauses < 2) → clarify shape: unknowns named, ⊥ request echoed into constraints/output_format, ⊥ scope-guard padding; still ⊥ invention (V33 holds) +- V45: surfaced v2 text → article agreement + fixed typo map normalized (∵ render copies user wording verbatim); ⊥ meaning change +- V46: A/B rubric = 8 items; R8 = ⊥ section restates `` (coverage-of-task ≥ 0.8), boilerplate-excluded set-wise on constraints; R8 false ⇒ disqualifying ∀ other scores; gate pass ≥ 7/8 ∧ R8 ∧ ⊥ invented - V40: model picker + installed list = app-styled elements (⊥ native ``/``|V40; custom app-styled combobox + list B32|2026-07-27|`app.js` set `models-title` = "Installed models — " ∴ overwrote merged card heading at runtime|V40 caught in browser; title → "Models — " B33|2026-07-27|new CSS block referenced undefined `--muted`/`--card`/`--bad` (pre-existing pattern elsewhere in stylesheet: 6+2+3 refs, never defined ∴ inert)|use defined tokens `--text-muted`/`--surface`/`--warn`; pre-existing refs left for separate cleanup +B38|2026-07-28|vague 1-clause request ("develop a app ... a proper PM would") → deterministic render echoed it as task ∧ sole constraint ∧ output_format + 2 scope guards, typos kept; scored 5/7 machine (7/7 lenient judge), ⊥ hard-fail ∴ gate green on a pure echo. `is_weak_prompt_echo` ⊥ fires ∵ V32 exempts structured output, and V33 finalizer always emits structure|V44,V45,V46; clarify shape + normalization + R8 +B39|2026-07-28|self-caught: first R8 metric = coverage-of-candidate ∴ flagged good CSV decomposition (bullets legitimately reuse task words)|invert → coverage-of-TASK +B40|2026-07-28|2nd R8 metric flagged long/debug cases ∵ task = sentence 1 ∴ one bullet legitimately restates it|judge constraints set-wise: fail iff ∅ remains after dropping restatements ∧ boilerplate B35|2026-07-27|`flm bench qwen3.6-moe:35b-a3b` died 4s in w/ driver `0xc01e0200` (page-in failed): 24.3GB weights + 32k-sweep KV > 25.6GB usable; ⊥ preflight ∴ user saw only a hex code|V41; catalog-footprint preflight w/ explanation B36|2026-07-27|keep-warm thread ⊥ aware of benchmarks: warms/reloads active model on 15min tick during a 10-20min bench ∴ NPU+mem contention mid-run|V42; `ffp_benchmark.is_running()` gate in `_warm_model_once` B37|2026-07-27|FLM returns HTTP **200** + `{"error":"Failed to load model!"}`; `_call_openai_compatible`/`ffp_chat` read only `choices` ∴ real cause discarded → "Local LLM returned no usable text"|V43; surface the error body diff --git a/config/grammar_hotkey.config.json.bak-20260728-083028 b/config/grammar_hotkey.config.json.bak-20260728-083028 new file mode 100644 index 0000000..6f6c4a3 --- /dev/null +++ b/config/grammar_hotkey.config.json.bak-20260728-083028 @@ -0,0 +1,189 @@ +{ + "enabled": true, + "llm": { + "provider": "fastflowlm", + "base_url": "http://127.0.0.1:52625", + "model": "qwen3.5:4b", + "auth_bearer": "flm", + "timeout_seconds": 96, + "auto_start": true + }, + "providers": { + "fastflowlm": { + "base_url": "http://127.0.0.1:52625", + "model": "qwen3.5:4b", + "auth_bearer": "flm", + "timeout_seconds": 96, + "auto_start": true + }, + "ollama": { + "base_url": "http://127.0.0.1:11434", + "model": "llama3.2:3b", + "auth_bearer": "ollama", + "timeout_seconds": 120, + "auto_start": false + } + }, + "flm_base_url": "http://127.0.0.1:52625", + "flm_model": "qwen3.5:4b", + "flm_timeout_seconds": 96, + "history_filename": "grammar_fix_history.jsonl", + "history_store_text": true, + "server": { + "auto_start": true, + "warm_on_start": true, + "keep_warm_minutes": 15, + "performance_mode": "max", + "startup_timeout_seconds": 25, + "serve_extra_args": [], + "log_to_file": false, + "log_file": "flm_server.log" + }, + "routing": { + "enabled": true, + "long_threshold_chars": 1400, + "chunk_size_chars": 1200, + "min_chunk_chars": 700 + }, + "prompt_builder": { + "prompt_version": "v2", + "target_agent": "claude_code", + "detail_level": "detailed", + "action_mode": "implement", + "structure": "agent_default", + "include_acceptance_criteria": true, + "include_verification": true, + "include_output_format": true, + "preserve_user_constraints": true, + "allow_user_suffix": true, + "user_suffix": "" + }, + "dictionary": { + "protected_words": [] + }, + "notifications": { + "enabled": true, + "dedupe_seconds": 5.0, + "dnd": false, + "log_enabled": true, + "quiet_hours": { + "enabled": false, + "start": "22:00", + "end": "07:00" + }, + "categories": { + "errors": { + "enabled": true + }, + "clipboard_suggestions": { + "enabled": true + }, + "updates": { + "enabled": true + }, + "diagnostics": { + "enabled": true + }, + "settings": { + "enabled": true + }, + "lifecycle": { + "enabled": true + }, + "action_result": { + "enabled": true + } + } + }, + "meetings": { + "enabled": true, + "mcp_url": "http://127.0.0.1:19532/mcp", + "source": "auto", + "max_context_tokens": 6000, + "batch": { + "enabled": true, + "start": "17:00", + "end": "21:00", + "only_when_idle": true, + "idle_minutes": 10, + "max_per_run": 10 + } + }, + "modes": { + "grammar": { + "label": "Grammar fix", + "shortcut": "Ctrl+Shift+G", + "description": "Fix grammar and wording while preserving meaning.", + "system_prompt": "Fix grammar, spelling, punctuation, capitalization, and obvious wording mistakes. Preserve the meaning and leave everything else exactly as written. Return only the corrected text, with no preamble and no notes." + }, + "prompt": { + "label": "Prompt fix (Claude)", + "description": "Rewrite rough text into a Claude-ready prompt (use prompt: prefix).", + "system_prompt": "Restate the user's requested coding deliverable as one concise imperative sentence. Use only details the user explicitly stated. Do not solve, infer, explain, or add requirements. Return only that sentence with no preamble, under 40 tokens." + }, + "summarize": { + "label": "Summarize", + "description": "3-bullet summary of selected text (use summarize: prefix).", + "system_prompt": "Summarize the user text as exactly 3 bullet points. Each bullet is one sentence, factual, no preamble or sign-off. Return only the bullets." + }, + "explain": { + "label": "Explain code/regex/SQL", + "description": "Plain-English explanation of selected code, regex, or query (use explain: prefix).", + "system_prompt": "Explain the selected code, regex, or SQL in 2-3 plain-English sentences. Call out one non-obvious edge case if any. No preamble. Return only the explanation." + }, + "tone": { + "label": "Tone shift", + "description": "Rewrite in selected tone (use tone: prefix). Active preset cycles from the tray.", + "preset": "friendly", + "presets": { + "formal": { + "system_prompt": "Rewrite the user text in a formal, professional tone. Preserve the meaning. Return only the rewritten text." + }, + "casual": { + "system_prompt": "Rewrite the user text in a casual, conversational tone. Preserve the meaning. Return only the rewritten text." + }, + "friendly": { + "system_prompt": "Rewrite the user text in a warm, friendly tone. Preserve the meaning. Return only the rewritten text." + } + }, + "system_prompt": "Rewrite the user text in a warm, friendly tone. Preserve the meaning. Return only the rewritten text." + } + }, + "notes": { + "vault_dir": "%USERPROFILE%\\Documents\\FastFlowPrompt Notes", + "categories": [ + "work/technical", + "work/managerial", + "work/career", + "research", + "personal", + "ideas" + ], + "fetch_timeout_seconds": 8, + "max_extracted_chars": 2000, + "low_confidence_to_inbox": true, + "generate_title": true, + "generate_summary": true + }, + "chat": { + "llm_auth_bearer": "flm", + "request_timeout_seconds": 240, + "temperature": 0.3, + "max_tokens": 1024, + "context_window_turns": 12, + "system_prompt": "You are a concise, helpful local assistant. Answer in Markdown. Keep replies short unless asked to elaborate.", + "window": { + "title": "Local LLM Chat", + "width": 520, + "height": 560, + "topmost": true, + "single_instance_port": 52640 + } + }, + "hotkeys": { + "grammar_fix": "^+g", + "open_chat": "^+1", + "capture_note": "^!n", + "ask_chat": "^+2" + } +} diff --git a/installer/installer.iss b/installer/installer.iss index c0486e2..9e5a1a2 100644 --- a/installer/installer.iss +++ b/installer/installer.iss @@ -41,7 +41,7 @@ #define AppURL "https://github.com/agr77one/Fastflow" #define AppExeName "Flowkey.exe" ; symbolic — actual launchers below ; Keep in lockstep with scripts\_version.py. -#define AppVersion "2.4.2" +#define AppVersion "2.4.3" [Setup] AppId={{8A4F1E6C-9B3D-4E62-9F7A-FASTFLOW140}} diff --git a/installer/sign.ps1 b/installer/sign.ps1 index 3427dc0..fdd8a31 100644 --- a/installer/sign.ps1 +++ b/installer/sign.ps1 @@ -52,7 +52,7 @@ .EXAMPLE # Sign the installer $env:FFP_SIGN_PFX_PASSWORD = "ChangeMe!" - .\sign.ps1 -FilePath ..\out\Flowkey-Setup-2.4.2.exe + .\sign.ps1 -FilePath ..\out\Flowkey-Setup-2.4.3.exe #> [CmdletBinding(DefaultParameterSetName = "Sign")] diff --git a/pyproject.toml b/pyproject.toml index a646838..f48d8c5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,7 @@ build-backend = "setuptools.build_meta" [project] name = "fastflowprompt" -version = "2.4.2" +version = "2.4.3" description = "Local-LLM-powered grammar fix, prompt rewrite, chat, and dashboard for Windows." readme = "README.md" requires-python = ">=3.11" diff --git a/scripts/_version.py b/scripts/_version.py index 8e3e2ae..b312e02 100644 --- a/scripts/_version.py +++ b/scripts/_version.py @@ -1,3 +1,3 @@ """Single source of truth for the app version. Read by grammar_fix.py.""" -__version__ = "2.4.2" +__version__ = "2.4.3" diff --git a/scripts/ffp_prompt_builder.py b/scripts/ffp_prompt_builder.py index f427c6b..db253a3 100644 --- a/scripts/ffp_prompt_builder.py +++ b/scripts/ffp_prompt_builder.py @@ -160,6 +160,73 @@ "Leave unspecified choices to the implementer.", ) +# Fewer real source clauses than this and the deterministic render has nothing to +# decompose: it would emit the request as the task, again as the single +# constraint, and again as the output format, padded with the scope guards above. +# That output is structurally valid but carries no information the user did not +# already type, so an underspecified request gets the clarify shape instead. +_MIN_SOURCE_CLAUSES = 2 + +# Dimensions a request can leave open. Deliberately abstract: naming concrete +# artifacts (an agenda, a database, a web UI) would invent requirements, which +# the v2 contract forbids. These only assert what was NOT stated. +_CLARIFY_UNKNOWNS = ( + "Target platform and language are unspecified - confirm before choosing.", + "The scope of the deliverable is not defined - agree it with the requester first.", + "No data source, storage, or integration was named - do not assume one.", + "Success criteria were not stated - establish them before implementing.", +) + +# Article agreement is deterministic; the typo map is a fixed, audited list of +# common slips. Applied to surfaced text because the v2 render copies the user's +# own words, so their typos would otherwise be repeated in every section. +_TYPO_FIXES = { + "perfrom": "perform", + "recieve": "receive", + "seperate": "separate", + "occured": "occurred", + "definately": "definitely", + "sucessful": "successful", + "sucessfully": "successfully", + "acommodate": "accommodate", + "existance": "existence", + "managment": "management", + "enviroment": "environment", + "sheduler": "scheduler", + "shedule": "schedule", + "teh": "the", + "adn": "and", + "wich": "which", + "alot": "a lot", +} +_TYPO_RE = re.compile(r"\b(" + "|".join(sorted(_TYPO_FIXES, key=len, reverse=True)) + r")\b", re.IGNORECASE) +# "a app" -> "an app". Only before a vowel SOUND we can detect safely, so the +# well-known exceptions (a user, a unique, a one-off, a European) are excluded. +_ARTICLE_A_RE = re.compile(r"\ba(?=\s+(?![uU]ni|[uU]se|[uU]ser|[uU]til|[oO]ne\b|[eE]uro)[aeiouAEIOU])", re.ASCII) +# "an schedule" -> "a schedule". +_ARTICLE_AN_RE = re.compile(r"\ban(?=\s+[b-df-hj-np-tv-zB-DF-HJ-NP-TV-Z])", re.ASCII) + + +def _match_case(original: str, replacement: str) -> str: + if original.isupper(): + return replacement.upper() + if original[:1].isupper(): + return replacement[:1].upper() + replacement[1:] + return replacement + + +def normalize_source_language(text: str) -> str: + """Fix article agreement and a fixed list of common typos. + + The v2 render surfaces the user's own wording, so "develop a app that allows + to perfrom ..." would otherwise appear verbatim in three sections. This only + corrects spelling/agreement; it never changes meaning or adds requirements.""" + cleaned = str(text or "") + cleaned = _TYPO_RE.sub(lambda m: _match_case(m.group(0), _TYPO_FIXES[m.group(0).lower()]), cleaned) + cleaned = _ARTICLE_A_RE.sub(lambda m: _match_case(m.group(0), "an"), cleaned) + cleaned = _ARTICLE_AN_RE.sub(lambda m: _match_case(m.group(0), "a"), cleaned) + return cleaned + @dataclass(frozen=True) class PromptBuilderSettings: @@ -503,6 +570,9 @@ def _render_xml(settings: PromptBuilderSettings, intent: PromptIntent, cleaned: def _render_prompt_v2_fallback(intent: PromptIntent, cleaned: str) -> str: + cleaned = normalize_source_language(cleaned) + if is_underspecified(cleaned): + return _render_prompt_v2_clarify(intent, cleaned) task = _prompt_v2_task(intent, cleaned) constraints = _prompt_v2_constraints(cleaned) constraint_body = "\n".join(f"- {item}" for item in constraints) @@ -522,6 +592,34 @@ def _render_prompt_v2_fallback(intent: PromptIntent, cleaned: str) -> str: ) +def _render_prompt_v2_clarify(intent: PromptIntent, cleaned: str) -> str: + """Render for a request with nothing to decompose. + + Instead of repeating the request as task, constraint, and output format (an + echo that passes the structural contract while telling the reader nothing), + this names what the request left open and instructs the agent to settle those + before building. It adds no requirements: every bullet asserts only that + something was *not* stated.""" + task = _prompt_v2_task(intent, cleaned) + unknowns = list(_CLARIFY_UNKNOWNS[:4]) + constraint_body = "\n".join(f"- {item}" for item in unknowns) + return ( + "\n" + f"{task}\n" + "\n" + "\n" + "The request states the goal only; no further detail was given.\n" + "\n" + "\n" + f"{constraint_body}\n" + "\n" + "\n" + "State assumptions and open questions first, then propose a minimal scope " + "for approval before implementing.\n" + "" + ) + + def _source_sentences(cleaned: str) -> list[str]: return [ part.strip(" \t\r\n.;") @@ -569,7 +667,12 @@ def _prompt_v2_task(intent: PromptIntent, cleaned: str) -> str: return task.rstrip(".!? ") + "." -def _prompt_v2_constraints(cleaned: str) -> list[str]: +def source_clauses(cleaned: str) -> list[str]: + """Distinct requirement clauses actually present in the source (no padding). + + This is the signal for whether the request can be decomposed at all: a + multi-requirement request yields several, while "develop an app that does + what a proper PM would" yields exactly one.""" clauses: list[str] = [] for sentence in _source_sentences(cleaned): for relative_part in _RELATIVE_SPLIT_RE.split(sentence, maxsplit=1): @@ -580,6 +683,19 @@ def _prompt_v2_constraints(cleaned: str) -> list[str]: clause = clause.rstrip(".!? ") + "." if clause.lower() not in {item.lower() for item in clauses}: clauses.append(clause) + return clauses + + +def is_underspecified(cleaned: str) -> bool: + """True when the source cannot be decomposed into distinct requirements. + + Such a request has no structure to surface, so the ordinary render would + repeat it as task, constraint, and output format (see _MIN_SOURCE_CLAUSES).""" + return len(source_clauses(cleaned)) < _MIN_SOURCE_CLAUSES + + +def _prompt_v2_constraints(cleaned: str) -> list[str]: + clauses = source_clauses(cleaned) for guard in _SAFE_SCOPE_GUARDS: if len(clauses) >= 3: break diff --git a/tests/test_prompt_builder.py b/tests/test_prompt_builder.py index d88ee10..8a1bfb8 100644 --- a/tests/test_prompt_builder.py +++ b/tests/test_prompt_builder.py @@ -139,7 +139,14 @@ def test_v33_default_v2_grounding_removes_model_inventions_and_splits_source_cla assert "\nA Python script.\n" in grounded -def test_v33_sparse_request_uses_only_fixed_scope_guards(): +def test_v33_sparse_request_uses_clarify_shape_without_inventing(): + """Sparse request -> clarify shape (V44), still inventing nothing (V33). + + Superseded the previous expectation that a sparse request be padded with the + fixed scope guards. That padding produced an output repeating the request as + task, constraint and output_format — structurally valid but carrying no + information the user had not typed (see B38). The anti-invention guarantee + this test was protecting is asserted below and unchanged.""" settings = ffp_prompt_builder.PromptBuilderSettings.from_config({}) source = "make the dashboard faster" intent = ffp_prompt_builder.resolve_intent(settings, source) @@ -147,11 +154,17 @@ def test_v33_sparse_request_uses_only_fixed_scope_guards(): assert ffp_prompt_builder.validate(grounded, settings).valid is True assert "\nMake the dashboard faster.\n" in grounded - assert "- Make the dashboard faster." in grounded - assert "- Preserve all stated requirements." in grounded - assert "- Do not add unstated requirements." in grounded - assert "browser" not in grounded.lower() - assert "\nNo output format specified.\n" in grounded + # Still no invention: the model's draft is discarded and nothing concrete is + # conjured (no browser, profiler, framework, or metric the user never named). + for invented in ("browser", "profiler", "webpack", "lighthouse", "redis"): + assert invented not in grounded.lower() + # The request is no longer echoed into constraints or output_format, and the + # guard padding is replaced by named unknowns. + constraints = grounded.split("")[1].split("")[0] + assert "- Make the dashboard faster." not in constraints + assert "- Preserve all stated requirements." not in constraints + assert "unspecified" in constraints + assert "No output format specified." not in grounded def test_v27_task_sentence_allows_identifiers_and_regex_punctuation(): diff --git a/tests/test_prompt_clarify_shape.py b/tests/test_prompt_clarify_shape.py new file mode 100644 index 0000000..a9a8749 --- /dev/null +++ b/tests/test_prompt_clarify_shape.py @@ -0,0 +1,222 @@ +"""Underspecified-request handling + the R8 echo gate (2.4.3). + +Reported 2026-07-28. The request + + develop a app that allows to perfrom a full schedule for my meeting a proper PM would + +produced an output that repeated it three times — as , as the only real + bullet, and as — padded with two scope guards, typos +included. It was structurally valid and invented nothing, so it scored 5/7 on the +machine-checkable rubric with no hard fail and would pass at 7/7 under a lenient +judge. The existing echo detector could never catch it: is_weak_prompt_echo() +returns False as soon as the output carries target structure (V32), and the v2 +finalizer always emits structure. + +Fixes: detect a source with nothing to decompose and render a clarify shape +instead; normalize typos/article agreement in surfaced text; add R8 (no section +restates the task) as a disqualifying rubric item. +""" +from __future__ import annotations + +import importlib.util +from pathlib import Path + +import ffp_prompt_builder as P +import pytest + +ROOT = Path(__file__).resolve().parents[1] +VAGUE = "develop a app that allows to perfrom a full schedule for my meeting a proper PM would" +CSV = ( + "build a python script that reads a folder of CSVs, validates rows against a schema, " + "writes an error report with file and line numbers" +) + + +def _eval_module(): + spec = importlib.util.spec_from_file_location("ev", ROOT / "tools" / "prompt_speed_quality_eval.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _render(source: str) -> str: + settings = P.PromptBuilderSettings.from_config(None) + intent = P.resolve_intent(settings, "") + return P.ground_prompt_v2_output("draft-is-discarded", settings, intent, P._normalize_text(source)) + + +# ---------- language normalization ------------------------------------------------------- + +@pytest.mark.parametrize(("raw", "expected"), [ + ("develop a app", "develop an app"), + ("perfrom a schedule", "perform a schedule"), + ("a app that allows to perfrom", "an app that allows to perform"), + ("an schedule", "a schedule"), + ("Perfrom the task", "Perform the task"), # case preserved + ("recieve teh data adn store", "receive the data and store"), +]) +def test_normalize_source_language(raw, expected): + assert P.normalize_source_language(raw) == expected + + +@pytest.mark.parametrize("text", [ + "a user story", # "user" starts with a consonant sound + "a unique id", + "a one-off script", + "a utility module", +]) +def test_article_exceptions_not_broken(text): + # "a user" must NOT become "an user". + assert P.normalize_source_language(text) == text + + +def test_normalization_does_not_touch_ordinary_text(): + assert P.normalize_source_language(CSV) == CSV + + +# ---------- underspecified detection ----------------------------------------------------- + +def test_the_reported_input_is_underspecified(): + assert P.is_underspecified(P.normalize_source_language(VAGUE)) is True + assert len(P.source_clauses(P.normalize_source_language(VAGUE))) == 1 + + +def test_decomposable_request_is_not_underspecified(): + assert P.is_underspecified(CSV) is False + assert len(P.source_clauses(CSV)) >= 3 + + +def test_source_clauses_excludes_scope_guard_padding(): + # source_clauses reports only real clauses; padding belongs to the renderer. + clauses = P.source_clauses(P.normalize_source_language(VAGUE)) + for guard in P._SAFE_SCOPE_GUARDS: + assert guard not in clauses + + +# ---------- clarify render --------------------------------------------------------------- + +def test_clarify_shape_replaces_the_echo(): + out = _render(VAGUE) + task = out.split("")[1].split("")[0].strip() + constraints = out.split("")[1].split("")[0] + out_fmt = out.split("")[1].split("")[0].strip() + + # Typos fixed in the surfaced task. + assert "an app" in task and "perform" in task + assert "a app" not in out and "perfrom" not in out + # The request is no longer pasted into constraints or output_format. + assert "full schedule for my meeting" not in constraints + assert "full schedule for my meeting" not in out_fmt + # Scope-guard padding is gone; real unknowns are named instead. + assert "Preserve all stated requirements." not in constraints + assert "unspecified" in constraints + assert "open questions" in out_fmt + + +def test_clarify_output_still_satisfies_the_v2_contract(): + out = _render(VAGUE) + settings = P.PromptBuilderSettings.from_config(None) + result = P.validate(out, settings) + assert result.valid, result.errors + + +def test_clarify_invents_no_requirements(): + # Every bullet must assert only that something was NOT stated - naming + # concrete artifacts (an agenda, a database, a web UI) would be invention. + out = _render(VAGUE).lower() + for invented in ("agenda", "attendee", "calendar", "database", "react", "sqlite", "rest api"): + assert invented not in out + + +def test_good_request_render_is_unchanged_by_the_clarify_path(): + out = _render(CSV) + assert "The request states the goal only" not in out # not the clarify shape + body = out.split("")[1].split("")[0] + items = [line for line in body.splitlines() if line.strip().startswith("-")] + assert len(items) == 4 # real decomposition preserved + + +# ---------- R8 rubric item --------------------------------------------------------------- + +OLD_ECHO = """ +Develop a app that allows to perfrom a full schedule for my meeting a proper PM would. + + +No additional context stated. + + +- Develop a app that allows to perfrom a full schedule for my meeting a proper PM would. +- Preserve all stated requirements. +- Do not add unstated requirements. + + +A app that allows to perfrom a full schedule for my meeting a proper PM would. +""" + +JUDGE_PASS = {"r3": True, "r4": True, "invented_requirement": False} + + +def test_r8_fails_the_reported_echo(): + ev = _eval_module() + scored = ev.score_output(OLD_ECHO, judgment=JUDGE_PASS) + assert scored["rubric"]["r8"] is False + assert scored["echoed_section"] in ("output_format", "constraints") + # Disqualifying even with every judged item passing - the old blind spot. + assert scored["passed"] is False + + +def test_r8_passes_every_fixed_case(): + ev = _eval_module() + for case in ev.FIXED_CASES: + scored = ev.score_output(_render(case["input"]), judgment=JUDGE_PASS) + assert scored["rubric"]["r8"] is True, f"{case['name']}: {scored['echoed_section']}" + + +def test_r8_allows_one_restating_bullet_among_distinct_ones(): + # A long request's task comes from its first sentence, so a bullet repeating + # that sentence is normal. Only a set with NOTHING else is a failure. + ev = _eval_module() + text = """ +Add a webhook import endpoint to the existing Python service. + + +No additional context stated. + + +- Add a webhook import endpoint to the existing Python service. +- Reject malformed JSON with 400. +- Authenticate using the existing X-Webhook-Key middleware. + + +A webhook endpoint. +""" + assert ev.score_output(text, judgment=JUDGE_PASS)["rubric"]["r8"] is True + + +def test_r8_ignores_boilerplate_when_deciding(): + ev = _eval_module() + text = """ +Build a reporting tool for quarterly numbers. + + +No additional context stated. + + +- Build a reporting tool for quarterly numbers. +- Preserve all stated requirements. +- Leave unspecified choices to the implementer. + + +A reporting tool. +""" + scored = ev.score_output(text, judgment=JUDGE_PASS) + assert scored["rubric"]["r8"] is False # only a restatement + padding + assert scored["echoed_section"] == "constraints" + + +def test_fixed_case_set_includes_the_reported_input(): + ev = _eval_module() + names = {case["name"] for case in ev.FIXED_CASES} + assert "vague_unparseable_scheduler" in names + case = next(c for c in ev.FIXED_CASES if c["name"] == "vague_unparseable_scheduler") + assert case["input"] == VAGUE diff --git a/tests/test_prompt_speed_quality_eval.py b/tests/test_prompt_speed_quality_eval.py index a47d072..9297e8f 100644 --- a/tests/test_prompt_speed_quality_eval.py +++ b/tests/test_prompt_speed_quality_eval.py @@ -44,15 +44,16 @@ def test_fixed_set_covers_required_spread(): assert len({case["name"] for case in cases}) == len(cases) -def test_machine_rubric_and_manual_judgment_combine_to_seven(): +def test_machine_rubric_and_manual_judgment_combine_to_eight(): result = prompt_eval.score_output( GOOD_OUTPUT, completion_tokens=92, judgment={"r3": True, "r4": True, "invented_requirement": False}, ) - assert result["rubric"] == {key: True for key in ("r1", "r2", "r3", "r4", "r5", "r6", "r7")} - assert result["score"] == 7 + # r8 (2.4.3): no section may restate ; see test_prompt_clarify_shape. + assert result["rubric"] == {key: True for key in ("r1", "r2", "r3", "r4", "r5", "r6", "r7", "r8")} + assert result["score"] == 8 assert result["passed"] is True assert result["pending_judge"] is False assert result["constraint_item_count"] == 3 @@ -68,7 +69,7 @@ def test_quality_stays_pending_without_semantic_judge_and_invention_hard_fails() assert pending["score"] is None assert pending["pending_judge"] is True - assert invented["score"] == 7 + assert invented["score"] == 8 assert invented["hard_fail"] is True assert invented["passed"] is False diff --git a/tests/test_version_sync.py b/tests/test_version_sync.py index 50ef3b5..b88b3fe 100644 --- a/tests/test_version_sync.py +++ b/tests/test_version_sync.py @@ -32,4 +32,4 @@ def test_v18_release_version_is_synchronized(): ), } - assert set(versions.values()) == {"2.4.2"}, versions + assert set(versions.values()) == {"2.4.3"}, versions diff --git a/tools/prompt_speed_quality_eval.py b/tools/prompt_speed_quality_eval.py index ccd2b73..869a2e5 100644 --- a/tools/prompt_speed_quality_eval.py +++ b/tools/prompt_speed_quality_eval.py @@ -91,6 +91,15 @@ "category": "vague", "input": "make the dashboard faster", }, + { + # Reported 2026-07-28. Harder than the case above: one clause, an + # unparseable semantic core ("a proper PM would"), and typos. The old + # render echoed it into task + constraints + output_format and still + # scored 5/7 machine-checkable with no hard fail, which is why R8 exists. + "name": "vague_unparseable_scheduler", + "category": "vague", + "input": "develop a app that allows to perfrom a full schedule for my meeting a proper PM would", + }, { "name": "long_webhook_import", "category": "long", @@ -300,6 +309,72 @@ def _constraint_items(text: str) -> list[str]: return items +def _content_words(text: str) -> set[str]: + return {w for w in re.findall(r"[a-z0-9']+", str(text or "").lower()) if w not in _STOPWORDS} + + +_STOPWORDS = frozenset( + "a an the and or of to for in on with that which is are be my your it its this these those " + "from into at as by all any".split() +) +# A section covering at least this much of is restating it, not refining it. +_R8_TASK_COVERAGE = 0.8 + + +# Padding phrases the renderer appends purely to reach the 3-item minimum. Held +# here rather than imported from ffp_prompt_builder ON PURPOSE: this gate must +# judge the shipped output independently, so it cannot silently follow a change +# in the code it is checking. Clarify-shape bullets ("... is unspecified - +# confirm before choosing") are NOT listed: they state something real about the +# request, whereas these say nothing situational at all. +_BOILERPLATE_CONSTRAINTS = frozenset({ + "preserve all stated requirements", + "do not add unstated requirements", + "leave unspecified choices to the implementer", +}) + + +def _is_boilerplate(item: str) -> bool: + return re.sub(r"[^a-z ]", "", str(item or "").lower()).strip() in _BOILERPLATE_CONSTRAINTS + + +def _section_repeats_task(sections: dict[str, str]) -> str: + """Name of the section proving the output is a restatement, or "" if none. + + This is the check the original rubric lacked. An output can carry all four + tags, invent nothing, and still be worthless if the request is pasted into + task, constraints and output_format — exactly what an underspecified request + produced. Structure alone is not evidence of conversion. + + Coverage is measured against the TASK, not the candidate section: genuine + decomposition necessarily reuses the task's words ("Write an error report + with file and line numbers." shares every word with its task line) but each + part covers only a fraction of the whole, while a restatement covers nearly + all of it. + + Constraints are judged as a set, not individually. A long request legitimately + yields one bullet restating its opening sentence (the task is derived from that + same sentence) alongside several distinct ones. The failure is when NOTHING + remains after discarding restatements and padding — then the section adds + nothing the task did not already say.""" + task_words = _content_words(sections.get("task", "")) + if len(task_words) < 3: + return "" + + def restates(text: str) -> bool: + words = _content_words(text) + return bool(words) and len(words & task_words) / len(task_words) >= _R8_TASK_COVERAGE + + # output_format must name an artifact/shape, never re-quote the request. + if restates(sections.get("output_format", "")): + return "output_format" + + items = _constraint_items(sections.get("constraints", "")) + if items and not [i for i in items if not restates(i) and not _is_boilerplate(i)]: + return "constraints" + return "" + + def score_output( text: str, *, @@ -331,15 +406,21 @@ def score_output( else None ) r3 = (len(constraint_items) in range(3, 6) and judged_r3) if judged_r3 is not None else None - rubric = {"r1": r1, "r2": r2, "r3": r3, "r4": judged_r4, "r5": r5, "r6": r6, "r7": r7} + # R8: no section may simply restate (machine-checkable, no judge). + echoed_section = _section_repeats_task(sections) if sections else "" + r8 = bool(sections) and not echoed_section + rubric = {"r1": r1, "r2": r2, "r3": r3, "r4": judged_r4, "r5": r5, "r6": r6, "r7": r7, "r8": r8} pending = any(value is None for value in rubric.values()) or invented is None score = None if any(value is None for value in rubric.values()) else sum(bool(v) for v in rubric.values()) hard_fail = invented is True return { "rubric": rubric, "score": score, - "passed": bool(score is not None and score >= 6 and not hard_fail), + # R8 is disqualifying on its own: an output that merely restates the + # request is not a converted prompt no matter what else it scores. + "passed": bool(score is not None and score >= 7 and not hard_fail and r8), "hard_fail": hard_fail, + "echoed_section": echoed_section, "pending_judge": pending, "invented_requirement": invented, "constraint_item_count": len(constraint_items),