diff --git a/README.md b/README.md index 1717ba4f..58d699c8 100644 --- a/README.md +++ b/README.md @@ -127,7 +127,7 @@ The dashboard below is generated from an append-only event ledger and immutable, - Benchmark evidence dashboard. Equivalent detailed tables follow: 6 suites have accepted checkpoints, 1 are current, and 4 shipped agents have no benchmark evidence. + Benchmark evidence dashboard. Equivalent detailed tables follow: 11 suites have accepted checkpoints, 6 are current, and 4 shipped agents have no benchmark evidence. The tracker separates lifecycle, evidence provenance, freshness, change type, and assessment. A stale score remains visible but is never presented as current. Current history is too sparse and heterogeneous to support exponential-growth or diminishing-return claims; the honest classification is **insufficient comparable evidence**. @@ -137,7 +137,11 @@ The tracker separates lifecycle, evidence provenance, freshness, change type, an | Feature critique | shipped | accepted | stale | methodology-reset | ? unknown | Gold finding recall: 23/25 (92.0%) · Clean-plan pass rate: 5/5 (100.0%) | | Feature completeness | shipped | accepted | stale | methodology-reset | ? unknown | Gold gap recall: 25/35 (71.4%) · Decoy false-flag rate: 2/27 (7.4%) | | Repository conventions | shipped | accepted | stale | quality | ? unknown | Gold gap recall: 18/18 (100.0%) · Decoy false-flag rate: 1/14 (7.1%) | -| Domain and correctness reviewers | shipped | evidence-only | unknown | coverage | → flat | Gold rows before catalog refresh: 24 · Gold rows after catalog refresh: 29 | +| API security reviewer | shipped | accepted | current | coverage | ? unknown | first-pass FAIL recall: 8/8 (100.0%) · first-pass clean pass: 5/6 (83.3%) · block recall: 8/8 (100.0%) · clean pass: 6/6 (100.0%) | +| Backend performance reviewer | shipped | accepted | current | coverage | ? unknown | first-pass FAIL recall: 7/7 (100.0%) · first-pass clean pass: 3/6 (50.0%) · block recall: 7/7 (100.0%) · clean pass: 6/6 (100.0%) | +| Frontend security reviewer | shipped | accepted | current | coverage | ? unknown | first-pass FAIL recall: 11/11 (100.0%) · first-pass clean pass: 7/8 (87.5%) · block recall: 10/11 (90.9%) · clean pass: 8/8 (100.0%) | +| Frontend performance reviewer | shipped | accepted | current | coverage | ? unknown | first-pass FAIL recall: 10/11 (90.9%) · first-pass clean pass: 7/8 (87.5%) · block recall: 10/11 (90.9%) · clean pass: 7/8 (87.5%) | +| Correctness reviewer | shipped | accepted | current | coverage | ? unknown | first-pass FAIL recall: 35/38 (92.1%) · first-pass clean pass: 24/28 (85.7%) | | Decision governance | shipped | evidence-only | unknown | quality | ↑ improved | Detect accuracy: 45/49 (91.8%) · DECISION recall: 8/9 (88.9%) | | Sentry capture judge | shipped | evidence-only | unknown | quality | ↑ improved | Commit-message F1: 56/100 (56.0%) · Focused-diff F1: 87/100 (87.0%) | | Edge-case autonomy | no-ship | accepted | stale | no-ship | ? unknown | Judge-free ceiling: 51.2% · Pre-registered target: 35.0% | @@ -152,17 +156,17 @@ The tracker separates lifecycle, evidence provenance, freshness, change type, an | Shipped agent | Lifecycle | Evidence mode | Suite(s) | | --- | --- | --- | --- | -| API security agent | shipped | evidence-only | reviewer-fleet | -| Backend performance agent | shipped | evidence-only | reviewer-fleet | +| API security agent | shipped | evidence-only | reviewer-api-security | +| Backend performance agent | shipped | evidence-only | reviewer-backend-performance | | Commit guard agent | shipped | evidence-only | commit-guard-retrieval | | Conventions agent | shipped | accepted | conventions | -| Correctness agent | shipped | evidence-only | reviewer-fleet | +| Correctness agent | shipped | evidence-only | reviewer-correctness | | Feature completeness agent | shipped | accepted | completeness | | Feature critique agent | shipped | accepted | critique | | Upstream-fix agent | shipped | none | — | | Frontend accessibility agent | shipped | none | — | -| Frontend performance agent | shipped | evidence-only | reviewer-fleet | -| Frontend security agent | shipped | evidence-only | reviewer-fleet | +| Frontend performance agent | shipped | evidence-only | reviewer-frontend-performance | +| Frontend security agent | shipped | evidence-only | reviewer-frontend-security | | Testing agent | shipped | none | — | | Testing reviewer agent | shipped | none | — | diff --git a/docs/benchmarks/README.md b/docs/benchmarks/README.md index 6ff044e6..c75a480c 100644 --- a/docs/benchmarks/README.md +++ b/docs/benchmarks/README.md @@ -75,7 +75,11 @@ Committed evidence rejects raw prompts, transcripts, absolute paths, email addre | Feature critique | shipped | accepted | stale | methodology-reset | ? unknown | Gold finding recall: 23/25 (92.0%) · Clean-plan pass rate: 5/5 (100.0%) | | Feature completeness | shipped | accepted | stale | methodology-reset | ? unknown | Gold gap recall: 25/35 (71.4%) · Decoy false-flag rate: 2/27 (7.4%) | | Repository conventions | shipped | accepted | stale | quality | ? unknown | Gold gap recall: 18/18 (100.0%) · Decoy false-flag rate: 1/14 (7.1%) | -| Domain and correctness reviewers | shipped | evidence-only | unknown | coverage | → flat | Gold rows before catalog refresh: 24 · Gold rows after catalog refresh: 29 | +| API security reviewer | shipped | accepted | current | coverage | ? unknown | first-pass FAIL recall: 8/8 (100.0%) · first-pass clean pass: 5/6 (83.3%) · block recall: 8/8 (100.0%) · clean pass: 6/6 (100.0%) | +| Backend performance reviewer | shipped | accepted | current | coverage | ? unknown | first-pass FAIL recall: 7/7 (100.0%) · first-pass clean pass: 3/6 (50.0%) · block recall: 7/7 (100.0%) · clean pass: 6/6 (100.0%) | +| Frontend security reviewer | shipped | accepted | current | coverage | ? unknown | first-pass FAIL recall: 11/11 (100.0%) · first-pass clean pass: 7/8 (87.5%) · block recall: 10/11 (90.9%) · clean pass: 8/8 (100.0%) | +| Frontend performance reviewer | shipped | accepted | current | coverage | ? unknown | first-pass FAIL recall: 10/11 (90.9%) · first-pass clean pass: 7/8 (87.5%) · block recall: 10/11 (90.9%) · clean pass: 7/8 (87.5%) | +| Correctness reviewer | shipped | accepted | current | coverage | ? unknown | first-pass FAIL recall: 35/38 (92.1%) · first-pass clean pass: 24/28 (85.7%) | | Decision governance | shipped | evidence-only | unknown | quality | ↑ improved | Detect accuracy: 45/49 (91.8%) · DECISION recall: 8/9 (88.9%) | | Sentry capture judge | shipped | evidence-only | unknown | quality | ↑ improved | Commit-message F1: 56/100 (56.0%) · Focused-diff F1: 87/100 (87.0%) | | Edge-case autonomy | no-ship | accepted | stale | no-ship | ? unknown | Judge-free ceiling: 51.2% · Pre-registered target: 35.0% | @@ -90,17 +94,17 @@ Committed evidence rejects raw prompts, transcripts, absolute paths, email addre | Subject | Kind | Lifecycle | Evidence | Suite(s) | | --- | --- | --- | --- | --- | -| API security agent | agent | shipped | evidence-only | reviewer-fleet | -| Backend performance agent | agent | shipped | evidence-only | reviewer-fleet | +| API security agent | agent | shipped | evidence-only | reviewer-api-security | +| Backend performance agent | agent | shipped | evidence-only | reviewer-backend-performance | | Commit guard agent | agent | shipped | evidence-only | commit-guard-retrieval | | Conventions agent | agent | shipped | accepted | conventions | -| Correctness agent | agent | shipped | evidence-only | reviewer-fleet | +| Correctness agent | agent | shipped | evidence-only | reviewer-correctness | | Feature completeness agent | agent | shipped | accepted | completeness | | Feature critique agent | agent | shipped | accepted | critique | | Upstream-fix agent | agent | shipped | none | — | | Frontend accessibility agent | agent | shipped | none | — | -| Frontend performance agent | agent | shipped | evidence-only | reviewer-fleet | -| Frontend security agent | agent | shipped | evidence-only | reviewer-fleet | +| Frontend performance agent | agent | shipped | evidence-only | reviewer-frontend-performance | +| Frontend security agent | agent | shipped | evidence-only | reviewer-frontend-security | | Testing agent | agent | shipped | none | — | | Testing reviewer agent | agent | shipped | none | — | | devkit CLI | bin | shipped | none | — | @@ -113,16 +117,16 @@ Committed evidence rejects raw prompts, transcripts, absolute paths, email addre | Folder fan-out ratchet | bin | shipped | none | — | | Prefix cache gate | bin | shipped | none | — | | qavis advisory gate | bin | shipped | external-required | qavis | -| Reviewer gate | bin | shipped | accepted | completeness, conventions, reviewer-fleet | +| Reviewer gate | bin | shipped | accepted | completeness, conventions, reviewer-api-security, reviewer-backend-performance, reviewer-frontend-security, reviewer-frontend-performance, reviewer-correctness | | Sentry judge gate | bin | shipped | evidence-only | sentry | | Size ratchet | bin | shipped | none | — | | Structure gate | bin | shipped | none | — | -| API security reviewer | reviewer | shipped | evidence-only | reviewer-fleet | -| Backend performance reviewer | reviewer | shipped | evidence-only | reviewer-fleet | -| Frontend security reviewer | reviewer | shipped | evidence-only | reviewer-fleet | -| Frontend performance reviewer | reviewer | shipped | evidence-only | reviewer-fleet | +| API security reviewer | reviewer | shipped | evidence-only | reviewer-api-security | +| Backend performance reviewer | reviewer | shipped | evidence-only | reviewer-backend-performance | +| Frontend security reviewer | reviewer | shipped | evidence-only | reviewer-frontend-security | +| Frontend performance reviewer | reviewer | shipped | evidence-only | reviewer-frontend-performance | | Commit guard reviewer | reviewer | shipped | evidence-only | commit-guard-retrieval | -| Correctness reviewer | reviewer | shipped | evidence-only | reviewer-fleet | +| Correctness reviewer | reviewer | shipped | evidence-only | reviewer-correctness | | Conventions reviewer | reviewer | shipped | accepted | conventions | | Feature critique judge | judge | shipped | accepted | critique | | Completeness judge | judge | shipped | accepted | completeness | diff --git a/docs/benchmarks/assets/dashboard-dark.svg b/docs/benchmarks/assets/dashboard-dark.svg index 197d9cad..c2abd3bc 100644 --- a/docs/benchmarks/assets/dashboard-dark.svg +++ b/docs/benchmarks/assets/dashboard-dark.svg @@ -1,26 +1,30 @@ - + devkit benchmark evidence dashboard -6 suites have accepted evidence. 4 shipped agents have no benchmark evidence. Rows use symbols and words as well as color. Full equivalent tables are in the README. +11 suites have accepted evidence. 4 shipped agents have no benchmark evidence. Rows use symbols and words as well as color. Full equivalent tables are in the README. - + Benchmark evidence, not benchmark vibes -6 accepted suites · 4 shipped-agent evidence gaps · growth curve: insufficient evidence +11 accepted suites · 4 shipped-agent evidence gaps · growth curve: insufficient evidence ✓ accepted ! stale × no-ship ? missing — status never relies on color alone Feature critique! staleGold finding recall: 23/25 (92.0%) · Clean-pl… Feature completeness! staleGold gap recall: 25/35 (71.4%) · Decoy false-… Repository conventions! staleGold gap recall: 18/18 (100.0%) · Decoy false… -Domain and correctness reviewers· evidence-onlyGold rows before catalog refresh: 24 · Gold r… -Decision governance· evidence-onlyDetect accuracy: 45/49 (91.8%) · DECISION rec… -Sentry capture judge· evidence-onlyCommit-message F1: 56/100 (56.0%) · Focused-d… -Edge-case autonomy!× stale no-shipJudge-free ceiling: 51.2% · Pre-registered ta… -Decision-log retrieval recall! staleGold axis retrieved: 12/12 (100.0%) · Multi-a… -Decision-log save quality✓ acceptedDefect recall (perturbation corpus): 13/13 (1… -Semantic search retrieval· evidence-onlyNo accepted local checkpoint -Duplication matcher· evidence-onlyNo accepted local checkpoint -Commit-guard retrieval· evidence-onlyClone retrieval recall at 10: 59.0% · Semanti… -qavis visual QA· externalNo accepted local checkpoint +API security reviewer✓ acceptedfirst-pass FAIL recall: 8/8 (100.0%) · first-… +Backend performance reviewer✓ acceptedfirst-pass FAIL recall: 7/7 (100.0%) · first-… +Frontend security reviewer✓ acceptedfirst-pass FAIL recall: 11/11 (100.0%) · firs… +Frontend performance reviewer✓ acceptedfirst-pass FAIL recall: 10/11 (90.9%) · first… +Correctness reviewer✓ acceptedfirst-pass FAIL recall: 35/38 (92.1%) · first… +Decision governance· evidence-onlyDetect accuracy: 45/49 (91.8%) · DECISION rec… +Sentry capture judge· evidence-onlyCommit-message F1: 56/100 (56.0%) · Focused-d… +Edge-case autonomy!× stale no-shipJudge-free ceiling: 51.2% · Pre-registered ta… +Decision-log retrieval recall! staleGold axis retrieved: 12/12 (100.0%) · Multi-a… +Decision-log save quality✓ acceptedDefect recall (perturbation corpus): 13/13 (1… +Semantic search retrieval· evidence-onlyNo accepted local checkpoint +Duplication matcher· evidence-onlyNo accepted local checkpoint +Commit-guard retrieval· evidence-onlyClone retrieval recall at 10: 59.0% · Semanti… +qavis visual QA· externalNo accepted local checkpoint diff --git a/docs/benchmarks/assets/dashboard-light.svg b/docs/benchmarks/assets/dashboard-light.svg index ea543563..2a00240a 100644 --- a/docs/benchmarks/assets/dashboard-light.svg +++ b/docs/benchmarks/assets/dashboard-light.svg @@ -1,26 +1,30 @@ - + devkit benchmark evidence dashboard -6 suites have accepted evidence. 4 shipped agents have no benchmark evidence. Rows use symbols and words as well as color. Full equivalent tables are in the README. +11 suites have accepted evidence. 4 shipped agents have no benchmark evidence. Rows use symbols and words as well as color. Full equivalent tables are in the README. - + Benchmark evidence, not benchmark vibes -6 accepted suites · 4 shipped-agent evidence gaps · growth curve: insufficient evidence +11 accepted suites · 4 shipped-agent evidence gaps · growth curve: insufficient evidence ✓ accepted ! stale × no-ship ? missing — status never relies on color alone Feature critique! staleGold finding recall: 23/25 (92.0%) · Clean-pl… Feature completeness! staleGold gap recall: 25/35 (71.4%) · Decoy false-… Repository conventions! staleGold gap recall: 18/18 (100.0%) · Decoy false… -Domain and correctness reviewers· evidence-onlyGold rows before catalog refresh: 24 · Gold r… -Decision governance· evidence-onlyDetect accuracy: 45/49 (91.8%) · DECISION rec… -Sentry capture judge· evidence-onlyCommit-message F1: 56/100 (56.0%) · Focused-d… -Edge-case autonomy!× stale no-shipJudge-free ceiling: 51.2% · Pre-registered ta… -Decision-log retrieval recall! staleGold axis retrieved: 12/12 (100.0%) · Multi-a… -Decision-log save quality✓ acceptedDefect recall (perturbation corpus): 13/13 (1… -Semantic search retrieval· evidence-onlyNo accepted local checkpoint -Duplication matcher· evidence-onlyNo accepted local checkpoint -Commit-guard retrieval· evidence-onlyClone retrieval recall at 10: 59.0% · Semanti… -qavis visual QA· externalNo accepted local checkpoint +API security reviewer✓ acceptedfirst-pass FAIL recall: 8/8 (100.0%) · first-… +Backend performance reviewer✓ acceptedfirst-pass FAIL recall: 7/7 (100.0%) · first-… +Frontend security reviewer✓ acceptedfirst-pass FAIL recall: 11/11 (100.0%) · firs… +Frontend performance reviewer✓ acceptedfirst-pass FAIL recall: 10/11 (90.9%) · first… +Correctness reviewer✓ acceptedfirst-pass FAIL recall: 35/38 (92.1%) · first… +Decision governance· evidence-onlyDetect accuracy: 45/49 (91.8%) · DECISION rec… +Sentry capture judge· evidence-onlyCommit-message F1: 56/100 (56.0%) · Focused-d… +Edge-case autonomy!× stale no-shipJudge-free ceiling: 51.2% · Pre-registered ta… +Decision-log retrieval recall! staleGold axis retrieved: 12/12 (100.0%) · Multi-a… +Decision-log save quality✓ acceptedDefect recall (perturbation corpus): 13/13 (1… +Semantic search retrieval· evidence-onlyNo accepted local checkpoint +Duplication matcher· evidence-onlyNo accepted local checkpoint +Commit-guard retrieval· evidence-onlyClone retrieval recall at 10: 59.0% · Semanti… +qavis visual QA· externalNo accepted local checkpoint diff --git a/docs/benchmarks/catalog.json b/docs/benchmarks/catalog.json index 80eb68ca..8df402e1 100644 --- a/docs/benchmarks/catalog.json +++ b/docs/benchmarks/catalog.json @@ -1,68 +1,966 @@ { "schemaVersion": 1, "subjects": [ - { "id": "agent-api-security-reviewer", "label": "API security agent", "kind": "agent", "lifecycle": "shipped", "evidence": "evidence-only", "suiteIds": ["reviewer-fleet"], "canonical": "agents/api-security-reviewer.md" }, - { "id": "agent-backend-performance-reviewer", "label": "Backend performance agent", "kind": "agent", "lifecycle": "shipped", "evidence": "evidence-only", "suiteIds": ["reviewer-fleet"], "canonical": "agents/backend-performance-reviewer.md" }, - { "id": "agent-commit-guard", "label": "Commit guard agent", "kind": "agent", "lifecycle": "shipped", "evidence": "evidence-only", "suiteIds": ["commit-guard-retrieval"], "canonical": "agents/commit-guard.md" }, - { "id": "agent-conventions-reviewer", "label": "Conventions agent", "kind": "agent", "lifecycle": "shipped", "evidence": "accepted", "suiteIds": ["conventions"], "canonical": "agents/conventions-reviewer.md" }, - { "id": "agent-correctness-reviewer", "label": "Correctness agent", "kind": "agent", "lifecycle": "shipped", "evidence": "evidence-only", "suiteIds": ["reviewer-fleet"], "canonical": "agents/correctness-reviewer.md" }, - { "id": "agent-feature-completeness-reviewer", "label": "Feature completeness agent", "kind": "agent", "lifecycle": "shipped", "evidence": "accepted", "suiteIds": ["completeness"], "canonical": "agents/feature-completeness-reviewer.md" }, - { "id": "agent-feature-critique", "label": "Feature critique agent", "kind": "agent", "lifecycle": "shipped", "evidence": "accepted", "suiteIds": ["critique"], "canonical": "agents/feature-critique.md" }, - { "id": "agent-fix-upstream-reviewer", "label": "Upstream-fix agent", "kind": "agent", "lifecycle": "shipped", "evidence": "none", "suiteIds": [], "canonical": "agents/fix-upstream-reviewer.md" }, - { "id": "agent-frontend-accessibility-reviewer", "label": "Frontend accessibility agent", "kind": "agent", "lifecycle": "shipped", "evidence": "none", "suiteIds": [], "canonical": "agents/frontend-accessibility-reviewer.md" }, - { "id": "agent-frontend-performance-reviewer", "label": "Frontend performance agent", "kind": "agent", "lifecycle": "shipped", "evidence": "evidence-only", "suiteIds": ["reviewer-fleet"], "canonical": "agents/frontend-performance-reviewer.md" }, - { "id": "agent-frontend-security-reviewer", "label": "Frontend security agent", "kind": "agent", "lifecycle": "shipped", "evidence": "evidence-only", "suiteIds": ["reviewer-fleet"], "canonical": "agents/frontend-security-reviewer.md" }, - { "id": "agent-testing-agent", "label": "Testing agent", "kind": "agent", "lifecycle": "shipped", "evidence": "none", "suiteIds": [], "canonical": "agents/testing-agent.md" }, - { "id": "agent-testing-reviewer", "label": "Testing reviewer agent", "kind": "agent", "lifecycle": "shipped", "evidence": "none", "suiteIds": [], "canonical": "agents/testing-reviewer.md" }, - - { "id": "bin-devkit", "label": "devkit CLI", "kind": "bin", "lifecycle": "shipped", "evidence": "none", "suiteIds": [], "canonical": "package.json#bin:devkit" }, - { "id": "bin-guard-clone", "label": "Clone gate", "kind": "bin", "lifecycle": "shipped", "evidence": "evidence-only", "suiteIds": ["co-occurrence"], "canonical": "package.json#bin:guard-clone" }, - { "id": "bin-guard-coverage", "label": "Coverage gate", "kind": "bin", "lifecycle": "shipped", "evidence": "none", "suiteIds": [], "canonical": "package.json#bin:guard-coverage" }, - { "id": "bin-guard-decisions", "label": "Decisions gate", "kind": "bin", "lifecycle": "shipped", "evidence": "evidence-only", "suiteIds": ["decisions", "decisions-recall", "decisions-save-quality"], "canonical": "package.json#bin:guard-decisions" }, - { "id": "bin-guard-deterministic", "label": "Deterministic orchestrator", "kind": "bin", "lifecycle": "shipped", "evidence": "none", "suiteIds": [], "canonical": "package.json#bin:guard-deterministic" }, - { "id": "bin-guard-dup", "label": "Semantic duplication gate", "kind": "bin", "lifecycle": "shipped", "evidence": "evidence-only", "suiteIds": ["co-occurrence"], "canonical": "package.json#bin:guard-dup" }, - { "id": "bin-guard-dup-allowlist", "label": "Duplication allowlist CLI", "kind": "bin", "lifecycle": "shipped", "evidence": "none", "suiteIds": [], "canonical": "package.json#bin:guard-dup-allowlist" }, - { "id": "bin-guard-fanout", "label": "Folder fan-out ratchet", "kind": "bin", "lifecycle": "shipped", "evidence": "none", "suiteIds": [], "canonical": "package.json#bin:guard-fanout" }, - { "id": "bin-guard-prefix", "label": "Prefix cache gate", "kind": "bin", "lifecycle": "shipped", "evidence": "none", "suiteIds": [], "canonical": "package.json#bin:guard-prefix" }, - { "id": "bin-guard-qavis-advisory", "label": "qavis advisory gate", "kind": "bin", "lifecycle": "shipped", "evidence": "external-required", "suiteIds": ["qavis"], "canonical": "package.json#bin:guard-qavis-advisory" }, - { "id": "bin-guard-review", "label": "Reviewer gate", "kind": "bin", "lifecycle": "shipped", "evidence": "accepted", "suiteIds": ["completeness", "conventions", "reviewer-fleet"], "canonical": "package.json#bin:guard-review" }, - { "id": "bin-guard-sentry", "label": "Sentry judge gate", "kind": "bin", "lifecycle": "shipped", "evidence": "evidence-only", "suiteIds": ["sentry"], "canonical": "package.json#bin:guard-sentry" }, - { "id": "bin-guard-size", "label": "Size ratchet", "kind": "bin", "lifecycle": "shipped", "evidence": "none", "suiteIds": [], "canonical": "package.json#bin:guard-size" }, - { "id": "bin-guard-structure", "label": "Structure gate", "kind": "bin", "lifecycle": "shipped", "evidence": "none", "suiteIds": [], "canonical": "package.json#bin:guard-structure" }, - - { "id": "reviewer-api-security-reviewer", "label": "API security reviewer", "kind": "reviewer", "lifecycle": "shipped", "evidence": "evidence-only", "suiteIds": ["reviewer-fleet"], "canonical": "REVIEWERS:api-security-reviewer" }, - { "id": "reviewer-backend-performance-reviewer", "label": "Backend performance reviewer", "kind": "reviewer", "lifecycle": "shipped", "evidence": "evidence-only", "suiteIds": ["reviewer-fleet"], "canonical": "REVIEWERS:backend-performance-reviewer" }, - { "id": "reviewer-frontend-security-reviewer", "label": "Frontend security reviewer", "kind": "reviewer", "lifecycle": "shipped", "evidence": "evidence-only", "suiteIds": ["reviewer-fleet"], "canonical": "REVIEWERS:frontend-security-reviewer" }, - { "id": "reviewer-frontend-performance-reviewer", "label": "Frontend performance reviewer", "kind": "reviewer", "lifecycle": "shipped", "evidence": "evidence-only", "suiteIds": ["reviewer-fleet"], "canonical": "REVIEWERS:frontend-performance-reviewer" }, - { "id": "reviewer-commit-guard", "label": "Commit guard reviewer", "kind": "reviewer", "lifecycle": "shipped", "evidence": "evidence-only", "suiteIds": ["commit-guard-retrieval"], "canonical": "REVIEWERS:commit-guard" }, - { "id": "reviewer-correctness-reviewer", "label": "Correctness reviewer", "kind": "reviewer", "lifecycle": "shipped", "evidence": "evidence-only", "suiteIds": ["reviewer-fleet"], "canonical": "REVIEWERS:correctness-reviewer" }, - { "id": "reviewer-conventions-reviewer", "label": "Conventions reviewer", "kind": "reviewer", "lifecycle": "shipped", "evidence": "accepted", "suiteIds": ["conventions"], "canonical": "REVIEWERS:conventions-reviewer" }, - - { "id": "judge-feature-critique", "label": "Feature critique judge", "kind": "judge", "lifecycle": "shipped", "evidence": "accepted", "suiteIds": ["critique"] }, - { "id": "judge-completeness", "label": "Completeness judge", "kind": "judge", "lifecycle": "shipped", "evidence": "accepted", "suiteIds": ["completeness"] }, - { "id": "judge-decisions", "label": "Decision detect/alignment/depth judges", "kind": "judge", "lifecycle": "shipped", "evidence": "evidence-only", "suiteIds": ["decisions"] }, - { "id": "judge-sentry", "label": "Sentry commit-message judge", "kind": "judge", "lifecycle": "shipped", "evidence": "evidence-only", "suiteIds": ["sentry"] }, - { "id": "judge-qavis", "label": "qavis visual QA", "kind": "judge", "lifecycle": "shipped", "evidence": "external-required", "suiteIds": ["qavis"], "externalUrl": "https://github.com/norvalbv/qavis" }, - { "id": "benchmark-decisions-recall", "label": "Decision-log retrieval recall", "kind": "benchmark", "lifecycle": "experimental", "evidence": "none", "suiteIds": ["decisions-recall"] }, - { "id": "benchmark-decisions-save-quality", "label": "Decision-log save quality", "kind": "benchmark", "lifecycle": "experimental", "evidence": "none", "suiteIds": ["decisions-save-quality"] }, - { "id": "benchmark-search-tool", "label": "Semantic search retrieval", "kind": "benchmark", "lifecycle": "experimental", "evidence": "evidence-only", "suiteIds": ["search-tool"] }, - { "id": "benchmark-edge-cases", "label": "Edge-case autonomy study", "kind": "benchmark", "lifecycle": "no-ship", "evidence": "accepted", "suiteIds": ["edge-cases"] } + { + "id": "agent-api-security-reviewer", + "label": "API security agent", + "kind": "agent", + "lifecycle": "shipped", + "evidence": "evidence-only", + "suiteIds": [ + "reviewer-api-security" + ], + "canonical": "agents/api-security-reviewer.md" + }, + { + "id": "agent-backend-performance-reviewer", + "label": "Backend performance agent", + "kind": "agent", + "lifecycle": "shipped", + "evidence": "evidence-only", + "suiteIds": [ + "reviewer-backend-performance" + ], + "canonical": "agents/backend-performance-reviewer.md" + }, + { + "id": "agent-commit-guard", + "label": "Commit guard agent", + "kind": "agent", + "lifecycle": "shipped", + "evidence": "evidence-only", + "suiteIds": [ + "commit-guard-retrieval" + ], + "canonical": "agents/commit-guard.md" + }, + { + "id": "agent-conventions-reviewer", + "label": "Conventions agent", + "kind": "agent", + "lifecycle": "shipped", + "evidence": "accepted", + "suiteIds": [ + "conventions" + ], + "canonical": "agents/conventions-reviewer.md" + }, + { + "id": "agent-correctness-reviewer", + "label": "Correctness agent", + "kind": "agent", + "lifecycle": "shipped", + "evidence": "evidence-only", + "suiteIds": [ + "reviewer-correctness" + ], + "canonical": "agents/correctness-reviewer.md" + }, + { + "id": "agent-feature-completeness-reviewer", + "label": "Feature completeness agent", + "kind": "agent", + "lifecycle": "shipped", + "evidence": "accepted", + "suiteIds": [ + "completeness" + ], + "canonical": "agents/feature-completeness-reviewer.md" + }, + { + "id": "agent-feature-critique", + "label": "Feature critique agent", + "kind": "agent", + "lifecycle": "shipped", + "evidence": "accepted", + "suiteIds": [ + "critique" + ], + "canonical": "agents/feature-critique.md" + }, + { + "id": "agent-fix-upstream-reviewer", + "label": "Upstream-fix agent", + "kind": "agent", + "lifecycle": "shipped", + "evidence": "none", + "suiteIds": [], + "canonical": "agents/fix-upstream-reviewer.md" + }, + { + "id": "agent-frontend-accessibility-reviewer", + "label": "Frontend accessibility agent", + "kind": "agent", + "lifecycle": "shipped", + "evidence": "none", + "suiteIds": [], + "canonical": "agents/frontend-accessibility-reviewer.md" + }, + { + "id": "agent-frontend-performance-reviewer", + "label": "Frontend performance agent", + "kind": "agent", + "lifecycle": "shipped", + "evidence": "evidence-only", + "suiteIds": [ + "reviewer-frontend-performance" + ], + "canonical": "agents/frontend-performance-reviewer.md" + }, + { + "id": "agent-frontend-security-reviewer", + "label": "Frontend security agent", + "kind": "agent", + "lifecycle": "shipped", + "evidence": "evidence-only", + "suiteIds": [ + "reviewer-frontend-security" + ], + "canonical": "agents/frontend-security-reviewer.md" + }, + { + "id": "agent-testing-agent", + "label": "Testing agent", + "kind": "agent", + "lifecycle": "shipped", + "evidence": "none", + "suiteIds": [], + "canonical": "agents/testing-agent.md" + }, + { + "id": "agent-testing-reviewer", + "label": "Testing reviewer agent", + "kind": "agent", + "lifecycle": "shipped", + "evidence": "none", + "suiteIds": [], + "canonical": "agents/testing-reviewer.md" + }, + { + "id": "bin-devkit", + "label": "devkit CLI", + "kind": "bin", + "lifecycle": "shipped", + "evidence": "none", + "suiteIds": [], + "canonical": "package.json#bin:devkit" + }, + { + "id": "bin-guard-clone", + "label": "Clone gate", + "kind": "bin", + "lifecycle": "shipped", + "evidence": "evidence-only", + "suiteIds": [ + "co-occurrence" + ], + "canonical": "package.json#bin:guard-clone" + }, + { + "id": "bin-guard-coverage", + "label": "Coverage gate", + "kind": "bin", + "lifecycle": "shipped", + "evidence": "none", + "suiteIds": [], + "canonical": "package.json#bin:guard-coverage" + }, + { + "id": "bin-guard-decisions", + "label": "Decisions gate", + "kind": "bin", + "lifecycle": "shipped", + "evidence": "evidence-only", + "suiteIds": [ + "decisions", + "decisions-recall", + "decisions-save-quality" + ], + "canonical": "package.json#bin:guard-decisions" + }, + { + "id": "bin-guard-deterministic", + "label": "Deterministic orchestrator", + "kind": "bin", + "lifecycle": "shipped", + "evidence": "none", + "suiteIds": [], + "canonical": "package.json#bin:guard-deterministic" + }, + { + "id": "bin-guard-dup", + "label": "Semantic duplication gate", + "kind": "bin", + "lifecycle": "shipped", + "evidence": "evidence-only", + "suiteIds": [ + "co-occurrence" + ], + "canonical": "package.json#bin:guard-dup" + }, + { + "id": "bin-guard-dup-allowlist", + "label": "Duplication allowlist CLI", + "kind": "bin", + "lifecycle": "shipped", + "evidence": "none", + "suiteIds": [], + "canonical": "package.json#bin:guard-dup-allowlist" + }, + { + "id": "bin-guard-fanout", + "label": "Folder fan-out ratchet", + "kind": "bin", + "lifecycle": "shipped", + "evidence": "none", + "suiteIds": [], + "canonical": "package.json#bin:guard-fanout" + }, + { + "id": "bin-guard-prefix", + "label": "Prefix cache gate", + "kind": "bin", + "lifecycle": "shipped", + "evidence": "none", + "suiteIds": [], + "canonical": "package.json#bin:guard-prefix" + }, + { + "id": "bin-guard-qavis-advisory", + "label": "qavis advisory gate", + "kind": "bin", + "lifecycle": "shipped", + "evidence": "external-required", + "suiteIds": [ + "qavis" + ], + "canonical": "package.json#bin:guard-qavis-advisory" + }, + { + "id": "bin-guard-review", + "label": "Reviewer gate", + "kind": "bin", + "lifecycle": "shipped", + "evidence": "accepted", + "suiteIds": [ + "completeness", + "conventions", + "reviewer-api-security", + "reviewer-backend-performance", + "reviewer-frontend-security", + "reviewer-frontend-performance", + "reviewer-correctness" + ], + "canonical": "package.json#bin:guard-review" + }, + { + "id": "bin-guard-sentry", + "label": "Sentry judge gate", + "kind": "bin", + "lifecycle": "shipped", + "evidence": "evidence-only", + "suiteIds": [ + "sentry" + ], + "canonical": "package.json#bin:guard-sentry" + }, + { + "id": "bin-guard-size", + "label": "Size ratchet", + "kind": "bin", + "lifecycle": "shipped", + "evidence": "none", + "suiteIds": [], + "canonical": "package.json#bin:guard-size" + }, + { + "id": "bin-guard-structure", + "label": "Structure gate", + "kind": "bin", + "lifecycle": "shipped", + "evidence": "none", + "suiteIds": [], + "canonical": "package.json#bin:guard-structure" + }, + { + "id": "reviewer-api-security-reviewer", + "label": "API security reviewer", + "kind": "reviewer", + "lifecycle": "shipped", + "evidence": "evidence-only", + "suiteIds": [ + "reviewer-api-security" + ], + "canonical": "REVIEWERS:api-security-reviewer" + }, + { + "id": "reviewer-backend-performance-reviewer", + "label": "Backend performance reviewer", + "kind": "reviewer", + "lifecycle": "shipped", + "evidence": "evidence-only", + "suiteIds": [ + "reviewer-backend-performance" + ], + "canonical": "REVIEWERS:backend-performance-reviewer" + }, + { + "id": "reviewer-frontend-security-reviewer", + "label": "Frontend security reviewer", + "kind": "reviewer", + "lifecycle": "shipped", + "evidence": "evidence-only", + "suiteIds": [ + "reviewer-frontend-security" + ], + "canonical": "REVIEWERS:frontend-security-reviewer" + }, + { + "id": "reviewer-frontend-performance-reviewer", + "label": "Frontend performance reviewer", + "kind": "reviewer", + "lifecycle": "shipped", + "evidence": "evidence-only", + "suiteIds": [ + "reviewer-frontend-performance" + ], + "canonical": "REVIEWERS:frontend-performance-reviewer" + }, + { + "id": "reviewer-commit-guard", + "label": "Commit guard reviewer", + "kind": "reviewer", + "lifecycle": "shipped", + "evidence": "evidence-only", + "suiteIds": [ + "commit-guard-retrieval" + ], + "canonical": "REVIEWERS:commit-guard" + }, + { + "id": "reviewer-correctness-reviewer", + "label": "Correctness reviewer", + "kind": "reviewer", + "lifecycle": "shipped", + "evidence": "evidence-only", + "suiteIds": [ + "reviewer-correctness" + ], + "canonical": "REVIEWERS:correctness-reviewer" + }, + { + "id": "reviewer-conventions-reviewer", + "label": "Conventions reviewer", + "kind": "reviewer", + "lifecycle": "shipped", + "evidence": "accepted", + "suiteIds": [ + "conventions" + ], + "canonical": "REVIEWERS:conventions-reviewer" + }, + { + "id": "judge-feature-critique", + "label": "Feature critique judge", + "kind": "judge", + "lifecycle": "shipped", + "evidence": "accepted", + "suiteIds": [ + "critique" + ] + }, + { + "id": "judge-completeness", + "label": "Completeness judge", + "kind": "judge", + "lifecycle": "shipped", + "evidence": "accepted", + "suiteIds": [ + "completeness" + ] + }, + { + "id": "judge-decisions", + "label": "Decision detect/alignment/depth judges", + "kind": "judge", + "lifecycle": "shipped", + "evidence": "evidence-only", + "suiteIds": [ + "decisions" + ] + }, + { + "id": "judge-sentry", + "label": "Sentry commit-message judge", + "kind": "judge", + "lifecycle": "shipped", + "evidence": "evidence-only", + "suiteIds": [ + "sentry" + ] + }, + { + "id": "judge-qavis", + "label": "qavis visual QA", + "kind": "judge", + "lifecycle": "shipped", + "evidence": "external-required", + "suiteIds": [ + "qavis" + ], + "externalUrl": "https://github.com/norvalbv/qavis" + }, + { + "id": "benchmark-decisions-recall", + "label": "Decision-log retrieval recall", + "kind": "benchmark", + "lifecycle": "experimental", + "evidence": "none", + "suiteIds": [ + "decisions-recall" + ] + }, + { + "id": "benchmark-decisions-save-quality", + "label": "Decision-log save quality", + "kind": "benchmark", + "lifecycle": "experimental", + "evidence": "none", + "suiteIds": [ + "decisions-save-quality" + ] + }, + { + "id": "benchmark-search-tool", + "label": "Semantic search retrieval", + "kind": "benchmark", + "lifecycle": "experimental", + "evidence": "evidence-only", + "suiteIds": [ + "search-tool" + ] + }, + { + "id": "benchmark-edge-cases", + "label": "Edge-case autonomy study", + "kind": "benchmark", + "lifecycle": "no-ship", + "evidence": "accepted", + "suiteIds": [ + "edge-cases" + ] + } ], "suites": [ - { "id": "critique", "label": "Feature critique", "adapter": "critique", "subjectIds": ["agent-feature-critique", "judge-feature-critique"], "runner": "gate-engine/critique/eval/bench.mts", "baseline": "gate-engine/critique/eval/results.baseline.json", "lifecycle": "shipped", "acceptance": "K=3 full tier, matcher K=3, zero outages", "hashes": { "implementation": ["agents/feature-critique.md", "skills/feature-critique/**"], "corpus": ["gate-engine/critique/eval/cases-*.jsonl"], "scorer": ["gate-engine/critique/eval/matcher.mts", "gate-engine/judge/matcher-core.mts", "gate-engine/critique/eval/matcher-audit.jsonl"], "runner": ["gate-engine/critique/eval/bench.mts", "gate-engine/critique/eval/run-critic.mts"] } }, - { "id": "completeness", "label": "Feature completeness", "adapter": "completeness", "subjectIds": ["agent-feature-completeness-reviewer", "judge-completeness", "bin-guard-review"], "runner": "gate-engine/review/eval/bench.mts", "baseline": "gate-engine/review/eval/results.baseline.json", "lifecycle": "shipped", "acceptance": "Full matcher tier, K=3, zero outages", "hashes": { "implementation": ["agents/feature-completeness-reviewer.md", "gate-engine/review/completeness.mts"], "corpus": ["gate-engine/review/eval/cases-completeness.jsonl"], "scorer": ["gate-engine/review/eval/matcher.mts", "gate-engine/judge/matcher-core.mts", "gate-engine/review/eval/matcher-audit.labels.jsonl"], "runner": ["gate-engine/review/eval/bench.mts"] } }, - { "id": "conventions", "label": "Repository conventions", "adapter": "conventions", "subjectIds": ["agent-conventions-reviewer", "reviewer-conventions-reviewer", "bin-guard-review"], "runner": "gate-engine/review/eval/conventions/bench.mts", "baseline": "gate-engine/review/eval/conventions/results.baseline.json", "lifecycle": "shipped", "acceptance": "Full matcher tier, K=3, zero outages", "hashes": { "implementation": ["agents/conventions-reviewer.md", "gate-engine/review/reviewers.mts"], "corpus": ["gate-engine/review/eval/conventions/cases-conventions.jsonl"], "scorer": ["gate-engine/review/eval/conventions/matcher.mts", "gate-engine/judge/matcher-core.mts", "gate-engine/review/eval/conventions/matcher-audit-conventions.labels.jsonl"], "runner": ["gate-engine/review/eval/conventions/bench.mts"] } }, - { "id": "reviewer-fleet", "label": "Domain and correctness reviewers", "adapter": "reviewer", "subjectIds": ["reviewer-api-security-reviewer", "reviewer-backend-performance-reviewer", "reviewer-frontend-security-reviewer", "reviewer-frontend-performance-reviewer", "reviewer-correctness-reviewer"], "runner": "gate-engine/review/eval/reviewers/bench.mts", "lifecycle": "shipped", "acceptance": "Complete model/cascade cohort with zero outages and suite-local floors", "hashes": { "implementation": ["gate-engine/review/run-review.mts", "gate-engine/review/reviewers.mts", "agents/*-reviewer.md", "skills/**"], "corpus": ["gate-engine/review/eval/reviewers/cases-*.jsonl"], "scorer": ["gate-engine/review/eval/reviewers/bench.mts"], "runner": ["gate-engine/review/eval/reviewers/bench.mts"] } }, - { "id": "decisions", "label": "Decision governance", "adapter": "decisions", "subjectIds": ["judge-decisions", "bin-guard-decisions"], "runner": "gate-engine/decisions/eval/bench.mts", "lifecycle": "shipped", "acceptance": "K=3 detect/depth, K=1 alignment with stable baseline-discordant retry, zero outages and suite hard floors", "hashes": { "implementation": ["gate-engine/decisions/detect.mts", "gate-engine/decisions/check-alignment.mts", "gate-engine/decisions/depth-judge.mts"], "corpus": ["gate-engine/decisions/eval/cases-*.jsonl"], "scorer": ["gate-engine/decisions/eval/bench.mts", "gate-engine/decisions/eval/acceptance.mts", "gate-engine/decisions/eval/compare.mts", "gate-engine/eval/statistics.mts"], "runner": ["gate-engine/decisions/eval/bench.mts", "gate-engine/decisions/eval/acceptance.mts", "gate-engine/decisions/eval/compare.mts", "gate-engine/decisions/eval/cases.mts", "gate-engine/decisions/eval/checkpoint.mts", "gate-engine/decisions/eval/report.mts", "gate-engine/eval/statistics.mts", "gate-engine/decisions/decisions.mts", "gate-engine/config.mts"] } }, - { "id": "sentry", "label": "Sentry capture judge", "adapter": "sentry", "subjectIds": ["judge-sentry", "bin-guard-sentry"], "runner": "gate-engine/sentry/eval/bench.mts", "lifecycle": "shipped", "acceptance": "Complete persisted precision/recall/F1 cell, zero outages", "hashes": { "implementation": ["gate-engine/sentry/check-sentry.mts"], "corpus": ["gate-engine/sentry/eval/cases.jsonl"], "scorer": ["gate-engine/sentry/eval/bench.mts"], "runner": ["gate-engine/sentry/eval/bench.mts"] } }, - { "id": "edge-cases", "label": "Edge-case autonomy", "adapter": "edge-cases", "subjectIds": ["benchmark-edge-cases"], "lifecycle": "no-ship", "acceptance": "Pre-registered full analysis with robust no-ship decision", "hashes": { "implementation": ["gate-engine/edge-cases/eval/prompts.mts"], "corpus": ["gate-engine/edge-cases/eval/cases.jsonl"], "scorer": ["gate-engine/edge-cases/eval/lib/**", "gate-engine/edge-cases/eval/counts.mts"], "runner": ["gate-engine/edge-cases/eval/bench.mts"] } }, - { "id": "decisions-recall", "label": "Decision-log retrieval recall", "adapter": "deterministic", "baseline": "gate-engine/decisions/eval/recall/results.baseline.json", "subjectIds": ["benchmark-decisions-recall", "bin-guard-decisions"], "runner": "gate-engine/decisions/eval/recall/bench.mts", "lifecycle": "experimental", "acceptance": "Deterministic set-arithmetic scoring, zero LLM calls; seed corpus only until gold cases are harvested from a real corpus", "hashes": { "implementation": ["gate-engine/decisions/retrieval.mts", "gate-engine/decisions/decisions.mts"], "corpus": ["gate-engine/decisions/eval/recall/cases-retrieval.jsonl", "gate-engine/decisions/eval/recall/corpus/**"], "scorer": ["gate-engine/decisions/eval/recall/scoring.mts"], "runner": ["gate-engine/decisions/eval/recall/bench.mts", "gate-engine/decisions/eval/recall/scoring.mts", "gate-engine/eval/statistics.mts"] } }, - { "id": "decisions-save-quality", "label": "Decision-log save quality", "adapter": "deterministic", "baseline": "gate-engine/decisions/eval/save-quality/results.baseline.json", "subjectIds": ["benchmark-decisions-save-quality", "bin-guard-decisions"], "runner": "gate-engine/decisions/eval/save-quality/bench.mts", "lifecycle": "experimental", "acceptance": "Deterministic structural checks, zero LLM calls; perturbation recall must clear the 80% floor before FPR@R80 is read, and the real corpus must produce no finding beyond the one named historical exception", "hashes": { "implementation": ["gate-engine/decisions/integrity/checks.mts", "gate-engine/decisions/integrity/scan.mts"], "corpus": ["gate-engine/decisions/eval/save-quality/cases-save.jsonl", "gate-engine/decisions/eval/save-quality/corpus/**"], "scorer": ["gate-engine/decisions/eval/save-quality/scoring.mts"], "runner": ["gate-engine/decisions/eval/save-quality/bench.mts", "gate-engine/decisions/eval/save-quality/scoring.mts", "gate-engine/decisions/integrity/perturb.mts"] } }, - { "id": "search-tool", "label": "Semantic search retrieval", "adapter": "reviewer", "subjectIds": ["benchmark-search-tool"], "runner": "gate-engine/search-tool/eval/eval.mts", "lifecycle": "experimental", "acceptance": "Evidence-only retrieval study", "hashes": { "implementation": ["gate-engine/search-tool/**"], "corpus": ["gate-engine/search-tool/eval/queries.json"], "scorer": ["gate-engine/search-tool/eval/eval.mts"], "runner": ["gate-engine/search-tool/eval/eval.mts"] } }, - { "id": "co-occurrence", "label": "Duplication matcher", "adapter": "reviewer", "subjectIds": ["bin-guard-dup", "bin-guard-clone"], "runner": "gate-engine/co-occurrence/matcher.mts", "lifecycle": "shipped", "acceptance": "Deterministic fixture benchmark", "hashes": { "implementation": ["gate-engine/co-occurrence/**"], "corpus": ["gate-engine/co-occurrence/fixtures/**"], "scorer": ["gate-engine/co-occurrence/matcher.mts"], "runner": ["gate-engine/co-occurrence/matcher.mts"] } }, - { "id": "commit-guard-retrieval", "label": "Commit-guard retrieval", "adapter": "reviewer", "subjectIds": ["agent-commit-guard", "reviewer-commit-guard"], "lifecycle": "shipped", "acceptance": "Evidence-only until a persisted suite exists", "hashes": { "implementation": ["agents/commit-guard.md", "skills/commit-guard/**"], "corpus": [], "scorer": [], "runner": [] } }, - { "id": "qavis", "label": "qavis visual QA", "adapter": "reviewer", "subjectIds": ["judge-qavis", "bin-guard-qavis-advisory"], "lifecycle": "shipped", "acceptance": "External evidence required", "hashes": { "implementation": ["gate-engine/qavis-advisory/**"], "corpus": [], "scorer": [], "runner": [] } } + { + "id": "critique", + "label": "Feature critique", + "adapter": "critique", + "subjectIds": [ + "agent-feature-critique", + "judge-feature-critique" + ], + "runner": "gate-engine/critique/eval/bench.mts", + "baseline": "gate-engine/critique/eval/results.baseline.json", + "lifecycle": "shipped", + "acceptance": "K=3 full tier, matcher K=3, zero outages", + "hashes": { + "implementation": [ + "agents/feature-critique.md", + "skills/feature-critique/**" + ], + "corpus": [ + "gate-engine/critique/eval/cases-*.jsonl" + ], + "scorer": [ + "gate-engine/critique/eval/matcher.mts", + "gate-engine/judge/matcher-core.mts", + "gate-engine/critique/eval/matcher-audit.jsonl" + ], + "runner": [ + "gate-engine/critique/eval/bench.mts", + "gate-engine/critique/eval/run-critic.mts" + ] + } + }, + { + "id": "completeness", + "label": "Feature completeness", + "adapter": "completeness", + "subjectIds": [ + "agent-feature-completeness-reviewer", + "judge-completeness", + "bin-guard-review" + ], + "runner": "gate-engine/review/eval/bench.mts", + "baseline": "gate-engine/review/eval/results.baseline.json", + "lifecycle": "shipped", + "acceptance": "Full matcher tier, K=3, zero outages", + "hashes": { + "implementation": [ + "agents/feature-completeness-reviewer.md", + "gate-engine/review/completeness.mts" + ], + "corpus": [ + "gate-engine/review/eval/cases-completeness.jsonl" + ], + "scorer": [ + "gate-engine/review/eval/matcher.mts", + "gate-engine/judge/matcher-core.mts", + "gate-engine/review/eval/matcher-audit.labels.jsonl" + ], + "runner": [ + "gate-engine/review/eval/bench.mts" + ] + } + }, + { + "id": "conventions", + "label": "Repository conventions", + "adapter": "conventions", + "subjectIds": [ + "agent-conventions-reviewer", + "reviewer-conventions-reviewer", + "bin-guard-review" + ], + "runner": "gate-engine/review/eval/conventions/bench.mts", + "baseline": "gate-engine/review/eval/conventions/results.baseline.json", + "lifecycle": "shipped", + "acceptance": "Full matcher tier, K=3, zero outages", + "hashes": { + "implementation": [ + "agents/conventions-reviewer.md", + "gate-engine/review/reviewers.mts" + ], + "corpus": [ + "gate-engine/review/eval/conventions/cases-conventions.jsonl" + ], + "scorer": [ + "gate-engine/review/eval/conventions/matcher.mts", + "gate-engine/judge/matcher-core.mts", + "gate-engine/review/eval/conventions/matcher-audit-conventions.labels.jsonl" + ], + "runner": [ + "gate-engine/review/eval/conventions/bench.mts" + ] + } + }, + { + "id": "reviewer-api-security", + "label": "API security reviewer", + "adapter": "reviewer", + "subjectIds": [ + "reviewer-api-security-reviewer", + "agent-api-security-reviewer", + "bin-guard-review" + ], + "runner": "gate-engine/review/eval/reviewers/bench.mts", + "sections": [ + "api-security-reviewer" + ], + "lifecycle": "shipped", + "acceptance": "Complete model/cascade cohort with zero outages and suite-local floors", + "hashes": { + "implementation": [ + "gate-engine/review/run-review.mts", + "gate-engine/review/reviewers.mts", + "agents/api-security-reviewer.md", + "skills/api-security/**" + ], + "corpus": [ + "gate-engine/review/eval/reviewers/cases-api-security.jsonl" + ], + "scorer": [ + "gate-engine/review/eval/reviewers/bench.mts" + ], + "runner": [ + "gate-engine/review/eval/reviewers/bench.mts" + ] + } + }, + { + "id": "reviewer-backend-performance", + "label": "Backend performance reviewer", + "adapter": "reviewer", + "subjectIds": [ + "reviewer-backend-performance-reviewer", + "agent-backend-performance-reviewer", + "bin-guard-review" + ], + "runner": "gate-engine/review/eval/reviewers/bench.mts", + "sections": [ + "backend-performance-reviewer" + ], + "lifecycle": "shipped", + "acceptance": "Complete model/cascade cohort with zero outages and suite-local floors", + "hashes": { + "implementation": [ + "gate-engine/review/run-review.mts", + "gate-engine/review/reviewers.mts", + "agents/backend-performance-reviewer.md", + "skills/backend-performance/**" + ], + "corpus": [ + "gate-engine/review/eval/reviewers/cases-backend-performance.jsonl" + ], + "scorer": [ + "gate-engine/review/eval/reviewers/bench.mts" + ], + "runner": [ + "gate-engine/review/eval/reviewers/bench.mts" + ] + } + }, + { + "id": "reviewer-frontend-security", + "label": "Frontend security reviewer", + "adapter": "reviewer", + "subjectIds": [ + "reviewer-frontend-security-reviewer", + "agent-frontend-security-reviewer", + "bin-guard-review" + ], + "runner": "gate-engine/review/eval/reviewers/bench.mts", + "sections": [ + "frontend-security-reviewer" + ], + "lifecycle": "shipped", + "acceptance": "Complete model/cascade cohort with zero outages and suite-local floors", + "hashes": { + "implementation": [ + "gate-engine/review/run-review.mts", + "gate-engine/review/reviewers.mts", + "agents/frontend-security-reviewer.md", + "skills/frontend-security/**" + ], + "corpus": [ + "gate-engine/review/eval/reviewers/cases-frontend-security.jsonl" + ], + "scorer": [ + "gate-engine/review/eval/reviewers/bench.mts" + ], + "runner": [ + "gate-engine/review/eval/reviewers/bench.mts" + ] + } + }, + { + "id": "reviewer-frontend-performance", + "label": "Frontend performance reviewer", + "adapter": "reviewer", + "subjectIds": [ + "reviewer-frontend-performance-reviewer", + "agent-frontend-performance-reviewer", + "bin-guard-review" + ], + "runner": "gate-engine/review/eval/reviewers/bench.mts", + "sections": [ + "frontend-performance-reviewer" + ], + "lifecycle": "shipped", + "acceptance": "Complete model/cascade cohort with zero outages and suite-local floors", + "hashes": { + "implementation": [ + "gate-engine/review/run-review.mts", + "gate-engine/review/reviewers.mts", + "agents/frontend-performance-reviewer.md", + "skills/frontend-performance/**" + ], + "corpus": [ + "gate-engine/review/eval/reviewers/cases-frontend-performance.jsonl" + ], + "scorer": [ + "gate-engine/review/eval/reviewers/bench.mts" + ], + "runner": [ + "gate-engine/review/eval/reviewers/bench.mts" + ] + } + }, + { + "id": "reviewer-correctness", + "label": "Correctness reviewer", + "adapter": "reviewer", + "subjectIds": [ + "reviewer-correctness-reviewer", + "agent-correctness-reviewer", + "bin-guard-review" + ], + "runner": "gate-engine/review/eval/reviewers/bench.mts", + "sections": [ + "correctness-reviewer" + ], + "lifecycle": "shipped", + "acceptance": "Complete model/cascade cohort with zero outages and suite-local floors", + "hashes": { + "implementation": [ + "gate-engine/review/run-review.mts", + "gate-engine/review/reviewers.mts", + "agents/correctness-reviewer.md", + "skills/correctness/**" + ], + "corpus": [ + "gate-engine/review/eval/reviewers/cases-correctness.jsonl" + ], + "scorer": [ + "gate-engine/review/eval/reviewers/bench.mts" + ], + "runner": [ + "gate-engine/review/eval/reviewers/bench.mts" + ] + } + }, + { + "id": "decisions", + "label": "Decision governance", + "adapter": "decisions", + "subjectIds": [ + "judge-decisions", + "bin-guard-decisions" + ], + "runner": "gate-engine/decisions/eval/bench.mts", + "lifecycle": "shipped", + "acceptance": "K=3 detect/depth, K=1 alignment with stable baseline-discordant retry, zero outages and suite hard floors", + "hashes": { + "implementation": [ + "gate-engine/decisions/detect.mts", + "gate-engine/decisions/check-alignment.mts", + "gate-engine/decisions/depth-judge.mts" + ], + "corpus": [ + "gate-engine/decisions/eval/cases-*.jsonl" + ], + "scorer": [ + "gate-engine/decisions/eval/bench.mts", + "gate-engine/decisions/eval/acceptance.mts", + "gate-engine/decisions/eval/compare.mts", + "gate-engine/eval/statistics.mts" + ], + "runner": [ + "gate-engine/decisions/eval/bench.mts", + "gate-engine/decisions/eval/acceptance.mts", + "gate-engine/decisions/eval/compare.mts", + "gate-engine/decisions/eval/cases.mts", + "gate-engine/decisions/eval/checkpoint.mts", + "gate-engine/decisions/eval/report.mts", + "gate-engine/eval/statistics.mts", + "gate-engine/decisions/decisions.mts", + "gate-engine/config.mts" + ] + } + }, + { + "id": "sentry", + "label": "Sentry capture judge", + "adapter": "sentry", + "subjectIds": [ + "judge-sentry", + "bin-guard-sentry" + ], + "runner": "gate-engine/sentry/eval/bench.mts", + "lifecycle": "shipped", + "acceptance": "Complete persisted precision/recall/F1 cell, zero outages", + "hashes": { + "implementation": [ + "gate-engine/sentry/check-sentry.mts" + ], + "corpus": [ + "gate-engine/sentry/eval/cases.jsonl" + ], + "scorer": [ + "gate-engine/sentry/eval/bench.mts" + ], + "runner": [ + "gate-engine/sentry/eval/bench.mts" + ] + } + }, + { + "id": "edge-cases", + "label": "Edge-case autonomy", + "adapter": "edge-cases", + "subjectIds": [ + "benchmark-edge-cases" + ], + "lifecycle": "no-ship", + "acceptance": "Pre-registered full analysis with robust no-ship decision", + "hashes": { + "implementation": [ + "gate-engine/edge-cases/eval/prompts.mts" + ], + "corpus": [ + "gate-engine/edge-cases/eval/cases.jsonl" + ], + "scorer": [ + "gate-engine/edge-cases/eval/lib/**", + "gate-engine/edge-cases/eval/counts.mts" + ], + "runner": [ + "gate-engine/edge-cases/eval/bench.mts" + ] + } + }, + { + "id": "decisions-recall", + "label": "Decision-log retrieval recall", + "adapter": "deterministic", + "baseline": "gate-engine/decisions/eval/recall/results.baseline.json", + "subjectIds": [ + "benchmark-decisions-recall", + "bin-guard-decisions" + ], + "runner": "gate-engine/decisions/eval/recall/bench.mts", + "lifecycle": "experimental", + "acceptance": "Deterministic set-arithmetic scoring, zero LLM calls; seed corpus only until gold cases are harvested from a real corpus", + "hashes": { + "implementation": [ + "gate-engine/decisions/retrieval.mts", + "gate-engine/decisions/decisions.mts" + ], + "corpus": [ + "gate-engine/decisions/eval/recall/cases-retrieval.jsonl", + "gate-engine/decisions/eval/recall/corpus/**" + ], + "scorer": [ + "gate-engine/decisions/eval/recall/scoring.mts" + ], + "runner": [ + "gate-engine/decisions/eval/recall/bench.mts", + "gate-engine/decisions/eval/recall/scoring.mts", + "gate-engine/eval/statistics.mts" + ] + } + }, + { + "id": "decisions-save-quality", + "label": "Decision-log save quality", + "adapter": "deterministic", + "baseline": "gate-engine/decisions/eval/save-quality/results.baseline.json", + "subjectIds": [ + "benchmark-decisions-save-quality", + "bin-guard-decisions" + ], + "runner": "gate-engine/decisions/eval/save-quality/bench.mts", + "lifecycle": "experimental", + "acceptance": "Deterministic structural checks, zero LLM calls; perturbation recall must clear the 80% floor before FPR@R80 is read, and the real corpus must produce no finding beyond the one named historical exception", + "hashes": { + "implementation": [ + "gate-engine/decisions/integrity/checks.mts", + "gate-engine/decisions/integrity/scan.mts" + ], + "corpus": [ + "gate-engine/decisions/eval/save-quality/cases-save.jsonl", + "gate-engine/decisions/eval/save-quality/corpus/**" + ], + "scorer": [ + "gate-engine/decisions/eval/save-quality/scoring.mts" + ], + "runner": [ + "gate-engine/decisions/eval/save-quality/bench.mts", + "gate-engine/decisions/eval/save-quality/scoring.mts", + "gate-engine/decisions/integrity/perturb.mts" + ] + } + }, + { + "id": "search-tool", + "label": "Semantic search retrieval", + "adapter": "reviewer", + "subjectIds": [ + "benchmark-search-tool" + ], + "runner": "gate-engine/search-tool/eval/eval.mts", + "lifecycle": "experimental", + "acceptance": "Evidence-only retrieval study", + "hashes": { + "implementation": [ + "gate-engine/search-tool/**" + ], + "corpus": [ + "gate-engine/search-tool/eval/queries.json" + ], + "scorer": [ + "gate-engine/search-tool/eval/eval.mts" + ], + "runner": [ + "gate-engine/search-tool/eval/eval.mts" + ] + } + }, + { + "id": "co-occurrence", + "label": "Duplication matcher", + "adapter": "reviewer", + "subjectIds": [ + "bin-guard-dup", + "bin-guard-clone" + ], + "runner": "gate-engine/co-occurrence/matcher.mts", + "lifecycle": "shipped", + "acceptance": "Deterministic fixture benchmark", + "hashes": { + "implementation": [ + "gate-engine/co-occurrence/**" + ], + "corpus": [ + "gate-engine/co-occurrence/fixtures/**" + ], + "scorer": [ + "gate-engine/co-occurrence/matcher.mts" + ], + "runner": [ + "gate-engine/co-occurrence/matcher.mts" + ] + } + }, + { + "id": "commit-guard-retrieval", + "label": "Commit-guard retrieval", + "adapter": "reviewer", + "subjectIds": [ + "agent-commit-guard", + "reviewer-commit-guard" + ], + "lifecycle": "shipped", + "acceptance": "Evidence-only until a persisted suite exists", + "hashes": { + "implementation": [ + "agents/commit-guard.md", + "skills/commit-guard/**" + ], + "corpus": [], + "scorer": [], + "runner": [] + } + }, + { + "id": "qavis", + "label": "qavis visual QA", + "adapter": "reviewer", + "subjectIds": [ + "judge-qavis", + "bin-guard-qavis-advisory" + ], + "lifecycle": "shipped", + "acceptance": "External evidence required", + "hashes": { + "implementation": [ + "gate-engine/qavis-advisory/**" + ], + "corpus": [], + "scorer": [], + "runner": [] + } + } + ], + "singletonJudges": [ + "judge-feature-critique", + "judge-completeness", + "judge-decisions", + "judge-sentry", + "judge-qavis", + "benchmark-search-tool", + "benchmark-edge-cases" ], - "singletonJudges": ["judge-feature-critique", "judge-completeness", "judge-decisions", "judge-sentry", "judge-qavis", "benchmark-search-tool", "benchmark-edge-cases"], "runnerExclusions": [] } diff --git a/docs/benchmarks/checkpoints/0f2540c72cc130841f6931619089832861a2265bd5c7e2a186cf371b52e46b82.json b/docs/benchmarks/checkpoints/0f2540c72cc130841f6931619089832861a2265bd5c7e2a186cf371b52e46b82.json new file mode 100644 index 00000000..80f200a1 --- /dev/null +++ b/docs/benchmarks/checkpoints/0f2540c72cc130841f6931619089832861a2265bd5c7e2a186cf371b52e46b82.json @@ -0,0 +1,173 @@ +{ + "acceptance": { + "accepted": true, + "reason": "Complete reviewer cohorts with row evidence, zero outages, and suite floors met" + }, + "adapter": "reviewer", + "capturedAt": "2026-08-01T11:19:02.000Z", + "comparisons": [], + "hashes": { + "corpus": "sha256:3186dcbf61807e7383f820bb74e519c4add0c399839107b2281b09255b7aefab", + "implementation": "sha256:a2be7c48718a353822f985e89713f0b475ce3e383891b3b9e92228409fd4c556", + "runner": "sha256:546a99e7239d201b9e017abd4b1662f133b387cb4bf553f1b40d6eaf7123045a", + "scorer": "sha256:546a99e7239d201b9e017abd4b1662f133b387cb4bf553f1b40d6eaf7123045a" + }, + "metrics": [ + { + "denominator": 7, + "direction": "higher", + "id": "backend-performance-reviewer@haiku@cascade-on:first-fail-recall", + "inferenceUnit": "row", + "interval": { + "lower": 0.6456695649333126, + "method": "wilson-95", + "upper": 1 + }, + "label": "backend-performance-reviewer@haiku@cascade-on first-pass FAIL recall", + "numerator": 7, + "unit": "ratio", + "value": 1 + }, + { + "denominator": 6, + "direction": "higher", + "id": "backend-performance-reviewer@haiku@cascade-on:first-clean-pass", + "inferenceUnit": "row", + "interval": { + "lower": 0.18761630648265054, + "method": "wilson-95", + "upper": 0.8123836935173494 + }, + "label": "backend-performance-reviewer@haiku@cascade-on first-pass clean pass", + "numerator": 3, + "unit": "ratio", + "value": 0.5 + }, + { + "denominator": 7, + "direction": "higher", + "id": "backend-performance-reviewer@haiku@cascade-on:block-recall", + "inferenceUnit": "row", + "interval": { + "lower": 0.6456695649333126, + "method": "wilson-95", + "upper": 1 + }, + "label": "backend-performance-reviewer@haiku@cascade-on block recall", + "numerator": 7, + "unit": "ratio", + "value": 1 + }, + { + "denominator": 6, + "direction": "higher", + "id": "backend-performance-reviewer@haiku@cascade-on:clean-pass", + "inferenceUnit": "row", + "interval": { + "lower": 0.6096657120978346, + "method": "wilson-95", + "upper": 1 + }, + "label": "backend-performance-reviewer@haiku@cascade-on clean pass", + "numerator": 6, + "unit": "ratio", + "value": 1 + } + ], + "rows": { + "backend-performance-reviewer@haiku@cascade-on:beperf-cache-key-never-hits": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "backend-performance-reviewer@haiku@cascade-on:beperf-decoy-batched-member-lookup": { + "expected": "PASS", + "finalStatus": "pass", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "backend-performance-reviewer@haiku@cascade-on:beperf-decoy-bounded-startup-loop": { + "expected": "PASS", + "finalStatus": "pass", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "backend-performance-reviewer@haiku@cascade-on:beperf-decoy-cache-aside-settings": { + "expected": "PASS", + "finalStatus": "pass", + "ok": true, + "okFinal": true, + "okFirst": false + }, + "backend-performance-reviewer@haiku@cascade-on:beperf-decoy-keyset-list": { + "expected": "PASS", + "finalStatus": "pass", + "ok": true, + "okFinal": true, + "okFirst": false + }, + "backend-performance-reviewer@haiku@cascade-on:beperf-decoy-lazy-singleton-pool": { + "expected": "PASS", + "finalStatus": "pass", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "backend-performance-reviewer@haiku@cascade-on:beperf-nplusone-statement-loop": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "backend-performance-reviewer@haiku@cascade-on:beperf-nplusone-statement-loop-fixed": { + "expected": "PASS", + "finalStatus": "pass", + "ok": true, + "okFinal": true, + "okFirst": false + }, + "backend-performance-reviewer@haiku@cascade-on:beperf-pagination-dropped-audit": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "backend-performance-reviewer@haiku@cascade-on:beperf-per-request-pg-client": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "backend-performance-reviewer@haiku@cascade-on:beperf-select-star-wide-table": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "backend-performance-reviewer@haiku@cascade-on:beperf-serial-webhook-fanout": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "backend-performance-reviewer@haiku@cascade-on:beperf-sync-io-handler": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + } + }, + "schemaVersion": 1, + "sourceCommit": "72abca9d81e84b8bbc15f75d1f85cfba1e624394", + "suiteId": "reviewer-backend-performance" +} diff --git a/docs/benchmarks/checkpoints/1ab5991c759be40ceefdb7e804d6ab2464c12a32554448a0a6052447f496287e.json b/docs/benchmarks/checkpoints/1ab5991c759be40ceefdb7e804d6ab2464c12a32554448a0a6052447f496287e.json new file mode 100644 index 00000000..1f59c1e5 --- /dev/null +++ b/docs/benchmarks/checkpoints/1ab5991c759be40ceefdb7e804d6ab2464c12a32554448a0a6052447f496287e.json @@ -0,0 +1,215 @@ +{ + "acceptance": { + "accepted": true, + "reason": "Complete reviewer cohorts with row evidence, zero outages, and suite floors met" + }, + "adapter": "reviewer", + "capturedAt": "2026-08-01T11:19:02.000Z", + "comparisons": [], + "hashes": { + "corpus": "sha256:a31b51b7343edce031dd43d79a0703118e8e5e96735a78cea2720c6cb5c38a14", + "implementation": "sha256:39f7131d35dc539e202d801bb6d2bd9e21064dad02c37b11d4348ced301df8d7", + "runner": "sha256:546a99e7239d201b9e017abd4b1662f133b387cb4bf553f1b40d6eaf7123045a", + "scorer": "sha256:546a99e7239d201b9e017abd4b1662f133b387cb4bf553f1b40d6eaf7123045a" + }, + "metrics": [ + { + "denominator": 11, + "direction": "higher", + "id": "frontend-security-reviewer@haiku@cascade-on:first-fail-recall", + "inferenceUnit": "row", + "interval": { + "lower": 0.7411670330319684, + "method": "wilson-95", + "upper": 1 + }, + "label": "frontend-security-reviewer@haiku@cascade-on first-pass FAIL recall", + "numerator": 11, + "unit": "ratio", + "value": 1 + }, + { + "denominator": 8, + "direction": "higher", + "id": "frontend-security-reviewer@haiku@cascade-on:first-clean-pass", + "inferenceUnit": "row", + "interval": { + "lower": 0.5291118177871464, + "method": "wilson-95", + "upper": 0.9775825085499433 + }, + "label": "frontend-security-reviewer@haiku@cascade-on first-pass clean pass", + "numerator": 7, + "unit": "ratio", + "value": 0.875 + }, + { + "denominator": 11, + "direction": "higher", + "id": "frontend-security-reviewer@haiku@cascade-on:block-recall", + "inferenceUnit": "row", + "interval": { + "lower": 0.6226415635484043, + "method": "wilson-95", + "upper": 0.9837678271141151 + }, + "label": "frontend-security-reviewer@haiku@cascade-on block recall", + "numerator": 10, + "unit": "ratio", + "value": 0.9090909090909091 + }, + { + "denominator": 8, + "direction": "higher", + "id": "frontend-security-reviewer@haiku@cascade-on:clean-pass", + "inferenceUnit": "row", + "interval": { + "lower": 0.6755924351161198, + "method": "wilson-95", + "upper": 1 + }, + "label": "frontend-security-reviewer@haiku@cascade-on clean pass", + "numerator": 8, + "unit": "ratio", + "value": 1 + } + ], + "rows": { + "frontend-security-reviewer@haiku@cascade-on:fesec-auth-callback-console-token": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "frontend-security-reviewer@haiku@cascade-on:fesec-changelog-dompurify": { + "expected": "PASS", + "finalStatus": "pass", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "frontend-security-reviewer@haiku@cascade-on:fesec-comment-richbody-raw-html": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "frontend-security-reviewer@haiku@cascade-on:fesec-event-reporter-redaction": { + "expected": "PASS", + "finalStatus": "pass", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "frontend-security-reviewer@haiku@cascade-on:fesec-expiry-hint-storage": { + "expected": "PASS", + "finalStatus": "pass", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "frontend-security-reviewer@haiku@cascade-on:fesec-footer-external-links": { + "expected": "PASS", + "finalStatus": "pass", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "frontend-security-reviewer@haiku@cascade-on:fesec-invite-post-no-csrf": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "frontend-security-reviewer@haiku@cascade-on:fesec-log-filter-user-regexp": { + "expected": "FAIL", + "finalStatus": "pass", + "ok": false, + "okFinal": false, + "okFirst": true + }, + "frontend-security-reviewer@haiku@cascade-on:fesec-log-filter-user-regexp-fixed": { + "expected": "PASS", + "finalStatus": "pass", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "frontend-security-reviewer@haiku@cascade-on:fesec-marketplace-window-open": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "frontend-security-reviewer@haiku@cascade-on:fesec-postmessage-wildcard": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "frontend-security-reviewer@haiku@cascade-on:fesec-profile-website-href": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "frontend-security-reviewer@haiku@cascade-on:fesec-profile-website-href-fixed": { + "expected": "PASS", + "finalStatus": "pass", + "ok": true, + "okFinal": true, + "okFirst": false + }, + "frontend-security-reviewer@haiku@cascade-on:fesec-refresh-token-localstorage": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "frontend-security-reviewer@haiku@cascade-on:fesec-release-notes-target-blank": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "frontend-security-reviewer@haiku@cascade-on:fesec-report-formula-new-function": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "frontend-security-reviewer@haiku@cascade-on:fesec-theme-preference-storage": { + "expected": "PASS", + "finalStatus": "pass", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "frontend-security-reviewer@haiku@cascade-on:fesec-toast-innerhtml-reset": { + "expected": "PASS", + "finalStatus": "pass", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "frontend-security-reviewer@haiku@cascade-on:fesec-widget-identity-secret": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + } + }, + "schemaVersion": 1, + "sourceCommit": "d855b7b0de8524021905688b617ac836fb652b4f", + "suiteId": "reviewer-frontend-security" +} diff --git a/docs/benchmarks/checkpoints/2520b632529db1c4341aa53d43f6f491e278aa86f4fd033f89be806a8d0f37db.json b/docs/benchmarks/checkpoints/2520b632529db1c4341aa53d43f6f491e278aa86f4fd033f89be806a8d0f37db.json new file mode 100644 index 00000000..49fa5a8d --- /dev/null +++ b/docs/benchmarks/checkpoints/2520b632529db1c4341aa53d43f6f491e278aa86f4fd033f89be806a8d0f37db.json @@ -0,0 +1,514 @@ +{ + "acceptance": { + "accepted": true, + "reason": "Complete reviewer cohorts with row evidence, zero outages, and suite floors met" + }, + "adapter": "reviewer", + "capturedAt": "2026-08-01T11:19:03.000Z", + "comparisons": [], + "hashes": { + "corpus": "sha256:db96ac3d606bde78aa3555707727076d1375c79401ca32e8f2bf36197c6b6b2f", + "implementation": "sha256:c1483e0a07cb5f2cf5eddcd13edb8efb3d38a556f6120a2bae13787ea5167d71", + "runner": "sha256:546a99e7239d201b9e017abd4b1662f133b387cb4bf553f1b40d6eaf7123045a", + "scorer": "sha256:546a99e7239d201b9e017abd4b1662f133b387cb4bf553f1b40d6eaf7123045a" + }, + "metrics": [ + { + "denominator": 38, + "direction": "higher", + "id": "correctness-reviewer@sonnet@cascade-off:first-fail-recall", + "inferenceUnit": "row", + "interval": { + "lower": 0.7920062854222853, + "method": "wilson-95", + "upper": 0.9727854044930122 + }, + "label": "correctness-reviewer@sonnet@cascade-off first-pass FAIL recall", + "numerator": 35, + "unit": "ratio", + "value": 0.9210526315789473 + }, + { + "denominator": 28, + "direction": "higher", + "id": "correctness-reviewer@sonnet@cascade-off:first-clean-pass", + "inferenceUnit": "row", + "interval": { + "lower": 0.685101978364747, + "method": "wilson-95", + "upper": 0.9430099468025059 + }, + "label": "correctness-reviewer@sonnet@cascade-off first-pass clean pass", + "numerator": 24, + "unit": "ratio", + "value": 0.8571428571428571 + } + ], + "rows": { + "correctness-reviewer@sonnet@cascade-off:corr-asymmetric-flip-classifier": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-broadcast-fanout-no-dedup": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-decoy-broadcast-targeted": { + "expected": "PASS", + "finalStatus": "pass", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-decoy-cas-guarded-write": { + "expected": "PASS", + "finalStatus": "pass", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-decoy-catch-rethrows": { + "expected": "PASS", + "finalStatus": "pass", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-decoy-clears-stale-artifact": { + "expected": "PASS", + "finalStatus": "fail", + "ok": false, + "okFinal": false, + "okFirst": false + }, + "correctness-reviewer@sonnet@cascade-off:corr-decoy-conditional-update": { + "expected": "PASS", + "finalStatus": "pass", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-decoy-early-exit-preserved": { + "expected": "PASS", + "finalStatus": "pass", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-decoy-empty-recompute-clears": { + "expected": "PASS", + "finalStatus": "pass", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-decoy-exit-code-fail-closed": { + "expected": "PASS", + "finalStatus": "pass", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-decoy-fs-recursive-mkdir": { + "expected": "PASS", + "finalStatus": "pass", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-decoy-lock-finally": { + "expected": "PASS", + "finalStatus": "pass", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-decoy-loop-break-preserved": { + "expected": "PASS", + "finalStatus": "pass", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-decoy-only-validated": { + "expected": "PASS", + "finalStatus": "pass", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-decoy-outage-exit-preserved": { + "expected": "PASS", + "finalStatus": "pass", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-decoy-parses-json-string-result": { + "expected": "PASS", + "finalStatus": "pass", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-decoy-reset-failure-reasons": { + "expected": "PASS", + "finalStatus": "pass", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-decoy-retry-clears-error": { + "expected": "PASS", + "finalStatus": "pass", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-decoy-retry-reset-counter": { + "expected": "PASS", + "finalStatus": "pass", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-decoy-return-checked-fails-closed": { + "expected": "PASS", + "finalStatus": "pass", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-decoy-select-for-update": { + "expected": "PASS", + "finalStatus": "pass", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-decoy-signature-callsite-updated": { + "expected": "PASS", + "finalStatus": "pass", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-decoy-targeted-broadcast": { + "expected": "PASS", + "finalStatus": "fail", + "ok": false, + "okFinal": false, + "okFirst": false + }, + "correctness-reviewer@sonnet@cascade-off:corr-decoy-tight-anchor-classifier": { + "expected": "PASS", + "finalStatus": "fail", + "ok": false, + "okFinal": false, + "okFirst": false + }, + "correctness-reviewer@sonnet@cascade-off:corr-decoy-timer-debounce-clears": { + "expected": "PASS", + "finalStatus": "pass", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-decoy-worktree-finally": { + "expected": "PASS", + "finalStatus": "pass", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-discarded-resume-return": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-early-exit-loop-break-dropped": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-early-exit-status-bypassed": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-empty-recompute-keeps-stale": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-execfile-cmd-shim-swallowed": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-exit-code-only-one-fails": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-field-validated-wrong-state": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-fs-check-then-act-race": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-group-map-last-write-wins": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-guarded-result-ignored-mode-flip": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-json-string-result-dropped": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-lock-not-released-on-throw": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-lock-timeout-runs-unlocked": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-loose-cmd-classifier-anchor": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-loose-json-error-classifier": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-manifest-unknown-version-wiped": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-nearexpiry-permanent-reauth": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-only-selector-silent-drop": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-outage-exit-code-downgraded": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-persist-unnormalized-signal": { + "expected": "FAIL", + "finalStatus": "pass", + "ok": false, + "okFinal": false, + "okFirst": false + }, + "correctness-reviewer@sonnet@cascade-off:corr-read-then-write-clobber-terminal": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-reminder-ignores-plan-mode": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-reopen-before-cas-strands": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-retry-broadcast-double-fire": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-retry-leaves-stale-error": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-retry-stuck-unclaimable": { + "expected": "FAIL", + "finalStatus": "pass", + "ok": false, + "okFinal": false, + "okFirst": false + }, + "correctness-reviewer@sonnet@cascade-off:corr-revive-user-cancelled": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-signature-change-callsite-missed": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-stale-checklist-artifact": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-stale-failure-reasons": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-submit-latch-stuck": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-swallowed-config-parse-bypass": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-sync-source-target-divergence": { + "expected": "FAIL", + "finalStatus": "pass", + "ok": false, + "okFinal": false, + "okFirst": false + }, + "correctness-reviewer@sonnet@cascade-off:corr-timer-debounce-double-fire": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-unguarded-unpark-clobber": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-worktree-leak-no-finally": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-xdomain-nplusone": { + "expected": "PASS", + "finalStatus": "pass", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-xdomain-render": { + "expected": "PASS", + "finalStatus": "fail", + "ok": false, + "okFinal": false, + "okFirst": false + }, + "correctness-reviewer@sonnet@cascade-off:corr-xdomain-sqli": { + "expected": "PASS", + "finalStatus": "pass", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "correctness-reviewer@sonnet@cascade-off:corr-xdomain-xss": { + "expected": "PASS", + "finalStatus": "pass", + "ok": true, + "okFinal": true, + "okFirst": true + } + }, + "schemaVersion": 1, + "sourceCommit": "4125ff02cfb655df76272edbb48b0fd7bfcafd33", + "suiteId": "reviewer-correctness" +} diff --git a/docs/benchmarks/checkpoints/a7b6e01854cc49ad7085e4ee30392ab8be43972dca4ca139f9461a9a72360653.json b/docs/benchmarks/checkpoints/a7b6e01854cc49ad7085e4ee30392ab8be43972dca4ca139f9461a9a72360653.json new file mode 100644 index 00000000..64dfe4d8 --- /dev/null +++ b/docs/benchmarks/checkpoints/a7b6e01854cc49ad7085e4ee30392ab8be43972dca4ca139f9461a9a72360653.json @@ -0,0 +1,215 @@ +{ + "acceptance": { + "accepted": true, + "reason": "Complete reviewer cohorts with row evidence, zero outages, and suite floors met" + }, + "adapter": "reviewer", + "capturedAt": "2026-08-01T11:19:03.000Z", + "comparisons": [], + "hashes": { + "corpus": "sha256:9818efd6009cb8f6080a56ecc00d2a7f47c78e59d2d268d63d4303d370e66f43", + "implementation": "sha256:c23535aa1da64b761dbf2b614e8f3f1dc7b7508f585ea83f870a2f1692fa48d4", + "runner": "sha256:546a99e7239d201b9e017abd4b1662f133b387cb4bf553f1b40d6eaf7123045a", + "scorer": "sha256:546a99e7239d201b9e017abd4b1662f133b387cb4bf553f1b40d6eaf7123045a" + }, + "metrics": [ + { + "denominator": 11, + "direction": "higher", + "id": "frontend-performance-reviewer@haiku@cascade-on:first-fail-recall", + "inferenceUnit": "row", + "interval": { + "lower": 0.6226415635484043, + "method": "wilson-95", + "upper": 0.9837678271141151 + }, + "label": "frontend-performance-reviewer@haiku@cascade-on first-pass FAIL recall", + "numerator": 10, + "unit": "ratio", + "value": 0.9090909090909091 + }, + { + "denominator": 8, + "direction": "higher", + "id": "frontend-performance-reviewer@haiku@cascade-on:first-clean-pass", + "inferenceUnit": "row", + "interval": { + "lower": 0.5291118177871464, + "method": "wilson-95", + "upper": 0.9775825085499433 + }, + "label": "frontend-performance-reviewer@haiku@cascade-on first-pass clean pass", + "numerator": 7, + "unit": "ratio", + "value": 0.875 + }, + { + "denominator": 11, + "direction": "higher", + "id": "frontend-performance-reviewer@haiku@cascade-on:block-recall", + "inferenceUnit": "row", + "interval": { + "lower": 0.6226415635484043, + "method": "wilson-95", + "upper": 0.9837678271141151 + }, + "label": "frontend-performance-reviewer@haiku@cascade-on block recall", + "numerator": 10, + "unit": "ratio", + "value": 0.9090909090909091 + }, + { + "denominator": 8, + "direction": "higher", + "id": "frontend-performance-reviewer@haiku@cascade-on:clean-pass", + "inferenceUnit": "row", + "interval": { + "lower": 0.5291118177871464, + "method": "wilson-95", + "upper": 0.9775825085499433 + }, + "label": "frontend-performance-reviewer@haiku@cascade-on clean pass", + "numerator": 7, + "unit": "ratio", + "value": 0.875 + } + ], + "rows": { + "frontend-performance-reviewer@haiku@cascade-on:feperf-activity-refetch-fixed": { + "expected": "PASS", + "finalStatus": "pass", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "frontend-performance-reviewer@haiku@cascade-on:feperf-activity-refetch-storm": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "frontend-performance-reviewer@haiku@cascade-on:feperf-audit-feed-index-keys": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "frontend-performance-reviewer@haiku@cascade-on:feperf-case-gallery-clean": { + "expected": "PASS", + "finalStatus": "pass", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "frontend-performance-reviewer@haiku@cascade-on:feperf-customer-lookup-clean": { + "expected": "PASS", + "finalStatus": "inconclusive", + "ok": false, + "okFinal": false, + "okFirst": true + }, + "frontend-performance-reviewer@haiku@cascade-on:feperf-entry-lodash-moment": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "frontend-performance-reviewer@haiku@cascade-on:feperf-export-dialog-inline-handlers": { + "expected": "PASS", + "finalStatus": "pass", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "frontend-performance-reviewer@haiku@cascade-on:feperf-head-blocking-script": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "frontend-performance-reviewer@haiku@cascade-on:feperf-integrations-table-clean": { + "expected": "PASS", + "finalStatus": "pass", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "frontend-performance-reviewer@haiku@cascade-on:feperf-member-avatar-fullsize": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "frontend-performance-reviewer@haiku@cascade-on:feperf-per-method-lodash": { + "expected": "PASS", + "finalStatus": "pass", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "frontend-performance-reviewer@haiku@cascade-on:feperf-relative-time-named-import": { + "expected": "PASS", + "finalStatus": "pass", + "ok": true, + "okFinal": true, + "okFirst": false + }, + "frontend-performance-reviewer@haiku@cascade-on:feperf-scroll-reflow": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "frontend-performance-reviewer@haiku@cascade-on:feperf-sku-rank-per-render": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "frontend-performance-reviewer@haiku@cascade-on:feperf-status-icons-namespace-import": { + "expected": "FAIL", + "finalStatus": "pass", + "ok": false, + "okFinal": false, + "okFirst": false + }, + "frontend-performance-reviewer@haiku@cascade-on:feperf-status-icons-namespace-import-fixed": { + "expected": "PASS", + "finalStatus": "pass", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "frontend-performance-reviewer@haiku@cascade-on:feperf-task-row-inline-style": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "frontend-performance-reviewer@haiku@cascade-on:feperf-transcript-memo-defeated": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "frontend-performance-reviewer@haiku@cascade-on:feperf-universal-transition-css": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + } + }, + "schemaVersion": 1, + "sourceCommit": "1fb073f3612a7bbf9cb58e6c0372abc1f7fe7311", + "suiteId": "reviewer-frontend-performance" +} diff --git a/docs/benchmarks/checkpoints/dd39350e605288a40e45bcfee85ac4d8e8a2aae1c91e38129129b471da4b4e10.json b/docs/benchmarks/checkpoints/dd39350e605288a40e45bcfee85ac4d8e8a2aae1c91e38129129b471da4b4e10.json new file mode 100644 index 00000000..fe6ee9e0 --- /dev/null +++ b/docs/benchmarks/checkpoints/dd39350e605288a40e45bcfee85ac4d8e8a2aae1c91e38129129b471da4b4e10.json @@ -0,0 +1,180 @@ +{ + "acceptance": { + "accepted": true, + "reason": "Complete reviewer cohorts with row evidence, zero outages, and suite floors met" + }, + "adapter": "reviewer", + "capturedAt": "2026-08-01T11:18:47.000Z", + "comparisons": [], + "hashes": { + "corpus": "sha256:b426f8dd7b93adf3f79876f6c7e2fcc6962e01ef083c7430f9fa91a1357aa4cf", + "implementation": "sha256:374599ed9af9c41276afda550a1a760953a6f7cc42e2c975e323c0640ed040e8", + "runner": "sha256:546a99e7239d201b9e017abd4b1662f133b387cb4bf553f1b40d6eaf7123045a", + "scorer": "sha256:546a99e7239d201b9e017abd4b1662f133b387cb4bf553f1b40d6eaf7123045a" + }, + "metrics": [ + { + "denominator": 8, + "direction": "higher", + "id": "api-security-reviewer@haiku@cascade-on:first-fail-recall", + "inferenceUnit": "row", + "interval": { + "lower": 0.6755924351161198, + "method": "wilson-95", + "upper": 1 + }, + "label": "api-security-reviewer@haiku@cascade-on first-pass FAIL recall", + "numerator": 8, + "unit": "ratio", + "value": 1 + }, + { + "denominator": 6, + "direction": "higher", + "id": "api-security-reviewer@haiku@cascade-on:first-clean-pass", + "inferenceUnit": "row", + "interval": { + "lower": 0.43649717781352987, + "method": "wilson-95", + "upper": 0.9699466302516935 + }, + "label": "api-security-reviewer@haiku@cascade-on first-pass clean pass", + "numerator": 5, + "unit": "ratio", + "value": 0.8333333333333334 + }, + { + "denominator": 8, + "direction": "higher", + "id": "api-security-reviewer@haiku@cascade-on:block-recall", + "inferenceUnit": "row", + "interval": { + "lower": 0.6755924351161198, + "method": "wilson-95", + "upper": 1 + }, + "label": "api-security-reviewer@haiku@cascade-on block recall", + "numerator": 8, + "unit": "ratio", + "value": 1 + }, + { + "denominator": 6, + "direction": "higher", + "id": "api-security-reviewer@haiku@cascade-on:clean-pass", + "inferenceUnit": "row", + "interval": { + "lower": 0.6096657120978346, + "method": "wilson-95", + "upper": 1 + }, + "label": "api-security-reviewer@haiku@cascade-on clean pass", + "numerator": 6, + "unit": "ratio", + "value": 1 + } + ], + "rows": { + "api-security-reviewer@haiku@cascade-on:apisec-authed-route-clean": { + "expected": "PASS", + "finalStatus": "pass", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "api-security-reviewer@haiku@cascade-on:apisec-command-injection": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "api-security-reviewer@haiku@cascade-on:apisec-endpoint-auth": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "api-security-reviewer@haiku@cascade-on:apisec-error-leak": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "api-security-reviewer@haiku@cascade-on:apisec-idor-lookup": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "api-security-reviewer@haiku@cascade-on:apisec-jwt-unrestricted": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "api-security-reviewer@haiku@cascade-on:apisec-jwt-verify-clean": { + "expected": "PASS", + "finalStatus": "pass", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "api-security-reviewer@haiku@cascade-on:apisec-log-credentials": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "api-security-reviewer@haiku@cascade-on:apisec-mass-assignment": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "api-security-reviewer@haiku@cascade-on:apisec-orm-search-clean": { + "expected": "PASS", + "finalStatus": "pass", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "api-security-reviewer@haiku@cascade-on:apisec-safe-path-nearmiss": { + "expected": "PASS", + "finalStatus": "pass", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "api-security-reviewer@haiku@cascade-on:apisec-sql-injection": { + "expected": "FAIL", + "finalStatus": "fail", + "ok": true, + "okFinal": true, + "okFirst": true + }, + "api-security-reviewer@haiku@cascade-on:apisec-sql-injection-fixed": { + "expected": "PASS", + "finalStatus": "pass", + "ok": true, + "okFinal": true, + "okFirst": false + }, + "api-security-reviewer@haiku@cascade-on:apisec-tagged-sql-nearmiss": { + "expected": "PASS", + "finalStatus": "pass", + "ok": true, + "okFinal": true, + "okFirst": true + } + }, + "schemaVersion": 1, + "sourceCommit": "d686014bb210189bbd493234f0577152e9c14998", + "suiteId": "reviewer-api-security" +} diff --git a/docs/benchmarks/history.jsonl b/docs/benchmarks/history.jsonl index 347713a1..e5bcf80f 100644 --- a/docs/benchmarks/history.jsonl +++ b/docs/benchmarks/history.jsonl @@ -16,3 +16,8 @@ {"assessment":"unknown","changeType":"methodology-reset","checkpoint":{"path":"docs/benchmarks/checkpoints/72e4f4dd0aacd09587f6a9d848e0384ec5f8f7dd541c066d99fcde9732941089.json","sha256":"72e4f4dd0aacd09587f6a9d848e0384ec5f8f7dd541c066d99fcde9732941089"},"comparisons":[{"method":"adapter comparability contract","note":"Corpus or scoring contract changed","predecessorEventId":"evt-2026-07-27-decisions-recall-26b766b73ccd","verdict":"not-comparable"}],"evidence":"accepted","freshness":"current","hashes":{"corpus":"sha256:eb96347d16b1398b18c746ca98f9f32f7985c3b0cfb286eb52a5a2e7fd1702c3","implementation":"sha256:2bd85c0eecccb7f9c6086494e0a331081ddb6fea796f685d4808beecf2277ab4","runner":"sha256:992f302a6885c33e8a42176c9872b4dfe85b768434d1cca0105c8ee689aa9698","scorer":"sha256:84d6fde229931d071eb801e42955235083e22eba1d54f5370b5097405fcc9370"},"id":"evt-2026-07-27-decisions-recall-e0440950043b","lifecycle":"experimental","metrics":[{"denominator":12,"direction":"higher","id":"containment-hybrid","inferenceUnit":"case","interval":{"lower":0.7575059933447592,"method":"wilson-95","upper":1},"label":"Gold axis retrieved","numerator":12,"unit":"ratio","value":1},{"denominator":4,"direction":"higher","id":"set-recall-hybrid","inferenceUnit":"case","interval":{"lower":0.30064184258240184,"method":"wilson-95","upper":0.9544127391902995},"label":"Multi-axis set recall","numerator":3,"unit":"ratio","value":0.75},{"denominator":12,"direction":"higher","id":"containment","inferenceUnit":"case","interval":{"lower":0.646120088858883,"method":"wilson-95","upper":0.9851349055950829},"label":"Gold axis retrieved (word-matching floor)","numerator":11,"unit":"ratio","value":0.9166666666666666},{"denominator":4,"direction":"higher","id":"set-recall","inferenceUnit":"case","interval":{"lower":0.15003898915214947,"method":"wilson-95","upper":0.8499610108478506},"label":"Multi-axis set recall (word-matching floor)","numerator":2,"unit":"ratio","value":0.5},{"denominator":6,"direction":"higher","id":"current-state-accuracy","inferenceUnit":"case","interval":{"lower":0.6096657120978346,"method":"wilson-95","upper":1},"label":"Current-state accuracy","numerator":6,"unit":"ratio","value":1},{"denominator":6,"direction":"lower","id":"stale-fact-error","inferenceUnit":"case","interval":{"lower":2.7755575615628914e-17,"method":"wilson-95","upper":0.3903342879021653},"label":"Stale-fact errors","numerator":0,"unit":"ratio","value":0},{"denominator":11,"direction":"lower","id":"false-answer-non-refusal","inferenceUnit":"case","interval":{"lower":0.7411670330319684,"method":"wilson-95","upper":1},"label":"Answered when it should abstain","numerator":11,"unit":"ratio","value":1},{"denominator":18,"direction":"lower","id":"false-abstention","inferenceUnit":"case","interval":{"lower":0,"method":"wilson-95","upper":0.17587922364665765},"label":"Abstained when it should answer","numerator":0,"unit":"ratio","value":0}],"note":"Deterministic suite: declared floors met, with per-case evidence","provenance":{"source":"committed sanitized checkpoint","sourceCommit":"738e9bbd6488daf20e9b0bfa5c1fcc1401911168","tier":"accepted"},"recordedAt":"2026-07-27T11:52:50.000Z","schemaVersion":1,"subjectIds":["benchmark-decisions-recall","bin-guard-decisions"],"suiteId":"decisions-recall"} {"assessment":"unknown","changeType":"methodology-reset","checkpoint":{"path":"docs/benchmarks/checkpoints/a11490329c3c985d824077e4e65a2a47e653081a538bc1821257fadfb466918c.json","sha256":"a11490329c3c985d824077e4e65a2a47e653081a538bc1821257fadfb466918c"},"comparisons":[{"method":"adapter comparability contract","note":"Corpus or scoring contract changed","predecessorEventId":"evt-2026-07-27-decisions-recall-e0440950043b","verdict":"not-comparable"}],"evidence":"accepted","freshness":"current","hashes":{"corpus":"sha256:344ff8fb36809cb84898830b77067345f594e532c500554e8aed054648c0defc","implementation":"sha256:2bd85c0eecccb7f9c6086494e0a331081ddb6fea796f685d4808beecf2277ab4","runner":"sha256:992f302a6885c33e8a42176c9872b4dfe85b768434d1cca0105c8ee689aa9698","scorer":"sha256:84d6fde229931d071eb801e42955235083e22eba1d54f5370b5097405fcc9370"},"id":"evt-2026-07-27-decisions-recall-ec9bf1beb15b","lifecycle":"experimental","metrics":[{"denominator":12,"direction":"higher","id":"containment-hybrid","inferenceUnit":"case","interval":{"lower":0.7575059933447592,"method":"wilson-95","upper":1},"label":"Gold axis retrieved","numerator":12,"unit":"ratio","value":1},{"denominator":4,"direction":"higher","id":"set-recall-hybrid","inferenceUnit":"case","interval":{"lower":0.30064184258240184,"method":"wilson-95","upper":0.9544127391902995},"label":"Multi-axis set recall","numerator":3,"unit":"ratio","value":0.75},{"denominator":12,"direction":"higher","id":"containment","inferenceUnit":"case","interval":{"lower":0.646120088858883,"method":"wilson-95","upper":0.9851349055950829},"label":"Gold axis retrieved (word-matching floor)","numerator":11,"unit":"ratio","value":0.9166666666666666},{"denominator":4,"direction":"higher","id":"set-recall","inferenceUnit":"case","interval":{"lower":0.15003898915214947,"method":"wilson-95","upper":0.8499610108478506},"label":"Multi-axis set recall (word-matching floor)","numerator":2,"unit":"ratio","value":0.5},{"denominator":6,"direction":"higher","id":"current-state-accuracy","inferenceUnit":"case","interval":{"lower":0.6096657120978346,"method":"wilson-95","upper":1},"label":"Current-state accuracy","numerator":6,"unit":"ratio","value":1},{"denominator":6,"direction":"lower","id":"stale-fact-error","inferenceUnit":"case","interval":{"lower":2.7755575615628914e-17,"method":"wilson-95","upper":0.3903342879021653},"label":"Stale-fact errors","numerator":0,"unit":"ratio","value":0},{"denominator":11,"direction":"lower","id":"false-answer-non-refusal","inferenceUnit":"case","interval":{"lower":0.7411670330319684,"method":"wilson-95","upper":1},"label":"Answered when it should abstain","numerator":11,"unit":"ratio","value":1},{"denominator":18,"direction":"lower","id":"false-abstention","inferenceUnit":"case","interval":{"lower":0,"method":"wilson-95","upper":0.17587922364665765},"label":"Abstained when it should answer","numerator":0,"unit":"ratio","value":0}],"note":"Deterministic suite: declared floors met, with per-case evidence","provenance":{"source":"committed sanitized checkpoint","sourceCommit":"e7bf145bf6db8fd22040a28cbe98112642900f57","tier":"accepted"},"recordedAt":"2026-07-27T12:49:55.000Z","schemaVersion":1,"subjectIds":["benchmark-decisions-recall","bin-guard-decisions"],"suiteId":"decisions-recall"} {"assessment":"unknown","changeType":"methodology-reset","checkpoint":{"path":"docs/benchmarks/checkpoints/abc89e38ebb42ac18e6adc580ec201b26a7a8be039712a73e17dc01a9ef25713.json","sha256":"abc89e38ebb42ac18e6adc580ec201b26a7a8be039712a73e17dc01a9ef25713"},"comparisons":[{"method":"adapter comparability contract","note":"Corpus or scoring contract changed","predecessorEventId":"evt-2026-07-06-critique-3307937110bb","verdict":"not-comparable"}],"evidence":"accepted","freshness":"current","hashes":{"corpus":"sha256:8825bb68f4a3ca000e45ca9fa956f90a88774c87f123f7d225c2594d39cd51c8","implementation":"sha256:c51cd1fcb7d5fdd9787d6257b4a81052bb99cb805fb8e1546d6537118c0e58bc","runner":"sha256:165d81637504f28edcefdc114e84ee8e5898d7d81b23fe2b2fd70a5190639165","scorer":"sha256:f293b5b4efec49a631fec3983837bc8df44ed93b5da93af89f2116321e4739a6"},"id":"evt-2026-07-27-critique-87ca17004612","lifecycle":"shipped","metrics":[{"denominator":25,"direction":"higher","floor":0.75,"id":"recall","inferenceUnit":"row","interval":{"lower":0.7503389104960468,"method":"wilson-95","upper":0.9777795987151524},"label":"Gold finding recall","numerator":23,"unit":"ratio","value":0.92},{"denominator":5,"direction":"higher","id":"clean-rate","inferenceUnit":"row","interval":{"lower":0.5655175352168251,"method":"wilson-95","upper":1},"label":"Clean-plan pass rate","numerator":5,"unit":"ratio","value":1},{"denominator":20,"direction":"lower","id":"decoy-flag-rate","inferenceUnit":"row","interval":{"lower":0.027866481213768224,"method":"wilson-95","upper":0.3010336452284873},"label":"Decoy flag rate","numerator":2,"unit":"ratio","value":0.1},{"denominator":33,"direction":"higher","id":"verdict-accuracy","inferenceUnit":"row","interval":{"lower":0.846812746178811,"method":"wilson-95","upper":0.9946305894631283},"label":"Verdict accuracy","numerator":32,"unit":"ratio","value":0.9696969696969697},{"denominator":12,"direction":"higher","id":"frame-accuracy","inferenceUnit":"row","interval":{"lower":0.646120088858883,"method":"wilson-95","upper":0.9851349055950829},"label":"Frame metadata accuracy","numerator":11,"unit":"ratio","value":0.9166666666666666},{"denominator":23,"direction":"higher","id":"severity-exact","inferenceUnit":"row","interval":{"lower":0.5809651698373599,"method":"wilson-95","upper":0.9033602197341379},"label":"Severity exact match","numerator":18,"unit":"ratio","value":0.782608695652174},{"denominator":99,"direction":"higher","id":"response-contract","inferenceUnit":"row","interval":{"lower":0.24823816410788008,"method":"wilson-95","upper":0.4308795732480624},"label":"Valid response contract","numerator":33,"unit":"ratio","value":0.3333333333333333},{"denominator":99,"direction":"higher","id":"semantic-input","inferenceUnit":"row","interval":{"lower":0.9293096883284993,"method":"wilson-95","upper":0.9944422798892423},"label":"Semantically scorable responses","numerator":97,"unit":"ratio","value":0.9797979797979798}],"note":"Plan-critique JSON contract and scorer corpus changed; K=3 completed with zero outages. Recall 23/25, clean 5/5, decoy flags 2/20, verdict accuracy 32/33; not directly comparable to the prior contract.","provenance":{"source":"committed sanitized checkpoint","sourceCommit":"f629b2607c813ece104a40d9c68765c98660e437","tier":"accepted"},"recordedAt":"2026-07-27T18:18:32.000Z","schemaVersion":1,"subjectIds":["agent-feature-critique","judge-feature-critique"],"suiteId":"critique"} +{"assessment":"unknown","changeType":"coverage","checkpoint":{"path":"docs/benchmarks/checkpoints/dd39350e605288a40e45bcfee85ac4d8e8a2aae1c91e38129129b471da4b4e10.json","sha256":"dd39350e605288a40e45bcfee85ac4d8e8a2aae1c91e38129129b471da4b4e10"},"comparisons":[],"evidence":"accepted","freshness":"current","hashes":{"corpus":"sha256:b426f8dd7b93adf3f79876f6c7e2fcc6962e01ef083c7430f9fa91a1357aa4cf","implementation":"sha256:374599ed9af9c41276afda550a1a760953a6f7cc42e2c975e323c0640ed040e8","runner":"sha256:546a99e7239d201b9e017abd4b1662f133b387cb4bf553f1b40d6eaf7123045a","scorer":"sha256:546a99e7239d201b9e017abd4b1662f133b387cb4bf553f1b40d6eaf7123045a"},"id":"evt-2026-08-01-reviewer-api-security-98dd09eb4642","lifecycle":"shipped","metrics":[{"denominator":8,"direction":"higher","id":"api-security-reviewer@haiku@cascade-on:first-fail-recall","inferenceUnit":"row","interval":{"lower":0.6755924351161198,"method":"wilson-95","upper":1},"label":"api-security-reviewer@haiku@cascade-on first-pass FAIL recall","numerator":8,"unit":"ratio","value":1},{"denominator":6,"direction":"higher","id":"api-security-reviewer@haiku@cascade-on:first-clean-pass","inferenceUnit":"row","interval":{"lower":0.43649717781352987,"method":"wilson-95","upper":0.9699466302516935},"label":"api-security-reviewer@haiku@cascade-on first-pass clean pass","numerator":5,"unit":"ratio","value":0.8333333333333334},{"denominator":8,"direction":"higher","id":"api-security-reviewer@haiku@cascade-on:block-recall","inferenceUnit":"row","interval":{"lower":0.6755924351161198,"method":"wilson-95","upper":1},"label":"api-security-reviewer@haiku@cascade-on block recall","numerator":8,"unit":"ratio","value":1},{"denominator":6,"direction":"higher","id":"api-security-reviewer@haiku@cascade-on:clean-pass","inferenceUnit":"row","interval":{"lower":0.6096657120978346,"method":"wilson-95","upper":1},"label":"api-security-reviewer@haiku@cascade-on clean pass","numerator":6,"unit":"ratio","value":1}],"note":"First accepted api-security reviewer cohort at the shipped config, 131-row reviewer corpus, zero outages.","provenance":{"source":"committed sanitized checkpoint","sourceCommit":"d686014bb210189bbd493234f0577152e9c14998","tier":"accepted"},"recordedAt":"2026-08-01T11:18:47.000Z","schemaVersion":1,"subjectIds":["reviewer-api-security-reviewer","agent-api-security-reviewer","bin-guard-review"],"suiteId":"reviewer-api-security"} +{"assessment":"unknown","changeType":"coverage","checkpoint":{"path":"docs/benchmarks/checkpoints/0f2540c72cc130841f6931619089832861a2265bd5c7e2a186cf371b52e46b82.json","sha256":"0f2540c72cc130841f6931619089832861a2265bd5c7e2a186cf371b52e46b82"},"comparisons":[],"evidence":"accepted","freshness":"current","hashes":{"corpus":"sha256:3186dcbf61807e7383f820bb74e519c4add0c399839107b2281b09255b7aefab","implementation":"sha256:a2be7c48718a353822f985e89713f0b475ce3e383891b3b9e92228409fd4c556","runner":"sha256:546a99e7239d201b9e017abd4b1662f133b387cb4bf553f1b40d6eaf7123045a","scorer":"sha256:546a99e7239d201b9e017abd4b1662f133b387cb4bf553f1b40d6eaf7123045a"},"id":"evt-2026-08-01-reviewer-backend-performance-2f54e540ef96","lifecycle":"shipped","metrics":[{"denominator":7,"direction":"higher","id":"backend-performance-reviewer@haiku@cascade-on:first-fail-recall","inferenceUnit":"row","interval":{"lower":0.6456695649333126,"method":"wilson-95","upper":1},"label":"backend-performance-reviewer@haiku@cascade-on first-pass FAIL recall","numerator":7,"unit":"ratio","value":1},{"denominator":6,"direction":"higher","id":"backend-performance-reviewer@haiku@cascade-on:first-clean-pass","inferenceUnit":"row","interval":{"lower":0.18761630648265054,"method":"wilson-95","upper":0.8123836935173494},"label":"backend-performance-reviewer@haiku@cascade-on first-pass clean pass","numerator":3,"unit":"ratio","value":0.5},{"denominator":7,"direction":"higher","id":"backend-performance-reviewer@haiku@cascade-on:block-recall","inferenceUnit":"row","interval":{"lower":0.6456695649333126,"method":"wilson-95","upper":1},"label":"backend-performance-reviewer@haiku@cascade-on block recall","numerator":7,"unit":"ratio","value":1},{"denominator":6,"direction":"higher","id":"backend-performance-reviewer@haiku@cascade-on:clean-pass","inferenceUnit":"row","interval":{"lower":0.6096657120978346,"method":"wilson-95","upper":1},"label":"backend-performance-reviewer@haiku@cascade-on clean pass","numerator":6,"unit":"ratio","value":1}],"note":"First accepted backend-performance reviewer cohort at the shipped config, 131-row reviewer corpus, zero outages.","provenance":{"source":"committed sanitized checkpoint","sourceCommit":"72abca9d81e84b8bbc15f75d1f85cfba1e624394","tier":"accepted"},"recordedAt":"2026-08-01T11:19:02.000Z","schemaVersion":1,"subjectIds":["reviewer-backend-performance-reviewer","agent-backend-performance-reviewer","bin-guard-review"],"suiteId":"reviewer-backend-performance"} +{"assessment":"unknown","changeType":"coverage","checkpoint":{"path":"docs/benchmarks/checkpoints/1ab5991c759be40ceefdb7e804d6ab2464c12a32554448a0a6052447f496287e.json","sha256":"1ab5991c759be40ceefdb7e804d6ab2464c12a32554448a0a6052447f496287e"},"comparisons":[],"evidence":"accepted","freshness":"current","hashes":{"corpus":"sha256:a31b51b7343edce031dd43d79a0703118e8e5e96735a78cea2720c6cb5c38a14","implementation":"sha256:39f7131d35dc539e202d801bb6d2bd9e21064dad02c37b11d4348ced301df8d7","runner":"sha256:546a99e7239d201b9e017abd4b1662f133b387cb4bf553f1b40d6eaf7123045a","scorer":"sha256:546a99e7239d201b9e017abd4b1662f133b387cb4bf553f1b40d6eaf7123045a"},"id":"evt-2026-08-01-reviewer-frontend-security-8c1cbfd57121","lifecycle":"shipped","metrics":[{"denominator":11,"direction":"higher","id":"frontend-security-reviewer@haiku@cascade-on:first-fail-recall","inferenceUnit":"row","interval":{"lower":0.7411670330319684,"method":"wilson-95","upper":1},"label":"frontend-security-reviewer@haiku@cascade-on first-pass FAIL recall","numerator":11,"unit":"ratio","value":1},{"denominator":8,"direction":"higher","id":"frontend-security-reviewer@haiku@cascade-on:first-clean-pass","inferenceUnit":"row","interval":{"lower":0.5291118177871464,"method":"wilson-95","upper":0.9775825085499433},"label":"frontend-security-reviewer@haiku@cascade-on first-pass clean pass","numerator":7,"unit":"ratio","value":0.875},{"denominator":11,"direction":"higher","id":"frontend-security-reviewer@haiku@cascade-on:block-recall","inferenceUnit":"row","interval":{"lower":0.6226415635484043,"method":"wilson-95","upper":0.9837678271141151},"label":"frontend-security-reviewer@haiku@cascade-on block recall","numerator":10,"unit":"ratio","value":0.9090909090909091},{"denominator":8,"direction":"higher","id":"frontend-security-reviewer@haiku@cascade-on:clean-pass","inferenceUnit":"row","interval":{"lower":0.6755924351161198,"method":"wilson-95","upper":1},"label":"frontend-security-reviewer@haiku@cascade-on clean pass","numerator":8,"unit":"ratio","value":1}],"note":"First accepted frontend-security reviewer cohort at the shipped config, 131-row reviewer corpus, zero outages.","provenance":{"source":"committed sanitized checkpoint","sourceCommit":"d855b7b0de8524021905688b617ac836fb652b4f","tier":"accepted"},"recordedAt":"2026-08-01T11:19:02.000Z","schemaVersion":1,"subjectIds":["reviewer-frontend-security-reviewer","agent-frontend-security-reviewer","bin-guard-review"],"suiteId":"reviewer-frontend-security"} +{"assessment":"unknown","changeType":"coverage","checkpoint":{"path":"docs/benchmarks/checkpoints/a7b6e01854cc49ad7085e4ee30392ab8be43972dca4ca139f9461a9a72360653.json","sha256":"a7b6e01854cc49ad7085e4ee30392ab8be43972dca4ca139f9461a9a72360653"},"comparisons":[],"evidence":"accepted","freshness":"current","hashes":{"corpus":"sha256:9818efd6009cb8f6080a56ecc00d2a7f47c78e59d2d268d63d4303d370e66f43","implementation":"sha256:c23535aa1da64b761dbf2b614e8f3f1dc7b7508f585ea83f870a2f1692fa48d4","runner":"sha256:546a99e7239d201b9e017abd4b1662f133b387cb4bf553f1b40d6eaf7123045a","scorer":"sha256:546a99e7239d201b9e017abd4b1662f133b387cb4bf553f1b40d6eaf7123045a"},"id":"evt-2026-08-01-reviewer-frontend-performance-50009f0eff89","lifecycle":"shipped","metrics":[{"denominator":11,"direction":"higher","id":"frontend-performance-reviewer@haiku@cascade-on:first-fail-recall","inferenceUnit":"row","interval":{"lower":0.6226415635484043,"method":"wilson-95","upper":0.9837678271141151},"label":"frontend-performance-reviewer@haiku@cascade-on first-pass FAIL recall","numerator":10,"unit":"ratio","value":0.9090909090909091},{"denominator":8,"direction":"higher","id":"frontend-performance-reviewer@haiku@cascade-on:first-clean-pass","inferenceUnit":"row","interval":{"lower":0.5291118177871464,"method":"wilson-95","upper":0.9775825085499433},"label":"frontend-performance-reviewer@haiku@cascade-on first-pass clean pass","numerator":7,"unit":"ratio","value":0.875},{"denominator":11,"direction":"higher","id":"frontend-performance-reviewer@haiku@cascade-on:block-recall","inferenceUnit":"row","interval":{"lower":0.6226415635484043,"method":"wilson-95","upper":0.9837678271141151},"label":"frontend-performance-reviewer@haiku@cascade-on block recall","numerator":10,"unit":"ratio","value":0.9090909090909091},{"denominator":8,"direction":"higher","id":"frontend-performance-reviewer@haiku@cascade-on:clean-pass","inferenceUnit":"row","interval":{"lower":0.5291118177871464,"method":"wilson-95","upper":0.9775825085499433},"label":"frontend-performance-reviewer@haiku@cascade-on clean pass","numerator":7,"unit":"ratio","value":0.875}],"note":"First accepted frontend-performance reviewer cohort at the shipped config, 131-row reviewer corpus, zero outages.","provenance":{"source":"committed sanitized checkpoint","sourceCommit":"1fb073f3612a7bbf9cb58e6c0372abc1f7fe7311","tier":"accepted"},"recordedAt":"2026-08-01T11:19:03.000Z","schemaVersion":1,"subjectIds":["reviewer-frontend-performance-reviewer","agent-frontend-performance-reviewer","bin-guard-review"],"suiteId":"reviewer-frontend-performance"} +{"assessment":"unknown","changeType":"coverage","checkpoint":{"path":"docs/benchmarks/checkpoints/2520b632529db1c4341aa53d43f6f491e278aa86f4fd033f89be806a8d0f37db.json","sha256":"2520b632529db1c4341aa53d43f6f491e278aa86f4fd033f89be806a8d0f37db"},"comparisons":[],"evidence":"accepted","freshness":"current","hashes":{"corpus":"sha256:db96ac3d606bde78aa3555707727076d1375c79401ca32e8f2bf36197c6b6b2f","implementation":"sha256:c1483e0a07cb5f2cf5eddcd13edb8efb3d38a556f6120a2bae13787ea5167d71","runner":"sha256:546a99e7239d201b9e017abd4b1662f133b387cb4bf553f1b40d6eaf7123045a","scorer":"sha256:546a99e7239d201b9e017abd4b1662f133b387cb4bf553f1b40d6eaf7123045a"},"id":"evt-2026-08-01-reviewer-correctness-75d2d15bab20","lifecycle":"shipped","metrics":[{"denominator":38,"direction":"higher","id":"correctness-reviewer@sonnet@cascade-off:first-fail-recall","inferenceUnit":"row","interval":{"lower":0.7920062854222853,"method":"wilson-95","upper":0.9727854044930122},"label":"correctness-reviewer@sonnet@cascade-off first-pass FAIL recall","numerator":35,"unit":"ratio","value":0.9210526315789473},{"denominator":28,"direction":"higher","id":"correctness-reviewer@sonnet@cascade-off:first-clean-pass","inferenceUnit":"row","interval":{"lower":0.685101978364747,"method":"wilson-95","upper":0.9430099468025059},"label":"correctness-reviewer@sonnet@cascade-off first-pass clean pass","numerator":24,"unit":"ratio","value":0.8571428571428571}],"note":"First accepted correctness reviewer cohort at the shipped config, 131-row reviewer corpus, zero outages.","provenance":{"source":"committed sanitized checkpoint","sourceCommit":"4125ff02cfb655df76272edbb48b0fd7bfcafd33","tier":"accepted"},"recordedAt":"2026-08-01T11:19:03.000Z","schemaVersion":1,"subjectIds":["reviewer-correctness-reviewer","agent-correctness-reviewer","bin-guard-review"],"suiteId":"reviewer-correctness"} diff --git a/gate-engine/eval/__tests__/adapters.test.mts b/gate-engine/eval/__tests__/adapters.test.mts index 7452dde7..91da00e5 100644 --- a/gate-engine/eval/__tests__/adapters.test.mts +++ b/gate-engine/eval/__tests__/adapters.test.mts @@ -93,6 +93,38 @@ describe('benchmark adapters', () => { }); }); + it('reads end-to-end ratios past the row counts a real cascade section carries', () => { + // Shape of an actual bench baseline: `gold`/`decoys` are ROW COUNTS sitting alongside the + // blockRecall/cleanPass ratios. Key-order resolution binds the count, emits no end-to-end + // metric, and leaves the cohort tallies at n=0 — failing the cascade floors it never read. + const parsed = parseReviewer({ + sections: { + 'api-security-reviewer@haiku@cascade-on': { + model: 'haiku', + cascade: true, + rows: { gold1: { expected: 'FAIL', okFirst: true, okFinal: true } }, + metrics: { + rows: 14, + gold: 8, + decoys: 6, + firstFailRecall: { k: 8, n: 8 }, + firstCleanPass: { k: 5, n: 6 }, + blockRecall: { k: 8, n: 8 }, + cleanPass: { k: 6, n: 6 }, + inconclusive: {}, + }, + }, + }, + }); + expect(parsed.metrics.map((metric) => metric.id)).toEqual([ + 'api-security-reviewer@haiku@cascade-on:first-fail-recall', + 'api-security-reviewer@haiku@cascade-on:first-clean-pass', + 'api-security-reviewer@haiku@cascade-on:block-recall', + 'api-security-reviewer@haiku@cascade-on:clean-pass', + ]); + expect(parsed.acceptance.accepted).toBe(true); + }); + it('parses native decisions, sentry, and a locked edge-case no-ship result', () => { const decisions = parseDecisions({ detect: { diff --git a/gate-engine/eval/__tests__/tracker.test.mts b/gate-engine/eval/__tests__/tracker.test.mts index f84533b3..6db6f2b5 100644 --- a/gate-engine/eval/__tests__/tracker.test.mts +++ b/gate-engine/eval/__tests__/tracker.test.mts @@ -489,7 +489,7 @@ describe('catalog and generated views', () => { expect(offsetPreferred['README.md']).not.toContain('Older offset metric'); }); - it('expands reviewer cohort metrics into per-reviewer suite sub-rows', () => { + it('keeps every cohort of a reviewer suite in the text alternative', () => { const source = repositorySource(ROOT, 'working'); const catalog = loadCatalog(source); const history = (source.read('docs/benchmarks/history.jsonl') ?? '') @@ -508,36 +508,44 @@ describe('catalog and generated views', () => { denominator: 9, inferenceUnit: 'row', }); - const fleet: BenchmarkEvent = { + // A re-bench at another model MERGES into the shared baseline, so one reviewer's suite can + // publish two cohorts at once. Neither may be truncated out of the tables. + const reviewer: BenchmarkEvent = { ...template, - id: 'evt-reviewer-fleet-cohorts', - suiteId: 'reviewer-fleet', + id: 'evt-reviewer-cohorts', + suiteId: 'reviewer-frontend-security', recordedAt: '2099-01-02T00:00:00Z', metrics: [ cohortMetric( - 'frontend-security-reviewer@sonnet@cascade-on', + 'frontend-security-reviewer@haiku@cascade-on', 'first-fail-recall', 'first-pass FAIL recall', ), - cohortMetric('frontend-security-reviewer@sonnet@cascade-on', 'clean-pass', 'clean pass'), + cohortMetric('frontend-security-reviewer@haiku@cascade-on', 'clean-pass', 'clean pass'), cohortMetric( - 'backend-performance-reviewer@sonnet@cascade-on', + 'frontend-security-reviewer@sonnet@cascade-on', 'first-fail-recall', 'first-pass FAIL recall', ), + cohortMetric('frontend-security-reviewer@sonnet@cascade-on', 'clean-pass', 'clean pass'), ], }; - const outputs = generatedOutputs(source, catalog, [...history, fleet]); + const outputs = generatedOutputs(source, catalog, [...history, reviewer]); const readme = outputs['README.md']; - expect(readme).toContain('↳ frontend-security-reviewer (sonnet, cascade-on)'); - expect(readme).toContain('↳ backend-performance-reviewer (sonnet, cascade-on)'); - expect(readme).toContain('first-pass FAIL recall: 9/9 (100.0%)'); - // A single-section or sectionless suite stays a flat row: the critique line keeps its - // original headline and gains no arrow prefix anywhere before the reviewer-fleet block. - const critiqueLine = readme.split('\n').find((line) => line.startsWith('| Feature critique ')); - expect(critiqueLine).toBeDefined(); - const arrowRows = readme.split('\n').filter((line) => line.startsWith('| ↳')); - expect(arrowRows).toHaveLength(2); // exactly the two cohort sections, nothing else - expect(outputs['docs/benchmarks/README.md']).toContain('↳ frontend-security-reviewer'); + const row = readme.split('\n').find((line) => line.startsWith('| Frontend security reviewer ')); + expect(row).toBeDefined(); + // Both cohorts, tagged, with every metric each carries — nothing sliced away. + expect(row).toContain('haiku/cascade-on'); + expect(row).toContain('sonnet/cascade-on'); + expect(row).toContain('first-pass FAIL recall: 9/9 (100.0%)'); + expect(row).toContain('clean pass: 9/9 (100.0%)'); + // The namespacing key itself is noise once the cohort is named. + expect(row).not.toContain('frontend-security-reviewer@haiku'); + // The graphic summarises, but must admit the cohorts it does not show. + expect(outputs['docs/benchmarks/assets/dashboard-light.svg']).toContain('+1 more cohorts'); + // A suite with no cohort structure keeps its two-metric headline. + expect(readme.split('\n').find((line) => line.startsWith('| Feature critique '))).toContain( + 'Gold finding recall', + ); }); }); diff --git a/gate-engine/eval/adapters.mts b/gate-engine/eval/adapters.mts index 41fcd5c2..aecef621 100644 --- a/gate-engine/eval/adapters.mts +++ b/gate-engine/eval/adapters.mts @@ -153,6 +153,14 @@ function parseOpenEnded(input: Json, key: 'completeness' | 'conventions'): Parse export const parseCompleteness = (input: Json) => parseOpenEnded(input, 'completeness'); export const parseConventions = (input: Json) => parseOpenEnded(input, 'conventions'); +/** One reviewer tally as {k,n} across k/n · hit/total · clean/total. A bare row COUNT (a number, + * e.g. `metrics.gold`) yields undefined, so it can't be mistaken for the ratio beside it. */ +function tally(c: Json | number | undefined): { k: number; n: number } | undefined { + if (typeof c !== 'object' || c === null) return undefined; + const [k, n] = [c.k ?? c.hit ?? c.clean, c.n ?? c.total]; + return k === undefined || n === undefined ? undefined : { k: Number(k), n: Number(n) }; +} + export function parseReviewer(input: Json): ParsedBaseline { const entries = Object.entries(input.sections ?? input.reviewers ?? input).filter( ([, value]) => value && typeof value === 'object', @@ -174,8 +182,10 @@ export function parseReviewer(input: Json): ParsedBaseline { for (const [key, raw] of entries) { const value = raw as Json; const summary = (value.metrics ?? value) as Json; - const gold = summary.gold ?? summary.blockRecall ?? summary.endToEnd?.gold; - const decoy = summary.decoys ?? summary.cleanPass ?? summary.endToEnd?.decoys; + // Order is a preference, not a binding — `gold`/`decoys` are row COUNTS in a native baseline. + const gold = tally(summary.gold) ?? tally(summary.blockRecall) ?? tally(summary.endToEnd?.gold); + const decoy = + tally(summary.decoys) ?? tally(summary.cleanPass) ?? tally(summary.endToEnd?.decoys); const firstFail = summary.firstFailRecall; const firstClean = summary.firstCleanPass; const cascade = value.cascade === true || key.endsWith('@cascade-on'); @@ -213,26 +223,16 @@ export function parseReviewer(input: Json): ParsedBaseline { ); sectionMetrics += 1; } - if (gold?.hit !== undefined && gold?.total !== undefined) { - metrics.push(ratio(`${key}:block-recall`, `${key} block recall`, gold.hit, gold.total)); - cohort.blockRecall.k += Number(gold.hit); - cohort.blockRecall.n += Number(gold.total); - sectionMetrics += 1; - } else if (gold?.k !== undefined && gold?.n !== undefined) { + if (gold) { metrics.push(ratio(`${key}:block-recall`, `${key} block recall`, gold.k, gold.n)); - cohort.blockRecall.k += Number(gold.k); - cohort.blockRecall.n += Number(gold.n); + cohort.blockRecall.k += gold.k; + cohort.blockRecall.n += gold.n; sectionMetrics += 1; } - if (decoy?.clean !== undefined && decoy?.total !== undefined) { - metrics.push(ratio(`${key}:clean-pass`, `${key} clean pass`, decoy.clean, decoy.total)); - cohort.cleanPass.k += Number(decoy.clean); - cohort.cleanPass.n += Number(decoy.total); - sectionMetrics += 1; - } else if (decoy?.k !== undefined && decoy?.n !== undefined) { + if (decoy) { metrics.push(ratio(`${key}:clean-pass`, `${key} clean pass`, decoy.k, decoy.n)); - cohort.cleanPass.k += Number(decoy.k); - cohort.cleanPass.n += Number(decoy.n); + cohort.cleanPass.k += decoy.k; + cohort.cleanPass.n += decoy.n; sectionMetrics += 1; } cohorts.set(cohortKey, cohort); diff --git a/gate-engine/eval/cli.mts b/gate-engine/eval/cli.mts index 85476ff9..aa2c470c 100644 --- a/gate-engine/eval/cli.mts +++ b/gate-engine/eval/cli.mts @@ -270,10 +270,27 @@ function readCheckpoint( return raw ? (JSON.parse(raw) as CheckpointEnvelope) : undefined; } -export function parsePublishBaseline(suiteId: string, adapter: string, raw: string) { +/** Keep only the sections a suite owns; a section key's leading segment names its subject. */ +function ownedSections(raw: string, suiteId: string, sections?: string[]): unknown { + const parsed = JSON.parse(raw) as { sections?: Record }; + if (!sections?.length || !parsed.sections) return parsed; + const owned = Object.entries(parsed.sections).filter(([key]) => + sections.includes(key.split('@')[0]), + ); + if (!owned.length) + throw new Error(`Baseline carries no section for ${suiteId} (expected ${sections.join(', ')})`); + return { ...parsed, sections: Object.fromEntries(owned) }; +} + +export function parsePublishBaseline( + suiteId: string, + adapter: string, + raw: string, + sections?: string[], +) { let baseline: ReturnType; try { - baseline = parseBaseline(adapter, JSON.parse(raw)); + baseline = parseBaseline(adapter, ownedSections(raw, suiteId, sections)); } catch (error) { throw new Error(`Adapter rejected ${suiteId}: ${(error as Error).message}`); } @@ -315,7 +332,7 @@ function publishLocked(args: string[], append: LockedAppend): void { if (!baselinePath) throw new Error(`Suite ${suiteId} has no baseline path; pass --baseline`); const raw = source.read(baselinePath); if (!raw) throw new Error(`Missing baseline ${baselinePath} at ${tree}`); - const baseline = parsePublishBaseline(suiteId, suite.adapter, raw); + const baseline = parsePublishBaseline(suiteId, suite.adapter, raw, suite.sections); const requestedChangeType = parsed.values['change-type'] ?? (suite.lifecycle === 'no-ship' ? 'no-ship' : 'quality'); if (!(CHANGE_TYPES as readonly string[]).includes(requestedChangeType)) diff --git a/gate-engine/eval/render.mts b/gate-engine/eval/render.mts index aee6116e..2232085d 100644 --- a/gate-engine/eval/render.mts +++ b/gate-engine/eval/render.mts @@ -79,18 +79,56 @@ function formatMetric(metric: MetricObservation): string { return Number.isInteger(metric.value) ? String(metric.value) : metric.value.toFixed(3); } +// A reviewer bench MERGES each run into its shared baseline, so one suite can hold several +// model × cascade cohorts at once (haiku today, sonnet after a re-bench). Metric ids carry the +// cohort that produced them; other adapters emit no such prefix and are unaffected. +const COHORT_METRIC = /^(.+):(first-fail-recall|first-clean-pass|block-recall|clean-pass)$/; +const cohortOf = (metric: MetricObservation) => COHORT_METRIC.exec(metric.id)?.[1]; + +/** Metric label without the cohort key the adapter namespaces it with. */ +function shortMetric(metric: MetricObservation): string { + const key = cohortOf(metric); + return key && metric.label.startsWith(`${key} `) + ? metric.label.slice(key.length + 1) + : metric.label; +} + +const cell = (metric: MetricObservation) => `${shortMetric(metric)}: ${formatMetric(metric)}`; + +function cohorts(event: BenchmarkEvent): Map { + const grouped = new Map(); + for (const metric of event.metrics) { + const key = cohortOf(metric); + if (key) grouped.set(key, [...(grouped.get(key) ?? []), metric]); + } + return grouped; +} + +/** + * The table's evidence cell. A cohort-structured suite lists EVERY metric of EVERY cohort, tagged + * once there is more than one: these tables are the graphic's complete text alternative, so a + * truncated cell would hide a regression in whichever cohort fell outside the cut. + */ function headline(event?: BenchmarkEvent): string { if (!event || event.metrics.length === 0) return 'No accepted local checkpoint'; - return event.metrics - .slice(0, 2) - .map((metric) => `${metric.label}: ${formatMetric(metric)}`) - .join(' · '); + const grouped = cohorts(event); + if (grouped.size === 0) return event.metrics.slice(0, 2).map(cell).join(' · '); + return [...grouped.entries()] + .map(([key, metrics]) => { + const cells = metrics.map(cell).join(' · '); + return grouped.size > 1 ? `${key.split('@').slice(1).join('/')} — ${cells}` : cells; + }) + .join(' | '); } +/** The graphic summarises; the table beside it carries the complete set. */ function svgHeadline(event?: BenchmarkEvent): string { - const value = headline(event); + if (!event || event.metrics.length === 0) return 'No accepted local checkpoint'; + const extra = cohorts(event).size; + const value = event.metrics.slice(0, 2).map(cell).join(' · '); const limit = 46; - return value.length > limit ? `${value.slice(0, limit - 1).trimEnd()}…` : value; + const clipped = value.length > limit ? `${value.slice(0, limit - 1).trimEnd()}…` : value; + return extra > 1 ? `${clipped} (+${extra - 1} more cohorts)` : clipped; } const ASSESSMENT_MARK: Record = { @@ -110,46 +148,11 @@ function markdownCell(value: string): string { .replaceAll(/\r?\n/g, ' '); } -// The reviewer adapter emits one metric set per cohort section (metric ids shaped -// "
:first-fail-recall" where section is e.g. "frontend-security-reviewer@sonnet@cascade-on"). -// A suite whose latest event carries two or more such sections gets one indented sub-row per -// section, so each reviewer's own numbers are readable without splitting the suite's single -// acceptance event. The id suffixes below are the reviewer adapter's metric vocabulary — other -// adapters never produce them, so no other suite grows sub-rows. -const COHORT_METRIC = /^(.+):(first-fail-recall|first-clean-pass|block-recall|clean-pass)$/; - -function cohortRows(event?: BenchmarkEvent): string[] { - if (!event) return []; - const sections = new Map(); - for (const metric of event.metrics) { - const section = COHORT_METRIC.exec(metric.id)?.[1]; - if (section) sections.set(section, [...(sections.get(section) ?? []), metric]); - } - if (sections.size < 2) return []; - return [...sections.entries()].map(([section, metrics]) => { - const [name, ...cohort] = section.split('@'); - const label = cohort.length ? `${name} (${cohort.join(', ')})` : name; - const cells = metrics - .slice(0, 3) - .map((metric) => { - const short = metric.label.startsWith(`${section} `) - ? metric.label.slice(section.length + 1) - : metric.label; - return `${short}: ${formatMetric(metric)}`; - }) - .join(' · '); - return `| ↳ ${markdownCell(label)} | | | | | | ${markdownCell(cells)} |`; - }); -} - function suiteTable(views: SuiteView[]): string { - const rows = views.flatMap(({ suite, event, evidence, freshness }) => { + const rows = views.map(({ suite, event, evidence, freshness }) => { const change = event?.changeType ?? '—'; const assessment = event ? ASSESSMENT_MARK[event.assessment] : '? unknown'; - return [ - `| ${markdownCell(suite.label)} | ${suite.lifecycle} | ${evidence} | ${freshness} | ${change} | ${assessment} | ${markdownCell(headline(event))} |`, - ...cohortRows(event), - ]; + return `| ${markdownCell(suite.label)} | ${suite.lifecycle} | ${evidence} | ${freshness} | ${change} | ${assessment} | ${markdownCell(headline(event))} |`; }); return [ '| Suite | Lifecycle | Evidence | Freshness | Change | Assessment | Latest evidence |', diff --git a/gate-engine/eval/types.mts b/gate-engine/eval/types.mts index 428ff62f..4249568a 100644 --- a/gate-engine/eval/types.mts +++ b/gate-engine/eval/types.mts @@ -114,6 +114,13 @@ export interface BenchmarkSuite { subjectIds: string[]; runner?: string; baseline?: string; + /** + * Sections of a shared baseline file this suite owns, when one runner scores several suites in + * one pass (the reviewer bench writes every reviewer into one file). Publishing keeps only these + * sections, so re-running ONE subject republishes only its own suite and never restates — or + * silently drops — the evidence of the others. + */ + sections?: string[]; lifecycle: Lifecycle; acceptance: string; hashes: { diff --git a/gate-engine/review/eval/reviewers/bench.mts b/gate-engine/review/eval/reviewers/bench.mts index 8ac97d54..a94e3cdb 100644 --- a/gate-engine/review/eval/reviewers/bench.mts +++ b/gate-engine/review/eval/reviewers/bench.mts @@ -12,9 +12,9 @@ * node bench.mts validate # 0 LLM calls: corpus linter (selection + expectItems + injection) * node bench.mts coverage # 0 LLM calls: catalog/type coverage of the corpus * - * Knobs: BENCH_MODEL first-pass model (default sonnet = production) · BENCH_CASCADE=off skips the - * opus escalation (first-pass metrics only, zero opus spend) · BENCH_CONCURRENCY rows in flight - * (default 2, the gate's own judge-contention default). + * Knobs: BENCH_MODEL first-pass model (default sonnet; the SHIPPED first pass is haiku — see + * GUARD_REVIEW_MODEL in run-review.mts — and a model-pinned reviewer ignores this knob) · + * BENCH_CASCADE=off skips the opus escalation (first-pass only, zero opus spend) · BENCH_CONCURRENCY. * * Scoring is DETERMINISTIC (no LLM matcher): expected verdict vs the captured first-pass verdict + * the end-to-end cascade outcome + the checklist state-file artifact snapshotted per judge pass