From d3909b8b7bcbe7be7df290d6c59aee22d142679b Mon Sep 17 00:00:00 2001 From: Sutu Sebastian Date: Mon, 16 Nov 2026 12:35:05 +0200 Subject: [PATCH 1/7] docs: the hashed prompt set is six files, and the changelog is CHANGELOG.md AGENTS.md listed five hashed skill files and named four prompts; the code hashes six, adding prompts/validate.md. It also promised that a prompt change moves only prompts_hash golden lines, but internal/provenance/prompts_test.go pins one literal hash, so the workflow named half the work. CONTRIBUTING.md pointed at a changelog at the bottom of docs/manual.md, which has no changelog section: changesets write the root CHANGELOG.md, and the release body is read back from it. internal/archtest/docdrift_test.go now holds AGENTS.md's sentence to provenance.promptFiles: the set, and the spelled count. --- AGENTS.md | 6 +- CONTRIBUTING.md | 6 +- internal/archtest/docdrift_test.go | 100 +++++++++++++++++++++++++++++ 3 files changed, 106 insertions(+), 6 deletions(-) create mode 100644 internal/archtest/docdrift_test.go diff --git a/AGENTS.md b/AGENTS.md index 8d5b6ca12..563263e62 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -43,9 +43,9 @@ skip-token ban and test determinism. Read both before writing or reviewing Go. argv a case did not register (`evals/README.md`, `evals/lib/hoststub.sh`). - **No `/tmp` test dirs.** Use `t.TempDir()` for files and `internal/testenv` for child processes; a hardcoded path leaks across parallel runs. -- **Exactly five skill files are hashed into goldens:** `skills/verdict/SKILL.md` - and `prompts/{system,review,diagrams,verifier}.md`. Changes must move only - `prompts_hash` lines in record goldens — prove it in the diff. +- **Exactly six skill files are hashed into goldens:** `skills/verdict/SKILL.md` + and `prompts/{system,review,diagrams,verifier,validate}.md`; a change moves + only `prompts_hash` lines and the literal pin in `provenance/prompts_test.go`. ## Git and commits diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7ba4b2a04..ba065ef43 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -86,7 +86,7 @@ verdict/ scripts/verdict-complete.sh # bash/zsh completion for verdict setup-verdict/ # GitHub Action that installs a pinned, verified binary release/ # contract.json: frozen artifact names - docs/manual.md # full manual: store schema, verbs, doctrines, changelog + docs/manual.md # full manual: store schema, verbs, doctrines docs/vision.md # what it is and where its craft comes from apps/docs/ # Blume docs site (deploys to /verdict/) evals/ # hermetic eval suite (make evals) @@ -159,8 +159,8 @@ comment-only merges ship none, and shipped work is never backfilled. Merging to `main` opens a `chore: version packages` PR. Merging that PR publishes to npm (OIDC trusted publishing, no tokens) and cuts the GitHub Release with your notes. Tags (`vX.Y.Z`) follow the releases. Store changes -within a major are additive only from `1.0` onward. The changelog is at the bottom of -[`docs/manual.md`](docs/manual.md). +within a major are additive only from `1.0` onward. The changelog is +[`CHANGELOG.md`](CHANGELOG.md), written by `changeset version`. ## Release train diff --git a/internal/archtest/docdrift_test.go b/internal/archtest/docdrift_test.go new file mode 100644 index 000000000..c2bd8cdab --- /dev/null +++ b/internal/archtest/docdrift_test.go @@ -0,0 +1,100 @@ +package archtest + +import ( + "go/ast" + "go/token" + "os" + "path/filepath" + "regexp" + "slices" + "strings" + "testing" +) + +// agentsPromptRule is the one AGENTS.md sentence that says how many skill files +// PromptsHash covers and which ones: the count word, the entry point, and the +// brace form naming the hashed prompts. +var agentsPromptRule = regexp.MustCompile( + "Exactly ([a-z]+) skill files are hashed into goldens:\\*\\* " + + "`skills/verdict/SKILL\\.md` and `prompts/\\{([a-z,]+)\\}\\.md`") + +// countWords names the English numerals the contributor docs spell counts with. +var countWords = map[string]int{ + "one": 1, "two": 2, "three": 3, "four": 4, "five": 5, + "six": 6, "seven": 7, "eight": 8, "nine": 9, "ten": 10, +} + +// TestAgentsHashedPromptSetMatchesProvenance holds the hand-written list in +// AGENTS.md to the files PromptsHash hashes, so a prompt added to one cannot +// leave the other naming the old set or the old count. +func TestAgentsHashedPromptSetMatchesProvenance(t *testing.T) { + t.Parallel() + root := moduleRoot(t) + match := agentsPromptRule.FindStringSubmatch(flattenDoc(t, filepath.Join(root, "AGENTS.md"))) + if match == nil { + t.Fatal("AGENTS.md no longer states the hashed skill files in the sentence this guard reads") + } + documented := []string{"SKILL.md"} + for _, name := range strings.Split(match[2], ",") { + documented = append(documented, "prompts/"+name+".md") + } + slices.Sort(documented) + hashed := hashedPromptFiles(t, root) + if !slices.Equal(documented, hashed) { + t.Errorf("AGENTS.md documents %v; PromptsHash hashes %v", documented, hashed) + } + want, ok := countWords[match[1]] + if !ok { + t.Fatalf("AGENTS.md says %q skill files; add that numeral to countWords", match[1]) + } + if want != len(hashed) { + t.Errorf("AGENTS.md says %s skill files; PromptsHash hashes %d", match[1], len(hashed)) + } +} + +// hashedPromptFiles reads the list PromptsHash walks from its declaration. +// promptFiles is unexported, and a copy re-typed here would pass while the +// hashed set moved. +func hashedPromptFiles(t *testing.T, root string) []string { + t.Helper() + path := filepath.Join(root, "internal", "provenance", "provenance.go") + for _, decl := range parseGo(t, path).Decls { + gen, ok := decl.(*ast.GenDecl) + if !ok || gen.Tok != token.VAR { + continue + } + for _, spec := range gen.Specs { + value, ok := spec.(*ast.ValueSpec) + if !ok || len(value.Names) != 1 || value.Names[0].Name != "promptFiles" { + continue + } + lit, ok := value.Values[0].(*ast.CompositeLit) + if !ok { + t.Fatalf("%s: promptFiles is not a composite literal", path) + } + names := make([]string, 0, len(lit.Elts)) + for _, elt := range lit.Elts { + name, ok := stringLiteral(elt) + if !ok { + t.Fatalf("%s: promptFiles holds a non-string entry", path) + } + names = append(names, name) + } + slices.Sort(names) + return names + } + } + t.Fatalf("%s declares no promptFiles list", path) + return nil +} + +// flattenDoc reads one Markdown file with its wrapped lines joined, so a guard +// matches a sentence however the paragraph is hard-wrapped. +func flattenDoc(t *testing.T, path string) string { + t.Helper() + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + return strings.Join(strings.Fields(string(raw)), " ") +} From a2cd1b600d7604bff7df64646a680b9add07a659 Mon Sep 17 00:00:00 2001 From: Sutu Sebastian Date: Mon, 16 Nov 2026 12:35:20 +0200 Subject: [PATCH 2/7] docs(vision): posting reaches GitLab and Forgejo, not just GitHub The vision said verdict posts "exactly one GitHub review" and that `gh` does GitHub I/O. `post` has GitLab and Forgejo adapters (internal/core/post), the README host table marks all three post-capable, and the manual documents the GitLab request-changes downgrade, so the one-line claim understated the tool. --- docs/vision.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/vision.md b/docs/vision.md index f92239ddd..417c2b53a 100644 --- a/docs/vision.md +++ b/docs/vision.md @@ -4,10 +4,10 @@ `verdict` is local, on-demand PR review for any coding agent. You point it at a pull request; it reads the complete diff, investigates with evidence-graded findings, and posts -exactly one GitHub review — then stops. The agent reviews; `gh` does GitHub I/O. No remote -servers, no external services, no daemons, no accounts. A local, read-only view is opt-in -(`verdict serve`): it binds loopback by default, and `--host` is the only way it leaves the -machine. +exactly one review — to GitHub, GitLab, or Forgejo — then stops. The agent reviews; the +host's own CLI (`gh`, `glab`, or `curl`) does host I/O. No remote servers, no external +services, no daemons, no accounts. A local, read-only view is opt-in (`verdict serve`): it +binds loopback by default, and `--host` is the only way it leaves the machine. One sentence: **PR reviews with receipts.** From 0a2cb5a46710054cfd98a2e93922a39dfd3e4987 Mon Sep 17 00:00:00 2001 From: Sutu Sebastian Date: Mon, 16 Nov 2026 12:36:07 +0200 Subject: [PATCH 3/7] docs(site): the roadmap lists every shipped verb, dashboard included The "Shipped today" table claimed to be the honest ledger of shipped verbs and left out backtest, index, pack, panel, serve, risk, handoff, ingest, run, and pr-wait, all of which the usage synopsis offers and the manual documents. The Next section still said the dashboard UI follows, but serve ships it today (inbox, review detail, threads, cost, precision, gate panel) and report --html exports the same views, so the dashboard is a shipped row now. TestRoadmapListsEveryShippedVerb reads the table's Commands column and compares it to the CLI synopsis, so a new verb and a stale row both fail here. --- apps/docs/content/04-roadmap/index.mdx | 28 +++++-------- internal/archtest/docdrift_test.go | 55 +++++++++++++++++++++++++- 2 files changed, 64 insertions(+), 19 deletions(-) diff --git a/apps/docs/content/04-roadmap/index.mdx b/apps/docs/content/04-roadmap/index.mdx index 42fd6ca22..351a9c8e5 100644 --- a/apps/docs/content/04-roadmap/index.mdx +++ b/apps/docs/content/04-roadmap/index.mdx @@ -18,15 +18,16 @@ Planned means it is not built yet, and says so on its own line. ## Shipped today -| Stage | Commands | What they own | -| ------------- | -------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | -| Bundle | `fetch`, `fetch-local` | One diff per review target, from a host PR or your working tree, into `.verdict/state/`. Chunking and a budget gate are opt-in. | -| Review | `record`, `adopt`, `fix` | Validate a review, merge it into findings, carry local findings onto a PR, and loop on one finding without escaping scope. | -| Decide | `report`, `triage`, `threads`, `next`, `show`, `trend`, `gate` | Render the bundle, gate on severity (`--check`), compose the gate dimensions (`gate`), emit SARIF (`--sarif`), and read trend across runs. | -| Post | `post` | Exactly one review, with anchor validation, stale-SHA and duplicate guards, `--dry-run`, and `--bind` for an approval. | -| Trust | `verify`, `check-tree`, `doctor`, `calibrate`, `ready` | Check the bundle, the tree, the host login, past calibration, and readiness in one place. | -| Agent surface | `skill`, `mcp` | The embedded skill (`skill`, `skill files`, `skill install`) and the review verbs as MCP tools over stdio. | -| Housekeeping | `init`, `config`, `learn`, `upgrade`, `uninstall` | Repo config, explain policy, project rules with provenance, and channel-aware upgrades and removals. | +| Stage | Commands | What they own | +| ------------- | ---------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Bundle | `fetch`, `fetch-local`, `index` | One diff per review target, from a host PR or your working tree, into `.verdict/state/`, plus the local code index (`index build`) the impact answers read. Chunking and a budget gate are opt-in. | +| Review | `run`, `record`, `panel`, `ingest`, `adopt`, `fix`, `handoff` | Launch your own agent on the bundle (`run`), validate its review and merge it into findings, plan independent disprove attempts (`panel`), ingest SARIF lint findings, carry local findings onto a PR, loop on one finding without escaping scope, and hand the fix brief to someone else. | +| Decide | `report`, `triage`, `threads`, `risk`, `next`, `show`, `trend`, `gate` | Render the bundle, gate on severity (`--check`), compose the gate dimensions (`gate`), emit SARIF (`--sarif`), queue the portfolio risk score (`risk`), and read trend across runs. | +| Post | `post`, `pr-wait` | Exactly one review, with anchor validation, stale-SHA and duplicate guards, `--dry-run`, and `--bind` for an approval; `pr-wait` waits on a PR's checks, review, or merge state without writing anything. | +| Trust | `verify`, `check-tree`, `doctor`, `calibrate`, `backtest`, `ready` | Check the bundle, the tree, the host login, past calibration, and readiness in one place, and mine, replay, and score bug-introducing candidates from your own history (`backtest`). | +| Agent surface | `skill`, `mcp` | The embedded skill (`skill`, `skill files`, `skill install`) and the review verbs as MCP tools over stdio. | +| Dashboard | `serve` | One read-only HTTP surface over `.verdict/state/`, with the browser inbox, review detail, threads, cost, precision, and gate panel; `report --html` exports the same views as static files. | +| Housekeeping | `init`, `config`, `pack`, `learn`, `upgrade`, `uninstall` | Repo config, explain policy, lens packs (`pack list`, `pack test`, `pack add`), project rules with provenance, and channel-aware upgrades and removals. | Clients and actions over those verbs: the Go SDK (`pkg/verdict`), the TypeScript client (`@stainless-code/verdict/client`), the JSON schemas in `skills/verdict/schemas/`, and @@ -40,15 +41,6 @@ client (`@stainless-code/verdict/client`), the JSON schemas in `skills/verdict/s ## Next -**A local dashboard.** One command that opens a read-only view over `.verdict/state/`: -the bundles, every finding with its evidence rung and anchor, the gate result, -cost per run, and the trend line. Started by a human who wants to look, or by an -agent that needs to show its work. Bound to localhost by default, reading the same -files the CLI writes, so nothing new is stored; `--host` is the explicit opt-in -that lets the surface leave the machine. The first slice is shipped: `verdict -serve` exposes `/v1` over one repository with the token exchange; the dashboard -UI follows. - **Incremental re-review on the remaining hosts.** `fetch --since` re-reviews only new commits today on GitHub and GitLab. GitCafe, Forgejo, Bitbucket, and Azure still get full fetches, and say so with an `incremental.diff.note` when diff --git a/internal/archtest/docdrift_test.go b/internal/archtest/docdrift_test.go index c2bd8cdab..ac09abfa2 100644 --- a/internal/archtest/docdrift_test.go +++ b/internal/archtest/docdrift_test.go @@ -91,10 +91,63 @@ func hashedPromptFiles(t *testing.T, root string) []string { // flattenDoc reads one Markdown file with its wrapped lines joined, so a guard // matches a sentence however the paragraph is hard-wrapped. func flattenDoc(t *testing.T, path string) string { + t.Helper() + return strings.Join(strings.Fields(readDoc(t, path)), " ") +} + +// readDoc reads one documentation file. +func readDoc(t *testing.T, path string) string { t.Helper() raw, err := os.ReadFile(path) if err != nil { t.Fatalf("read %s: %v", path, err) } - return strings.Join(strings.Fields(string(raw)), " ") + return string(raw) +} + +// TestRoadmapListsEveryShippedVerb holds the docs-site roadmap to the CLI +// synopsis: its "Shipped today" table is the page's claim about what ships, so +// a verb the binary offers and the table omits is a stale page, and a verb the +// table names that the synopsis dropped is a stale row. +func TestRoadmapListsEveryShippedVerb(t *testing.T) { + t.Parallel() + root := moduleRoot(t) + documented := roadmapCommands(t, readDoc(t, filepath.Join(root, filepath.FromSlash(roadmapPage)))) + slices.Sort(documented) + offered := cliVerbs(t, root) + slices.Sort(offered) + if !slices.Equal(documented, offered) { + t.Errorf("the roadmap's Shipped today table lists %v; the CLI synopsis offers %v", documented, offered) + } +} + +// roadmapPage is the docs-site page whose table claims what ships today. +const roadmapPage = "apps/docs/content/04-roadmap/index.mdx" + +// backtickedVerb matches one command name a table cell names in backticks. +var backtickedVerb = regexp.MustCompile("`([a-z][a-z-]*)`") + +// roadmapCommands reads the commands the roadmap's "Shipped today" table lists +// in its Commands column. +func roadmapCommands(t *testing.T, page string) []string { + t.Helper() + start := strings.Index(page, "\n## Shipped today\n") + end := strings.Index(page, "\n## Next\n") + if start < 0 || end < start { + t.Fatalf("%s has no \"## Shipped today\" section before \"## Next\"", roadmapPage) + } + var commands []string + for _, line := range strings.Split(page[start:end], "\n") { + columns := strings.Split(line, "|") + if !strings.HasPrefix(line, "|") || len(columns) < 4 { + continue + } + for _, match := range backtickedVerb.FindAllStringSubmatch(columns[2], -1) { + commands = append(commands, match[1]) + } + } + if len(commands) == 0 { + t.Fatalf("%s: the Shipped today table lists no commands", roadmapPage) + } + return commands } From a8e094ea7654a3937efd60838275421a72841297 Mon Sep 17 00:00:00 2001 From: Sutu Sebastian Date: Mon, 16 Nov 2026 12:48:12 +0200 Subject: [PATCH 4/7] docs(benchmark): the corpus is 50 cases, 32 real ones with fixed defects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The page called bench/corpus/ "eight seeded bug cases, two seeded clean controls, and two fake agent outputs per case". That described the seeded half: the directory holds 50 cases — 8 seeded bug cases, 2 seeded clean controls, 32 real-PR cases, and 8 real clean controls copied from merged fixes — and only the ten seeded cases carry fixture outputs. TestBenchmarkCorpusCountMatchesTree reads the case directories and each manifest's kind, so the total and the real-case count both move with the tree. The walkers stay under the repo's lint complexity budget. --- docs/benchmark.md | 11 +- internal/archtest/docdrift_test.go | 159 +++++++++++++++++++++-------- 2 files changed, 123 insertions(+), 47 deletions(-) diff --git a/docs/benchmark.md b/docs/benchmark.md index 93173f3da..e742de9fd 100644 --- a/docs/benchmark.md +++ b/docs/benchmark.md @@ -5,11 +5,12 @@ of real and seeded pull requests with known bugs, graded offline by `calibrate` and aggregated by `trend`. This page is the construction rule, the 06b operator runbook, and the citation contract for the comparison page. -The checked-in starter corpus is [`bench/corpus/`](../bench/corpus/README.md): -eight seeded bug cases, two seeded clean controls, and two fake agent outputs -per case. It proves the -reporting path with exact, reproducible numbers; it is not a model-quality -claim. +The checked-in corpus is [`bench/corpus/`](../bench/corpus/README.md): 50 case +directories — eight seeded bug cases and two seeded clean controls, each with +two fake agent outputs, plus 32 real-PR cases and eight real clean controls +copied from merged fixes. The seeded cases prove the reporting path with +exact, reproducible numbers; the real ones are scored by live runs. Neither is +a model-quality claim. ## What the numbers mean diff --git a/internal/archtest/docdrift_test.go b/internal/archtest/docdrift_test.go index ac09abfa2..110e13586 100644 --- a/internal/archtest/docdrift_test.go +++ b/internal/archtest/docdrift_test.go @@ -1,8 +1,9 @@ package archtest import ( + "encoding/json" + "fmt" "go/ast" - "go/token" "os" "path/filepath" "regexp" @@ -35,7 +36,7 @@ func TestAgentsHashedPromptSetMatchesProvenance(t *testing.T) { t.Fatal("AGENTS.md no longer states the hashed skill files in the sentence this guard reads") } documented := []string{"SKILL.md"} - for _, name := range strings.Split(match[2], ",") { + for name := range strings.SplitSeq(match[2], ",") { documented = append(documented, "prompts/"+name+".md") } slices.Sort(documented) @@ -58,41 +59,37 @@ func TestAgentsHashedPromptSetMatchesProvenance(t *testing.T) { func hashedPromptFiles(t *testing.T, root string) []string { t.Helper() path := filepath.Join(root, "internal", "provenance", "provenance.go") - for _, decl := range parseGo(t, path).Decls { - gen, ok := decl.(*ast.GenDecl) - if !ok || gen.Tok != token.VAR { - continue - } - for _, spec := range gen.Specs { - value, ok := spec.(*ast.ValueSpec) - if !ok || len(value.Names) != 1 || value.Names[0].Name != "promptFiles" { - continue - } - lit, ok := value.Values[0].(*ast.CompositeLit) - if !ok { - t.Fatalf("%s: promptFiles is not a composite literal", path) - } - names := make([]string, 0, len(lit.Elts)) - for _, elt := range lit.Elts { - name, ok := stringLiteral(elt) - if !ok { - t.Fatalf("%s: promptFiles holds a non-string entry", path) - } - names = append(names, name) - } - slices.Sort(names) - return names + lit := promptFilesLiteral(t, path) + names := make([]string, 0, len(lit.Elts)) + for _, elt := range lit.Elts { + name, ok := stringLiteral(elt) + if !ok { + t.Fatalf("%s: promptFiles holds a non-string entry", path) } + names = append(names, name) } - t.Fatalf("%s declares no promptFiles list", path) - return nil + slices.Sort(names) + return names } -// flattenDoc reads one Markdown file with its wrapped lines joined, so a guard -// matches a sentence however the paragraph is hard-wrapped. -func flattenDoc(t *testing.T, path string) string { +// promptFilesLiteral finds the composite literal promptFiles is declared with. +func promptFilesLiteral(t *testing.T, path string) *ast.CompositeLit { t.Helper() - return strings.Join(strings.Fields(readDoc(t, path)), " ") + var lit *ast.CompositeLit + ast.Inspect(parseGo(t, path), func(node ast.Node) bool { + value, ok := node.(*ast.ValueSpec) + if !ok || len(value.Names) != 1 || value.Names[0].Name != "promptFiles" { + return true + } + if composite, ok := value.Values[0].(*ast.CompositeLit); ok { + lit = composite + } + return false + }) + if lit == nil { + t.Fatalf("%s declares no promptFiles composite literal", path) + } + return lit } // readDoc reads one documentation file. @@ -105,6 +102,19 @@ func readDoc(t *testing.T, path string) string { return string(raw) } +// flattenDoc reads one Markdown file with its wrapped lines joined, so a guard +// matches a sentence however the paragraph is hard-wrapped. +func flattenDoc(t *testing.T, path string) string { + t.Helper() + return strings.Join(strings.Fields(readDoc(t, path)), " ") +} + +// roadmapPage is the docs-site page whose table claims what ships today. +const roadmapPage = "apps/docs/content/04-roadmap/index.mdx" + +// backtickedVerb matches one command name a table cell names in backticks. +var backtickedVerb = regexp.MustCompile("`([a-z][a-z-]*)`") + // TestRoadmapListsEveryShippedVerb holds the docs-site roadmap to the CLI // synopsis: its "Shipped today" table is the page's claim about what ships, so // a verb the binary offers and the table omits is a stale page, and a verb the @@ -121,23 +131,34 @@ func TestRoadmapListsEveryShippedVerb(t *testing.T) { } } -// roadmapPage is the docs-site page whose table claims what ships today. -const roadmapPage = "apps/docs/content/04-roadmap/index.mdx" - -// backtickedVerb matches one command name a table cell names in backticks. -var backtickedVerb = regexp.MustCompile("`([a-z][a-z-]*)`") - // roadmapCommands reads the commands the roadmap's "Shipped today" table lists // in its Commands column. func roadmapCommands(t *testing.T, page string) []string { + t.Helper() + commands := tableCommands(roadmapSection(t, page)) + if len(commands) == 0 { + t.Fatalf("%s: the Shipped today table lists no commands", roadmapPage) + } + return commands +} + +// roadmapSection returns the roadmap page between "## Shipped today" and +// "## Next". +func roadmapSection(t *testing.T, page string) string { t.Helper() start := strings.Index(page, "\n## Shipped today\n") end := strings.Index(page, "\n## Next\n") if start < 0 || end < start { t.Fatalf("%s has no \"## Shipped today\" section before \"## Next\"", roadmapPage) } + return page[start:end] +} + +// tableCommands collects the backticked commands in the Commands column of +// every table row in section. +func tableCommands(section string) []string { var commands []string - for _, line := range strings.Split(page[start:end], "\n") { + for line := range strings.SplitSeq(section, "\n") { columns := strings.Split(line, "|") if !strings.HasPrefix(line, "|") || len(columns) < 4 { continue @@ -146,8 +167,62 @@ func roadmapCommands(t *testing.T, page string) []string { commands = append(commands, match[1]) } } - if len(commands) == 0 { - t.Fatalf("%s: the Shipped today table lists no commands", roadmapPage) - } return commands } + +// TestBenchmarkCorpusCountMatchesTree holds docs/benchmark.md's case counts to +// the case directories under bench/corpus/: the page describes the corpus an +// operator and the evals both work from, so a case added without it leaves the +// page describing a corpus that no longer exists. +func TestBenchmarkCorpusCountMatchesTree(t *testing.T) { + t.Parallel() + root := moduleRoot(t) + dirs := corpusCaseDirs(t, root) + realCases := 0 + for _, dir := range dirs { + if caseKind(t, filepath.Join(dir, "case.json")) == "real" { + realCases++ + } + } + doc := flattenDoc(t, filepath.Join(root, "docs", "benchmark.md")) + for _, want := range []string{ + fmt.Sprintf("%d case directories", len(dirs)), + fmt.Sprintf("%d real-PR cases", realCases), + } { + if !strings.Contains(doc, want) { + t.Errorf("docs/benchmark.md never says %q; bench/corpus/ holds that many", want) + } + } +} + +// corpusCaseDirs lists the case directories bench/corpus/ holds. +func corpusCaseDirs(t *testing.T, root string) []string { + t.Helper() + corpus := filepath.Join(root, "bench", "corpus") + entries, err := os.ReadDir(corpus) + if err != nil { + t.Fatalf("read %s: %v", corpus, err) + } + dirs := make([]string, 0, len(entries)) + for _, entry := range entries { + if entry.IsDir() { + dirs = append(dirs, filepath.Join(corpus, entry.Name())) + } + } + if len(dirs) == 0 { + t.Fatalf("%s holds no case directories; the guard would pass vacuously", corpus) + } + return dirs +} + +// caseKind reads the kind one corpus manifest declares. +func caseKind(t *testing.T, path string) string { + t.Helper() + var manifest struct { + Kind string `json:"kind"` + } + if err := json.Unmarshal([]byte(readDoc(t, path)), &manifest); err != nil { + t.Fatalf("decode %s: %v", path, err) + } + return manifest.Kind +} From 398c404ef78075b15cde5c10779f6fdea7b6713f Mon Sep 17 00:00:00 2001 From: Sutu Sebastian Date: Mon, 16 Nov 2026 12:55:49 +0200 Subject: [PATCH 5/7] docs(bench): retitle the corpus README to match what it holds The page documented only its seeded half. It carries the "Adding a real case" rule for the 32 real-PR cases and 8 real clean controls checked in beside the seeded fixtures, so "Seeded benchmark corpus" named half the directory. --- bench/corpus/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/corpus/README.md b/bench/corpus/README.md index b317e0c54..29bd240ec 100644 --- a/bench/corpus/README.md +++ b/bench/corpus/README.md @@ -1,4 +1,4 @@ -# Seeded benchmark corpus +# Benchmark corpus This directory is the checked-in half of the benchmark corpus: deterministic seeded cases plus fake agent outputs, used by the `bench-corpus` eval and as From b7fa0f2126d9cc6dd2a767f1f68f175f60312a34 Mon Sep 17 00:00:00 2001 From: Sutu Sebastian Date: Mon, 16 Nov 2026 12:59:46 +0200 Subject: [PATCH 6/7] docs(bench): the corpus README intro names the real cases too The opening sentence still called the seeded cases "the template for the operator's real-PR corpus". The real cases are checked in beside them now, so the intro says what the directory holds: seeded fixtures for the bench-corpus eval, plus real-PR cases and clean controls copied from merged fixes. --- bench/corpus/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bench/corpus/README.md b/bench/corpus/README.md index 29bd240ec..29fa8b9af 100644 --- a/bench/corpus/README.md +++ b/bench/corpus/README.md @@ -1,9 +1,9 @@ # Benchmark corpus -This directory is the checked-in half of the benchmark corpus: deterministic -seeded cases plus fake agent outputs, used by the `bench-corpus` eval and as -the template for the operator's real-PR corpus. Every case is a real-shaped -review bundle with ground truth: +This directory is the checked-in benchmark corpus: deterministic seeded cases +with fake agent outputs for the `bench-corpus` eval, plus real-PR cases and +clean controls copied from merged fixes. Every case is a real-shaped review +bundle with ground truth: ``` bench/corpus// From 8c8070d4c14d7fb51ce62ff327f6dee5a7adbb30 Mon Sep 17 00:00:00 2001 From: Sutu Sebastian Date: Mon, 16 Nov 2026 12:59:46 +0200 Subject: [PATCH 7/7] test(archtest): drop the narration comments from the doc guards The doc guards carried a comment on every unexported var, helper and test. Only the reason hashedPromptFiles reads the list out of Go source survives; the Commands column is named in the code instead of explained above it. --- internal/archtest/docdrift_test.go | 33 ++---------------------------- 1 file changed, 2 insertions(+), 31 deletions(-) diff --git a/internal/archtest/docdrift_test.go b/internal/archtest/docdrift_test.go index 110e13586..ee905a168 100644 --- a/internal/archtest/docdrift_test.go +++ b/internal/archtest/docdrift_test.go @@ -12,22 +12,15 @@ import ( "testing" ) -// agentsPromptRule is the one AGENTS.md sentence that says how many skill files -// PromptsHash covers and which ones: the count word, the entry point, and the -// brace form naming the hashed prompts. var agentsPromptRule = regexp.MustCompile( "Exactly ([a-z]+) skill files are hashed into goldens:\\*\\* " + "`skills/verdict/SKILL\\.md` and `prompts/\\{([a-z,]+)\\}\\.md`") -// countWords names the English numerals the contributor docs spell counts with. var countWords = map[string]int{ "one": 1, "two": 2, "three": 3, "four": 4, "five": 5, "six": 6, "seven": 7, "eight": 8, "nine": 9, "ten": 10, } -// TestAgentsHashedPromptSetMatchesProvenance holds the hand-written list in -// AGENTS.md to the files PromptsHash hashes, so a prompt added to one cannot -// leave the other naming the old set or the old count. func TestAgentsHashedPromptSetMatchesProvenance(t *testing.T) { t.Parallel() root := moduleRoot(t) @@ -53,7 +46,6 @@ func TestAgentsHashedPromptSetMatchesProvenance(t *testing.T) { } } -// hashedPromptFiles reads the list PromptsHash walks from its declaration. // promptFiles is unexported, and a copy re-typed here would pass while the // hashed set moved. func hashedPromptFiles(t *testing.T, root string) []string { @@ -72,7 +64,6 @@ func hashedPromptFiles(t *testing.T, root string) []string { return names } -// promptFilesLiteral finds the composite literal promptFiles is declared with. func promptFilesLiteral(t *testing.T, path string) *ast.CompositeLit { t.Helper() var lit *ast.CompositeLit @@ -92,7 +83,6 @@ func promptFilesLiteral(t *testing.T, path string) *ast.CompositeLit { return lit } -// readDoc reads one documentation file. func readDoc(t *testing.T, path string) string { t.Helper() raw, err := os.ReadFile(path) @@ -102,23 +92,15 @@ func readDoc(t *testing.T, path string) string { return string(raw) } -// flattenDoc reads one Markdown file with its wrapped lines joined, so a guard -// matches a sentence however the paragraph is hard-wrapped. func flattenDoc(t *testing.T, path string) string { t.Helper() return strings.Join(strings.Fields(readDoc(t, path)), " ") } -// roadmapPage is the docs-site page whose table claims what ships today. const roadmapPage = "apps/docs/content/04-roadmap/index.mdx" -// backtickedVerb matches one command name a table cell names in backticks. var backtickedVerb = regexp.MustCompile("`([a-z][a-z-]*)`") -// TestRoadmapListsEveryShippedVerb holds the docs-site roadmap to the CLI -// synopsis: its "Shipped today" table is the page's claim about what ships, so -// a verb the binary offers and the table omits is a stale page, and a verb the -// table names that the synopsis dropped is a stale row. func TestRoadmapListsEveryShippedVerb(t *testing.T) { t.Parallel() root := moduleRoot(t) @@ -131,8 +113,6 @@ func TestRoadmapListsEveryShippedVerb(t *testing.T) { } } -// roadmapCommands reads the commands the roadmap's "Shipped today" table lists -// in its Commands column. func roadmapCommands(t *testing.T, page string) []string { t.Helper() commands := tableCommands(roadmapSection(t, page)) @@ -142,8 +122,6 @@ func roadmapCommands(t *testing.T, page string) []string { return commands } -// roadmapSection returns the roadmap page between "## Shipped today" and -// "## Next". func roadmapSection(t *testing.T, page string) string { t.Helper() start := strings.Index(page, "\n## Shipped today\n") @@ -154,8 +132,6 @@ func roadmapSection(t *testing.T, page string) string { return page[start:end] } -// tableCommands collects the backticked commands in the Commands column of -// every table row in section. func tableCommands(section string) []string { var commands []string for line := range strings.SplitSeq(section, "\n") { @@ -163,17 +139,14 @@ func tableCommands(section string) []string { if !strings.HasPrefix(line, "|") || len(columns) < 4 { continue } - for _, match := range backtickedVerb.FindAllStringSubmatch(columns[2], -1) { + commandCell := columns[2] + for _, match := range backtickedVerb.FindAllStringSubmatch(commandCell, -1) { commands = append(commands, match[1]) } } return commands } -// TestBenchmarkCorpusCountMatchesTree holds docs/benchmark.md's case counts to -// the case directories under bench/corpus/: the page describes the corpus an -// operator and the evals both work from, so a case added without it leaves the -// page describing a corpus that no longer exists. func TestBenchmarkCorpusCountMatchesTree(t *testing.T) { t.Parallel() root := moduleRoot(t) @@ -195,7 +168,6 @@ func TestBenchmarkCorpusCountMatchesTree(t *testing.T) { } } -// corpusCaseDirs lists the case directories bench/corpus/ holds. func corpusCaseDirs(t *testing.T, root string) []string { t.Helper() corpus := filepath.Join(root, "bench", "corpus") @@ -215,7 +187,6 @@ func corpusCaseDirs(t *testing.T, root string) []string { return dirs } -// caseKind reads the kind one corpus manifest declares. func caseKind(t *testing.T, path string) string { t.Helper() var manifest struct {