diff --git a/AGENTS.md b/AGENTS.md index 8d5b6ca12..563263e62 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -43,9 +43,9 @@ skip-token ban and test determinism. Read both before writing or reviewing Go. argv a case did not register (`evals/README.md`, `evals/lib/hoststub.sh`). - **No `/tmp` test dirs.** Use `t.TempDir()` for files and `internal/testenv` for child processes; a hardcoded path leaks across parallel runs. -- **Exactly five skill files are hashed into goldens:** `skills/verdict/SKILL.md` - and `prompts/{system,review,diagrams,verifier}.md`. Changes must move only - `prompts_hash` lines in record goldens — prove it in the diff. +- **Exactly six skill files are hashed into goldens:** `skills/verdict/SKILL.md` + and `prompts/{system,review,diagrams,verifier,validate}.md`; a change moves + only `prompts_hash` lines and the literal pin in `provenance/prompts_test.go`. ## Git and commits diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7ba4b2a04..ba065ef43 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -86,7 +86,7 @@ verdict/ scripts/verdict-complete.sh # bash/zsh completion for verdict setup-verdict/ # GitHub Action that installs a pinned, verified binary release/ # contract.json: frozen artifact names - docs/manual.md # full manual: store schema, verbs, doctrines, changelog + docs/manual.md # full manual: store schema, verbs, doctrines docs/vision.md # what it is and where its craft comes from apps/docs/ # Blume docs site (deploys to /verdict/) evals/ # hermetic eval suite (make evals) @@ -159,8 +159,8 @@ comment-only merges ship none, and shipped work is never backfilled. Merging to `main` opens a `chore: version packages` PR. Merging that PR publishes to npm (OIDC trusted publishing, no tokens) and cuts the GitHub Release with your notes. Tags (`vX.Y.Z`) follow the releases. Store changes -within a major are additive only from `1.0` onward. The changelog is at the bottom of -[`docs/manual.md`](docs/manual.md). +within a major are additive only from `1.0` onward. The changelog is +[`CHANGELOG.md`](CHANGELOG.md), written by `changeset version`. ## Release train diff --git a/apps/docs/content/04-roadmap/index.mdx b/apps/docs/content/04-roadmap/index.mdx index 42fd6ca22..351a9c8e5 100644 --- a/apps/docs/content/04-roadmap/index.mdx +++ b/apps/docs/content/04-roadmap/index.mdx @@ -18,15 +18,16 @@ Planned means it is not built yet, and says so on its own line. ## Shipped today -| Stage | Commands | What they own | -| ------------- | -------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | -| Bundle | `fetch`, `fetch-local` | One diff per review target, from a host PR or your working tree, into `.verdict/state/`. Chunking and a budget gate are opt-in. | -| Review | `record`, `adopt`, `fix` | Validate a review, merge it into findings, carry local findings onto a PR, and loop on one finding without escaping scope. | -| Decide | `report`, `triage`, `threads`, `next`, `show`, `trend`, `gate` | Render the bundle, gate on severity (`--check`), compose the gate dimensions (`gate`), emit SARIF (`--sarif`), and read trend across runs. | -| Post | `post` | Exactly one review, with anchor validation, stale-SHA and duplicate guards, `--dry-run`, and `--bind` for an approval. | -| Trust | `verify`, `check-tree`, `doctor`, `calibrate`, `ready` | Check the bundle, the tree, the host login, past calibration, and readiness in one place. | -| Agent surface | `skill`, `mcp` | The embedded skill (`skill`, `skill files`, `skill install`) and the review verbs as MCP tools over stdio. | -| Housekeeping | `init`, `config`, `learn`, `upgrade`, `uninstall` | Repo config, explain policy, project rules with provenance, and channel-aware upgrades and removals. | +| Stage | Commands | What they own | +| ------------- | ---------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Bundle | `fetch`, `fetch-local`, `index` | One diff per review target, from a host PR or your working tree, into `.verdict/state/`, plus the local code index (`index build`) the impact answers read. Chunking and a budget gate are opt-in. | +| Review | `run`, `record`, `panel`, `ingest`, `adopt`, `fix`, `handoff` | Launch your own agent on the bundle (`run`), validate its review and merge it into findings, plan independent disprove attempts (`panel`), ingest SARIF lint findings, carry local findings onto a PR, loop on one finding without escaping scope, and hand the fix brief to someone else. | +| Decide | `report`, `triage`, `threads`, `risk`, `next`, `show`, `trend`, `gate` | Render the bundle, gate on severity (`--check`), compose the gate dimensions (`gate`), emit SARIF (`--sarif`), queue the portfolio risk score (`risk`), and read trend across runs. | +| Post | `post`, `pr-wait` | Exactly one review, with anchor validation, stale-SHA and duplicate guards, `--dry-run`, and `--bind` for an approval; `pr-wait` waits on a PR's checks, review, or merge state without writing anything. | +| Trust | `verify`, `check-tree`, `doctor`, `calibrate`, `backtest`, `ready` | Check the bundle, the tree, the host login, past calibration, and readiness in one place, and mine, replay, and score bug-introducing candidates from your own history (`backtest`). | +| Agent surface | `skill`, `mcp` | The embedded skill (`skill`, `skill files`, `skill install`) and the review verbs as MCP tools over stdio. | +| Dashboard | `serve` | One read-only HTTP surface over `.verdict/state/`, with the browser inbox, review detail, threads, cost, precision, and gate panel; `report --html` exports the same views as static files. | +| Housekeeping | `init`, `config`, `pack`, `learn`, `upgrade`, `uninstall` | Repo config, explain policy, lens packs (`pack list`, `pack test`, `pack add`), project rules with provenance, and channel-aware upgrades and removals. | Clients and actions over those verbs: the Go SDK (`pkg/verdict`), the TypeScript client (`@stainless-code/verdict/client`), the JSON schemas in `skills/verdict/schemas/`, and @@ -40,15 +41,6 @@ client (`@stainless-code/verdict/client`), the JSON schemas in `skills/verdict/s ## Next -**A local dashboard.** One command that opens a read-only view over `.verdict/state/`: -the bundles, every finding with its evidence rung and anchor, the gate result, -cost per run, and the trend line. Started by a human who wants to look, or by an -agent that needs to show its work. Bound to localhost by default, reading the same -files the CLI writes, so nothing new is stored; `--host` is the explicit opt-in -that lets the surface leave the machine. The first slice is shipped: `verdict -serve` exposes `/v1` over one repository with the token exchange; the dashboard -UI follows. - **Incremental re-review on the remaining hosts.** `fetch --since` re-reviews only new commits today on GitHub and GitLab. GitCafe, Forgejo, Bitbucket, and Azure still get full fetches, and say so with an `incremental.diff.note` when diff --git a/bench/corpus/README.md b/bench/corpus/README.md index b317e0c54..29fa8b9af 100644 --- a/bench/corpus/README.md +++ b/bench/corpus/README.md @@ -1,9 +1,9 @@ -# Seeded benchmark corpus +# Benchmark corpus -This directory is the checked-in half of the benchmark corpus: deterministic -seeded cases plus fake agent outputs, used by the `bench-corpus` eval and as -the template for the operator's real-PR corpus. Every case is a real-shaped -review bundle with ground truth: +This directory is the checked-in benchmark corpus: deterministic seeded cases +with fake agent outputs for the `bench-corpus` eval, plus real-PR cases and +clean controls copied from merged fixes. Every case is a real-shaped review +bundle with ground truth: ``` bench/corpus// diff --git a/docs/benchmark.md b/docs/benchmark.md index 93173f3da..e742de9fd 100644 --- a/docs/benchmark.md +++ b/docs/benchmark.md @@ -5,11 +5,12 @@ of real and seeded pull requests with known bugs, graded offline by `calibrate` and aggregated by `trend`. This page is the construction rule, the 06b operator runbook, and the citation contract for the comparison page. -The checked-in starter corpus is [`bench/corpus/`](../bench/corpus/README.md): -eight seeded bug cases, two seeded clean controls, and two fake agent outputs -per case. It proves the -reporting path with exact, reproducible numbers; it is not a model-quality -claim. +The checked-in corpus is [`bench/corpus/`](../bench/corpus/README.md): 50 case +directories — eight seeded bug cases and two seeded clean controls, each with +two fake agent outputs, plus 32 real-PR cases and eight real clean controls +copied from merged fixes. The seeded cases prove the reporting path with +exact, reproducible numbers; the real ones are scored by live runs. Neither is +a model-quality claim. ## What the numbers mean diff --git a/docs/vision.md b/docs/vision.md index f92239ddd..417c2b53a 100644 --- a/docs/vision.md +++ b/docs/vision.md @@ -4,10 +4,10 @@ `verdict` is local, on-demand PR review for any coding agent. You point it at a pull request; it reads the complete diff, investigates with evidence-graded findings, and posts -exactly one GitHub review — then stops. The agent reviews; `gh` does GitHub I/O. No remote -servers, no external services, no daemons, no accounts. A local, read-only view is opt-in -(`verdict serve`): it binds loopback by default, and `--host` is the only way it leaves the -machine. +exactly one review — to GitHub, GitLab, or Forgejo — then stops. The agent reviews; the +host's own CLI (`gh`, `glab`, or `curl`) does host I/O. No remote servers, no external +services, no daemons, no accounts. A local, read-only view is opt-in (`verdict serve`): it +binds loopback by default, and `--host` is the only way it leaves the machine. One sentence: **PR reviews with receipts.** diff --git a/internal/archtest/docdrift_test.go b/internal/archtest/docdrift_test.go new file mode 100644 index 000000000..ee905a168 --- /dev/null +++ b/internal/archtest/docdrift_test.go @@ -0,0 +1,199 @@ +package archtest + +import ( + "encoding/json" + "fmt" + "go/ast" + "os" + "path/filepath" + "regexp" + "slices" + "strings" + "testing" +) + +var agentsPromptRule = regexp.MustCompile( + "Exactly ([a-z]+) skill files are hashed into goldens:\\*\\* " + + "`skills/verdict/SKILL\\.md` and `prompts/\\{([a-z,]+)\\}\\.md`") + +var countWords = map[string]int{ + "one": 1, "two": 2, "three": 3, "four": 4, "five": 5, + "six": 6, "seven": 7, "eight": 8, "nine": 9, "ten": 10, +} + +func TestAgentsHashedPromptSetMatchesProvenance(t *testing.T) { + t.Parallel() + root := moduleRoot(t) + match := agentsPromptRule.FindStringSubmatch(flattenDoc(t, filepath.Join(root, "AGENTS.md"))) + if match == nil { + t.Fatal("AGENTS.md no longer states the hashed skill files in the sentence this guard reads") + } + documented := []string{"SKILL.md"} + for name := range strings.SplitSeq(match[2], ",") { + documented = append(documented, "prompts/"+name+".md") + } + slices.Sort(documented) + hashed := hashedPromptFiles(t, root) + if !slices.Equal(documented, hashed) { + t.Errorf("AGENTS.md documents %v; PromptsHash hashes %v", documented, hashed) + } + want, ok := countWords[match[1]] + if !ok { + t.Fatalf("AGENTS.md says %q skill files; add that numeral to countWords", match[1]) + } + if want != len(hashed) { + t.Errorf("AGENTS.md says %s skill files; PromptsHash hashes %d", match[1], len(hashed)) + } +} + +// promptFiles is unexported, and a copy re-typed here would pass while the +// hashed set moved. +func hashedPromptFiles(t *testing.T, root string) []string { + t.Helper() + path := filepath.Join(root, "internal", "provenance", "provenance.go") + lit := promptFilesLiteral(t, path) + names := make([]string, 0, len(lit.Elts)) + for _, elt := range lit.Elts { + name, ok := stringLiteral(elt) + if !ok { + t.Fatalf("%s: promptFiles holds a non-string entry", path) + } + names = append(names, name) + } + slices.Sort(names) + return names +} + +func promptFilesLiteral(t *testing.T, path string) *ast.CompositeLit { + t.Helper() + var lit *ast.CompositeLit + ast.Inspect(parseGo(t, path), func(node ast.Node) bool { + value, ok := node.(*ast.ValueSpec) + if !ok || len(value.Names) != 1 || value.Names[0].Name != "promptFiles" { + return true + } + if composite, ok := value.Values[0].(*ast.CompositeLit); ok { + lit = composite + } + return false + }) + if lit == nil { + t.Fatalf("%s declares no promptFiles composite literal", path) + } + return lit +} + +func readDoc(t *testing.T, path string) string { + t.Helper() + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + return string(raw) +} + +func flattenDoc(t *testing.T, path string) string { + t.Helper() + return strings.Join(strings.Fields(readDoc(t, path)), " ") +} + +const roadmapPage = "apps/docs/content/04-roadmap/index.mdx" + +var backtickedVerb = regexp.MustCompile("`([a-z][a-z-]*)`") + +func TestRoadmapListsEveryShippedVerb(t *testing.T) { + t.Parallel() + root := moduleRoot(t) + documented := roadmapCommands(t, readDoc(t, filepath.Join(root, filepath.FromSlash(roadmapPage)))) + slices.Sort(documented) + offered := cliVerbs(t, root) + slices.Sort(offered) + if !slices.Equal(documented, offered) { + t.Errorf("the roadmap's Shipped today table lists %v; the CLI synopsis offers %v", documented, offered) + } +} + +func roadmapCommands(t *testing.T, page string) []string { + t.Helper() + commands := tableCommands(roadmapSection(t, page)) + if len(commands) == 0 { + t.Fatalf("%s: the Shipped today table lists no commands", roadmapPage) + } + return commands +} + +func roadmapSection(t *testing.T, page string) string { + t.Helper() + start := strings.Index(page, "\n## Shipped today\n") + end := strings.Index(page, "\n## Next\n") + if start < 0 || end < start { + t.Fatalf("%s has no \"## Shipped today\" section before \"## Next\"", roadmapPage) + } + return page[start:end] +} + +func tableCommands(section string) []string { + var commands []string + for line := range strings.SplitSeq(section, "\n") { + columns := strings.Split(line, "|") + if !strings.HasPrefix(line, "|") || len(columns) < 4 { + continue + } + commandCell := columns[2] + for _, match := range backtickedVerb.FindAllStringSubmatch(commandCell, -1) { + commands = append(commands, match[1]) + } + } + return commands +} + +func TestBenchmarkCorpusCountMatchesTree(t *testing.T) { + t.Parallel() + root := moduleRoot(t) + dirs := corpusCaseDirs(t, root) + realCases := 0 + for _, dir := range dirs { + if caseKind(t, filepath.Join(dir, "case.json")) == "real" { + realCases++ + } + } + doc := flattenDoc(t, filepath.Join(root, "docs", "benchmark.md")) + for _, want := range []string{ + fmt.Sprintf("%d case directories", len(dirs)), + fmt.Sprintf("%d real-PR cases", realCases), + } { + if !strings.Contains(doc, want) { + t.Errorf("docs/benchmark.md never says %q; bench/corpus/ holds that many", want) + } + } +} + +func corpusCaseDirs(t *testing.T, root string) []string { + t.Helper() + corpus := filepath.Join(root, "bench", "corpus") + entries, err := os.ReadDir(corpus) + if err != nil { + t.Fatalf("read %s: %v", corpus, err) + } + dirs := make([]string, 0, len(entries)) + for _, entry := range entries { + if entry.IsDir() { + dirs = append(dirs, filepath.Join(corpus, entry.Name())) + } + } + if len(dirs) == 0 { + t.Fatalf("%s holds no case directories; the guard would pass vacuously", corpus) + } + return dirs +} + +func caseKind(t *testing.T, path string) string { + t.Helper() + var manifest struct { + Kind string `json:"kind"` + } + if err := json.Unmarshal([]byte(readDoc(t, path)), &manifest); err != nil { + t.Fatalf("decode %s: %v", path, err) + } + return manifest.Kind +}