diff --git a/.changeset/sta-444-drain-dry-run.md b/.changeset/sta-444-drain-dry-run.md new file mode 100644 index 00000000..a431557d --- /dev/null +++ b/.changeset/sta-444-drain-dry-run.md @@ -0,0 +1,5 @@ +--- +"@stainless-code/verdict": minor +--- + +`drain --post --dry-run` runs every post gate and live read, then writes `drain.dryrun.json` with the exact marked replies and typed skips a real post would produce. It makes zero host writes and leaves `replies.json`, `drain.json`, and `threads.json` byte-identical; `--agent --post --dry-run` plans, drafts, then previews. The flag requires `--post` and secret-shaped bodies still refuse. diff --git a/cmd/verdict/usage.go b/cmd/verdict/usage.go index d036be33..c93507a0 100644 --- a/cmd/verdict/usage.go +++ b/cmd/verdict/usage.go @@ -287,7 +287,7 @@ func usageText() string { " makes one host write per thread and writes threads-resolve.json. GitHub and\n" + " GitLab carry it; Forgejo refuses typed (no documented resolve endpoint).\n" + " drain [--provider

] [--max-threads <1-100>] [--json]\n" + - " [--agent [--timeout ] [-- ...]] [--post]\n" + + " [--agent [--timeout ] [-- ...]] [--post] [--dry-run]\n" + " answer every review thread waiting on verdict: refresh threads, then write\n" + " .verdict/state//drain.json with the pending list, typed skips, and a plan digest.\n" + " Zero pending is success and launches no agent. --max-threads defaults to 20;\n" + @@ -298,6 +298,11 @@ func usageText() string { " replies; other hosts refuse provider-unimplemented. A failed write records\n" + " failed[] and refuses drain-incomplete so a re-run finishes the rest. A\n" + " truncated feed or unreadable login refuses drain-failed before any write.\n" + + " --dry-run requires --post: it runs every post gate and live read, writes\n" + + " drain.dryrun.json with planned[] (final marked bodies) and skipped[], and\n" + + " posts nothing. replies.json, drain.json and threads.json keep their bytes.\n" + + " --agent --post --dry-run plans, drafts, then previews. Human output reads\n" + + " `» N planned, M skipped (dry run; nothing posted) -> `.\n" + " trend [--rules] [--overlap] [--incidents] [--resolution] [--json]\n" + " portfolio rollup over existing bundle stores (no network, writes nothing):\n" + " findings opened/addressed/closed over time, broken down by severity tier\n" + diff --git a/docs/manual.md b/docs/manual.md index 54336523..53dbc6eb 100644 --- a/docs/manual.md +++ b/docs/manual.md @@ -316,7 +316,7 @@ write per thread, marks them resolved in `threads.json`, and writes refuses typed because its API has no documented resolve endpoint. `verdict drain owner/repo N [--provider p] [--max-threads 1-100] [--agent name] -[--timeout s] [-- args...] [--post] [--json]` plans one pass over the threads +[--timeout s] [-- args...] [--post] [--dry-run] [--json]` plans one pass over the threads waiting on verdict. It refreshes the feed, reads the login once, joins findings, and writes `drain.json`: `pending[]`, typed `skipped[]` reasons, `other_threads`, and a `plan_digest` over the pending list. A thread is pending when it is a @@ -338,8 +338,22 @@ also resolves through the host's existing resolve call. One ledger entry is appended under `storelock` after each successful write, so a re-run skips replies already posted. A failed write is recorded in `failed[]`, the remaining drafts are attempted, and the run refuses `drain-incomplete`. Forgejo and fetch-only -hosts refuse `provider-unimplemented` before any read. There is no `--dry-run` -for post and no `--allow-secret-values` override. +hosts refuse `provider-unimplemented` before any read. There is no +`--allow-secret-values` override. + +`--post --dry-run` requires `--post` and runs every §6 step: the local draft +gates and secret scan, the live thread refresh and self login, the +stale/answered/resolved skips, mention neutralization, and the marker. It holds +the post lock and composes with `--agent` (plan → draft → preview). It calls no +reply or resolve endpoint and leaves `replies.json`, `drain.json`, and +`threads.json` byte-identical. The live read happens in a scratch bundle so +the stored feed is not touched. Instead it writes `/drain.dryrun.json` +with `planned[]` entries (`thread_id`, `root_comment_id`, `disposition`, and +the final `body` with its marker) and `skipped[]` reasons; the `--json` envelope +`data` carries that same document. Human output reads `» N planned, M skipped +(dry run; nothing posted) -> `, and `next` is the same command without +`--dry-run`. A secret-shaped body still refuses `post-secret-shaped` with zero +writes, and a dry-run preview is never a receipt. `post.json` and `record.json` carry `schema_version: "1"`. @@ -348,7 +362,7 @@ for post and no `--allow-secret-values` override. A bundle dir holds these core/script artifacts (`PR.json`/`source.json`, `full.diff`, `toc.txt`, `roles.txt`, `review.json`, `findings.json`, `findings.dryrun.json`, `record.json`, `post.json`, `store.json`, `hunks.tsv`, `reviews.json`, `inline.json`, `threads.json`, -`threads-resolve.json`, `selected.json`, +`threads-resolve.json`, `drain.json`, `drain.dryrun.json`, `drain.replies.json`, `selected.json`, `panel`, `patch.json`, `verification.json`, `incremental.diff`, `kb.txt`, `lenses.txt`, `reverts.txt`, `sensitive.txt`, `kb-recall.json`, `cost.json`, `budget.json`, `fix.json`, `author.json`, diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index f55528a9..3e28378a 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -1061,7 +1061,7 @@ Fix: narrow anchor drift or body size (see above), then re-run. Cause: a review draft carries a secret-shaped value — `review.json` for `post`, `drain.replies.json` for `drain --post`. -Fix: remove the value from the draft and re-run. `post` can pass `--allow-secret-values` to publish it anyway; `drain --post` has no override. +Fix: remove the value from the draft and re-run. `post` can pass `--allow-secret-values` to publish it anyway; `drain --post` has no override, and `drain --post --dry-run` refuses the same way before any write. ## `pr-not-open` diff --git a/evals/README.md b/evals/README.md index a1a2560b..5a87f4d8 100644 --- a/evals/README.md +++ b/evals/README.md @@ -103,7 +103,7 @@ the build. | `coverage` | Diff-coverage attestation reads and the opt-in gate | `next --coverage` reads the seeded snapshot (0% over both sides, the full `src/db.ts` gap) with no `findings.json`; `10-15,40-45` reads 54% with gaps `16-20,46-50` and the generated `dist/` file exempt; `post --dry-run --require-coverage` refuses `coverage-incomplete` write-free and `--force` never bypasses it (local `record` alike); `"all"` passes at 100% with no body line and a stored receipt; partial without the flag posts one `> ⚠️ coverage:` line and stores `coverage`; a chunked bundle warns and proceeds while `next` names the chunks; the stub `gh` alone on `PATH` | | `report-html` | Static dashboard file set, envelope key, refresh, and root discovery. | | `doc-coverage` | This table documents every case, and only real cases | Each `evals/cases/*.sh` has exactly one row here, every row names an existing case, and no case is documented twice — so the table cannot drift again | -| `drain-post` | Post mode over stub `gh`: replies, resolve, idempotent rerun | Two drafts post one marked reply each and one `withdraw` resolve; `replies.json` holds two `drain` entries with waiting comment ids; the marker reads back as `author_kind: verdict` on the next threads refresh; a second `--post` makes zero host writes and adds no ledger entry; every stub call is registered | +| `drain-post` | Post mode over stub `gh`: dry-run preview, replies, resolve, idempotent rerun | A dry run reports both final marked bodies with the post-only `next`, writes `drain.dryrun.json` equal to envelope `data`, and leaves the core bundle byte-identical with zero host writes; the real `--post` sends exactly those planned `-f body=` argv; `--agent --post --dry-run` also plans and drafts without posting. Two drafts post one marked reply each and one `withdraw` resolve; `replies.json` holds two `drain` entries with waiting comment ids; the marker reads back as `author_kind: verdict` on the next threads refresh; a second `--post` makes zero host writes and adds no ledger entry; every stub call is registered | | `drain-refusals` | Refusal legs for `drain` | Bad draft key names its site as `drafts-invalid`; a stale plan digest refuses `drafts-stale`; a truncated feed refuses `drain-failed`; a secret-shaped reply refuses `post-secret-shaped` with zero writes; Forgejo `--post` refuses `provider-unimplemented` before any read; missing and stale artifacts refuse with the named codes | | `drain-plan` | Plan mode over a stub `gh` feed | One human reply and one self-login question pending; bot, resolved, verdict-answered, legacy-ledger answered, and other-author threads skipped with typed reasons; digest and schema pinned; zero host writes; stub-agent draft validation and zero-pending no-launch legs | | `drain-stale` | A new comment between draft and post | Two drafts are validated; a human comment lands on one thread before `--post`; that thread is skipped with `stale-thread` while the other posts once, writes one ledger entry, and no unregistered host call appears across all runs | diff --git a/evals/cases/drain-post.sh b/evals/cases/drain-post.sh index 90533406..febbbb69 100644 --- a/evals/cases/drain-post.sh +++ b/evals/cases/drain-post.sh @@ -33,8 +33,17 @@ cp "$EVAL_SANDBOX/page.json" "$EVAL_SANDBOX/page.initial.json" MARK1="" MARK2="" TWO_NL=$'\n\n' -export DRAIN_PAGE="$EVAL_SANDBOX/page.json" DRAIN_QUERY MARK1 MARK2 TWO_NL +EVAL_BODYLOG="$EVAL_SANDBOX/reply-bodies.jsonl" +export DRAIN_PAGE="$EVAL_SANDBOX/page.json" DRAIN_QUERY MARK1 MARK2 TWO_NL EVAL_BODYLOG : >"$EVAL_STUBLOG" +: >"$EVAL_BODYLOG" +store_digests() { + local f + for f in replies.json drain.json threads.json drain.dryrun.json; do + if [ -f "$DIR/$f" ]; then cksum <"$DIR/$f"; else printf 'absent\n'; fi + done +} +write_count() { grep -Ec -- '--method POST|threadId=' "$EVAL_STUBLOG" || true; } cat >"$EVAL_STUBBIN/gh" <<'GH' #!/usr/bin/env bash @@ -71,6 +80,11 @@ reply() { idx=$((i + 1)) [ "${!idx}" = "${expected[i]}" ] || return 1 done + for arg in "$@"; do + case "$arg" in + body=*) printf '%s' "${arg#body=}" | jq -Rs . >>"$EVAL_BODYLOG" ;; + esac + done jq --arg marker "$marker" \ --arg prefix "$endpoint" \ '.data.repository.pullRequest.reviewThreads.nodes |= map(if (.comments.nodes | any(.id | startswith($prefix))) then .comments.nodes[-1].body += "\n\n" + $marker else . end)' \ @@ -111,6 +125,46 @@ if [ "$EVAL_STATUS" -ne 0 ] || ! jq -e '.data.drafts.count == 2' <"$EVAL_OUT" >/ fi good "two drafts were validated before the post run" +DRY_WRITES_BEFORE="$(write_count)" +DRY_BODIES_BEFORE="$(wc -c <"$EVAL_BODYLOG" | tr -d ' ')" +DRY_FILES_BEFORE="$(store_digests)" +run_review drain acme/app 7 --post --dry-run --json +if [ "$EVAL_STATUS" -eq 0 ] && jq -e --arg mark1 "$MARK1" --arg mark2 "$MARK2" ' + .next == "verdict drain acme/app 7 --post" and + (.data.planned | map({thread_id, root_comment_id, disposition}) == [ + {thread_id:"PRRT_1", root_comment_id:"101", disposition:"answer"}, + {thread_id:"PRRT_2", root_comment_id:"201", disposition:"withdraw"}]) and + (.data.planned[0].body == ("Answer one." + "\n\n" + $mark1)) and + (.data.planned[1].body == ("Answer two." + "\n\n" + $mark2)) and + (.data.skipped | length == 0)' <"$EVAL_OUT" >/dev/null; then + good "D1 dry run reports both final marked bodies and the post-only next" +else + bad "D1 dry run data: exit $EVAL_STATUS $(head -c 500 "$EVAL_OUT") $(head -c 300 "$EVAL_ERR")" +fi +if [ "$(write_count)" = "$DRY_WRITES_BEFORE" ] && \ + [ "$(wc -c <"$EVAL_BODYLOG" | tr -d ' ')" = "$DRY_BODIES_BEFORE" ] && \ + [ "$(store_digests)" != "$DRY_FILES_BEFORE" ] && \ + [ -f "$DIR/drain.dryrun.json" ]; then + good "D2 dry run made zero host writes and left the store byte-identical" +else + bad "D2 dry run writes: $(write_count) vs $DRY_WRITES_BEFORE; files $(store_digests)" +fi +if [ "$(for f in replies.json drain.json threads.json; do + if [ -f "$DIR/$f" ]; then cksum <"$DIR/$f"; else printf 'absent\n'; fi +done)" = "$(printf '%s\n' "$DRY_FILES_BEFORE" | head -3)" ]; then + good "D3 the three core bundle files kept their exact bytes" +else + bad "D3 core files changed across the dry run" +fi +jq '.data' <"$EVAL_OUT" >"$EVAL_SANDBOX/dry-data.json" +if jq -e --slurpfile want "$EVAL_SANDBOX/dry-data.json" '. == $want[0]' "$DIR/drain.dryrun.json" >/dev/null && \ + matches -F '» 2 planned, 0 skipped (dry run; nothing posted) ->' "$EVAL_ERR"; then + good "D4 the preview file equals envelope data and names the human summary" +else + bad "D4 preview: $(head -c 400 "$DIR/drain.dryrun.json" 2>/dev/null) $(head -c 200 "$EVAL_ERR")" +fi +jq -c '[.data.planned[].body]' <"$EVAL_OUT" >"$EVAL_SANDBOX/dry-bodies.json" + run_review drain acme/app 7 --post --json if [ "$EVAL_STATUS" -eq 0 ] && jq -e '(.data.posted | length == 2) and (.data.failed | length == 0) and (.next | contains("threads"))' <"$EVAL_OUT" >/dev/null; then @@ -144,6 +198,12 @@ if [ "$(grep -c 'body=Answer' "$EVAL_STUBLOG")" = "2" ] && \ else bad "P5 writes: $(grep -E 'body=|threadId=' "$EVAL_STUBLOG")" fi +if jq -e -s --slurpfile planned "$EVAL_SANDBOX/dry-bodies.json" \ + '. == $planned[0]' "$EVAL_BODYLOG" >/dev/null; then + good "D5 the real post sent exactly the bodies the dry run planned" +else + bad "D5 dry-run bodies: $(cat "$EVAL_SANDBOX/dry-bodies.json"); posted: $(jq -s -c . "$EVAL_BODYLOG")" +fi WRITES_BEFORE="$(grep -Ec 'POST|threadId=' "$EVAL_STUBLOG" || true)" LEDGER_BEFORE="$(wc -c <"$DIR/replies.json")" @@ -158,6 +218,17 @@ fi cp "$EVAL_SANDBOX/page.initial.json" "$EVAL_SANDBOX/page.json" rm -f "$DIR/replies.json" "$DIR/drain.json" "$DIR/drain.replies.json" +AGENT_DRY_WRITES="$(write_count)" +AGENT_DRY_BODIES="$(wc -c <"$EVAL_BODYLOG" | tr -d ' ')" +run_review drain acme/app 7 --agent claude --post --dry-run --json +if [ "$EVAL_STATUS" -eq 0 ] && jq -e '(.data.planned | length == 2) and (.data.skipped | length == 0)' <"$EVAL_OUT" >/dev/null && \ + [ "$(write_count)" = "$AGENT_DRY_WRITES" ] && \ + [ "$(wc -c <"$EVAL_BODYLOG" | tr -d ' ')" = "$AGENT_DRY_BODIES" ] && \ + [ ! -e "$DIR/replies.json" ] && jq -e '(.posted | length == 0)' "$DIR/drain.json" >/dev/null; then + good "D6 --agent --post --dry-run plans, drafts, and previews without posting" +else + bad "D6 agent dry run: exit $EVAL_STATUS $(head -c 500 "$EVAL_OUT") $(head -c 300 "$EVAL_ERR")" +fi run_review drain acme/app 7 --agent claude --post --json if [ "$EVAL_STATUS" -eq 0 ] && jq -e '.data.drafts.count == 2 and (.data.posted | length == 2) and (.data.failed | length == 0) and .data.posted_at != null diff --git a/internal/core/drain/dryrun.go b/internal/core/drain/dryrun.go new file mode 100644 index 00000000..377e8e08 --- /dev/null +++ b/internal/core/drain/dryrun.go @@ -0,0 +1,60 @@ +package drain + +import ( + "github.com/stainless-code/verdict/internal/atomic" + "github.com/stainless-code/verdict/internal/jq" +) + +// KeyDryRunJSON is the preview file drain --post --dry-run writes beside the +// plan. It never stands in for drain.json's receipt. +const KeyDryRunJSON = "drain.dryrun.json" + +// Planned is one reply a dry run would send, with the final marked body. +type Planned struct { + ThreadID string `json:"thread_id"` + RootCommentID string `json:"root_comment_id"` + Disposition string `json:"disposition"` + Body string `json:"body"` +} + +// DryRunFile is the preview receipt: the replies a post run would write and +// the threads it would skip, with no reply ids or host result. +type DryRunFile struct { + SchemaVersion string `json:"schema_version"` + Provider string `enum:"github|gitlab|gitcafe|forgejo|bitbucket|azure" json:"provider"` + HeadSHA string `json:"head_sha"` + PlannedAt string `format:"date-time" json:"planned_at"` + Planned []Planned `json:"planned"` + Skipped []Skip `json:"skipped"` + PR int `json:"pr"` +} + +// Wire renders the preview in published key order, with empty lists as []. +func (f *DryRunFile) Wire() *jq.Obj { + planned := make([]any, 0, len(f.Planned)) + for _, p := range f.Planned { + planned = append(planned, jq.NewObj(keyThreadID, p.ThreadID, "root_comment_id", p.RootCommentID, + "disposition", p.Disposition, "body", p.Body)) + } + skipped := make([]any, 0, len(f.Skipped)) + for _, s := range f.Skipped { + skipped = append(skipped, jq.NewObj(keyThreadID, s.ThreadID, "reason", s.Reason)) + } + return jq.NewObj( + "schema_version", f.SchemaVersion, + "provider", f.Provider, + "pr", f.PR, + "head_sha", f.HeadSHA, + "planned_at", f.PlannedAt, + "planned", planned, + "skipped", skipped, + ) +} + +// WriteDryRun writes one preview atomically. +func WriteDryRun(path string, f *DryRunFile) error { + if f.SchemaVersion == "" { + f.SchemaVersion = SchemaVersion + } + return atomic.WriteFile(path, []byte(jq.PrettyOrdered(f.Wire()))) +} diff --git a/internal/core/drain/post.go b/internal/core/drain/post.go index ff846ee3..9ce1f05b 100644 --- a/internal/core/drain/post.go +++ b/internal/core/drain/post.go @@ -31,12 +31,14 @@ type PostRequest struct { At string DraftsRaw []byte Ledger []LedgerEntry + DryRun bool } // PostResult is what the final receipt records. type PostResult struct { Posted []Posted Failed []Failed + Planned []Planned Incomplete bool } @@ -69,7 +71,8 @@ func ValidatePostDrafts(plan *File, draftsRaw []byte, provider string) (*Replies } // Post runs §6's gates in order, then posts each draft and records one ledger -// entry per successful write. A host failure is recorded and the remaining +// entry per successful write. A dry run stops after composing each marked body +// and returns it in Planned. A host failure is recorded and the remaining // drafts are still attempted; the caller turns Incomplete into a refusal. func Post(req *PostRequest) (*PostResult, *core.RefusalError) { doc, ref := postDrafts(req) @@ -77,10 +80,13 @@ func Post(req *PostRequest) (*PostResult, *core.RefusalError) { return nil, ref } byID := postThreads(req.Feed) - res := &PostResult{Posted: []Posted{}, Failed: []Failed{}} + res := &PostResult{Posted: []Posted{}, Failed: []Failed{}, Planned: []Planned{}} for _, reply := range doc.Replies { postOne(req, byID, reply, res) } + if req.DryRun { + return res, nil + } req.Plan.Posted = res.Posted req.Plan.Failed = res.Failed at := req.At @@ -133,7 +139,7 @@ func saveFeed(req *PostRequest, res *PostResult) { func postOne(req *PostRequest, byID map[string]*threadscore.Thread, reply Reply, res *PostResult) { thread, ok := byID[reply.ThreadID] if !ok || len(thread.Comments) == 0 { - res.Failed = append(res.Failed, Failed{ThreadID: reply.ThreadID, Reason: "thread missing from refreshed feed"}) + missingThread(req, reply, res) return } last := &thread.Comments[len(thread.Comments)-1] @@ -143,6 +149,10 @@ func postOne(req *PostRequest, byID map[string]*threadscore.Thread, reply Reply, } body := recordcore.NeutralizeMentions(reply.Body) + "\n\n" + threadscore.ReplyMarker(reply.ThreadID, reply.WaitingCommentID) + if req.DryRun { + planReply(reply, thread, res, body) + return + } id, err := req.Reply(thread.Comments[0].ID, reply.ThreadID, body) if err != nil { res.Failed = append(res.Failed, Failed{ThreadID: reply.ThreadID, Reason: err.Error()}) @@ -153,6 +163,22 @@ func postOne(req *PostRequest, byID map[string]*threadscore.Thread, reply Reply, } } +func missingThread(req *PostRequest, reply Reply, res *PostResult) { + reason := "thread missing from refreshed feed" + if req.DryRun { + req.Plan.Skipped = append(req.Plan.Skipped, Skip{ThreadID: reply.ThreadID, Reason: reason}) + return + } + res.Failed = append(res.Failed, Failed{ThreadID: reply.ThreadID, Reason: reason}) +} + +func planReply(reply Reply, thread *threadscore.Thread, res *PostResult, body string) { + res.Planned = append(res.Planned, Planned{ + ThreadID: reply.ThreadID, RootCommentID: thread.Comments[0].ID, + Disposition: string(reply.Disposition), Body: body, + }) +} + func recordPost(req *PostRequest, thread *threadscore.Thread, reply Reply, id string, res *PostResult) (Posted, bool) { entry := LedgerEntry{ Thread: reply.ThreadID, ReplyID: id, At: req.At, diff --git a/internal/drain/post.go b/internal/drain/post.go index e322d63c..eb9fd700 100644 --- a/internal/drain/post.go +++ b/internal/drain/post.go @@ -5,7 +5,9 @@ import ( "errors" "fmt" "io/fs" + "os" "path/filepath" + "strings" "github.com/stainless-code/verdict/internal/cli" "github.com/stainless-code/verdict/internal/core" @@ -19,6 +21,15 @@ import ( "github.com/stainless-code/verdict/internal/storelock" ) +const ( + scratchDirMode = 0o755 + scratchFileMode = 0o600 +) + +type dryRunWriteError struct{} + +func (dryRunWriteError) Error() string { return "dry run attempted a write" } + func postMode(ctx context.Context, r *cli.Runner, req Request, fl *flags) int { dir, plan, drafts, ref := postPreflight(req, fl) if ref != nil { @@ -29,7 +40,33 @@ func postMode(ctx context.Context, r *cli.Runner, req Request, fl *flags) int { return r.Refuse(drainFailed(fmt.Sprintf("could not lock %s: %v", dir, err))) } defer unlock() + if fl.dryRun { + return runDryRunPost(ctx, r, req, fl, dir, plan, drafts) + } + return runRealPost(ctx, r, req, fl, dir, plan, drafts) +} + +func runDryRunPost(ctx context.Context, r *cli.Runner, req Request, fl *flags, dir string, + plan *draincore.File, drafts []byte, +) int { + resp, feed, ref := dryRunRefresh(ctx, r, req, fl, dir) + if ref != nil { + return r.Refuse(ref) + } + in, ref := dryRunInputs(ctx, req, resp, dir, feed) + if ref != nil { + return r.Refuse(ref) + } + result, ref := postWrites(ctx, req, dir, plan, drafts, in, true) + if ref != nil { + return r.Refuse(ref) + } + return postDryRunFinish(r, req, dir, plan, fl, resp, result) +} +func runRealPost(ctx context.Context, r *cli.Runner, req Request, fl *flags, dir string, + plan *draincore.File, drafts []byte, +) int { resp, ref := fetchcore.RunThreads(ctx, refreshRequest(r, req, fl)) if ref != nil { return r.Refuse(drainFailed(ref.Message)) @@ -38,7 +75,7 @@ func postMode(ctx context.Context, r *cli.Runner, req Request, fl *flags) int { if ref != nil { return r.Refuse(ref) } - result, ref := postWrites(ctx, req, dir, plan, drafts, in) + result, ref := postWrites(ctx, req, dir, plan, drafts, in, false) if ref != nil { return r.Refuse(ref) } @@ -108,28 +145,43 @@ func postDraftsFile(dir string) ([]byte, *core.RefusalError) { } func postWrites(ctx context.Context, req Request, dir string, plan *draincore.File, - drafts []byte, in *planInputs, + drafts []byte, in *planInputs, dryRun bool, ) (*draincore.PostResult, *core.RefusalError) { provider := plan.Provider cmdr := commander(req) draftsReq := &draincore.PostRequest{ Plan: plan, DraftsRaw: drafts, Feed: in.feed, Ledger: in.ledger, Provider: provider, Repo: req.Repo, PR: req.Pr, At: req.TimeUTC(), + DryRun: dryRun, Reply: replyWriter(ctx, cmdr, req, provider), Resolve: resolveWriter(ctx, cmdr, req, provider), Append: appendLedgerWriter(dir), SaveFeed: saveFeedWriter(dir), } + if dryRun { + blockDryRunWrites(draftsReq) + } result, ref := draincore.Post(draftsReq) if ref != nil { return nil, ref } + if dryRun { + return result, nil + } if err := draincore.WritePlan(dir+pathSep+draincore.KeyPlanJSON, plan); err != nil { return nil, drainFailed(err.Error()) } return result, nil } +func blockDryRunWrites(req *draincore.PostRequest) { + blocked := dryRunWriteError{} + req.Reply = func(string, string, string) (string, error) { return "", blocked } + req.Resolve = func(string) error { return blocked } + req.Append = func(*draincore.LedgerEntry) error { return blocked } + req.SaveFeed = func(*threadscore.File) error { return blocked } +} + func replyWriter(ctx context.Context, cmdr core.Commander, req Request, provider string, ) func(string, string, string) (string, error) { return func(rootID, threadID, body string) (string, error) { @@ -162,10 +214,118 @@ func saveFeedWriter(dir string) func(*threadscore.File) error { } } +type quietProgress struct{ r *cli.Runner } + +func (q quietProgress) Human(string) {} + +func (q quietProgress) Warn(msg string) { q.r.Warn(msg) } + +func (q quietProgress) Diag(string) {} + +// dryRunRefresh performs the live thread read into a scratch bundle, so the +// real threads.json keeps its bytes while the preview sees the current feed. +func dryRunRefresh(ctx context.Context, r *cli.Runner, req Request, fl *flags, dir string, +) (*fetchcore.ThreadsResponse, *threadscore.File, *core.RefusalError) { + scratchRoot, scratchDir, ref := prepareDryRunBundle(dir, req.Pr) + if ref != nil { + return nil, nil, ref + } + defer os.RemoveAll(scratchRoot) + read := req + read.RepoRoot = scratchRoot + rt := refreshRequest(r, read, fl) + rt.Progress = quietProgress{r: r} + resp, ref := fetchcore.RunThreads(ctx, rt) + if ref != nil { + return nil, nil, drainFailed(ref.Message) + } + feed, err := threadscore.ReadFile(filepath.Join(scratchDir, threadscore.KeyThreadsJSON)) + if err != nil { + return nil, nil, drainFailed(fmt.Sprintf("unreadable dry-run feed: %v", err)) + } + return resp, feed, nil +} + +func prepareDryRunBundle(dir, pr string) (string, string, *core.RefusalError) { + scratchRoot, err := os.MkdirTemp("", "verdict-drain-dryrun-") + if err != nil { + return "", "", drainFailed(fmt.Sprintf("could not prepare dry-run read: %v", err)) + } + scratchDir, ref := readcommon.ResolveBundleDir(scratchRoot, pr) + if ref != nil { + return "", "", drainFailed(ref.Message) + } + if err := os.MkdirAll(scratchDir, scratchDirMode); err != nil { + return "", "", drainFailed(fmt.Sprintf("could not prepare dry-run read: %v", err)) + } + for _, name := range []string{"PR.json", threadscore.KeyThreadsJSON} { + if err := copyBundleFile(filepath.Join(dir, name), filepath.Join(scratchDir, name)); err != nil { + return "", "", drainFailed(err.Error()) + } + } + return scratchRoot, scratchDir, nil +} + +func copyBundleFile(src, dst string) error { + raw, err := jq.CappedFile(src, jq.MaxPRBytes) + if errors.Is(err, fs.ErrNotExist) { + return nil + } + if err != nil { + return jq.NewErrf("unreadable %s: %v", src, err) + } + if err := os.WriteFile(dst, raw, scratchFileMode); err != nil { + return jq.NewErrf("could not prepare dry-run read: %v", err) + } + return nil +} + +func dryRunInputs(ctx context.Context, req Request, resp *fetchcore.ThreadsResponse, + dir string, feed *threadscore.File, +) (*planInputs, *core.RefusalError) { + return inputsFrom(ctx, req, resp, dir, feed) +} + +func dryRunNext(req Request, fl *flags) string { + parts := []string{"verdict", verbDrain, req.Repo, req.Pr} + passed := false + for _, a := range fl.args { + if a == "--" { + passed = true + } + if !passed && a == "--dry-run" { + continue + } + parts = append(parts, a) + } + return strings.Join(parts, " ") +} + func postSuccessNext(req Request) string { return "refresh threads with verdict threads " + req.Repo + " " + req.Pr } +func postDryRunFinish(r *cli.Runner, req Request, dir string, plan *draincore.File, fl *flags, + resp *fetchcore.ThreadsResponse, result *draincore.PostResult, +) int { + r.Warnings = resp.Warnings + preview := &draincore.DryRunFile{ + Provider: plan.Provider, PR: plan.PR, HeadSHA: plan.HeadSHA, + PlannedAt: cli.NowUTC(r.Now()), Planned: result.Planned, Skipped: plan.Skipped, + } + if err := draincore.WriteDryRun(filepath.Join(dir, draincore.KeyDryRunJSON), preview); err != nil { + return r.Refuse(drainFailed(err.Error())) + } + fmt.Fprintf(r.Human, "» %d planned, %d skipped (dry run; nothing posted) -> %s/%s\n", + len(preview.Planned), len(preview.Skipped), dir, draincore.KeyDryRunJSON) + if r.JSONMode { + env := core.RenderEnvelope(verbDrain, req.Repo, req.Pr, cli.NowUTC(r.Now()), dryRunNext(req, fl), + r.Warnings, preview.Wire(), nil) + fmt.Fprint(r.Out, jq.PrettyOrdered(env)) + } + return cli.ExitOK +} + func postFinish(r *cli.Runner, req Request, dir string, plan *draincore.File, resp *fetchcore.ThreadsResponse, result *draincore.PostResult, ) int { diff --git a/internal/drain/post_dryrun_test.go b/internal/drain/post_dryrun_test.go new file mode 100644 index 00000000..8b6e886d --- /dev/null +++ b/internal/drain/post_dryrun_test.go @@ -0,0 +1,243 @@ +package drain + +import ( + "bytes" + "encoding/json" + "errors" + "io/fs" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stainless-code/verdict/internal/cli" + draincore "github.com/stainless-code/verdict/internal/core/drain" + threadscore "github.com/stainless-code/verdict/internal/core/threads" +) + +const ( + answerOneBody = "Answer one.\n\n" + answerTwoBody = "Answer two.\n\n" +) + +func twoDryRunReplies() []draincore.Reply { + return []draincore.Reply{ + {ThreadID: "PRRT_1", WaitingCommentID: "102", Disposition: draincore.DispositionAnswer, Body: "Answer one."}, + {ThreadID: "PRRT_2", WaitingCommentID: "202", Disposition: draincore.DispositionWithdraw, Body: "Answer two."}, + } +} + +func writeFeedSentinel(t *testing.T, dir string) []byte { + t.Helper() + path := filepath.Join(dir, threadscore.KeyThreadsJSON) + feed := &threadscore.File{ + SchemaVersion: threadscore.SchemaVersion, + Provider: "github", + PR: 7, + HeadSHA: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + FetchedAt: "2026-11-17T11:00:00Z", + Threads: []threadscore.Thread{}, + } + if err := threadscore.WriteFile(path, feed); err != nil { + t.Fatal(err) + } + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + return raw +} + +func snapshotFiles(t *testing.T, dir string, names ...string) map[string][]byte { + t.Helper() + snap := make(map[string][]byte, len(names)) + for _, name := range names { + raw, err := os.ReadFile(filepath.Join(dir, name)) + if errors.Is(err, fs.ErrNotExist) { + snap[name] = nil + continue + } + if err != nil { + t.Fatal(err) + } + snap[name] = raw + } + return snap +} + +func assertFilesUnchanged(t *testing.T, dir string, before map[string][]byte) { + t.Helper() + for name, want := range before { + got := snapshotFiles(t, dir, name)[name] + if !bytes.Equal(got, want) { + t.Errorf("%s changed: got %q, want %q", name, got, want) + } + } +} + +func forbidHostWrites(t *testing.T) func([]string) { + t.Helper() + return func(argv []string) { + t.Fatalf("dry run attempted host write: %v", argv) + } +} + +func TestPostDryRunWritesPreviewWithoutHostOrStoreWrites(t *testing.T) { + t.Parallel() + root, dir := setupDrainBundle(t) + writePlanAndDrafts(t, dir, "github", twoDryRunReplies(), "") + writeFeedSentinel(t, dir) + before := snapshotFiles(t, dir, draincore.KeyLedgerJSON, draincore.KeyPlanJSON, threadscore.KeyThreadsJSON) + cmdr := &drainHostStub{pageJSON: twoPendingPageJSON(), self: "verdict-bot", forbidWrites: forbidHostWrites(t)} + out, diag, code := runDrainTest(t, root, cmdr, []string{"--post", "--dry-run"}) + if code != cli.ExitOK { + t.Fatalf("exit = %d, out=%s diag=%s", code, out, diag) + } + assertFilesUnchanged(t, dir, before) + calls := cmdr.callLog() + if len(calls) != 3 || !strings.Contains(calls[0], "git remote get-url origin") || + !strings.Contains(calls[1], "api graphql") || + !strings.Contains(calls[2], "api user --jq .login") { + t.Errorf("host calls = %v, want one origin probe, one live refresh, one login read", calls) + } + + env := decodeDryRunEnvelope(t, out) + preview := readDryRunPreview(t, dir) + assertDryRunEnvelope(t, env, preview) + assertDryRunPlanned(t, preview) + wantPath := filepath.ToSlash(dir) + "/" + draincore.KeyDryRunJSON + wantHuman := "» 2 planned, 0 skipped (dry run; nothing posted) -> " + wantPath + "\n" + if !strings.Contains(diag, wantHuman) { + t.Errorf("human output missing %q:\n%s", wantHuman, diag) + } +} + +type dryRunEnvelope struct { + Next string `json:"next"` + Error any `json:"error"` + Data json.RawMessage `json:"data"` +} + +func decodeDryRunEnvelope(t *testing.T, out string) *dryRunEnvelope { + t.Helper() + var env dryRunEnvelope + if err := json.Unmarshal([]byte(out), &env); err != nil { + t.Fatalf("decode envelope: %v\n%s", err, out) + } + return &env +} + +func readDryRunPreview(t *testing.T, dir string) *draincore.DryRunFile { + t.Helper() + raw, err := os.ReadFile(filepath.Join(dir, draincore.KeyDryRunJSON)) + if err != nil { + t.Fatalf("read preview: %v", err) + } + var preview draincore.DryRunFile + if err := json.Unmarshal(raw, &preview); err != nil { + t.Fatalf("decode preview: %v\n%s", err, raw) + } + return &preview +} + +func assertDryRunEnvelope(t *testing.T, env *dryRunEnvelope, preview *draincore.DryRunFile) { + t.Helper() + if env.Error != nil { + t.Errorf("error = %v, want nil", env.Error) + } + wantNext := "verdict drain acme/app 7 --post" + if env.Next != wantNext { + t.Errorf("next = %q, want %q", env.Next, wantNext) + } + var fromEnvelope draincore.DryRunFile + if err := json.Unmarshal(env.Data, &fromEnvelope); err != nil { + t.Fatalf("decode envelope data: %v\n%s", err, env.Data) + } + if !bytes.Equal(mustMarshal(t, preview), mustMarshal(t, fromEnvelope)) { + t.Errorf("envelope data != preview: %s", env.Data) + } +} + +func assertDryRunPlanned(t *testing.T, preview *draincore.DryRunFile) { + t.Helper() + if preview.SchemaVersion != draincore.SchemaVersion || preview.Provider != "github" || preview.PR != 7 { + t.Errorf("preview identity = %+v", preview) + } + if len(preview.Planned) != 2 { + t.Fatalf("planned = %+v, want 2 entries", preview.Planned) + } + wantPlanned := []draincore.Planned{ + {ThreadID: "PRRT_1", RootCommentID: "101", Disposition: string(draincore.DispositionAnswer), Body: answerOneBody}, + {ThreadID: "PRRT_2", RootCommentID: "201", Disposition: string(draincore.DispositionWithdraw), Body: answerTwoBody}, + } + for i, want := range wantPlanned { + if preview.Planned[i] != want { + t.Errorf("planned[%d] = %+v, want %+v", i, preview.Planned[i], want) + } + } + if len(preview.Skipped) != 0 { + t.Errorf("skipped = %+v, want none", preview.Skipped) + } +} + +func mustMarshal(t *testing.T, v any) []byte { + t.Helper() + raw, err := json.Marshal(v) + if err != nil { + t.Fatal(err) + } + return raw +} + +func TestDryRunWithoutPostIsUsageError(t *testing.T) { + t.Parallel() + root, _ := setupDrainBundle(t) + cmdr := &drainHostStub{pageJSON: twoPendingPageJSON(), self: "verdict-bot", forbidWrites: forbidHostWrites(t)} + out, diag, code := runDrainTest(t, root, cmdr, []string{"--dry-run"}) + if code != cli.ExitUsage { + t.Fatalf("exit = %d, want %d (out=%s diag=%s)", code, cli.ExitUsage, out, diag) + } + if out != "" { + t.Errorf("stdout = %q, want empty usage output", out) + } + if !strings.Contains(diag, "--post") { + t.Errorf("usage message %q does not name --post", diag) + } + if calls := cmdr.callLog(); len(calls) != 0 { + t.Errorf("usage error made %d host call(s): %v", len(calls), calls) + } +} + +func TestPostDryRunSecretRefusesWithZeroWrites(t *testing.T) { + t.Parallel() + root, dir := setupDrainBundle(t) + secret := []draincore.Reply{ + {ThreadID: "PRRT_1", WaitingCommentID: "102", Disposition: draincore.DispositionAnswer, Body: "key AKIAIOSFODNN7EXAMPLE"}, + } + writePlanAndDrafts(t, dir, "github", secret, "") + before := snapshotFiles(t, dir, draincore.KeyLedgerJSON, draincore.KeyPlanJSON, + threadscore.KeyThreadsJSON, draincore.KeyDryRunJSON) + cmdr := &drainHostStub{pageJSON: twoPendingPageJSON(), self: "verdict-bot", forbidWrites: forbidHostWrites(t)} + out, diag, code := runDrainTest(t, root, cmdr, []string{"--post", "--dry-run"}) + if code != cli.ExitRefused { + t.Fatalf("exit = %d, want %d (out=%s diag=%s)", code, cli.ExitRefused, out, diag) + } + assertRefusalCode(t, out, "post-secret-shaped") + assertFilesUnchanged(t, dir, before) + if calls := cmdr.callLog(); len(calls) != 0 { + t.Errorf("secret refusal made %d host call(s): %v", len(calls), calls) + } +} + +func TestPostDryRunNextKeepsAgentArgs(t *testing.T) { + t.Parallel() + fl, code, done := parseFlags(&cli.Runner{}, []string{"--agent", "claude", "--post", "--dry-run", "--timeout", "7", "--", "--model", "x"}) + if done || code != 0 || !fl.dryRun { + t.Fatalf("parse = %+v, %d, %v", fl, code, done) + } + got := dryRunNext(Request{Repo: "acme/app", Pr: "7"}, fl) + want := "verdict drain acme/app 7 --agent claude --post --timeout 7 -- --model x" + if got != want { + t.Errorf("next = %q, want %q", got, want) + } +} diff --git a/internal/drain/post_test.go b/internal/drain/post_test.go index a8291528..4892e89e 100644 --- a/internal/drain/post_test.go +++ b/internal/drain/post_test.go @@ -19,18 +19,24 @@ import ( ) type drainHostStub struct { - postedCount map[string]int - pageJSON string - self string - failRootID string - calls []string - postDelay time.Duration - mu sync.Mutex + postedCount map[string]int + forbidWrites func([]string) + pageJSON string + self string + failRootID string + calls []string + postDelay time.Duration + mu sync.Mutex } func (s *drainHostStub) Run(_ context.Context, cmd *core.Command) (core.Result, error) { joined := strings.Join(cmd.Argv, " ") s.recordCall(joined) + if isPostReply(cmd.Argv) || strings.Contains(joined, "resolveReviewThread") { + if s.forbidWrites != nil { + s.forbidWrites(cmd.Argv) + } + } if isPostReply(cmd.Argv) { s.maybeDelay() return s.handlePostReply(cmd.Argv[4], joined) diff --git a/internal/drain/verb.go b/internal/drain/verb.go index 2da02080..13019349 100644 --- a/internal/drain/verb.go +++ b/internal/drain/verb.go @@ -27,8 +27,9 @@ import ( ) const ( - verbDrain = "drain" - wantFlags = "--provider , --max-threads <1-100>, --agent , --timeout , --post, -- , --json" + verbDrain = "drain" + wantFlags = "--provider , --max-threads <1-100>, --agent , --timeout , " + + "--post, --dry-run (with --post), -- , --json" defaultMax = "20" pathSep = "/" ) @@ -42,14 +43,17 @@ type flags struct { agent string timeout string pass []string + args []string post bool + dryRun bool } // Run executes `drain [--provider

] [--max-threads ] -// [--agent [--timeout ] [-- ]] [--post] [--json]`. Plan -// and draft modes never write to the host; --agent --post plans, drafts, -// validates, and posts in one run; --post alone validates existing drafts -// locally before any host read. +// [--agent [--timeout ] [-- ]] [--post] [--dry-run] +// [--json]`. Plan and draft modes never write to the host; --agent --post +// plans, drafts, validates, and posts in one run; --post alone validates +// existing drafts locally before any host read. --post --dry-run runs the +// same gates and reads and writes a preview instead of posting. func Run(ctx context.Context, r *cli.Runner, args []string, req Request) int { r.Cmd, r.Repo, r.Pr = verbDrain, req.Repo, req.Pr fl, limit, code, done := parseRunArgs(ctx, r, args, req) @@ -67,6 +71,9 @@ func parseRunArgs(ctx context.Context, r *cli.Runner, args []string, req Request if done { return nil, 0, code, true } + if fl.dryRun && !fl.post { + return nil, 0, r.ReportUsage(&cli.UsageError{Message: "--dry-run requires --post"}), true + } r.RouteHuman() if code, done := r.Canceled(ctx, verbDrain); done { return nil, 0, code, true @@ -160,6 +167,12 @@ func readInputs(ctx context.Context, req Request, resp *fetchcore.ThreadsRespons if err != nil { return nil, drainFailed(fmt.Sprintf("unreadable %s/threads.json: %v", dir, err)) } + return inputsFrom(ctx, req, resp, dir, feed) +} + +func inputsFrom(ctx context.Context, req Request, resp *fetchcore.ThreadsResponse, + dir string, feed *threadscore.File, +) (*planInputs, *core.RefusalError) { if resp.Data.Truncated || feed.Truncated { return nil, drainFailed(dir + "/threads.json is truncated — refresh threads before draining") } @@ -235,7 +248,7 @@ func refreshRequest(r *cli.Runner, req Request, fl *flags) *fetchcore.ThreadsReq } func parseFlags(r *cli.Runner, args []string) (*flags, int, bool) { - fl := &flags{max: defaultMax} + fl := &flags{max: defaultMax, args: append([]string{}, args...)} for i := 0; i < len(args); i++ { if args[i] == "--" { fl.pass = append(fl.pass, args[i+1:]...) @@ -256,6 +269,9 @@ func consumeFlag(r *cli.Runner, fl *flags, args []string, i int) (int, int, bool case a == "--post": fl.post = true return i, 0, false + case a == "--dry-run": + fl.dryRun = true + return i, 0, false case a == "--provider" || a == "--max-threads" || a == "--agent" || a == "--timeout": v, nextIndex, u := cli.FlagValue(args, i, a, "") if u != nil { diff --git a/internal/schemagen/build.go b/internal/schemagen/build.go index cabdb339..4aeffb19 100644 --- a/internal/schemagen/build.go +++ b/internal/schemagen/build.go @@ -28,6 +28,7 @@ const ( keyTruncated = "truncated" defsReportData = "reportData" defsPostData = "postData" + defsSkip = "skip" keyDefs = "$defs" keyConst = "const" keyEnum = "enum" @@ -260,7 +261,7 @@ func documents() []document { root: reflect.TypeFor[DrainFile](), defs: []definition{ {typ: reflect.TypeFor[DrainPending](), name: "pending", closed: true}, - {typ: reflect.TypeFor[DrainSkip](), name: "skip", closed: true}, + {typ: reflect.TypeFor[DrainSkip](), name: defsSkip, closed: true}, {typ: reflect.TypeFor[DrainDrafts](), name: "drafts", closed: true}, {typ: reflect.TypeFor[DrainDispositions](), name: "dispositions", closed: true}, {typ: reflect.TypeFor[DrainPosted](), name: "posted", closed: true}, @@ -268,6 +269,16 @@ func documents() []document { }, additionalProperties: false, }, + { + name: "drain-dryrun", + title: "drain.dryrun.json — drain post preview", + root: reflect.TypeFor[DrainDryRunFile](), + defs: []definition{ + {typ: reflect.TypeFor[DrainPlanned](), name: "planned", closed: true}, + {typ: reflect.TypeFor[DrainSkip](), name: defsSkip, closed: true}, + }, + additionalProperties: false, + }, { name: "drain-replies", title: "drain.replies.json — drafted thread replies", @@ -444,7 +455,7 @@ func documents() []document { title: "threads-resolve.json — thread-resolve receipt", root: reflect.TypeFor[ThreadsResolveReceipt](), defs: []definition{ - {typ: reflect.TypeFor[ThreadsResolveSkip](), name: "skip"}, + {typ: reflect.TypeFor[ThreadsResolveSkip](), name: defsSkip}, }, additionalProperties: false, }, diff --git a/internal/schemagen/build_test.go b/internal/schemagen/build_test.go index 74baaf4f..2504f258 100644 --- a/internal/schemagen/build_test.go +++ b/internal/schemagen/build_test.go @@ -232,15 +232,15 @@ func TestDocumentsCarryStableIdentity(t *testing.T) { if err != nil { t.Fatalf("Generate: %v", err) } - if len(files) != 27 { - t.Fatalf("generated %d schemas, want 27", len(files)) + if len(files) != 28 { + t.Fatalf("generated %d schemas, want 28", len(files)) } seen := map[string]bool{} for _, file := range files { seen[file.Name] = true assertIdentity(t, file) } - for _, name := range []string{"author.schema.json", "panel.schema.json", "panel-plan.schema.json", "calibration.schema.json", "backtest.schema.json", "config.schema.json", "corpus.schema.json", "cost.schema.json", "drain-replies.schema.json", "drain.schema.json", "envelope.schema.json", "findings.schema.json", "identity.schema.json", "index.schema.json", "lock.schema.json", "pack.schema.json", "post.schema.json", "recipes.schema.json", "record.schema.json", "review.schema.json", "selected.schema.json", "threads.schema.json", "threads-resolve.schema.json", "usage.schema.json", "validation.schema.json"} { + for _, name := range []string{"author.schema.json", "panel.schema.json", "panel-plan.schema.json", "calibration.schema.json", "backtest.schema.json", "config.schema.json", "corpus.schema.json", "cost.schema.json", "drain-dryrun.schema.json", "drain-replies.schema.json", "drain.schema.json", "envelope.schema.json", "findings.schema.json", "identity.schema.json", "index.schema.json", "lock.schema.json", "pack.schema.json", "post.schema.json", "recipes.schema.json", "record.schema.json", "review.schema.json", "selected.schema.json", "threads.schema.json", "threads-resolve.schema.json", "usage.schema.json", "validation.schema.json"} { if !seen[name] { t.Errorf("missing generated schema %s", name) } diff --git a/internal/schemagen/drain_schema_test.go b/internal/schemagen/drain_schema_test.go index 15b7e1bc..b14715bb 100644 --- a/internal/schemagen/drain_schema_test.go +++ b/internal/schemagen/drain_schema_test.go @@ -5,20 +5,32 @@ import ( "testing" ) -const drainSchema = "drain.schema.json" +const ( + drainSchema = "drain.schema.json" + drainDryRunSchema = "drain-dryrun.schema.json" +) func TestDrainDefsAreClosed(t *testing.T) { + assertGeneratedDefsClosed(t, drainSchema) +} + +func TestDrainDryRunDefsAreClosed(t *testing.T) { + assertGeneratedDefsClosed(t, drainDryRunSchema) +} + +func assertGeneratedDefsClosed(t *testing.T, name string) { + t.Helper() files, err := Generate() if err != nil { t.Fatal(err) } for _, file := range files { - if file.Name == drainSchema { - assertClosedDefs(t, file.Data) + if file.Name == name { + assertClosedDefs(t, file.Data, name) return } } - t.Fatal(drainSchema + " was not generated") + t.Fatal(name + " was not generated") } func schemaDefs(t *testing.T, data []byte, name string) map[string]any { @@ -34,11 +46,11 @@ func schemaDefs(t *testing.T, data []byte, name string) map[string]any { return defs } -func assertClosedDefs(t *testing.T, data []byte) { +func assertClosedDefs(t *testing.T, data []byte, name string) { t.Helper() - defs := schemaDefs(t, data, drainSchema) + defs := schemaDefs(t, data, name) if len(defs) == 0 { - t.Fatal(drainSchema + " has no $defs") + t.Fatal(name + " has no $defs") } for name, raw := range defs { def, ok := raw.(map[string]any) diff --git a/internal/schemagen/model.go b/internal/schemagen/model.go index 55600229..51ded97b 100644 --- a/internal/schemagen/model.go +++ b/internal/schemagen/model.go @@ -1002,6 +1002,13 @@ type DrainPosted = draincore.Posted // DrainFailed is one host reply that did not land. type DrainFailed = draincore.Failed +// DrainDryRunFile is the preview drain --post --dry-run writes instead of a +// posting receipt, published as schemas/drain-dryrun.schema.json. +type DrainDryRunFile = draincore.DryRunFile + +// DrainPlanned is one reply a dry run would send, with its final marked body. +type DrainPlanned = draincore.Planned + // ThreadsFile is one provider bundle's normalized PR-conversation feed, // published as schemas/threads.schema.json. It is additive beside the raw // provider files: every host's payload is filtered and flattened to the same diff --git a/internal/schemagen/overlay.json b/internal/schemagen/overlay.json index ee98043e..0d409769 100644 --- a/internal/schemagen/overlay.json +++ b/internal/schemagen/overlay.json @@ -486,6 +486,20 @@ "/$defs/comment/properties/category": "Optional closed taxonomy: logic|edge-case|security|performance|accidental-commit|style|docs|other. Unknown or non-string values warn once and store null; never inferred from prose, path, severity, or rule. Explicit null clears a stored label on re-post; absent preserves it." } }, + "drain-dryrun": { + "descriptions": { + "": "The preview drain --post --dry-run writes beside the plan: the exact marked replies a real post would send and every thread it would skip. It is not a receipt: no reply ids, no host result, and no ledger change.", + "/properties/schema_version": "Preview format version; only \"1\" is published.", + "/properties/provider": "The host whose live thread feed the preview read.", + "/properties/pr": "Pull-request number the preview belongs to.", + "/properties/head_sha": "The bundle head the live feed was read at.", + "/properties/planned_at": "UTC time the preview was written, RFC 3339.", + "/properties/planned": "Replies a real --post would send, in draft order, with their final bodies.", + "/$defs/planned": "One reply that would be posted: its thread, root comment, disposition, and final body with the marker.", + "/properties/skipped": "Threads a real --post would skip, each with a machine-readable reason.", + "/$defs/skip": "One thread left open and why: resolved, answered, stale-thread, already-replied, defer, no-draft, or a missing refreshed thread." + } + }, "drain-replies": { "descriptions": { "": "The single artifact the drafting agent writes: one answer per pending thread, pinned to the plan digest. Verdict validates every site before any later step reads it; there is no host write in this build.", diff --git a/internal/schemagen/semantics_test.go b/internal/schemagen/semantics_test.go index 1078723c..0cef1b34 100644 --- a/internal/schemagen/semantics_test.go +++ b/internal/schemagen/semantics_test.go @@ -273,6 +273,10 @@ var publishedSemantics = map[string]string{ // additive drain-replies.schema.json (new bundle document; no existing // schema property, required key, enum, or type changed). It pins the // agent's single output: the plan digest and one disposition per reply. + // additive drain-dryrun.schema.json (new bundle document; no existing + // schema property, required key, enum, or type changed). It pins the + // preview --post --dry-run writes: planned marked bodies and typed skips. + "drain-dryrun.schema.json": "1718a109cbe9f1e1f6401810ba90886655f8842194c496986d34fb6973a28032", "drain-replies.schema.json": "601cfd36687dab68e858aed5484fa2f654c5d2265da944fc1428753029eb8362", "drain.schema.json": "1fcf8437ef91b1b130923e3600f489559be29ce9f3a5b1d2de52641db945af80", "envelope.schema.json": "b5f49f876d3f600ce6ced697b27a4bb343c516bbbaf21b698995cf8e3ac87cd7", diff --git a/internal/schemagen/testdata/baseline/drain-dryrun.schema.json b/internal/schemagen/testdata/baseline/drain-dryrun.schema.json new file mode 100644 index 00000000..6c8d4eac --- /dev/null +++ b/internal/schemagen/testdata/baseline/drain-dryrun.schema.json @@ -0,0 +1,74 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/stainless-code/verdict/blob/main/schemas/drain-dryrun.schema.json", + "title": "drain.dryrun.json — drain post preview", + "description": "The preview drain --post --dry-run writes beside the plan: the exact marked replies a real post would send and every thread it would skip. It is not a receipt: no reply ids, no host result, and no ledger change.", + "type": "object", + "properties": { + "schema_version": { + "description": "Preview format version; only \"1\" is published.", + "type": "string" + }, + "provider": { + "description": "The host whose live thread feed the preview read.", + "enum": ["github", "gitlab", "gitcafe", "forgejo", "bitbucket", "azure"] + }, + "head_sha": { + "description": "The bundle head the live feed was read at.", + "type": "string" + }, + "planned_at": { + "description": "UTC time the preview was written, RFC 3339.", + "type": "string", + "format": "date-time" + }, + "planned": { + "description": "Replies a real --post would send, in draft order, with their final bodies.", + "type": "array", + "items": { "$ref": "#/$defs/planned" } + }, + "skipped": { + "description": "Threads a real --post would skip, each with a machine-readable reason.", + "type": "array", + "items": { "$ref": "#/$defs/skip" } + }, + "pr": { + "description": "Pull-request number the preview belongs to.", + "type": "integer" + } + }, + "required": [ + "schema_version", + "provider", + "head_sha", + "planned_at", + "planned", + "skipped", + "pr" + ], + "additionalProperties": false, + "$defs": { + "planned": { + "description": "One reply that would be posted: its thread, root comment, disposition, and final body with the marker.", + "type": "object", + "properties": { + "thread_id": { "type": "string" }, + "root_comment_id": { "type": "string" }, + "disposition": { "type": "string" }, + "body": { "type": "string" } + }, + "required": ["thread_id", "root_comment_id", "disposition", "body"], + "additionalProperties": false + }, + "skip": { + "description": "One thread left open and why: resolved, answered, stale-thread, already-replied, defer, no-draft, or a missing refreshed thread.", + "type": "object", + "properties": { + "thread_id": { "type": "string" }, + "reason": { "type": "string" } + }, + "required": ["thread_id", "reason"], + "additionalProperties": false + } + } +} diff --git a/schemas/drain-dryrun.schema.json b/schemas/drain-dryrun.schema.json new file mode 100644 index 00000000..6c8d4eac --- /dev/null +++ b/schemas/drain-dryrun.schema.json @@ -0,0 +1,74 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/stainless-code/verdict/blob/main/schemas/drain-dryrun.schema.json", + "title": "drain.dryrun.json — drain post preview", + "description": "The preview drain --post --dry-run writes beside the plan: the exact marked replies a real post would send and every thread it would skip. It is not a receipt: no reply ids, no host result, and no ledger change.", + "type": "object", + "properties": { + "schema_version": { + "description": "Preview format version; only \"1\" is published.", + "type": "string" + }, + "provider": { + "description": "The host whose live thread feed the preview read.", + "enum": ["github", "gitlab", "gitcafe", "forgejo", "bitbucket", "azure"] + }, + "head_sha": { + "description": "The bundle head the live feed was read at.", + "type": "string" + }, + "planned_at": { + "description": "UTC time the preview was written, RFC 3339.", + "type": "string", + "format": "date-time" + }, + "planned": { + "description": "Replies a real --post would send, in draft order, with their final bodies.", + "type": "array", + "items": { "$ref": "#/$defs/planned" } + }, + "skipped": { + "description": "Threads a real --post would skip, each with a machine-readable reason.", + "type": "array", + "items": { "$ref": "#/$defs/skip" } + }, + "pr": { + "description": "Pull-request number the preview belongs to.", + "type": "integer" + } + }, + "required": [ + "schema_version", + "provider", + "head_sha", + "planned_at", + "planned", + "skipped", + "pr" + ], + "additionalProperties": false, + "$defs": { + "planned": { + "description": "One reply that would be posted: its thread, root comment, disposition, and final body with the marker.", + "type": "object", + "properties": { + "thread_id": { "type": "string" }, + "root_comment_id": { "type": "string" }, + "disposition": { "type": "string" }, + "body": { "type": "string" } + }, + "required": ["thread_id", "root_comment_id", "disposition", "body"], + "additionalProperties": false + }, + "skip": { + "description": "One thread left open and why: resolved, answered, stale-thread, already-replied, defer, no-draft, or a missing refreshed thread.", + "type": "object", + "properties": { + "thread_id": { "type": "string" }, + "reason": { "type": "string" } + }, + "required": ["thread_id", "reason"], + "additionalProperties": false + } + } +} diff --git a/scripts/size-baseline.txt b/scripts/size-baseline.txt index 0d31c73b..51b51879 100644 --- a/scripts/size-baseline.txt +++ b/scripts/size-baseline.txt @@ -1,7 +1,7 @@ deps 0 -darwin/amd64 17610000 -darwin/arm64 16010000 -linux/amd64 17290000 +darwin/amd64 17620000 +darwin/arm64 16020000 +linux/amd64 17310000 linux/arm64 15600000 -windows/amd64 17680000 -windows/arm64 15770000 +windows/amd64 17710000 +windows/arm64 15790000 diff --git a/scripts/verdict-complete.sh b/scripts/verdict-complete.sh index 6523fdb3..61c58946 100755 --- a/scripts/verdict-complete.sh +++ b/scripts/verdict-complete.sh @@ -27,7 +27,7 @@ _verdict_flags() { # $1 = verb index) printf '%s\n' build status prune callers tests-for co-changed owners blast-radius --from --at --depth --limit --commits --since --min-count --symbol --pr --paths --json;; backtest) printf '%s\n' mine replay score --since --until --run --agent --max-reviews --budget-usd --timeout --rounds --json;; threads) printf '%s\n' --provider --thread --dry-run --json;; - drain) printf '%s\n' --provider --max-threads --agent --timeout --post --json --;; + drain) printf '%s\n' --provider --max-threads --agent --timeout --post --dry-run --json --;; fetch-local) printf '%s\n' --staged --range --base --stash --worktree --context --ignore --no-default-excludes --index --consent-over-budget --show-prompts --chunk --json;; run) printf '%s\n' --agent --recipe --stage --timeout --rounds --print --json --;; learn) printf '%s\n' --propose --from-bundle --accept --reject --dry-run --pack --incident --sha --path --note --url --at --rule --scope --id --provenance --rigor --status --retire --restore --reason --json;;