From 39f361e7c53f6e75c0228abfa41bb1b9879caaad Mon Sep 17 00:00:00 2001 From: REPPL <77722411+REPPL@users.noreply.github.com> Date: Tue, 7 Jul 2026 12:31:32 +0100 Subject: [PATCH] feat: self-discovering CLI smoke harness (evals/) + eval intent (itd-75) A smoke test that builds the real abcd binary, walks the Cobra command tree in-process via cli.NewRootCommand() (so new commands are covered with no edit), and exercises each against the binary: every command's --help must render without panicking, read-only verbs (version, status board) run for real, and an unknown flag degrades to a clean non-zero. Catches "compiles but panics on invocation", which the unit lane misses. Behind the `smoke` build tag so it stays out of `go test ./...`. Wired into a dedicated CI `smoke` job, the release verify gate (smokes the tagged-commit binary before publish), and `make smoke`. evals/data/ is reserved for the v2 fixture-driven scenarios. itd-75 captures the generalisation: abcd manages this harness for any repo it configures, with fixture-driven scenarios auto-discovered from evals/data/. This repo is the v1 dogfood. --- .../intents/drafts/itd-75-cli-eval-harness.md | 39 +++++ .github/workflows/ci.yml | 19 +++ .github/workflows/release.yml | 3 + Makefile | 9 +- evals/README.md | 35 ++++ evals/data/README.md | 9 ++ evals/smoke_test.go | 150 ++++++++++++++++++ 7 files changed, 263 insertions(+), 1 deletion(-) create mode 100644 .abcd/development/intents/drafts/itd-75-cli-eval-harness.md create mode 100644 evals/README.md create mode 100644 evals/data/README.md create mode 100644 evals/smoke_test.go diff --git a/.abcd/development/intents/drafts/itd-75-cli-eval-harness.md b/.abcd/development/intents/drafts/itd-75-cli-eval-harness.md new file mode 100644 index 00000000..60009563 --- /dev/null +++ b/.abcd/development/intents/drafts/itd-75-cli-eval-harness.md @@ -0,0 +1,39 @@ +--- +id: itd-75 +slug: cli-eval-harness +spec_id: null +kind: standalone +suggested_kind: null +reclassification_history: [] +--- + +# abcd Proves Your CLI Actually Runs — From Fixtures You Drop In + +## Press Release + +> **Every command a repo exposes gets exercised, and you extend the coverage by dropping a folder — never by writing test plumbing.** abcd builds the binary, discovers the whole command surface by walking it (no hand-kept list), and smokes each command: help renders, nothing panics, read-only verbs run. Then it goes further — a fixtures folder of user-specified and synthetic inputs is auto-discovered and replayed against the matching commands, asserting the *shape* of what comes back. Adding a scenario is dropping an input and an expected-shape file into `evals/data//`; the harness picks it up on the next run, in local dev, in CI, and in the release gate against the very binary about to ship. +> +> "I used to find out a command crashed only when a user hit it," said Dev, a maintainer. "Now every command is smoked on every push, and when I want a real scenario I drop a sample corpus in a folder and abcd runs my tool against it. My confidence in a release stopped being a feeling." + +## Why This Matters + +Unit tests prove functions behave; they do not prove the *assembled binary* runs. A command can compile, pass its unit tests, and still panic the moment it is invoked — a broken wiring, a nil dependency, a flag that explodes on parse. The only honest check is to run the real binary through its real command surface. Doing that by hand rots: someone must remember to add each new command to the list. Discovering the surface from the binary itself removes that decay, and a fixtures-folder convention lets coverage grow by contribution rather than by editing a harness. + +This is also the natural home for the "wired or it isn't done" rule: a command that is not reachable and runnable from the built binary is not done, and the eval harness is what makes that enforceable rather than aspirational. + +## What It Looks Like + +- **Self-discovering smoke (shipped as the v1 dogfood).** Walk the command tree, run every command's `--help` and the read-only verbs against the built binary, assert no panic and graceful exit. New commands are covered automatically. +- **Fixture-driven scenarios.** `evals/data//` holds inputs (user-specified real samples and synthetic generators) plus an expected-shape assertion. The harness matches folders to commands and replays them — `memory ingest` over a corpus, `capture` round-trips, `launch --dry-run` over a scratch tree — with no Go edit per scenario. +- **Any abcd-managed repo inherits it.** `abcd ahoy` scaffolds the harness and an empty `evals/data/` with instructions, and wires the smoke into the repo's CI and release gate. A repo becomes self-smoking by being abcd-managed. +- **Release-artefact smoke.** The gate runs the harness against the binary built from the tagged commit, so the thing that ships is the thing that was exercised. + +## Dogfood (v1 running on this repo) + +`evals/` in abcd-cli is the working prototype: a `smoke`-tagged Go harness that builds `abcd`, discovers the command tree via the exported root command, and smokes every command; wired into a `smoke` CI job, the release verify gate, and `make smoke`. This intent lifts that from a repo-local harness into an abcd capability every managed repo inherits, and adds the fixtures layer. Relates to the install surface (`ahoy`) and the "wired or it isn't done" principle. + +## Open Questions + +- Fixture format: one folder per command with an input corpus + an expected-shape file, or a single manifest describing scenarios? +- How synthetic data is generated and kept deterministic (seeded generators vs committed samples), and how a repo marks a fixture as containing only safe, non-sensitive data. +- Which commands are safe to run for real by default versus opt-in, and how a mutating command declares a scratch sandbox. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 870804ad..bb2e4dbd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -137,3 +137,22 @@ jobs: persist-credentials: false - name: Reviews-charter discipline (RD001-RD003) run: bash scripts/check-reviews.sh + + # Self-discovering smoke harness (evals/): builds the binary, walks the Cobra + # command tree in-process, and runs every command's --help plus the read-only + # verbs against the built binary. Catches a command that compiles but panics when + # actually invoked — which the unit lane (go test ./...) misses, because the + # harness is behind the `smoke` build tag. + smoke: + timeout-minutes: 10 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - name: Set up Go + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 + with: + go-version: '1.25' + - name: Smoke every command + run: make smoke diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f39f9739..d66cd32f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -94,6 +94,9 @@ jobs: - name: Reviews-charter discipline (RD001-RD003) run: bash scripts/check-reviews.sh + - name: Smoke every command (self-discovering harness) + run: make smoke + # Build and publish ONLY after verify is green. Binaries are cross-compiled from # the same pushed commit (github.sha) that verify just gated, so the shipped # artefacts match the code that passed — even if the tag were re-pointed since. diff --git a/Makefile b/Makefile index 5fa66cba..6026ecb6 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ VERSION ?= # for public distribution. LDFLAGS := -s -w$(if $(VERSION), -X github.com/REPPL/abcd-cli/internal/core.Version=$(VERSION),) -.PHONY: build test vet clean preflight lint-reviews record-lint docs-lint +.PHONY: build test vet clean preflight lint-reviews record-lint docs-lint smoke # Cross-compile every supported target to bin/abcd--. # Pass VERSION=vX.Y.Z to stamp the version (release builds); omit for a dev build. @@ -26,6 +26,13 @@ build: test: go test ./... +# Self-discovering smoke harness (evals/): build the binary, walk the Cobra tree, +# run every command's --help + the read-only verbs. Behind the `smoke` build tag so +# it stays out of the unit-test lane; run explicitly here, in CI's smoke job, and in +# the release verify gate. +smoke: + go test -tags smoke ./evals/... + vet: go vet ./... diff --git a/evals/README.md b/evals/README.md new file mode 100644 index 00000000..ee7b956d --- /dev/null +++ b/evals/README.md @@ -0,0 +1,35 @@ +# evals — smoke harness + +A self-discovering smoke test for the `abcd` binary. It builds the real binary, +walks the Cobra command tree **in-process** (via `cli.NewRootCommand()`) to +discover every command and flag, and exercises each against the built binary — so +a command added tomorrow is covered here with no edit. + +## What it checks (v1) + +- **Every** command and subcommand: `abcd --help` exits 0, produces output, + and never panics. This catches the failure unit tests miss — a command that + compiles but crashes when actually invoked. +- **Read-only, no-argument verbs** (`version`, the bare status board) run for real + to a graceful exit. +- **Flag hygiene:** an unknown flag is a clean non-zero error, not a panic. + +## Running it + +Gated behind the `smoke` build tag so it stays out of the fast unit-test lane: + +```bash +make smoke +# or +go test -tags smoke ./evals/... +``` + +CI runs it as the dedicated `smoke` job, and the release workflow smokes the +binary built from the tagged commit before publishing. + +## `data/` (reserved for v2) + +Fixture-driven, per-command scenarios — user-specified and synthetic inputs the +harness auto-discovers to drive richer smokes (e.g. `memory ingest` over a sample +corpus, `capture` round-trips). Deferred; the generalisation into an abcd-managed +eval framework is captured as intent **itd-75**. diff --git a/evals/data/README.md b/evals/data/README.md new file mode 100644 index 00000000..84f6e12a --- /dev/null +++ b/evals/data/README.md @@ -0,0 +1,9 @@ +# evals/data — fixtures (reserved for v2) + +Empty by design. This folder will hold user-specified and synthetic fixtures that +the smoke harness auto-discovers to drive richer, per-command scenarios beyond the +v1 structural smoke (help renders, no panic, read-only verbs run). + +Shape is deliberately undecided until v2 (see intent **itd-75**): likely one +subdirectory per command with an input corpus and an expected-shape assertion, so +adding a scenario is dropping a folder here — never editing Go. diff --git a/evals/smoke_test.go b/evals/smoke_test.go new file mode 100644 index 00000000..c4ad6fef --- /dev/null +++ b/evals/smoke_test.go @@ -0,0 +1,150 @@ +//go:build smoke + +// Package evals holds abcd's self-discovering smoke harness. It builds the real +// `abcd` binary, walks the Cobra command tree in-process to discover every +// command and flag (so a command added tomorrow is covered with no edit here), +// and exercises each one against the built binary. Gated behind the `smoke` build +// tag so it does not slow the unit-test lane; run it with: +// +// go test -tags smoke ./evals/... +// make smoke +// +// v1 smokes structure only (help renders, no panic, flags parse, read-only verbs +// run). Fixture-driven per-command scenarios (evals/data/) are future work — see +// intent itd-75. +package evals + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/REPPL/abcd-cli/internal/surface/cli" + "github.com/spf13/cobra" +) + +// abcdBin is the freshly-built binary under test, set once by TestMain. +var abcdBin string + +func TestMain(m *testing.M) { + dir, err := os.MkdirTemp("", "abcd-smoke") + if err != nil { + panic("smoke: mktemp: " + err.Error()) + } + defer os.RemoveAll(dir) + + abcdBin = filepath.Join(dir, "abcd") + // Build from the module root (this package lives at /evals). + build := exec.Command("go", "build", "-o", abcdBin, "./cmd/abcd") + build.Dir = ".." + build.Stdout, build.Stderr = os.Stderr, os.Stderr + if err := build.Run(); err != nil { + panic("smoke: build abcd: " + err.Error()) + } + os.Exit(m.Run()) +} + +// run executes the built binary and returns combined output + exit code. A +// non-zero exit is returned, not fataled — callers decide whether it is expected. +// Any failure to launch the process at all is fatal. +func run(t *testing.T, args ...string) (string, int) { + t.Helper() + out, err := exec.Command(abcdBin, args...).CombinedOutput() + if err == nil { + return string(out), 0 + } + if ee, ok := err.(*exec.ExitError); ok { + return string(out), ee.ExitCode() + } + t.Fatalf("could not launch `abcd %s`: %v", strings.Join(args, " "), err) + return "", -1 +} + +// commandArgs walks the Cobra tree from the real root command and returns each +// command as the arg slice needed to invoke it (root program name excluded). +// Includes hidden and auto-generated (help/completion) commands — their --help +// must render too. +func commandArgs() [][]string { + var paths [][]string + var walk func(c *cobra.Command, prefix []string) + walk = func(c *cobra.Command, prefix []string) { + for _, sub := range c.Commands() { + p := append(append([]string(nil), prefix...), sub.Name()) + paths = append(paths, p) + walk(sub, p) + } + } + walk(cli.NewRootCommand(), nil) + return paths +} + +// panicked reports whether output carries a Go runtime panic/stack trace — the +// failure mode this harness exists to catch (a command that compiles but crashes +// when actually invoked). +func panicked(out string) bool { + return strings.Contains(out, "panic:") || strings.Contains(out, "goroutine ") +} + +// TestEveryCommandHelpRenders is the core smoke: for every discovered command, +// `--help` must exit 0, produce output, and never panic. --help short-circuits +// before arg/flag validation in Cobra, so this is safe for commands with required +// args, yet still proves the command is wired and its help text builds. +func TestEveryCommandHelpRenders(t *testing.T) { + cmds := commandArgs() + if len(cmds) == 0 { + t.Fatal("discovered zero commands — the command tree walk is broken") + } + for _, p := range cmds { + p := p + t.Run(strings.Join(p, "/"), func(t *testing.T) { + args := append(append([]string(nil), p...), "--help") + out, code := run(t, args...) + if panicked(out) { + t.Fatalf("`abcd %s --help` panicked:\n%s", strings.Join(p, " "), out) + } + if code != 0 { + t.Errorf("`abcd %s --help` exit=%d\n%s", strings.Join(p, " "), code, out) + } + if strings.TrimSpace(out) == "" { + t.Errorf("`abcd %s --help` produced no output", strings.Join(p, " ")) + } + }) + } +} + +// TestReadOnlyVerbsRun executes the safe, no-argument, read-only verbs for real +// (not just --help), asserting they run to a graceful exit without panicking. +func TestReadOnlyVerbsRun(t *testing.T) { + cases := []struct { + args []string + wantZero bool // version/help must be 0; the bare status board may report non-zero + }{ + {[]string{"--help"}, true}, + {[]string{"version"}, true}, + {[]string{}, false}, // bare status board: no panic, any exit + } + for _, tc := range cases { + out, code := run(t, tc.args...) + label := "abcd " + strings.Join(tc.args, " ") + if panicked(out) { + t.Errorf("`%s` panicked:\n%s", label, out) + } + if tc.wantZero && code != 0 { + t.Errorf("`%s` exit=%d, want 0\n%s", label, code, out) + } + } +} + +// TestUnknownFlagIsGraceful proves an unknown flag is a clean non-zero error, not +// a panic — flag parsing must degrade gracefully on bad input. +func TestUnknownFlagIsGraceful(t *testing.T) { + out, code := run(t, "version", "--definitely-not-a-real-flag") + if panicked(out) { + t.Fatalf("unknown flag panicked:\n%s", out) + } + if code == 0 { + t.Errorf("unknown flag unexpectedly succeeded (exit 0):\n%s", out) + } +}