diff --git a/README.md b/README.md index f9d0cee..410e5f3 100644 --- a/README.md +++ b/README.md @@ -389,9 +389,10 @@ Planning walks each scenario through the expectations: `Acceptance::with_coverage()` and `expectation plan --coverage` add generated tests that follow those routes, starting from the expectation's own section setup. An expectation with no route is tested on its own: its Given arranged on the section setup, then its When. - They also add a test for each state of the state map a move from a start that no other - test passes through ("Visiting the state after «…»"), so the map's first view has a - screenshot of every state. + `Acceptance::with_state_pictures()` and `expectation plan --state-pictures` add a test for + each state of the state map a move from a start that no other test passes through ("Visiting + the state after «…»"), so the map's first view has a screenshot of every state. They add + pictures, not verdicts, so a quick run can leave them out. - A test starts from its section's setup, with its scenario's Givens written over it. ## Running against the system under test diff --git a/book/src/cli.md b/book/src/cli.md index 93abbe1..bed4ae2 100644 --- a/book/src/cli.md +++ b/book/src/cli.md @@ -64,11 +64,13 @@ expectations behind each move, and the ways in. It works without JavaScript too. ```bash expectation plan expectation plan --coverage +expectation plan --coverage --state-pictures ``` Prints the acceptance test each story compiles to: what's arranged, each step, which expectations fire and what's checked. `--coverage` adds the generated tests for expectations -no story reaches. +no story reaches. `--state-pictures` adds the ones that visit a state only so the state map has +a picture of it (see [Stories](stories.md)). ## docs diff --git a/book/src/remote.md b/book/src/remote.md index cf742db..1fca2a5 100644 --- a/book/src/remote.md +++ b/book/src/remote.md @@ -159,6 +159,8 @@ expectation-run expectations --platform SwiftUI --docs docs --runner app `--stylesheet CSS` and `--state-shot-ratio W:H` style the generated docs as they do for `expectation docs` (see [the manual](manual.md#your-own-theme)); bad values fail before the run. `--command-timeout SECONDS` sets that per-command deadline (`--wait` only covers connecting). +`--no-state-pictures` leaves out the tests that only visit a state for the state map's picture +of it: a quicker run with the same verdicts (see [Stories](stories.md)). It listens, waits for the app, runs the suite for that platform, prints the results, and with `--docs` saves them and writes the manual. `--screenshot-cmd 'xcrun simctl io booted screenshot "$1"'` takes screenshots from outside, for apps that don't take their own. diff --git a/book/src/stories.md b/book/src/stories.md index 5641c6d..7bea0d5 100644 --- a/book/src/stories.md +++ b/book/src/stories.md @@ -62,6 +62,22 @@ each section that `Follows:` a setup the expectations get from, taking that rout checking the section's setup at the end. Generated tests are named "Reaching «…»". They appear in the test report, but not as tutorials. +`Acceptance::with_state_pictures()` (and `expectation plan --state-pictures`) adds another +kind, named "Visiting the state after «…»": one for each state of the state map that is a move +from a start and that no other test passes through. They add no verdicts. They are there so the +state map has a screenshot of every state it shows at first. Each one arranges a start of its +own, so on a system that is slow to set up they can be most of a run: leave them out of a +quick run, and add them, after `with_coverage()`, when the run is for the manual. + +```rust +let quick = Acceptance::load("expectations")?.with_coverage(); +let for_the_manual = Acceptance::load("expectations")?.with_coverage().with_state_pictures(); +``` + +`expectation-run` has both, and `--no-state-pictures` leaves the second out. A runner's results +file holds its last run, so give a quick run a `--runner` name of its own if a full run's +pictures should stay in the manual. + With coverage, every expectation gets a **verdict**: | Verdict | Means | diff --git a/crates/cli/src/lib.rs b/crates/cli/src/lib.rs index 750919f..1475856 100644 --- a/crates/cli/src/lib.rs +++ b/crates/cli/src/lib.rs @@ -20,8 +20,11 @@ commands: print the map of the app's states and the moves between them, as a Mermaid state diagram or Graphviz DOT; --by names the states that tell them apart, comma separated (default: every enumerated one) - plan [--coverage] print the acceptance tests the stories compile to - (--coverage adds generated tests for untested expectations) + plan [--coverage] [--state-pictures] + print the acceptance tests the stories compile to + (--coverage adds generated tests for untested expectations, + --state-pictures the ones that visit a state for the state map's + picture of it) docs [--approved-only] [--reader] [--results ] [--stylesheet ] [--state-shot-ratio ] [--screenshots ] [--section ]... [--base-url ] [--expect-app-revision ] [--expect-data-mode ] [--expect-fixture ] @@ -193,6 +196,8 @@ pub fn main() { if flag("--coverage") { let extra = expectation_story::coverage_plans(&c.suite, &c.analysis, &c.plans); c.plans.extend(extra); + } + if flag("--state-pictures") { let visits = expectation_story::visit_plans(&c.suite, &c.analysis, &c.plans); c.plans.extend(visits); } diff --git a/crates/harness/src/lib.rs b/crates/harness/src/lib.rs index 7d01209..883ac3c 100644 --- a/crates/harness/src/lib.rs +++ b/crates/harness/src/lib.rs @@ -150,12 +150,21 @@ impl Acceptance { }) } - /// Add generated tests for the expectations no story exercises. + /// Add generated tests for the expectations no story exercises ("Reaching «…»"): with + /// them, every expectation has a verdict. For the state map's pictures as well, add + /// [`Acceptance::with_state_pictures`] after it. pub fn with_coverage(mut self) -> Self { let extra = expectation_story::coverage_plans(&self.suite, &self.analysis, &self.plans); self.plans.extend(extra); - // And a test for each state a move from a start that no test passes through, so the - // state map has a screenshot of every state it shows at first. + self + } + + /// Add a test for each state a move from a start that no test passes through ("Visiting + /// the state after «…»"), so the state map has a screenshot of every state it shows at + /// first. They add pictures, not verdicts, and each arranges a start of its own, so a quick + /// run can leave them out and a docs build can have them. Call it last: it adds only the + /// states the tests there are already don't pass through. + pub fn with_state_pictures(mut self) -> Self { let visits = expectation_story::visit_plans(&self.suite, &self.analysis, &self.plans); self.plans.extend(visits); self diff --git a/crates/harness/tests/state_pictures.rs b/crates/harness/tests/state_pictures.rs new file mode 100644 index 0000000..2846a3f --- /dev/null +++ b/crates/harness/tests/state_pictures.rs @@ -0,0 +1,117 @@ +//! The tests that only visit a state, for the state map's picture of it, are asked for apart +//! from the ones that give every expectation a verdict. + +use expectation_harness::{Acceptance, Bindings, RunOptions}; + +/// The story reaches both expectations, but flips the switch with the dimmer high: the lamp on +/// with the dimmer low is a state one move from the start that no test passes through. +const LAMP: &str = " +Vocabulary: + State: the lamp is off | on + State: the dimmer is low | high + Event: the user flips the switch + Event: the user turns the dimmer + Event: the user turns it back +Background: + Given the lamp is off + And the dimmer is low +Feature: Lamp + Expectation: Flipping turns it on + Given the lamp is off + When the user flips the switch + Then the lamp is on + Expectation: Turning the dimmer brightens it + Given the dimmer is low + When the user turns the dimmer + Then the dimmer is high + Expectation: Turning it back dims it + Given the dimmer is high + When the user turns it back + Then the dimmer is low +Story: Bright light + When the user turns the dimmer + Then the dimmer is high + When the user flips the switch + Then the lamp is on +"; + +fn lamp() -> Acceptance { + let loaded = expectation_language::load_sources("lamp", &[("lamp.expect", LAMP)]); + Acceptance::from_loaded(loaded).unwrap_or_else(|e| panic!("{e}")) +} + +fn driver() -> Bindings<(bool, bool)> { + Bindings::new((false, false)) + .observe( + "the lamp", + |w: &(bool, bool)| if w.0 { "on" } else { "off" }, + ) + .observe( + "the dimmer", + |w: &(bool, bool)| { + if w.1 { "high" } else { "low" } + }, + ) + .arrange("the lamp", |w, v| w.0 = v.to_string() == "on") + .arrange("the dimmer", |w, v| w.1 = v.to_string() == "high") + .on("the user flips the switch", |w| w.0 = true) + .on("the user turns the dimmer", |w| w.1 = true) + .on("the user turns it back", |w| w.1 = false) + .on_reset(|w| *w = (false, false)) +} + +fn titles(a: &Acceptance) -> Vec<&str> { + a.plans.iter().map(|p| p.title.as_str()).collect() +} + +#[test] +fn coverage_gives_the_verdicts_and_state_pictures_are_asked_for_apart() { + let stories = lamp(); + assert_eq!(titles(&stories), ["Bright light"]); + + // Coverage: a test for the expectation no story reaches, and no more. + let covered = lamp().with_coverage(); + assert_eq!( + titles(&covered), + ["Bright light", "Reaching «Turning it back dims it»"] + ); + + // The pictures: the states a move from the start that those tests don't pass through. + let pictured = lamp().with_coverage().with_state_pictures(); + let visits: Vec<&str> = titles(&pictured)[covered.plans.len()..].to_vec(); + assert!(!visits.is_empty()); + assert!( + visits + .iter() + .all(|t| t.starts_with("Visiting the state after «")), + "{visits:?}" + ); + // The tests they are added to are the same ones, in the same order. + assert_eq!( + titles(&pictured)[..covered.plans.len()], + titles(&covered)[..] + ); + // Asked for twice, there is nothing more to visit. + let again = lamp() + .with_coverage() + .with_state_pictures() + .with_state_pictures(); + assert_eq!(titles(&again), titles(&pictured)); + + // They add pictures, not verdicts: every expectation's is the same without them. + let quick = covered.run(&mut driver(), &RunOptions::default()); + let full = pictured.run(&mut driver(), &RunOptions::default()); + assert!(quick.passed() && full.passed()); + assert_eq!( + quick.verdicts(&covered.suite, &covered.plans), + full.verdicts(&pictured.suite, &pictured.plans) + ); +} + +#[test] +fn state_pictures_alone_visit_what_the_stories_dont() { + let a = lamp().with_state_pictures(); + assert_eq!(a.plans[0].title, "Bright light"); + assert!(a.plans.len() > 1); + assert!(a.plans[1..].iter().all(|p| p.title.starts_with("Visiting"))); +} diff --git a/crates/remote/src/bin/expectation-run.rs b/crates/remote/src/bin/expectation-run.rs index d94291c..3a12bdc 100644 --- a/crates/remote/src/bin/expectation-run.rs +++ b/crates/remote/src/bin/expectation-run.rs @@ -4,6 +4,7 @@ //! expectation-run [--platform NAME] [--addr 127.0.0.1:4477] [--wait SECONDS] //! [--docs DIR] [--runner NAME] [--settle MS] [--screenshot-cmd "COMMAND"] //! [--attachments DIR] [--screenshot-store DIR] [--strict[=all]] +//! [--no-state-pictures] //! [--app-revision REV] [--data-mode MODE] [--fixture NAME] //! ``` //! @@ -16,6 +17,8 @@ //! `DIR////`; keep it outside `--docs`, which is for publishing. //! `--screenshot-store` keeps screenshots by content in `DIR` (`.png`), so identical //! frames are stored once and unchanged tests write nothing new; the docs take them from there. +//! `--no-state-pictures` leaves out the tests that only visit a state for the state map's +//! picture of it: a quicker run with the same verdicts. //! `--app-revision`, `--data-mode` and `--fixture` say what the run is against (a commit, `fixtures`, //! the fixture's name): they are saved with the results and shown in the manual, which is //! written for the same, so results other runs saved for another build are labelled as such. @@ -34,7 +37,7 @@ use std::time::Duration; const USAGE: &str = "usage: expectation-run [--platform NAME] [--addr HOST:PORT] [--wait SECONDS] \ [--command-timeout SECONDS] \ - [--stylesheet CSS] [--reader] [--state-shot-ratio W:H] [--docs DIR] [--runner NAME] [--settle MS] [--screenshot-cmd COMMAND] [--attachments DIR] [--screenshot-store DIR] [--strict[=all]] [--app-revision REV] [--data-mode MODE] [--fixture NAME]"; + [--stylesheet CSS] [--reader] [--state-shot-ratio W:H] [--docs DIR] [--runner NAME] [--settle MS] [--screenshot-cmd COMMAND] [--attachments DIR] [--screenshot-store DIR] [--strict[=all]] [--no-state-pictures] [--app-revision REV] [--data-mode MODE] [--fixture NAME]"; fn main() { let mut args = std::env::args().skip(1); @@ -63,6 +66,7 @@ fn main() { let mut reader = false; let mut provenance = expectation_harness::Provenance::default(); let mut policy = Policy::Lenient; + let mut state_pictures = true; while let Some(a) = args.next() { let mut value = || { args.next() @@ -97,6 +101,7 @@ fn main() { Some(expectation_harness::parse_ratio(&value()).unwrap_or_else(|e| fail(&e))) } "--reader" => reader = true, + "--no-state-pictures" => state_pictures = false, "--app-revision" => provenance.app_revision = Some(value()), "--data-mode" => provenance.data_mode = Some(value()), "--fixture" => provenance.fixture = Some(value()), @@ -135,6 +140,9 @@ fn main() { }) .with_coverage(); let mut acceptance = acceptance; + if state_pictures { + acceptance = acceptance.with_state_pictures(); + } if let Some(p) = &stylesheet { acceptance = acceptance.with_stylesheet(p).unwrap_or_else(|e| fail(&e)); } diff --git a/crates/remote/tests/state_pictures.rs b/crates/remote/tests/state_pictures.rs new file mode 100644 index 0000000..b2411b8 --- /dev/null +++ b/crates/remote/tests/state_pictures.rs @@ -0,0 +1,108 @@ +//! `expectation-run --no-state-pictures`: the same verdicts, without the tests that only visit +//! a state for the state map's picture of it. + +use expectation_remote::client::{Client, Handlers}; +use std::io::{BufRead, BufReader, Read}; +use std::path::Path; +use std::process::{Command, Stdio}; +use std::time::Duration; + +/// The story flips the switch with the dimmer high, so the lamp on with the dimmer low is a +/// state one move from the start that no story passes through. +const SUITE: &str = " +Vocabulary: + State: the lamp is off | on + State: the dimmer is low | high + Event: the user flips the switch + Event: the user turns the dimmer +Background: + Given the lamp is off + And the dimmer is low +Feature: Lamp + Expectation: Flipping turns it on + Given the lamp is off + When the user flips the switch + Then the lamp is on + Expectation: Turning the dimmer brightens it + Given the dimmer is low + When the user turns the dimmer + Then the dimmer is high +Story: Bright light + When the user turns the dimmer + Then the dimmer is high + When the user flips the switch + Then the lamp is on +"; + +/// The lines the runner prints, one per test. +fn run(suite: &Path, args: &[&str]) -> Vec { + let mut runner = Command::new(env!("CARGO_BIN_EXE_expectation-run")) + .arg(suite) + .args(["--addr", "127.0.0.1:0", "--settle", "0", "--wait", "20"]) + .args(args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let mut said = BufReader::new(runner.stderr.take().unwrap()); + let mut line = String::new(); + said.read_line(&mut line).unwrap(); + let addr = line + .split("http://") + .nth(1) + .and_then(|s| s.split_whitespace().next()) + .unwrap_or_else(|| panic!("no address in {line:?}")) + .to_string(); + let h: Handlers<(bool, bool)> = Handlers::new() + .observe( + "the lamp", + |w: &(bool, bool)| if w.0 { "on" } else { "off" }, + ) + .observe( + "the dimmer", + |w: &(bool, bool)| { + if w.1 { "high" } else { "low" } + }, + ) + .arrange("the lamp", |w, v| w.0 = v.to_string() == "on") + .arrange("the dimmer", |w, v| w.1 = v.to_string() == "high") + .on("the user flips the switch", |w| w.0 = true) + .on("the user turns the dimmer", |w| w.1 = true) + .on_reset(|w| *w = (false, false)); + let client = Client::connect(&addr, h.hello("test", "Lamp")); + std::thread::spawn(move || { + let mut world = (false, false); + while let Some(r) = client.recv_timeout(Duration::from_secs(20)) { + h.handle(&mut world, r); + } + }); + let mut out = String::new(); + runner + .stdout + .take() + .unwrap() + .read_to_string(&mut out) + .unwrap(); + assert!(runner.wait().unwrap().success(), "{line}"); + out.lines().map(String::from).collect() +} + +#[test] +fn a_run_can_leave_the_state_pictures_out() { + let root = Path::new(env!("CARGO_TARGET_TMPDIR")).join("state-pictures-run"); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(&root).unwrap(); + std::fs::write(root.join("lamp.expect"), SUITE).unwrap(); + + let full = run(&root, &[]); + let quick = run(&root, &["--no-state-pictures", "--strict"]); + assert_eq!(quick, ["✓ Bright light"]); + assert_eq!(full[..quick.len()], quick[..]); + assert!(full.len() > quick.len()); + assert!( + full[quick.len()..] + .iter() + .all(|l| l.starts_with("✓ Visiting the state after «")), + "{full:?}" + ); +}