From 94968d72e151b1711f884ff913340470b28959c4 Mon Sep 17 00:00:00 2001 From: Patrick Tescher Date: Thu, 12 Nov 2026 07:40:38 -0800 Subject: [PATCH] A run can be held to every test having run A run passes when nothing failed, so tests the driver can't run yet, pending ones and ones that weren't run let a release gate through. A policy asks for more, and the default is what it was: - Policy::Required (expectation-run --strict): every test that isn't pending ran and passed - Policy::All (--strict=all): every test ran and passed, nothing pending - Report::meets and Report::incomplete, which lists each test that didn't run with its status and why, and the pending ones Required let by, apart - Acceptance::assert_passes_with for tests - a run of no tests meets neither: it verified nothing - expectation-run judges its own run's report, so a pass an earlier run saved doesn't stand in for a test that didn't run now Closes #78 Co-Authored-By: Claude --- book/src/drivers.md | 30 ++++ book/src/remote.md | 2 +- crates/harness/src/lib.rs | 29 +++- crates/harness/tests/strict.rs | 200 +++++++++++++++++++++++ crates/remote/src/bin/expectation-run.rs | 25 ++- crates/remote/tests/strict.rs | 122 ++++++++++++++ crates/story/src/lib.rs | 2 +- crates/story/src/report.rs | 94 +++++++++++ 8 files changed, 493 insertions(+), 11 deletions(-) create mode 100644 crates/harness/tests/strict.rs create mode 100644 crates/remote/tests/strict.rs diff --git a/book/src/drivers.md b/book/src/drivers.md index 7b14211..1bad593 100644 --- a/book/src/drivers.md +++ b/book/src/drivers.md @@ -162,6 +162,36 @@ A test whose start can't be set up (the reset or an arrangement fails) fails the other steps are skipped. The runner still asks for a screenshot, so the report shows what the system looked like when it went wrong. If there is none, the failure stands as it is. +### A run as a gate + +A run passes when nothing failed. Tests the driver can't run yet, pending ones and ones that +weren't run don't count against it, so a suite can be adopted a binding at a time. A release +gate needs more: that the tests ran. Ask for it with a policy. + +| Policy | Passes when | `expectation-run` | +|---|---|---| +| `Policy::Lenient` | nothing failed | the default | +| `Policy::Required` | every test that isn't pending ran and passed | `--strict` | +| `Policy::All` | every test ran and passed, with nothing pending | `--strict=all` | + +```rust +acceptance.assert_passes_with(&mut driver, &opts, Policy::Required); + +// Or with the report: +let report = acceptance.run(&mut driver, &opts); +if !report.meets(Policy::Required) { + eprint!("{}", report.failures(&acceptance.suite, &acceptance.plans)); + eprint!("{}", report.incomplete(&acceptance.plans, Policy::Required)); +} +``` + +- `report.incomplete` lists each test that didn't run with its status and why, and under + `Required` the pending ones it let by, apart. +- A run of no tests doesn't meet `Required` or `All`: it verified nothing. +- A policy judges the run that was just made. The report `publish` returns is merged with what + earlier runs saved, and a pass saved earlier doesn't stand in for a test that didn't run now. +- An expectation's approval is about the spec, not the run. It plays no part. + ### Several runners `Acceptance::publish(dir, runner, &report)` saves a run's results as diff --git a/book/src/remote.md b/book/src/remote.md index 3c55123..cf742db 100644 --- a/book/src/remote.md +++ b/book/src/remote.md @@ -166,7 +166,7 @@ screenshot "$1"'` takes screenshots from outside, for apps that don't take their (see [Screenshots in large projects](manual.md#screenshots-in-large-projects)). It exits 1 if a test failed and 0 otherwise, **including when every test was "not supported -yet"**. A failed screenshot is only a warning on stderr. Read the per-plan lines it prints +yet"**, unless the run is `--strict` (see [A run as a gate](drivers.md#a-run-as-a-gate)). A failed screenshot is only a warning on stderr. Read the per-plan lines it prints (`✓` passed, `✗` failed, `–` anything else) or the `plan` rows in `docs/results/@.tsv`, and check the ones you expect to pass, before calling anything verified. diff --git a/crates/harness/src/lib.rs b/crates/harness/src/lib.rs index d61a2c1..0f59334 100644 --- a/crates/harness/src/lib.rs +++ b/crates/harness/src/lib.rs @@ -25,7 +25,7 @@ use expectation_graph::Analysis; use expectation_language::Loaded; use expectation_model::diagnostic::{has_errors, render_all}; use expectation_model::{Diagnostic, Suite}; -pub use expectation_story::Control; +pub use expectation_story::{Control, Policy}; use expectation_story::{Plan, Report}; use std::path::Path; @@ -286,8 +286,22 @@ impl Acceptance { } /// Run every plan, write the docs to `opts.docs_dir` (if set) whatever happened, and panic - /// with what failed. For `#[test]`s. + /// with what failed. For `#[test]`s. Tests that aren't supported yet, are pending or weren't + /// run don't fail it: see [`Acceptance::assert_passes_with`] for a gate that wants them run. pub fn assert_passes(&self, driver: &mut dyn Driver, opts: &RunOptions) -> Report { + self.assert_passes_with(driver, opts, Policy::Lenient) + } + + /// As [`Acceptance::assert_passes`], held to `policy`: with [`Policy::Required`] it also + /// panics when a test that isn't pending didn't run (the driver can't yet, a service is + /// missing), and with [`Policy::All`] when anything is pending too. It judges this run: + /// results saved by earlier ones don't stand in for a test that didn't run now. + pub fn assert_passes_with( + &self, + driver: &mut dyn Driver, + opts: &RunOptions, + policy: Policy, + ) -> Report { let report = self.run(driver, opts); if let Some(dir) = &opts.docs_dir { self.docs(Some(&report)) @@ -296,8 +310,15 @@ impl Acceptance { } if !report.passed() { panic!( - "acceptance tests failed:\n{}", - report.failures(&self.suite, &self.plans) + "acceptance tests failed:\n{}{}", + report.failures(&self.suite, &self.plans), + report.incomplete(&self.plans, policy) + ); + } + if !report.meets(policy) { + panic!( + "acceptance tests didn't all run:\n{}", + report.incomplete(&self.plans, policy) ); } report diff --git a/crates/harness/tests/strict.rs b/crates/harness/tests/strict.rs new file mode 100644 index 0000000..c7042fd --- /dev/null +++ b/crates/harness/tests/strict.rs @@ -0,0 +1,200 @@ +//! A gate that wants the tests to have run: what isn't supported yet, or wasn't run, stops it, +//! where an ordinary run lets it by. + +use expectation_harness::{Acceptance, Bindings, Policy, RunOptions}; +use expectation_story::{Report, Status}; + +const LAMP: &str = " +Vocabulary: + State: the lamp is off | on + State: the dimmer is low | high + Event: the user flips the switch + Event: the user turns the dimmer +Background: + Given the lamp is off + And the dimmer is low +Feature: Lamp + Expectation: Flipping turns it on + Given the lamp is off + When the user flips the switch + Then the lamp is on + Expectation: Turning the dimmer brightens it + Given the dimmer is low + When the user turns the dimmer + Then the dimmer is high +Story: Light + When the user flips the switch + Then the lamp is on +Story: Bright + When the user turns the dimmer + Then the dimmer is high +"; + +const PENDING: &str = " +Feature: Dimming + Expectation: Turning the dimmer again dims it + Status: pending + Given the dimmer is high + When the user turns the dimmer + Then the dimmer is low +Story: Dim again + Given the dimmer is high + When the user turns the dimmer + Then the dimmer is low +"; + +fn lamp(src: &str) -> Acceptance { + let loaded = expectation_language::load_sources("lamp", &[("lamp.expect", src)]); + Acceptance::from_loaded(loaded).unwrap_or_else(|e| panic!("{e}")) +} + +/// A lamp whose dimmer the driver can turn, or not yet. +fn driver(dimmer: bool, broken: bool) -> Bindings<(bool, bool)> { + let b = Bindings::new((false, false)) + .observe( + "the lamp", + |w: &(bool, bool)| if w.0 { "on" } else { "off" }, + ) + .observe( + "the dimmer", + |w: &(bool, bool)| if w.1 { "high" } else { "low" }, + ) + .arrange("the lamp", |w, v| w.0 = v.to_string() == "on") + .arrange("the dimmer", |w, v| w.1 = v.to_string() == "high") + .on("the user flips the switch", move |w| w.0 = !broken) + .on_reset(|w| *w = (false, false)); + if dimmer { + b.on("the user turns the dimmer", |w| w.1 = true) + } else { + b + } +} + +fn status(a: &Acceptance, report: &Report, title: &str) -> Status { + let i = a.plans.iter().position(|p| p.title == title).unwrap(); + report.results[i].status +} + +#[test] +fn a_test_the_driver_cant_run_passes_an_ordinary_run_and_stops_a_strict_one() { + let a = lamp(LAMP); + let report = a.run(&mut driver(false, false), &RunOptions::default()); + assert_eq!(status(&a, &report, "Light"), Status::Passed); + assert_eq!(status(&a, &report, "Bright"), Status::Unsupported); + + // As it always was. + assert!(report.passed()); + assert!(report.meets(Policy::Lenient)); + assert_eq!(report.incomplete(&a.plans, Policy::Lenient), ""); + // Required execution is incomplete, and it says which test and why. + assert!(!report.meets(Policy::Required)); + assert!(!report.meets(Policy::All)); + let why = report.incomplete(&a.plans, Policy::Required); + assert!( + why.contains("– unsupported: Bright (The driver can't"), + "{why}" + ); + assert!(!why.contains("Light"), "{why}"); + + // With the dimmer bound, everything runs and every policy is met. + let report = a.run(&mut driver(true, false), &RunOptions::default()); + for policy in [Policy::Lenient, Policy::Required, Policy::All] { + assert!(report.meets(policy)); + assert_eq!(report.incomplete(&a.plans, policy), ""); + } +} + +#[test] +fn pending_is_let_by_unless_everything_is_asked_for_and_is_listed_apart() { + let a = lamp(&format!("{LAMP}{PENDING}")); + let report = a.run(&mut driver(true, false), &RunOptions::default()); + assert_eq!(status(&a, &report, "Dim again"), Status::Pending); + assert!(report.meets(Policy::Lenient)); + assert!(report.meets(Policy::Required)); + assert_eq!(report.incomplete(&a.plans, Policy::Required), ""); + assert!(!report.meets(Policy::All)); + let why = report.incomplete(&a.plans, Policy::All); + assert!( + why.contains("– pending: Dim again (Pending: it depends on"), + "{why}" + ); + + // With something else missing, the pending test is named, but not as what is missing. + let report = a.run(&mut driver(false, false), &RunOptions::default()); + let why = report.incomplete(&a.plans, Policy::Required); + let (missing, pending) = why.split_once("Pending, and not required:\n").expect(&why); + assert!(missing.contains("– unsupported: Bright"), "{why}"); + assert!(!missing.contains("Dim again"), "{why}"); + assert!(pending.contains("– pending: Dim again"), "{why}"); +} + +#[test] +fn a_failure_fails_every_policy_and_is_told_as_a_failure() { + let a = lamp(LAMP); + let report = a.run(&mut driver(true, true), &RunOptions::default()); + assert_eq!(status(&a, &report, "Light"), Status::Failed); + for policy in [Policy::Lenient, Policy::Required, Policy::All] { + assert!(!report.meets(policy)); + // What failed is in `failures`, once. + assert_eq!(report.incomplete(&a.plans, policy), ""); + } + assert!(report.failures(&a.suite, &a.plans).contains("✗ Light")); +} + +#[test] +fn what_other_runs_saved_doesnt_stand_in_for_a_test_that_didnt_run() { + let a = lamp(LAMP); + let before = a.run(&mut driver(true, false), &RunOptions::default()); + assert!(before.meets(Policy::Required)); + let dir = std::path::Path::new(env!("CARGO_TARGET_TMPDIR")).join("strict-merged"); + let _ = std::fs::remove_dir_all(&dir); + + // An earlier run saved a pass for every test; this one can't run the dimmer's. + a.publish(&dir, "earlier", &before).unwrap(); + let now = a.run(&mut driver(false, false), &RunOptions::default()); + let merged = a.publish(&dir, "now", &now).unwrap(); + // The manual has the earlier pass, and the gate still sees that this run didn't make it. + assert_eq!(status(&a, &merged, "Bright"), Status::Passed); + assert!(!now.meets(Policy::Required)); +} + +#[test] +fn a_run_of_no_tests_verified_nothing() { + let none = Report::default(); + assert!(none.passed()); + assert!(none.meets(Policy::Lenient)); + for policy in [Policy::Required, Policy::All] { + assert!(!none.meets(policy)); + assert!( + none.incomplete(&[], policy) + .contains("No tests were selected") + ); + } + // And tests that were never run aren't complete either. + let a = lamp(LAMP); + let not_run = Report::not_run(&a.plans); + assert!(not_run.passed()); + assert!(!not_run.meets(Policy::Required)); + assert!( + not_run + .incomplete(&a.plans, Policy::Required) + .contains("– not-run: Light") + ); +} + +#[test] +#[should_panic(expected = "acceptance tests didn't all run")] +fn a_strict_assertion_panics_when_a_test_didnt_run() { + lamp(LAMP).assert_passes_with( + &mut driver(false, false), + &RunOptions::default(), + Policy::Required, + ); +} + +#[test] +fn an_ordinary_assertion_still_lets_it_by() { + let a = lamp(LAMP); + let report = a.assert_passes(&mut driver(false, false), &RunOptions::default()); + assert_eq!(report.count(Status::Unsupported), 1); +} diff --git a/crates/remote/src/bin/expectation-run.rs b/crates/remote/src/bin/expectation-run.rs index 254ad59..8d7f5d8 100644 --- a/crates/remote/src/bin/expectation-run.rs +++ b/crates/remote/src/bin/expectation-run.rs @@ -3,7 +3,7 @@ //! ```text //! expectation-run [--platform NAME] [--addr 127.0.0.1:4477] [--wait SECONDS] //! [--docs DIR] [--runner NAME] [--settle MS] [--screenshot-cmd "COMMAND"] -//! [--attachments DIR] [--screenshot-store DIR] +//! [--attachments DIR] [--screenshot-store DIR] [--strict[=all]] //! ``` //! //! It listens, waits for the app to say hello, runs every story and a generated test for each @@ -15,9 +15,12 @@ //! `DIR////`; keep it outside `--docs`, which is for publishing. //! `--screenshot-store` keeps screenshots by content in `DIR` (`.png`), so identical //! frames are stored once and unchanged tests write nothing new; the docs take them from there. -//! Exits 1 if anything failed. +//! Exits 1 if anything failed. Tests that aren't supported yet, are pending or didn't run don't +//! fail it, unless it is `--strict`: then every test that isn't pending must have run and passed +//! in this run (`--strict=all`: every test, pending or not), and there must be at least one. +//! Results saved by earlier runs don't count towards it. -use expectation_harness::{Acceptance, RunOptions, Screenshot}; +use expectation_harness::{Acceptance, Policy, RunOptions, Screenshot}; use expectation_remote::{DEFAULT_ADDR, RemoteDriver}; use expectation_story::Status; use std::path::PathBuf; @@ -26,7 +29,7 @@ use std::time::Duration; const USAGE: &str = "usage: expectation-run [--platform NAME] [--addr HOST:PORT] [--wait SECONDS] \ [--command-timeout SECONDS] \ - [--stylesheet CSS] [--reader] [--state-shot-ratio W:H] [--docs DIR] [--runner NAME] [--settle MS] [--screenshot-cmd COMMAND] [--attachments DIR] [--screenshot-store DIR]"; + [--stylesheet CSS] [--reader] [--state-shot-ratio W:H] [--docs DIR] [--runner NAME] [--settle MS] [--screenshot-cmd COMMAND] [--attachments DIR] [--screenshot-store DIR] [--strict[=all]]"; fn main() { let mut args = std::env::args().skip(1); @@ -53,6 +56,7 @@ fn main() { ); let (mut stylesheet, mut ratio, mut attachments, mut store) = (None, None, None, None); let mut reader = false; + let mut policy = Policy::Lenient; while let Some(a) = args.next() { let mut value = || { args.next() @@ -87,6 +91,12 @@ fn main() { Some(expectation_harness::parse_ratio(&value()).unwrap_or_else(|e| fail(&e))) } "--reader" => reader = true, + "--strict" => policy = Policy::Required, + _ if a.starts_with("--strict=") => { + policy = Policy::parse(&a["--strict=".len()..]).unwrap_or_else(|| { + fail("--strict is --strict, --strict=required or --strict=all") + }) + } "--docs" => docs = Some(PathBuf::from(value())), "--runner" => runner = Some(value()), "--settle" => { @@ -196,11 +206,16 @@ fn main() { )), } } - if !report.passed() { + // This run's own report: what earlier runs saved doesn't stand in for a test that didn't run. + if !report.meets(policy) { eprint!( "\n{}", report.failures(&acceptance.suite, &acceptance.plans) ); + let incomplete = report.incomplete(&acceptance.plans, policy); + if !incomplete.is_empty() { + eprint!("\nNot every test ran and passed (--strict):\n{incomplete}"); + } exit(1); } } diff --git a/crates/remote/tests/strict.rs b/crates/remote/tests/strict.rs new file mode 100644 index 0000000..dccfe91 --- /dev/null +++ b/crates/remote/tests/strict.rs @@ -0,0 +1,122 @@ +//! `expectation-run --strict`: a test the app can't run yet exits 0 as it always did, and 1 +//! when the run is a gate. + +use expectation_remote::client::{Client, Handlers}; +use std::io::{BufRead, BufReader, Read}; +use std::path::Path; +use std::process::{Command, Stdio}; +use std::time::Duration; + +const SUITE: &str = " +Vocabulary: + State: the lamp is off | on + State: the dimmer is low | high + Event: the user flips the switch + Event: the user turns the dimmer +Background: + Given the lamp is off + And the dimmer is low +Feature: Lamp + Expectation: Flipping turns it on + Given the lamp is off + When the user flips the switch + Then the lamp is on + Expectation: Turning the dimmer brightens it + Given the dimmer is low + When the user turns the dimmer + Then the dimmer is high +Story: Light + When the user flips the switch + Then the lamp is on +Story: Bright + When the user turns the dimmer + Then the dimmer is high +"; + +/// Run the suite with the real runner and `args`, against a lamp that has a dimmer or not. +/// Says how it exited and what it wrote to stderr. +fn run(suite: &Path, dimmer: bool, args: &[&str]) -> (Option, String) { + let mut runner = Command::new(env!("CARGO_BIN_EXE_expectation-run")) + .arg(suite) + .args(["--addr", "127.0.0.1:0", "--settle", "0", "--wait", "20"]) + .args(args) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let mut said = BufReader::new(runner.stderr.take().unwrap()); + let mut line = String::new(); + said.read_line(&mut line).unwrap(); + let addr = line + .split("http://") + .nth(1) + .and_then(|s| s.split_whitespace().next()) + .unwrap_or_else(|| panic!("no address in {line:?}")) + .to_string(); + let mut h: Handlers<(bool, bool)> = Handlers::new() + .observe( + "the lamp", + |w: &(bool, bool)| if w.0 { "on" } else { "off" }, + ) + .observe( + "the dimmer", + |w: &(bool, bool)| { + if w.1 { "high" } else { "low" } + }, + ) + .arrange("the lamp", |w, v| w.0 = v.to_string() == "on") + .arrange("the dimmer", |w, v| w.1 = v.to_string() == "high") + .on("the user flips the switch", |w| w.0 = true) + .on_reset(|w| *w = (false, false)); + if dimmer { + h = h.on("the user turns the dimmer", |w| w.1 = true); + } + let client = Client::connect(&addr, h.hello("test", "Lamp")); + std::thread::spawn(move || { + let mut world = (false, false); + while let Some(r) = client.recv_timeout(Duration::from_secs(20)) { + h.handle(&mut world, r); + } + }); + let mut rest = String::new(); + said.read_to_string(&mut rest).unwrap(); + (runner.wait().unwrap().code(), rest) +} + +#[test] +fn a_strict_run_exits_1_when_a_test_didnt_run() { + let root = Path::new(env!("CARGO_TARGET_TMPDIR")).join("strict-run"); + let _ = std::fs::remove_dir_all(&root); + let (suite, docs) = (root.join("suite"), root.join("docs")); + std::fs::create_dir_all(&suite).unwrap(); + std::fs::write(suite.join("lamp.expect"), SUITE).unwrap(); + let docs = docs.to_str().unwrap(); + + // Everything runs: an earlier run saves a pass for every test. + for strict in [&[][..], &["--strict"], &["--strict=all"]] { + let (code, said) = run(&suite, true, &[strict, &["--docs", docs]].concat()); + assert_eq!(code, Some(0), "{said}"); + } + + // The app can't turn the dimmer. As it always was, that is not a failure. + let (code, said) = run(&suite, false, &["--docs", docs]); + assert_eq!(code, Some(0), "{said}"); + assert!(!said.contains("--strict"), "{said}"); + // A gate says which test didn't run, and the passes saved before don't stand in for it. + for strict in ["--strict", "--strict=required", "--strict=all"] { + let (code, said) = run(&suite, false, &[strict, "--docs", docs]); + assert_eq!(code, Some(1), "{said}"); + assert!( + said.contains("– unsupported: Bright (The driver can't"), + "{said}" + ); + assert!(!said.contains("Light"), "{said}"); + } + // A policy it doesn't know is a mistake in the command, before anything runs. + let unknown = Command::new(env!("CARGO_BIN_EXE_expectation-run")) + .arg(&suite) + .arg("--strict=sometimes") + .output() + .unwrap(); + assert_eq!(unknown.status.code(), Some(2)); +} diff --git a/crates/story/src/lib.rs b/crates/story/src/lib.rs index 4686348..481dd64 100644 --- a/crates/story/src/lib.rs +++ b/crates/story/src/lib.rs @@ -24,7 +24,7 @@ pub use plan::{ reaching_plans, untested, visit_plans, }; pub use report::{ - Attachment, CheckResult, PlanResult, Report, Run, Status, StepResult, Verdict, Window, + Attachment, CheckResult, PlanResult, Policy, Report, Run, Status, StepResult, Verdict, Window, fingerprint, load_results, load_results_for, now_ms, platform_shots, results_file, screenshots_named, }; diff --git a/crates/story/src/report.rs b/crates/story/src/report.rs index 7ae6b35..2c4a4ce 100644 --- a/crates/story/src/report.rs +++ b/crates/story/src/report.rs @@ -205,6 +205,47 @@ impl PlanResult { } } +/// What a run must show to count as done. [`Report::passed`] is the first: fine while a suite +/// is being adopted, when tests the driver can't run yet shouldn't stop anyone. The others are +/// for a gate (a release, a merge) that needs the tests to have run. +/// +/// A policy is about the report of the run that was just made. Results loaded from files and +/// merged say what other runs showed, so don't judge a merged report with one. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Policy { + /// Nothing failed. What isn't supported yet, is pending or wasn't run doesn't count against it. + #[default] + Lenient, + /// Every test ran and passed, but for the ones that depend on an expectation marked + /// `Status: pending`: those are behaviour that is known not to be built, and are listed + /// apart. A run of no tests doesn't meet it. + Required, + /// Every test ran and passed, with nothing pending. A run of no tests doesn't meet it. + All, +} + +impl Policy { + /// `required` or `all`, as `expectation-run --strict` takes them; `lenient` is the default. + pub fn parse(s: &str) -> Option { + match s { + "lenient" => Some(Policy::Lenient), + "required" => Some(Policy::Required), + "all" => Some(Policy::All), + _ => None, + } + } + + /// Does a test with this status count against the policy? + fn rejects(self, status: Status) -> bool { + match (self, status) { + (_, Status::Failed) => true, + (Policy::Lenient, _) | (_, Status::Passed) => false, + (Policy::Required, Status::Pending) => false, + _ => true, + } + } +} + /// One result per plan, in plan order. #[derive(Clone, Debug, Default)] pub struct Report { @@ -284,6 +325,59 @@ impl Report { !self.results.iter().any(|r| r.status == Status::Failed) } + /// Does this run meet `policy`? [`Policy::Lenient`] is [`Report::passed`]; the others also + /// want every test (or every test that isn't pending) to have run, and at least one test. + pub fn meets(&self, policy: Policy) -> bool { + (policy == Policy::Lenient || !self.results.is_empty()) + && !self.results.iter().any(|r| policy.rejects(r.status)) + } + + /// Why this run doesn't meet `policy`, for a failure message: each test that didn't run + /// (unsupported, pending, not run), with its status and the reason it gave, and then the + /// pending ones [`Policy::Required`] lets by, apart. Empty when the policy is met, and when + /// all that is wrong is what failed: that is in [`Report::failures`]. + pub fn incomplete(&self, plans: &[Plan], policy: Policy) -> String { + let mut s = String::new(); + if self.meets(policy) { + return s; + } + if self.results.is_empty() { + return "No tests were selected, so nothing was verified.\n".into(); + } + let line = |s: &mut String, plan: &Plan, r: &PlanResult| { + let _ = write!(s, "– {}: {}", r.status.name(), plan.title); + match r + .start + .error + .as_deref() + .filter(|_| r.status != Status::Failed) + { + Some(why) => { + let _ = writeln!(s, " ({why})"); + } + None => s.push('\n'), + } + }; + for (plan, r) in plans.iter().zip(&self.results) { + if policy.rejects(r.status) && r.status != Status::Failed { + line(&mut s, plan, r); + } + } + let allowed: Vec<_> = plans + .iter() + .zip(&self.results) + .filter(|(_, r)| r.status == Status::Pending && !policy.rejects(r.status)) + .filter(|_| !s.is_empty()) + .collect(); + if !allowed.is_empty() { + s.push_str("Pending, and not required:\n"); + for (plan, r) in allowed { + line(&mut s, plan, r); + } + } + s + } + pub fn count(&self, status: Status) -> usize { self.results.iter().filter(|r| r.status == status).count() }