From a191c98160e52a26d3d217021d150ad0bac018ac Mon Sep 17 00:00:00 2001 From: Patrick Tescher Date: Thu, 12 Nov 2026 11:37:04 -0800 Subject: [PATCH] A test starts where the last one ended when its Givens hold there When a test has passed and everything the next one is given is observed to hold already, the runner skips the reset and the arrangement. It picks a test that starts where the last one ended when there is one; results stay in plan order. Each result records how it began, and the manual says when a test continued. Driver::may_continue, Bindings::always_reset, RunOptions::share_starts and expectation-run --no-shared-starts turn it off. Refs #67 Co-Authored-By: Claude --- book/src/drivers.md | 32 ++- book/src/remote.md | 3 + crates/docs/src/html.rs | 3 + crates/docs/src/lib.rs | 3 + crates/docs/src/markdown.rs | 3 + crates/harness/src/bindings.rs | 13 ++ crates/harness/src/driver.rs | 8 + crates/harness/src/lib.rs | 2 +- crates/harness/src/run.rs | 190 +++++++++++++---- crates/harness/tests/shared_starts.rs | 260 +++++++++++++++++++++++ crates/remote/src/bin/expectation-run.rs | 16 +- crates/remote/tests/shared_starts.rs | 114 ++++++++++ crates/story/src/lib.rs | 2 +- crates/story/src/report.rs | 37 ++++ 14 files changed, 635 insertions(+), 51 deletions(-) create mode 100644 crates/harness/tests/shared_starts.rs create mode 100644 crates/remote/tests/shared_starts.rs diff --git a/book/src/drivers.md b/book/src/drivers.md index 1bad593..a70f693 100644 --- a/book/src/drivers.md +++ b/book/src/drivers.md @@ -125,6 +125,7 @@ pub trait Driver { fn needs(&self) -> Vec { vec![] } fn fresh(&self) -> Vec { vec![] } fn reset(&mut self) -> Result<(), String> { Ok(()) } + fn may_continue(&self) -> bool { true } fn arrange(&mut self, var: &Variable, value: &Value) -> Result<(), String> { self.set(var, value) } fn set(&mut self, var: &Variable, value: &Value) -> Result<(), String>; fn perform(&mut self, event: &Event) -> Result<(), String>; @@ -162,6 +163,34 @@ A test whose start can't be set up (the reset or an arrangement fails) fails the other steps are skipped. The runner still asks for a screenshot, so the report shows what the system looked like when it went wrong. If there is none, the failure stands as it is. +### Starting where the last test ended + +Resetting and arranging a start can be the slow part of a test. So when a test has passed, and +everything the next one is given already holds, the runner starts the next one there: no +reset, nothing arranged. It decides by looking. It observes each thing the test is given, +once, and only continues if all of them hold as they are. It never continues from a test that +failed, into a test that is given nothing, or across a `Launch:` that starts the program +again. + +To make the most of this, the runner picks the next test: one that starts where the last one +ended if there is one, and otherwise the next in order. The report stays in the order of the +tests. Each result says how the test began (`PlanResult::began`, a `began` line in the saved +results), and the manual's page for a test that continued says so. If a share looks wrong, +that is where to look first. + +The check only sees what the suite describes. Say no when a test leaves something behind that +no state of the suite covers and that the next test could notice: a log, a clock, a cache, a +counter. + +```rust,ignore +fn may_continue(&self) -> bool { false } // a Driver +Bindings::new(world).always_reset() // Bindings +RunOptions { share_starts: false, ..Default::default() } // one run +``` + +A driver that asks for services afresh before each test (`fresh`) never continues: those +services must not keep what the last test left. + ### A run as a gate A run passes when nothing failed. Tests the driver can't run yet, pending ones and ones that @@ -218,5 +247,6 @@ let report = acceptance.run_parallel(4, &RunOptions::default(), |n| { }); ``` -Workers take the next test as they finish one, and the report is in the order of the tests. +Workers take a test as they finish one (one that starts where their last one ended, if there +is one), and the report is in the order of the tests. The closure runs on the worker's thread, so a driver need not be `Send`. diff --git a/book/src/remote.md b/book/src/remote.md index 1fca2a5..6dd4869 100644 --- a/book/src/remote.md +++ b/book/src/remote.md @@ -161,6 +161,9 @@ expectation-run expectations --platform SwiftUI --docs docs --runner app `--command-timeout SECONDS` sets that per-command deadline (`--wait` only covers connecting). `--no-state-pictures` leaves out the tests that only visit a state for the state map's picture of it: a quicker run with the same verdicts (see [Stories](stories.md)). +A test starts where the last one ended when what it is given already holds there, and is +printed with `(continued)`; `--no-shared-starts` resets before every test (see +[Drivers](drivers.md#starting-where-the-last-test-ended)). It listens, waits for the app, runs the suite for that platform, prints the results, and with `--docs` saves them and writes the manual. `--screenshot-cmd 'xcrun simctl io booted screenshot "$1"'` takes screenshots from outside, for apps that don't take their own. diff --git a/crates/docs/src/html.rs b/crates/docs/src/html.rs index bf68d4a..cd3ae6f 100644 --- a/crates/docs/src/html.rs +++ b/crates/docs/src/html.rs @@ -1856,6 +1856,9 @@ impl Site<'_> { } let _ = write!(o, r#"

{}

"#, line.trim_end()); } + if result.and_then(|r| r.began) == Some(expectation_story::Began::Continued) { + let _ = write!(o, r#"

{}

"#, esc(crate::CONTINUED)); + } if let Some(e) = result.and_then(|r| r.start.error.as_ref()) { let class = if status == Status::Unsupported { "note warn" diff --git a/crates/docs/src/lib.rs b/crates/docs/src/lib.rs index 0c381ab..f72c715 100644 --- a/crates/docs/src/lib.rs +++ b/crates/docs/src/lib.rs @@ -28,6 +28,9 @@ mod states; mod thumbs; mod voice; +/// What a test's page says when it began where another ended (`Began::Continued`). +pub(crate) const CONTINUED: &str = "This run started where the test that ran before it had ended: everything it is given was seen to hold there, so nothing was reset or arranged."; + use expectation_graph::Analysis; use expectation_model::{ExpId, Feature, StepText, Suite, VarKind, Variable}; use expectation_story::{Because, Plan, Report, Status, Verdict, coverage}; diff --git a/crates/docs/src/markdown.rs b/crates/docs/src/markdown.rs index c29a24a..6cdf9cb 100644 --- a/crates/docs/src/markdown.rs +++ b/crates/docs/src/markdown.rs @@ -1067,6 +1067,9 @@ fn test_page(d: &Docs, f: &Facts, i: usize) -> String { } let _ = writeln!(o, "{}\n", line.trim_end()); } + if result.and_then(|r| r.began) == Some(expectation_story::Began::Continued) { + let _ = writeln!(o, "{}\n", crate::CONTINUED); + } if let Some(e) = result.and_then(|r| r.start.error.as_ref()) { let _ = writeln!(o, "> {e}\n"); } diff --git a/crates/harness/src/bindings.rs b/crates/harness/src/bindings.rs index 7ea7530..3824bf4 100644 --- a/crates/harness/src/bindings.rs +++ b/crates/harness/src/bindings.rs @@ -39,6 +39,7 @@ pub struct Bindings { attachments: Option>, needs: Vec, fresh: Vec, + always_reset: bool, } impl Bindings { @@ -58,9 +59,17 @@ impl Bindings { attachments: None, needs: vec![], fresh: vec![], + always_reset: false, } } + /// Reset before every test, even one that could start where the last one ended (see + /// [`Driver::may_continue`]): for a world that keeps what no state of the suite describes. + pub fn always_reset(mut self) -> Self { + self.always_reset = true; + self + } + /// The services to start afresh before each test (a subset of those it needs). pub fn fresh(mut self, services: &[&str]) -> Self { self.fresh = services.iter().map(|s| s.to_string()).collect(); @@ -156,6 +165,10 @@ impl Driver for Bindings { self.fresh.clone() } + fn may_continue(&self) -> bool { + !self.always_reset + } + fn reset(&mut self) -> Result<(), String> { if let Some(f) = &mut self.reset { f(&mut self.world); diff --git a/crates/harness/src/driver.rs b/crates/harness/src/driver.rs index 744e329..1b720e4 100644 --- a/crates/harness/src/driver.rs +++ b/crates/harness/src/driver.rs @@ -105,6 +105,14 @@ pub trait Driver { Ok(()) } + /// May a test start where the last one ended, without a reset? The runner only does so when + /// the last test passed and everything this one is given is observed to hold already, so + /// say no when that isn't enough: when what a test leaves behind that no state of the + /// suite describes (a log, a clock, a cache) could change what the next one sees. + fn may_continue(&self) -> bool { + true + } + /// Make `var` hold `value` before the first step ("Given …"). fn arrange(&mut self, var: &Variable, value: &Value) -> Result<(), String> { self.set(var, value) diff --git a/crates/harness/src/lib.rs b/crates/harness/src/lib.rs index 883ac3c..989115c 100644 --- a/crates/harness/src/lib.rs +++ b/crates/harness/src/lib.rs @@ -25,7 +25,7 @@ use expectation_graph::Analysis; use expectation_language::Loaded; use expectation_model::diagnostic::{has_errors, render_all}; use expectation_model::{Diagnostic, Suite}; -pub use expectation_story::{Control, Policy, Provenance}; +pub use expectation_story::{Began, Control, Policy, Provenance}; use expectation_story::{Plan, Report}; use std::path::Path; diff --git a/crates/harness/src/run.rs b/crates/harness/src/run.rs index 77bcaf8..2147c87 100644 --- a/crates/harness/src/run.rs +++ b/crates/harness/src/run.rs @@ -4,8 +4,8 @@ use crate::driver::{Driver, Need, Progress}; use crate::service::Services; use expectation_model::{Suite, Trigger}; use expectation_story::{ - Attachment, Check, CheckResult, Control, Plan, PlanResult, Report, Run, Status, StepResult, - Window, now_ms, + Attachment, Began, Check, CheckResult, Control, Plan, PlanResult, Report, Run, Status, + StepResult, Window, now_ms, }; use std::path::PathBuf; use std::sync::Mutex; @@ -43,6 +43,13 @@ pub struct RunOptions { /// a manual can say which build a saved pass was about. Say only what may be published; /// nothing is filled in for you. pub provenance: expectation_story::Provenance, + /// Let a test start where the last one ended, when that test passed and everything this one + /// is given is observed to hold there already: no reset, nothing arranged. Tests are then + /// run in an order that makes the most of it (the results stay in the plans' order), and + /// each result says how it began ([`PlanResult::began`]). On by default; a driver can + /// refuse it for itself ([`Driver::may_continue`]), and one that wants services afresh + /// for each test ([`Driver::fresh`]) never continues. + pub share_starts: bool, } impl Default for RunOptions { @@ -55,21 +62,94 @@ impl Default for RunOptions { controls: true, attachments_dir: None, provenance: Default::default(), + share_starts: true, } } } -pub fn run(suite: &Suite, plans: &[Plan], driver: &mut dyn Driver, opts: &RunOptions) -> Report { - let mut launched = None; - let run = Run::start().against(opts.provenance.clone()); +/// What a driver has done so far in a run, as far as its next test is concerned. +#[derive(Default)] +struct Session { + /// How the program was last started (a section's `Launch:`), so it is started once for the + /// plans that share it and again when a section says to start it differently. + launched: Option, + /// The last test it ran, if that test passed: the system is where that test ends. + last: Option, +} + +impl Session { + /// Would `plan` be started the way the program is running now? + fn launched_for(&self, suite: &Suite, plan: &Plan) -> bool { + suite + .launch_of(plan.section) + .is_none_or(|how| self.launched.as_deref() == Some(how)) + } + + /// After `plans[at]` gave `result`: a pass leaves the system where that test ends, a test + /// that wasn't run leaves it where it was, and after anything else it could be anywhere. + fn after(&mut self, at: usize, result: &PlanResult) { + match result.status { + Status::Passed => self.last = Some(at), + Status::Pending | Status::Unsupported => {} + _ => self.last = None, + } + } +} + +/// The test to run next, taken out of `left`: one that starts where the last one ended, so it +/// may need no reset, or else the first that is left. +fn take( + suite: &Suite, + plans: &[Plan], + left: &mut Vec, + session: &Session, + share: bool, +) -> Option { + let follows = session.last.filter(|_| share).and_then(|last| { + let last = &plans[last]; + let end = last.steps.last().map_or(&last.start, |s| &s.state); + left.iter().position(|&i| { + let p = &plans[i]; + !p.pending + && !p.start_checks.is_empty() + && session.launched_for(suite, p) + && end.implies(&p.start) + }) + }); + (!left.is_empty()).then(|| left.remove(follows.unwrap_or(0))) +} + +/// Run every plan, each chosen by [`take`], and give the results in the plans' order. +fn run_each( + suite: &Suite, + plans: &[Plan], + driver: &mut dyn Driver, + opts: &RunOptions, + mut one: impl FnMut(&Plan, &mut dyn Driver, &mut Session) -> PlanResult, +) -> Report { + let mut session = Session::default(); + let mut left: Vec = (0..plans.len()).collect(); + let mut results: Vec> = plans.iter().map(|_| None).collect(); + while let Some(i) = { + let share = opts.share_starts && driver.may_continue(); + take(suite, plans, &mut left, &session, share) + } { + let result = one(&plans[i], driver, &mut session); + session.after(i, &result); + results[i] = Some(result); + } Report { - results: plans - .iter() - .map(|p| run_plan(suite, p, driver, opts, &mut launched, &run)) - .collect(), + results: results.into_iter().flatten().collect(), } } +pub fn run(suite: &Suite, plans: &[Plan], driver: &mut dyn Driver, opts: &RunOptions) -> Report { + let run = Run::start().against(opts.provenance.clone()); + run_each(suite, plans, driver, opts, |plan, driver, session| { + run_plan(suite, plan, driver, opts, session, &run) + }) +} + /// As [`run`], with what the driver needs from outside started before each plan: a service that /// is up stays, one that died is started again, and when one cannot be had the plan is not /// supported yet, with what to do about it. @@ -80,14 +160,10 @@ pub fn run_with( opts: &RunOptions, services: &mut Services, ) -> Report { - let mut launched = None; let run = Run::start().against(opts.provenance.clone()); - Report { - results: plans - .iter() - .map(|plan| run_served(suite, plan, driver, opts, services, &mut launched, &run)) - .collect(), - } + run_each(suite, plans, driver, opts, |plan, driver, session| { + run_served(suite, plan, driver, opts, services, session, &run) + }) } /// What one worker of [`run_parallel`] runs with: a driver of its own, and the services that @@ -114,7 +190,8 @@ impl Worker { /// As [`run_with`], on `workers` threads that take the next plan as they finish one. `make` is /// called once on each worker's thread, with the worker's number (from 0), to build its /// [`Worker`], so a driver need not be `Send` and two workers never share one: give each its own -/// app, port, database and screen. Results are in the order of `plans`. `workers` is at least 1 +/// app, port, database and screen. A worker takes a plan that starts where its last one ended +/// when there is one (see [`RunOptions::share_starts`]). Results are in the order of `plans`. `workers` is at least 1 /// and at most the number of plans. pub fn run_parallel( suite: &Suite, @@ -126,20 +203,23 @@ pub fn run_parallel( where F: Fn(usize) -> Worker + Sync, { - let next = AtomicUsize::new(0); + let left: Mutex> = Mutex::new((0..plans.len()).collect()); // One run, however many workers it has. let run = Run::start().against(opts.provenance.clone()); let run = &run; let done: Mutex> = Mutex::new(Vec::with_capacity(plans.len())); std::thread::scope(|scope| { for n in 0..workers.clamp(1, plans.len().max(1)) { - let (next, done, make) = (&next, &done, &make); + let (left, done, make) = (&left, &done, &make); scope.spawn(move || { - let mut worker = None; - let mut launched = None; + let mut worker: Option = None; + let mut session = Session::default(); loop { - let i = next.fetch_add(1, Ordering::SeqCst); - let Some(plan) = plans.get(i) else { break }; + let share = opts.share_starts + && worker.as_ref().is_some_and(|w| w.driver.may_continue()); + let taken = take(suite, plans, &mut left.lock().unwrap(), &session, share); + let Some(i) = taken else { break }; + let plan = &plans[i]; // Built when there is a plan to run, so a worker with nothing to do starts nothing. let w = worker.get_or_insert_with(|| make(n)); let result = run_served( @@ -148,9 +228,10 @@ where &mut *w.driver, opts, &mut w.services, - &mut launched, + &mut session, run, ); + session.after(i, &result); done.lock().unwrap().push((i, result)); } }); @@ -169,7 +250,7 @@ fn run_served( driver: &mut dyn Driver, opts: &RunOptions, services: &mut Services, - launched: &mut Option, + session: &mut Session, run: &Run, ) -> PlanResult { if let Some(r) = pending(suite, plan) { @@ -187,8 +268,10 @@ fn run_served( }, |_| Ok(()), ) { - Ok(()) => run_plan(suite, plan, driver, opts, launched, run), + Ok(()) => run_plan(suite, plan, driver, opts, session, run), Err(why) => { + // Whatever was started afresh before that has lost what the last test left. + session.last = None; let mut r = PlanResult::with(Status::Unsupported, plan); r.start.error = Some(format!("It needs {why}.")); ran_in(r, run) @@ -274,17 +357,15 @@ fn unsupported(suite: &Suite, plan: &Plan, driver: &dyn Driver) -> Option, + session: &mut Session, run: &Run, ) -> PlanResult { - ran_in(run_steps(suite, plan, driver, opts, launched, run), run) + ran_in(run_steps(suite, plan, driver, opts, session, run), run) } /// A result, marked with the run it came from. @@ -298,7 +379,7 @@ fn run_steps( plan: &Plan, driver: &mut dyn Driver, opts: &RunOptions, - launched: &mut Option, + session: &mut Session, run: &Run, ) -> PlanResult { let failed = |error: String| StepResult { @@ -358,17 +439,20 @@ fn run_steps( controls: vec![], used: vec![], run: None, + began: None, attachments, stale: false, }; if let Some(how) = suite.launch_of(plan.section) - && launched.as_deref() != Some(how) + && !session.launched_for(suite, plan) { + // Started again, it isn't where the last test left it. + session.last = None; if let Err(e) = driver.launch(how) { return skip_all(failed(format!("couldn't launch: {e}")), vec![]); } - *launched = Some(how.to_string()); + session.launched = Some(how.to_string()); } let began = now_ms(); let setup_failed = @@ -388,20 +472,31 @@ fn run_steps( keep(driver, 0); start }; - if let Err(e) = driver.reset() { - let start = setup_failed(format!("couldn't reset: {e}"), driver, &mut keep); - return skip_all(start, attachments); - } - for (x, c) in plan.start.iter() { - let var = suite.vocabulary.var(x); - let value = var - .witness(c) - .expect("plans only arrange satisfiable states"); - if let Err(e) = driver.arrange(var, &value) { - let error = format!("couldn't arrange {}: {e}", var.describe(c)); - let start = setup_failed(error, driver, &mut keep); + // Where the last test ended, if that is where this one starts: looked at once, as it is, + // without waiting for it to become so. A test that is given nothing has nothing to tell + // it by, and one whose driver wants services afresh must not find what the last one left. + let continued = opts.share_starts + && session.last.is_some() + && !plan.start_checks.is_empty() + && driver.may_continue() + && driver.fresh().is_empty() + && observe(suite, driver, &plan.start_checks, 0).status == Status::Passed; + if !continued { + if let Err(e) = driver.reset() { + let start = setup_failed(format!("couldn't reset: {e}"), driver, &mut keep); return skip_all(start, attachments); } + for (x, c) in plan.start.iter() { + let var = suite.vocabulary.var(x); + let value = var + .witness(c) + .expect("plans only arrange satisfiable states"); + if let Err(e) = driver.arrange(var, &value) { + let error = format!("couldn't arrange {}: {e}", var.describe(c)); + let start = setup_failed(error, driver, &mut keep); + return skip_all(start, attachments); + } + } } driver.settle(); // Setting the test up is not using the controls. @@ -451,6 +546,11 @@ fn run_steps( controls: seen.controls, used: seen.used, run: None, + began: Some(if continued { + Began::Continued + } else { + Began::Arranged + }), attachments, stale: false, } diff --git a/crates/harness/tests/shared_starts.rs b/crates/harness/tests/shared_starts.rs new file mode 100644 index 0000000..352a625 --- /dev/null +++ b/crates/harness/tests/shared_starts.rs @@ -0,0 +1,260 @@ +//! A test starts where the last one ended, without a reset, when that test passed and what this +//! one is given is observed to hold there already. + +use expectation_harness::{Acceptance, Began, Bindings, RunOptions}; +use expectation_story::{Report, Status}; + +/// «Bright light» ends with the lamp on and the dimmer high, which is where «Dim again» and +/// «Off again» start. «Just the dimmer» starts from the background. +const LAMP: &str = " +Vocabulary: + State: the lamp is off | on + State: the dimmer is low | high + Event: the user flips the switch + Event: the user flips it back + Event: the user turns the dimmer + Event: the user turns it back +Background: + Given the lamp is off + And the dimmer is low +Feature: Lamp + Expectation: Flipping turns it on + Given the lamp is off + When the user flips the switch + Then the lamp is on + Expectation: Flipping back turns it off + Given the lamp is on + When the user flips it back + Then the lamp is off + Expectation: Turning the dimmer brightens it + Given the dimmer is low + When the user turns the dimmer + Then the dimmer is high + Expectation: Turning it back dims it + Given the dimmer is high + When the user turns it back + Then the dimmer is low +Story: Bright light + When the user turns the dimmer + Then the dimmer is high + When the user flips the switch + Then the lamp is on +Story: Just the dimmer + When the user turns the dimmer + Then the dimmer is high +Story: Dim again + Given the lamp is on + And the dimmer is high + When the user turns it back + Then the dimmer is low +Story: Off again + Given the lamp is on + And the dimmer is high + When the user flips it back + Then the lamp is off +"; + +#[derive(Default)] +struct Lamp { + on: bool, + high: bool, + /// What was asked of it, in order: `reset`, and each event. + log: Vec<&'static str>, + /// Turning the dimmer back does nothing. + stuck: bool, + /// The bulb is about to go: once flipped on, it is seen on this many times and then off. + fragile: Option, + looks: std::cell::Cell, + /// It was turned on at the switch, not arranged on (with a new bulb). + flipped: bool, +} + +fn lamp() -> Acceptance { + let loaded = expectation_language::load_sources("lamp", &[("lamp.expect", LAMP)]); + Acceptance::from_loaded(loaded).unwrap_or_else(|e| panic!("{e}")) +} + +fn seen_on(w: &Lamp) -> bool { + let looks = w.looks.get(); + w.looks.set(looks + 1); + w.on && !(w.flipped && w.fragile.is_some_and(|n| looks >= n)) +} + +fn driver(stuck: bool) -> Bindings { + let world = Lamp { + stuck, + ..Default::default() + }; + Bindings::new(world) + .observe("the lamp", |w: &Lamp| if seen_on(w) { "on" } else { "off" }) + .observe("the dimmer", |w: &Lamp| if w.high { "high" } else { "low" }) + .arrange("the lamp", |w, v| { + w.flipped = false; + w.on = v.to_string() == "on" + }) + .arrange("the dimmer", |w, v| w.high = v.to_string() == "high") + .on("the user flips the switch", |w| { + w.log.push("flip"); + w.looks.set(0); + w.flipped = true; + w.on = true + }) + .on("the user flips it back", |w| { + w.log.push("flip back"); + w.on = false + }) + .on("the user turns the dimmer", |w| { + w.log.push("turn"); + w.high = true + }) + .on("the user turns it back", |w| { + w.log.push("turn back"); + w.high = w.stuck + }) + .on_reset(|w| { + w.log.push("reset"); + w.on = false; + w.high = false + }) +} + +fn began(report: &Report) -> Vec> { + report.results.iter().map(|r| r.began).collect() +} + +fn titles(a: &Acceptance) -> Vec<&str> { + a.plans.iter().map(|p| p.title.as_str()).collect() +} + +const ARRANGED: Option = Some(Began::Arranged); +const CONTINUED: Option = Some(Began::Continued); + +#[test] +fn a_test_starts_where_the_last_one_ended_when_its_givens_hold_there() { + let a = lamp(); + assert_eq!( + titles(&a), + ["Bright light", "Just the dimmer", "Dim again", "Off again"] + ); + let mut d = driver(false); + let report = a.run(&mut d, &RunOptions::default()); + assert!(report.passed()); + // «Dim again» is run next, where «Bright light» ended, and the results are still in the + // plans' order. It leaves the dimmer low, so «Off again» has to be arranged. + assert_eq!(began(&report), [ARRANGED, ARRANGED, CONTINUED, ARRANGED]); + assert_eq!( + d.world.log, + [ + "reset", + "turn", + "flip", + "turn back", + "reset", + "turn", + "reset", + "flip back" + ] + ); + + // The same verdicts as resetting before each one. + let mut d = driver(false); + let opts = RunOptions { + share_starts: false, + ..Default::default() + }; + let each = a.run(&mut d, &opts); + assert_eq!(began(&each), [ARRANGED; 4]); + assert_eq!(d.world.log.iter().filter(|l| **l == "reset").count(), 4); + assert_eq!( + report.verdicts(&a.suite, &a.plans), + each.verdicts(&a.suite, &a.plans) + ); +} + +#[test] +fn a_driver_can_refuse_to_continue() { + let a = lamp(); + let mut d = driver(false).always_reset(); + let report = a.run(&mut d, &RunOptions::default()); + assert!(report.passed()); + assert_eq!(began(&report), [ARRANGED; 4]); + // And then the tests are run in the order they are in. + assert_eq!( + d.world.log, + [ + "reset", + "turn", + "flip", + "reset", + "turn", + "reset", + "turn back", + "reset", + "flip back" + ] + ); +} + +#[test] +fn nothing_continues_from_a_test_that_failed() { + // Only the two that start bright: with the dimmer stuck, «Dim again» fails and leaves the + // system where «Off again» starts, which is reset all the same. + let mut a = lamp(); + a.plans.retain(|p| p.title.ends_with("again")); + let mut d = driver(true); + let report = a.run(&mut d, &RunOptions::default()); + let status: Vec = report.results.iter().map(|r| r.status).collect(); + assert_eq!(status, [Status::Failed, Status::Passed]); + assert_eq!(began(&report), [ARRANGED, ARRANGED]); +} + +#[test] +fn what_a_test_is_given_is_observed_not_assumed() { + // «Bright light» passes, but the lamp goes out after it was seen on, before the next + // test: «Dim again» should start there by the model, and is arranged because it doesn't. + let mut a = lamp(); + a.plans + .retain(|p| ["Bright light", "Dim again"].contains(&p.title.as_str())); + let first = a.run(&mut driver(false), &RunOptions::default()); + assert_eq!(began(&first), [ARRANGED, CONTINUED]); + + let mut d = driver(false); + d.world.fragile = Some(1); + let report = a.run(&mut d, &RunOptions::default()); + assert!(report.passed(), "{}", report.failures(&a.suite, &a.plans)); + assert_eq!(began(&report), [ARRANGED, ARRANGED]); +} + +#[test] +fn a_test_given_nothing_is_always_reset() { + // Nothing to observe says nothing about where the system is. + let mut a = lamp(); + a.plans.retain(|p| p.title == "Just the dimmer"); + a.plans.push(a.plans[0].clone()); + for p in &mut a.plans { + p.start_checks.clear(); + } + let report = a.run(&mut driver(false), &RunOptions::default()); + assert_eq!(began(&report), [ARRANGED, ARRANGED]); +} + +#[test] +fn saved_results_and_the_manual_say_how_a_test_began() { + let a = lamp(); + let report = a.run(&mut driver(false), &RunOptions::default()); + let saved = report.save(&a.plans); + assert_eq!(saved.matches("\nbegan\tcontinued\n").count(), 1); + assert_eq!(saved.matches("\nbegan\tarranged\n").count(), 3); + let loaded = Report::load(&saved, &a.plans); + assert_eq!(began(&loaded), began(&report)); + + let dir = std::env::temp_dir().join(format!("expectation-shared-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + a.publish(&dir, "test", &report).unwrap(); + let page = |name: &str| std::fs::read_to_string(a.tree(&dir).join(name)).unwrap(); + let said = "started where the test that ran before it had ended"; + assert!(page("tests/dim-again.md").contains(said)); + assert!(!page("tests/off-again.md").contains(said)); + assert!(page("site/test-dim-again.html").contains(said)); + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/crates/remote/src/bin/expectation-run.rs b/crates/remote/src/bin/expectation-run.rs index 3a12bdc..9419d61 100644 --- a/crates/remote/src/bin/expectation-run.rs +++ b/crates/remote/src/bin/expectation-run.rs @@ -4,7 +4,7 @@ //! expectation-run [--platform NAME] [--addr 127.0.0.1:4477] [--wait SECONDS] //! [--docs DIR] [--runner NAME] [--settle MS] [--screenshot-cmd "COMMAND"] //! [--attachments DIR] [--screenshot-store DIR] [--strict[=all]] -//! [--no-state-pictures] +//! [--no-state-pictures] [--no-shared-starts] //! [--app-revision REV] [--data-mode MODE] [--fixture NAME] //! ``` //! @@ -19,6 +19,9 @@ //! frames are stored once and unchanged tests write nothing new; the docs take them from there. //! `--no-state-pictures` leaves out the tests that only visit a state for the state map's //! picture of it: a quicker run with the same verdicts. +//! A test starts where the last one ended, without a reset, when that test passed and what this +//! one is given already holds there; such a test is marked `(continued)`. `--no-shared-starts` +//! resets before every test. //! `--app-revision`, `--data-mode` and `--fixture` say what the run is against (a commit, `fixtures`, //! the fixture's name): they are saved with the results and shown in the manual, which is //! written for the same, so results other runs saved for another build are labelled as such. @@ -37,7 +40,7 @@ use std::time::Duration; const USAGE: &str = "usage: expectation-run [--platform NAME] [--addr HOST:PORT] [--wait SECONDS] \ [--command-timeout SECONDS] \ - [--stylesheet CSS] [--reader] [--state-shot-ratio W:H] [--docs DIR] [--runner NAME] [--settle MS] [--screenshot-cmd COMMAND] [--attachments DIR] [--screenshot-store DIR] [--strict[=all]] [--no-state-pictures] [--app-revision REV] [--data-mode MODE] [--fixture NAME]"; + [--stylesheet CSS] [--reader] [--state-shot-ratio W:H] [--docs DIR] [--runner NAME] [--settle MS] [--screenshot-cmd COMMAND] [--attachments DIR] [--screenshot-store DIR] [--strict[=all]] [--no-state-pictures] [--no-shared-starts] [--app-revision REV] [--data-mode MODE] [--fixture NAME]"; fn main() { let mut args = std::env::args().skip(1); @@ -67,6 +70,7 @@ fn main() { let mut provenance = expectation_harness::Provenance::default(); let mut policy = Policy::Lenient; let mut state_pictures = true; + let mut share_starts = true; while let Some(a) = args.next() { let mut value = || { args.next() @@ -102,6 +106,7 @@ fn main() { } "--reader" => reader = true, "--no-state-pictures" => state_pictures = false, + "--no-shared-starts" => share_starts = false, "--app-revision" => provenance.app_revision = Some(value()), "--data-mode" => provenance.data_mode = Some(value()), "--fixture" => provenance.fixture = Some(value()), @@ -200,6 +205,7 @@ fn main() { screenshots_under: acceptance.screenshots_under(), attachments_dir: attachments, provenance, + share_starts, screenshot_store: store.map(PathBuf::from), ..Default::default() }, @@ -210,7 +216,11 @@ fn main() { Status::Failed => "✗", _ => "–", }; - println!("{mark} {}", plan.title); + let how = match r.began { + Some(expectation_harness::Began::Continued) => " (continued)", + _ => "", + }; + println!("{mark} {}{how}", plan.title); } if let Some(e) = driver.screenshot_error() { eprintln!("warning: some screenshots failed: {e}"); diff --git a/crates/remote/tests/shared_starts.rs b/crates/remote/tests/shared_starts.rs new file mode 100644 index 0000000..234630d --- /dev/null +++ b/crates/remote/tests/shared_starts.rs @@ -0,0 +1,114 @@ +//! `expectation-run` starts a test where the last one ended when what it is given holds there, +//! says so, and resets before every test with `--no-shared-starts`. + +use expectation_remote::client::{Client, Handlers}; +use std::io::{BufRead, BufReader, Read}; +use std::path::Path; +use std::process::{Command, Stdio}; +use std::time::Duration; + +/// «Bright light» ends with the lamp on and the dimmer high, where «Dim again» starts. +const SUITE: &str = " +Vocabulary: + State: the lamp is off | on + State: the dimmer is low | high + Event: the user flips the switch + Event: the user turns the dimmer + Event: the user turns it back +Background: + Given the lamp is off + And the dimmer is low +Feature: Lamp + Expectation: Flipping turns it on + Given the lamp is off + When the user flips the switch + Then the lamp is on + Expectation: Turning the dimmer brightens it + Given the dimmer is low + When the user turns the dimmer + Then the dimmer is high + Expectation: Turning it back dims it + Given the dimmer is high + When the user turns it back + Then the dimmer is low +Story: Bright light + When the user turns the dimmer + Then the dimmer is high + When the user flips the switch + Then the lamp is on +Story: Dim again + Given the lamp is on + And the dimmer is high + When the user turns it back + Then the dimmer is low +"; + +/// The lines the runner prints, one per test. +fn run(suite: &Path, args: &[&str]) -> Vec { + let mut runner = Command::new(env!("CARGO_BIN_EXE_expectation-run")) + .arg(suite) + .args(["--addr", "127.0.0.1:0", "--settle", "0", "--wait", "20"]) + .args(args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let mut said = BufReader::new(runner.stderr.take().unwrap()); + let mut line = String::new(); + said.read_line(&mut line).unwrap(); + let addr = line + .split("http://") + .nth(1) + .and_then(|s| s.split_whitespace().next()) + .unwrap_or_else(|| panic!("no address in {line:?}")) + .to_string(); + let h: Handlers<(bool, bool)> = Handlers::new() + .observe( + "the lamp", + |w: &(bool, bool)| if w.0 { "on" } else { "off" }, + ) + .observe( + "the dimmer", + |w: &(bool, bool)| { + if w.1 { "high" } else { "low" } + }, + ) + .arrange("the lamp", |w, v| w.0 = v.to_string() == "on") + .arrange("the dimmer", |w, v| w.1 = v.to_string() == "high") + .on("the user flips the switch", |w| w.0 = true) + .on("the user turns the dimmer", |w| w.1 = true) + .on("the user turns it back", |w| w.1 = false) + .on_reset(|w| *w = (false, false)); + let client = Client::connect(&addr, h.hello("test", "Lamp")); + std::thread::spawn(move || { + let mut world = (false, false); + while let Some(r) = client.recv_timeout(Duration::from_secs(20)) { + h.handle(&mut world, r); + } + }); + let mut out = String::new(); + runner + .stdout + .take() + .unwrap() + .read_to_string(&mut out) + .unwrap(); + assert!(runner.wait().unwrap().success(), "{line}"); + out.lines().map(String::from).collect() +} + +#[test] +fn a_run_says_which_tests_continued_and_can_reset_before_each() { + let root = Path::new(env!("CARGO_TARGET_TMPDIR")).join("shared-starts-run"); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(&root).unwrap(); + std::fs::write(root.join("lamp.expect"), SUITE).unwrap(); + + let shared = run(&root, &["--no-state-pictures", "--strict"]); + assert_eq!(shared, ["✓ Bright light", "✓ Dim again (continued)"]); + let each = run( + &root, + &["--no-state-pictures", "--strict", "--no-shared-starts"], + ); + assert_eq!(each, ["✓ Bright light", "✓ Dim again"]); +} diff --git a/crates/story/src/lib.rs b/crates/story/src/lib.rs index fa57621..a29bfdb 100644 --- a/crates/story/src/lib.rs +++ b/crates/story/src/lib.rs @@ -24,7 +24,7 @@ pub use plan::{ reaching_plans, untested, visit_plans, }; pub use report::{ - Agreement, Attachment, CheckResult, PlanResult, Policy, Provenance, Report, Run, Status, + Agreement, Attachment, Began, CheckResult, PlanResult, Policy, Provenance, Report, Run, Status, StepResult, Verdict, Window, fingerprint, load_results, load_results_for, now_ms, platform_shots, results_file, screenshots_named, }; diff --git a/crates/story/src/report.rs b/crates/story/src/report.rs index 6e9cf0f..661636a 100644 --- a/crates/story/src/report.rs +++ b/crates/story/src/report.rs @@ -196,6 +196,31 @@ pub struct Attachment { pub path: String, } +/// How a test got to its start. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Began { + /// The system was reset and the start arranged. + Arranged, + /// The test before it on the same driver had passed and left the system where this one + /// starts: everything it is given was observed to hold, so nothing was reset or arranged. + Continued, +} + +impl Began { + pub fn name(self) -> &'static str { + match self { + Began::Arranged => "arranged", + Began::Continued => "continued", + } + } + + pub fn parse(s: &str) -> Option { + [Began::Arranged, Began::Continued] + .into_iter() + .find(|b| b.name() == s) + } +} + #[derive(Clone, Debug)] pub struct StepResult { pub status: Status, @@ -242,6 +267,8 @@ pub struct PlanResult { pub used: Vec, /// The run this result came from, if it was saved by a version of Expectation that says. pub run: Option, + /// How it got to its start, if it got that far and the results say. + pub began: Option, /// What the driver handed over while it ran. pub attachments: Vec, /// Not run since the test changed: saved results exist, but for an older version of it. @@ -263,6 +290,7 @@ impl PlanResult { controls: vec![], used: vec![], run: None, + began: None, attachments: vec![], stale: false, } @@ -585,6 +613,9 @@ impl Report { r.status.name(), fingerprint(plan) ); + if let Some(b) = r.began { + let _ = writeln!(out, "began\t{}", b.name()); + } for c in &r.controls { let _ = writeln!( out, @@ -707,6 +738,7 @@ impl Report { controls: vec![], used: vec![], run: run.clone(), + began: None, attachments: vec![], stale: false, }, @@ -719,6 +751,11 @@ impl Report { r.controls.push(c); } } + ["began", how, ..] => { + if let Some((_, r)) = current.as_mut() { + r.began = Began::parse(how); + } + } ["used", key] => { if let Some((_, r)) = current.as_mut() { r.used.push(key.to_string());