diff --git a/book/src/cli.md b/book/src/cli.md index 26d7726..93abbe1 100644 --- a/book/src/cli.md +++ b/book/src/cli.md @@ -98,6 +98,10 @@ error. (see [Screenshots in large projects](manual.md#screenshots-in-large-projects)). Give a directory and the ones the pages show are copied in. Give a URL prefix and the pages link there. +- `--expect-app-revision `, `--expect-data-mode ` and `--expect-fixture ` say + what the manual is for. Each test's page then says whether its result was run against the + same, something else, or didn't say (see + [What a run was against](evidence.md#what-a-run-was-against)). No verdict changes. ## screenshots prune diff --git a/book/src/evidence.md b/book/src/evidence.md index 6dd892b..6f0f14b 100644 --- a/book/src/evidence.md +++ b/book/src/evidence.md @@ -24,6 +24,7 @@ its screenshot. Both are saved in the results file, as lines that older readers ```text expectation-results 1 run +provenance plan step window @@ -35,6 +36,52 @@ A `run` line applies to the plans after it. Step `0` is the test's start; `1` is step. Results saved before runs had ids still load. Their tests have no run, so no evidence can count for them. +## What a run was against + +A test's fingerprint says the test hasn't changed. It says nothing about the program: a pass +saved for one build still stands when the manual is written for the next. So a run can say +what it was against, and the results keep it: + +```rust +let opts = RunOptions { + provenance: Provenance { + app_revision: Some("3f2a9c1".into()), // a commit, a version, a build number + data_mode: Some("fixtures".into()), // fixtures, live, recorded + fixture: Some("seed-7".into()), // which one, by a name + }, + ..Default::default() +}; +``` + +```bash +expectation-run expectations --docs docs --app-revision 3f2a9c1 --data-mode fixtures --fixture seed-7 +``` + +- Each part is optional. A part nobody gave is unknown, and is never guessed. +- Nothing is read from the environment. The results and the manual publish what you give, so + give a name, not a credential or the data itself. +- The results file has a `provenance` line for each part, after the run's line. Older files + have none, and still load. +- A manual merged from several runs keeps each result's own run, so two tests on one page can + be about two builds. Each test's page says what its result was run against. + +Say what the manual is for, and each test's page says whether its result is about that: + +```bash +expectation docs expectations docs --expect-app-revision 4b7d210 +``` + +| The run | The page says | +|---|---| +| said the same | it is what this manual is for | +| said something else | it is what another run saw, and not a failure of this build | +| didn't say | it isn't known | + +The test report's first page counts the three. A result is never changed, hidden or failed +because of what it was run against: the verdict is still the run's. `expectation-run --docs` +writes the manual for what it was run against, and `Acceptance::with_expected` does the same +from Rust. + ## Attachments: what the driver captured A driver can hand over files after the test's start and after every step, such as the network diff --git a/crates/cli/src/lib.rs b/crates/cli/src/lib.rs index b29d0e2..750919f 100644 --- a/crates/cli/src/lib.rs +++ b/crates/cli/src/lib.rs @@ -24,6 +24,7 @@ commands: (--coverage adds generated tests for untested expectations) docs [--approved-only] [--reader] [--results ] [--stylesheet ] [--state-shot-ratio ] [--screenshots ] [--section ]... [--base-url ] + [--expect-app-revision ] [--expect-data-mode ] [--expect-fixture ] write the docs (Markdown and an HTML site), with the results runs have saved in /results; --stylesheet restyles the site with a CSS file, --state-shot-ratio shapes the state map's pictures; @@ -35,7 +36,10 @@ commands: (the ones shown are copied in) or a URL prefix the pages link to; --section (repeatable) writes just those sections, for a preview, with links elsewhere going to --base-url - or, without one, as plain text + or, without one, as plain text; + --expect-app-revision, --expect-data-mode and --expect-fixture say + what the manual is for: each test's page says whether its result + was run against the same, something else, or didn't say screenshots prune ... [--dry-run] remove the screenshots in a content-addressed store that no results file in the given directories names, and their thumbnails @@ -123,6 +127,11 @@ pub fn main() { take("--screenshots"), ); let results = take("--results"); + let expected = expectation_story::Provenance { + app_revision: take("--expect-app-revision"), + data_mode: take("--expect-data-mode"), + fixture: take("--expect-fixture"), + }; // `--section` may be given more than once. let mut sections = vec![]; while let Some(v) = take("--section") { @@ -198,6 +207,7 @@ pub fn main() { (approved_only, reader), results, (sections, base_url), + &expected, ), ["screenshots", "prune", store, dirs @ ..] if !dirs.is_empty() => { prune_screenshots(store, dirs, flag("--dry-run")) @@ -225,6 +235,7 @@ fn docs( (approved_only, reader): (bool, bool), results: Option, (sections, base_url): (Vec, Option), + expected: &expectation_story::Provenance, ) { let fail = |e: String| -> ! { eprintln!("error: {e}"); @@ -242,6 +253,7 @@ fn docs( reader, shared: None, code: &[], + expected: (!expected.is_empty()).then_some(expected), }; if let Some(expectation_docs::Screenshots::Dir(d)) = shots.as_deref().map(expectation_docs::Screenshots::parse) diff --git a/crates/docs/src/html.rs b/crates/docs/src/html.rs index 95c9ca2..bf68d4a 100644 --- a/crates/docs/src/html.rs +++ b/crates/docs/src/html.rs @@ -1531,6 +1531,9 @@ impl Site<'_> { esc(&self.summary()), self.tally(&all) ); + if let Some(text) = self.d.against_summary() { + let _ = write!(o, r#"

{}

"#, esc(&text)); + } o.push_str(r#"

The tests use the states, actions and shorthand of the test vocabulary. The map shows which behaviour leads to which, the state map shows the states the app gets into and how to get between them, and the theme shows the colours and type the docs are drawn in.

"#); let controls = self.d.control_rows(); if !controls.is_empty() { @@ -1834,6 +1837,25 @@ impl Site<'_> { ); } } + if let Some(a) = self.d.against(i) { + let said: Vec = a + .said + .iter() + .map(|(label, v)| format!("{label} {}", esc(v))) + .collect(); + let mut line = String::new(); + if !said.is_empty() { + let _ = write!(line, "Run against {}. ", said.join(", ")); + } + let class = match &a.verdict { + Some((expectation_story::Agreement::Matching, _)) | None => "note", + Some(_) => "note warn", + }; + if let Some((_, text)) = &a.verdict { + line.push_str(&esc(text)); + } + let _ = write!(o, r#"

{}

"#, line.trim_end()); + } if let Some(e) = result.and_then(|r| r.start.error.as_ref()) { let class = if status == Status::Unsupported { "note warn" diff --git a/crates/docs/src/lib.rs b/crates/docs/src/lib.rs index f99340d..0c381ab 100644 --- a/crates/docs/src/lib.rs +++ b/crates/docs/src/lib.rs @@ -83,6 +83,9 @@ pub struct Look<'a> { /// Code coverage files (see [`expectation_story::CodeCoverage`]): shown on the test pages /// as observed execution, apart from the results. pub code: &'a [expectation_story::CodeCoverage], + /// What this manual is written for (a build, its data): each test page says whether the + /// run its result is from was against the same. `None` compares nothing. + pub expected: Option<&'a expectation_story::Provenance>, } /// Where the screenshots are that the results name as `screenshots/`. @@ -277,7 +280,93 @@ fn prune(dir: &Path, written: &std::collections::HashSet<&Path>) -> std::io::Res Ok(()) } +/// What a test page says about what its result was run against. +pub(crate) struct Against { + /// The parts the run said, by what the manual calls them. + pub said: Vec<(&'static str, String)>, + /// How that compares with what the manual is for, in a sentence, if it is for something. + pub verdict: Option<(expectation_story::Agreement, String)>, +} + impl Docs<'_> { + /// What test `i`'s result was run against, and whether that is what this manual is for. + /// `None` when it didn't run, or there is nothing to say: no part was given and the manual + /// isn't for anything in particular. + pub(crate) fn against(&self, i: usize) -> Option { + use expectation_story::Agreement; + let result = self.report?.results.get(i)?; + if !result.status.ran() { + return None; + } + // A result from before runs said what they were against says nothing: unknown. + let none = expectation_story::Provenance::default(); + let run = result.run.as_ref().map_or(&none, |r| &r.provenance); + let said: Vec<(&'static str, String)> = run + .parts() + .into_iter() + .filter_map(|(_, label, v)| Some((label, v?.to_string()))) + .collect(); + let verdict = self.look.expected.and_then(|want| { + let agreement = run.agreement(want)?; + let pairs = run.parts().into_iter().zip(want.parts()); + let text = match agreement { + Agreement::Matching => "This is what this manual is for.".to_string(), + Agreement::Mismatched => { + let differs: Vec = pairs + .filter_map(|((_, label, mine), (_, _, want))| { + let (mine, want) = (mine?, want?); + (mine != want).then(|| format!("{label} {want}")) + }) + .collect(); + format!( + "This manual is for {}, so this is what an earlier or other run saw. It \ + is not a result for this one, and not a failure of it.", + expectation_model::text::join_and(&differs) + ) + } + Agreement::Unknown => { + let missing: Vec = pairs + .filter(|((_, _, mine), (_, _, want))| mine.is_none() && want.is_some()) + .map(|((_, label, _), _)| label.to_string()) + .collect(); + format!( + "The run didn't say its {}, so it isn't known whether this is about \ + what this manual is for.", + expectation_model::text::join_and(&missing) + ) + } + }; + Some((agreement, text)) + }); + (!said.is_empty() || verdict.is_some()).then_some(Against { said, verdict }) + } + + /// For the test report's first page, when the manual is for something: how many of the + /// results that ran are about it. + pub(crate) fn against_summary(&self) -> Option { + use expectation_story::Agreement; + let want = self.look.expected.filter(|w| !w.is_empty())?; + let (mut same, mut other, mut unknown) = (0, 0, 0); + for i in 0..self.plans.len() { + match self.against(i).and_then(|a| a.verdict) { + Some((Agreement::Matching, _)) => same += 1, + Some((Agreement::Mismatched, _)) => other += 1, + Some((Agreement::Unknown, _)) => unknown += 1, + None => {} + } + } + let for_what: Vec = want + .parts() + .into_iter() + .filter_map(|(_, label, v)| Some(format!("{label} {}", v?))) + .collect(); + Some(format!( + "This manual is for {}. Of the tests that ran, {same} ran against it, {other} against \ + something else, and {unknown} didn't say. Each test's page says which; a result from \ + another run is what that run saw, and is never counted as a failure.", + expectation_model::text::join_and(&for_what) + )) + } /// Is this the reader edition? pub fn reader(&self) -> bool { self.look.reader || self.outline.is_some_and(|o| o.reader) @@ -1519,6 +1608,7 @@ Feature: Paying r.run = Some(expectation_story::Run { id: "r1".into(), started_ms: 1, + provenance: Default::default(), }); } let (slug, print) = (&plans[0].slug, expectation_story::fingerprint(&plans[0])); diff --git a/crates/docs/src/markdown.rs b/crates/docs/src/markdown.rs index c6f7291..c29a24a 100644 --- a/crates/docs/src/markdown.rs +++ b/crates/docs/src/markdown.rs @@ -684,6 +684,9 @@ fn report(d: &Docs, f: &Facts) -> String { [test vocabulary](vocabulary.md).\n", s.name ); + if let Some(text) = d.against_summary() { + let _ = writeln!(o, "{text}\n"); + } match (d.report, &f.verdicts) { (Some(r), Some(v)) => { let count = |x: Verdict| v.iter().filter(|&&y| y == x).count(); @@ -1049,6 +1052,21 @@ fn test_page(d: &Docs, f: &Facts, i: usize) -> String { } } let _ = writeln!(o, "**Result:** {}\n", status_mark(f.status[i])); + if let Some(a) = d.against(i) { + let said: Vec = a + .said + .iter() + .map(|(label, v)| format!("{label} `{}`", v.replace('`', "'"))) + .collect(); + let mut line = String::new(); + if !said.is_empty() { + let _ = write!(line, "**Run against:** {}. ", said.join(", ")); + } + if let Some((_, text)) = &a.verdict { + line.push_str(text); + } + let _ = writeln!(o, "{}\n", line.trim_end()); + } if let Some(e) = result.and_then(|r| r.start.error.as_ref()) { let _ = writeln!(o, "> {e}\n"); } diff --git a/crates/harness/src/lib.rs b/crates/harness/src/lib.rs index 0f59334..7d01209 100644 --- a/crates/harness/src/lib.rs +++ b/crates/harness/src/lib.rs @@ -25,7 +25,7 @@ use expectation_graph::Analysis; use expectation_language::Loaded; use expectation_model::diagnostic::{has_errors, render_all}; use expectation_model::{Diagnostic, Suite}; -pub use expectation_story::{Control, Policy}; +pub use expectation_story::{Control, Policy, Provenance}; use expectation_story::{Plan, Report}; use std::path::Path; @@ -51,6 +51,8 @@ pub struct Acceptance { pub screenshots: Option, /// Write the reader edition of the manual (see [`Acceptance::with_reader`]). pub reader: bool, + /// What the manual is for (see [`Acceptance::with_expected`]). + pub expected: Option, } impl Acceptance { @@ -144,6 +146,7 @@ impl Acceptance { shot_ratio: None, screenshots: None, reader: false, + expected: None, }) } @@ -195,6 +198,14 @@ impl Acceptance { self } + /// Say what the manual is for (a build, its data), so each test's page says whether its + /// result was run against the same: results merged from earlier runs, or other runners, + /// may be about another build. A result that is isn't changed or hidden, only labelled. + pub fn with_expected(mut self, provenance: expectation_story::Provenance) -> Self { + self.expected = (!provenance.is_empty()).then_some(provenance); + self + } + pub fn render_diagnostics(&self) -> String { render_all(&self.diagnostics, &self.suite.sources) } @@ -244,6 +255,7 @@ impl Acceptance { reader: self.reader, shared: None, code: &[], + expected: self.expected.as_ref(), }, } } diff --git a/crates/harness/src/run.rs b/crates/harness/src/run.rs index 310c3f6..77bcaf8 100644 --- a/crates/harness/src/run.rs +++ b/crates/harness/src/run.rs @@ -39,6 +39,10 @@ pub struct RunOptions { /// `////`; `None` keeps nothing. Keep it out of the docs /// directory: a capture can hold what a manual must not publish. pub attachments_dir: Option, + /// What this run is against (the program's revision, its data), kept with every result so + /// a manual can say which build a saved pass was about. Say only what may be published; + /// nothing is filled in for you. + pub provenance: expectation_story::Provenance, } impl Default for RunOptions { @@ -50,13 +54,14 @@ impl Default for RunOptions { patience: 100, controls: true, attachments_dir: None, + provenance: Default::default(), } } } pub fn run(suite: &Suite, plans: &[Plan], driver: &mut dyn Driver, opts: &RunOptions) -> Report { let mut launched = None; - let run = Run::start(); + let run = Run::start().against(opts.provenance.clone()); Report { results: plans .iter() @@ -76,7 +81,7 @@ pub fn run_with( services: &mut Services, ) -> Report { let mut launched = None; - let run = Run::start(); + let run = Run::start().against(opts.provenance.clone()); Report { results: plans .iter() @@ -123,7 +128,7 @@ where { let next = AtomicUsize::new(0); // One run, however many workers it has. - let run = Run::start(); + let run = Run::start().against(opts.provenance.clone()); let run = &run; let done: Mutex> = Mutex::new(Vec::with_capacity(plans.len())); std::thread::scope(|scope| { diff --git a/crates/harness/tests/provenance.rs b/crates/harness/tests/provenance.rs new file mode 100644 index 0000000..ae4c34a --- /dev/null +++ b/crates/harness/tests/provenance.rs @@ -0,0 +1,246 @@ +//! What a run was against (the program's revision, its data) is kept with its results, so a +//! manual made of several runs says which build each result is about. + +use expectation_harness::{Acceptance, Bindings, Provenance, RunOptions}; +use expectation_story::{Agreement, Report, Status}; +use std::path::{Path, PathBuf}; + +const LAMP: &str = " +Vocabulary: + State: the lamp is off | on + State: the dimmer is low | high + Event: the user flips the switch + Event: the user turns the dimmer +Background: + Given the lamp is off + And the dimmer is low +Feature: Lamp + Expectation: Flipping turns it on + Given the lamp is off + When the user flips the switch + Then the lamp is on + Expectation: Turning the dimmer brightens it + Given the dimmer is low + When the user turns the dimmer + Then the dimmer is high +Story: Light + When the user flips the switch + Then the lamp is on +Story: Bright + When the user turns the dimmer + Then the dimmer is high +"; + +fn lamp() -> Acceptance { + let loaded = expectation_language::load_sources("lamp", &[("lamp.expect", LAMP)]); + Acceptance::from_loaded(loaded).unwrap_or_else(|e| panic!("{e}")) +} + +/// A lamp, with a dimmer the driver can turn or not. +fn driver(dimmer: bool) -> Bindings<(bool, bool)> { + let b = Bindings::new((false, false)) + .observe( + "the lamp", + |w: &(bool, bool)| if w.0 { "on" } else { "off" }, + ) + .observe( + "the dimmer", + |w: &(bool, bool)| { + if w.1 { "high" } else { "low" } + }, + ) + .arrange("the lamp", |w, v| w.0 = v.to_string() == "on") + .arrange("the dimmer", |w, v| w.1 = v.to_string() == "high") + .on("the user flips the switch", |w| w.0 = true) + .on_reset(|w| *w = (false, false)); + if dimmer { + b.on("the user turns the dimmer", |w| w.1 = true) + } else { + b + } +} + +fn build(revision: &str) -> Provenance { + Provenance { + app_revision: Some(revision.into()), + data_mode: Some("fixtures".into()), + fixture: Some("seed-7".into()), + } +} + +fn run(a: &Acceptance, dimmer: bool, provenance: Provenance) -> Report { + a.run( + &mut driver(dimmer), + &RunOptions { + provenance, + ..Default::default() + }, + ) +} + +fn fresh(name: &str) -> PathBuf { + let dir = Path::new(env!("CARGO_TARGET_TMPDIR")).join(name); + let _ = std::fs::remove_dir_all(&dir); + dir +} + +fn of<'a>(a: &Acceptance, report: &'a Report, title: &str) -> &'a expectation_story::PlanResult { + &report.results[a.plans.iter().position(|p| p.title == title).unwrap()] +} + +fn page(dir: &Path, a: &Acceptance, title: &str, html: bool) -> String { + let slug = &a.plans.iter().find(|p| p.title == title).unwrap().slug; + let path = if html { + dir.join("site").join(format!("test-{slug}.html")) + } else { + dir.join("tests").join(format!("{slug}.md")) + }; + std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("{}: {e}", path.display())) +} + +#[test] +fn what_a_run_was_against_is_saved_with_its_results_and_loads_back() { + let a = lamp(); + let report = run(&a, true, build("abc123")); + let saved = report.save(&a.plans); + assert!(saved.contains("provenance\tapp-revision\tabc123\n")); + assert!(saved.contains("provenance\tdata-mode\tfixtures\n")); + assert!(saved.contains("provenance\tfixture\tseed-7\n")); + let loaded = Report::load(&saved, &a.plans); + for (was, is) in report.results.iter().zip(&loaded.results) { + assert_eq!(was.run, is.run); + assert_eq!(is.run.as_ref().unwrap().provenance, build("abc123")); + } + // The rest of the run is what it was. + assert_eq!(loaded.save(&a.plans), saved); + + // Only what was said is written, and nothing is filled in. + let partial = Provenance { + app_revision: Some("abc123".into()), + ..Default::default() + }; + let saved = run(&a, true, partial.clone()).save(&a.plans); + assert!(!saved.contains("data-mode") && !saved.contains("fixture")); + let loaded = Report::load(&saved, &a.plans); + assert_eq!(loaded.results[0].run.as_ref().unwrap().provenance, partial); + let saved = run(&a, true, Provenance::default()).save(&a.plans); + assert!(!saved.contains("provenance"), "{saved}"); +} + +#[test] +fn results_from_before_runs_said_are_unknown_not_guessed() { + let a = lamp(); + let saved = run(&a, true, build("abc123")).save(&a.plans); + let legacy: String = saved + .lines() + .filter(|l| !l.starts_with("provenance\t")) + .map(|l| format!("{l}\n")) + .collect(); + let loaded = Report::load(&legacy, &a.plans); + assert!(loaded.results.iter().all(|r| r.status == Status::Passed)); + let run = loaded.results[0].run.as_ref().unwrap(); + assert!(run.provenance.is_empty()); + assert_eq!( + run.provenance.agreement(&build("abc123")), + Some(Agreement::Unknown) + ); + // A part a later version adds is skipped, and the parts it knows still load. + let newer = saved.replace( + "provenance\tfixture\tseed-7\n", + "provenance\tfixture\tseed-7\nprovenance\tregion\teu\n", + ); + let loaded = Report::load(&newer, &a.plans); + assert_eq!( + loaded.results[0].run.as_ref().unwrap().provenance, + build("abc123") + ); +} + +#[test] +fn a_manual_of_several_runs_says_which_build_each_result_is_about() { + let a = lamp().with_expected(build("bbb222")); + let docs = fresh("provenance-mixed"); + // Build A ran everything. Build B's run can't turn the dimmer, so its test keeps A's result. + let first = run(&a, true, build("aaa111")); + a.publish(&docs, "nightly", &first).unwrap(); + let second = run(&a, false, build("bbb222")); + let merged = a.publish(&docs, "ci", &second).unwrap(); + + // Each result keeps where it came from; the older pass is still a pass. + let (light, bright) = (of(&a, &merged, "Light"), of(&a, &merged, "Bright")); + assert_eq!(bright.status, Status::Passed); + let against = |r: &expectation_story::PlanResult| r.run.as_ref().unwrap().provenance.clone(); + assert_eq!(against(bright), build("aaa111")); + assert!([build("aaa111"), build("bbb222")].contains(&against(light))); + assert_eq!( + against(bright).agreement(&build("bbb222")), + Some(Agreement::Mismatched) + ); + + for html in [false, true] { + let page = page(&docs, &a, "Bright", html); + assert!(page.contains("aaa111"), "{page}"); + assert!(page.contains("fixtures") && page.contains("seed-7")); + assert!(page.contains("This manual is for application revision bbb222")); + assert!(page.contains("not a failure of it")); + } + let index = std::fs::read_to_string(docs.join("tests.md")).unwrap(); + assert!( + index.contains("This manual is for application revision bbb222"), + "{index}" + ); + assert!(index.contains("against something else")); + // The verdicts are the runs': nothing failed because a build differs. + assert!(merged.passed()); + assert!(index.contains("2 passed, 0 failed"), "{index}"); +} + +#[test] +fn a_manual_says_matching_unknown_and_nothing_when_it_isnt_for_anything() { + // The same build: it says so. + let a = lamp().with_expected(build("abc123")); + let docs = fresh("provenance-matching"); + a.publish(&docs, "ci", &run(&a, true, build("abc123"))) + .unwrap(); + for html in [false, true] { + let page = page(&docs, &a, "Light", html); + assert!(page.contains("abc123") && page.contains("This is what this manual is for.")); + } + + // A run that didn't say: unknown, by name. + let docs = fresh("provenance-unknown"); + a.publish(&docs, "ci", &run(&a, true, Provenance::default())) + .unwrap(); + for html in [false, true] { + let page = page(&docs, &a, "Light", html); + assert!( + page.contains("The run didn't say its application revision, data mode and fixture"), + "{page}" + ); + } + + // A manual for nothing in particular shows what was said, and compares nothing. + let a = lamp(); + let docs = fresh("provenance-shown"); + a.publish(&docs, "ci", &run(&a, true, build("abc123"))) + .unwrap(); + let page = page(&docs, &a, "Light", false); + assert!(page.contains("**Run against:** application revision `abc123`")); + assert!(!page.contains("This manual is for") && !page.contains("didn't say")); + assert!( + !std::fs::read_to_string(docs.join("tests.md")) + .unwrap() + .contains("This manual is for") + ); + + // And with nothing said and nothing expected, the page is as it always was. + let docs = fresh("provenance-none"); + a.publish(&docs, "ci", &run(&a, true, Provenance::default())) + .unwrap(); + assert!(!page_has_against(&docs, &a)); +} + +fn page_has_against(docs: &Path, a: &Acceptance) -> bool { + page(docs, a, "Light", false).contains("Run against") + || page(docs, a, "Light", true).contains("against") +} diff --git a/crates/remote/src/bin/expectation-run.rs b/crates/remote/src/bin/expectation-run.rs index 8d7f5d8..d94291c 100644 --- a/crates/remote/src/bin/expectation-run.rs +++ b/crates/remote/src/bin/expectation-run.rs @@ -4,6 +4,7 @@ //! expectation-run [--platform NAME] [--addr 127.0.0.1:4477] [--wait SECONDS] //! [--docs DIR] [--runner NAME] [--settle MS] [--screenshot-cmd "COMMAND"] //! [--attachments DIR] [--screenshot-store DIR] [--strict[=all]] +//! [--app-revision REV] [--data-mode MODE] [--fixture NAME] //! ``` //! //! It listens, waits for the app to say hello, runs every story and a generated test for each @@ -15,6 +16,10 @@ //! `DIR////`; keep it outside `--docs`, which is for publishing. //! `--screenshot-store` keeps screenshots by content in `DIR` (`.png`), so identical //! frames are stored once and unchanged tests write nothing new; the docs take them from there. +//! `--app-revision`, `--data-mode` and `--fixture` say what the run is against (a commit, `fixtures`, +//! the fixture's name): they are saved with the results and shown in the manual, which is +//! written for the same, so results other runs saved for another build are labelled as such. +//! Nothing is read from the environment; say only what may be published. //! Exits 1 if anything failed. Tests that aren't supported yet, are pending or didn't run don't //! fail it, unless it is `--strict`: then every test that isn't pending must have run and passed //! in this run (`--strict=all`: every test, pending or not), and there must be at least one. @@ -29,7 +34,7 @@ use std::time::Duration; const USAGE: &str = "usage: expectation-run [--platform NAME] [--addr HOST:PORT] [--wait SECONDS] \ [--command-timeout SECONDS] \ - [--stylesheet CSS] [--reader] [--state-shot-ratio W:H] [--docs DIR] [--runner NAME] [--settle MS] [--screenshot-cmd COMMAND] [--attachments DIR] [--screenshot-store DIR] [--strict[=all]]"; + [--stylesheet CSS] [--reader] [--state-shot-ratio W:H] [--docs DIR] [--runner NAME] [--settle MS] [--screenshot-cmd COMMAND] [--attachments DIR] [--screenshot-store DIR] [--strict[=all]] [--app-revision REV] [--data-mode MODE] [--fixture NAME]"; fn main() { let mut args = std::env::args().skip(1); @@ -56,6 +61,7 @@ fn main() { ); let (mut stylesheet, mut ratio, mut attachments, mut store) = (None, None, None, None); let mut reader = false; + let mut provenance = expectation_harness::Provenance::default(); let mut policy = Policy::Lenient; while let Some(a) = args.next() { let mut value = || { @@ -91,6 +97,9 @@ fn main() { Some(expectation_harness::parse_ratio(&value()).unwrap_or_else(|e| fail(&e))) } "--reader" => reader = true, + "--app-revision" => provenance.app_revision = Some(value()), + "--data-mode" => provenance.data_mode = Some(value()), + "--fixture" => provenance.fixture = Some(value()), "--strict" => policy = Policy::Required, _ if a.starts_with("--strict=") => { policy = Policy::parse(&a["--strict=".len()..]).unwrap_or_else(|| { @@ -135,6 +144,7 @@ fn main() { if reader { acceptance = acceptance.with_reader(); } + acceptance = acceptance.with_expected(provenance.clone()); if let Some(s) = &store { acceptance = acceptance.with_screenshots(s.clone()); } @@ -181,6 +191,7 @@ fn main() { docs_dir: docs.clone(), screenshots_under: acceptance.screenshots_under(), attachments_dir: attachments, + provenance, screenshot_store: store.map(PathBuf::from), ..Default::default() }, diff --git a/crates/remote/tests/provenance.rs b/crates/remote/tests/provenance.rs new file mode 100644 index 0000000..66cad09 --- /dev/null +++ b/crates/remote/tests/provenance.rs @@ -0,0 +1,132 @@ +//! `expectation-run --app-revision` saves what the run was against, and `expectation docs +//! --expect-app-revision` says whether a saved result is about the build the manual is for. + +use expectation_remote::client::{Client, Handlers}; +use std::io::{BufRead, BufReader, Read}; +use std::path::Path; +use std::process::{Command, Stdio}; +use std::time::Duration; + +const SUITE: &str = " +Vocabulary: + State: the lamp is off | on + Event: the user flips the switch +Background: + Given the lamp is off +Feature: Lamp + Expectation: Flipping turns it on + Given the lamp is off + When the user flips the switch + Then the lamp is on +Story: Light + When the user flips the switch + Then the lamp is on +"; + +fn run(suite: &Path, args: &[&str]) { + let mut runner = Command::new(env!("CARGO_BIN_EXE_expectation-run")) + .arg(suite) + .args(["--addr", "127.0.0.1:0", "--settle", "0", "--wait", "20"]) + .args(args) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let mut said = BufReader::new(runner.stderr.take().unwrap()); + let mut line = String::new(); + said.read_line(&mut line).unwrap(); + let addr = line + .split("http://") + .nth(1) + .and_then(|s| s.split_whitespace().next()) + .unwrap_or_else(|| panic!("no address in {line:?}")) + .to_string(); + let h: Handlers = Handlers::new() + .observe("the lamp", |on: &bool| if *on { "on" } else { "off" }) + .arrange("the lamp", |on, v| *on = v.to_string() == "on") + .on("the user flips the switch", |on| *on = true) + .on_reset(|on| *on = false); + let client = Client::connect(&addr, h.hello("test", "Lamp")); + std::thread::spawn(move || { + let mut on = false; + while let Some(r) = client.recv_timeout(Duration::from_secs(20)) { + h.handle(&mut on, r); + } + }); + let mut rest = String::new(); + said.read_to_string(&mut rest).unwrap(); + assert!(runner.wait().unwrap().success(), "{line}{rest}"); +} + +fn docs(suite: &Path, out: &Path, args: &[&str]) { + let manifest = Path::new(env!("CARGO_MANIFEST_DIR")).join("../cli/Cargo.toml"); + let written = Command::new(env!("CARGO")) + .args(["run", "-q", "--manifest-path"]) + .arg(manifest) + .args(["--bin", "expectation", "--", "docs"]) + .arg(suite) + .arg(out) + .args(args) + .output() + .unwrap(); + assert!( + written.status.success(), + "{}", + String::from_utf8_lossy(&written.stderr) + ); +} + +#[test] +fn a_run_says_what_it_was_against_and_the_docs_say_whether_that_is_this_build() { + let root = Path::new(env!("CARGO_TARGET_TMPDIR")).join("provenance-run"); + let _ = std::fs::remove_dir_all(&root); + let (suite, out) = (root.join("suite"), root.join("docs")); + std::fs::create_dir_all(&suite).unwrap(); + std::fs::write(suite.join("lamp.expect"), SUITE).unwrap(); + let page = || std::fs::read_to_string(out.join("tests/light.md")).unwrap(); + + run( + &suite, + &["--runner", "app", "--docs", out.to_str().unwrap()], + ); + // Nothing was said, so nothing is saved or shown. + let saved = std::fs::read_to_string(out.join("results/app.tsv")).unwrap(); + assert!(!saved.contains("provenance"), "{saved}"); + assert!(!page().contains("Run against")); + + let args = ["--runner", "app", "--docs", out.to_str().unwrap()]; + let build = ["--app-revision", "aaa111", "--data-mode", "fixtures"]; + run( + &suite, + &[&args[..], &build[..], &["--fixture", "seed-7"]].concat(), + ); + let saved = std::fs::read_to_string(out.join("results/app.tsv")).unwrap(); + assert!(saved.contains("provenance\tapp-revision\taaa111\n")); + assert!(saved.contains("provenance\tdata-mode\tfixtures\n")); + assert!(saved.contains("provenance\tfixture\tseed-7\n")); + // The run's own manual is for what it ran against. + assert!(page().contains("application revision `aaa111`")); + assert!(page().contains("This is what this manual is for.")); + + // The manual for the next build, written without running again: the pass is build A's. + docs(&suite, &out, &["--expect-app-revision", "bbb222"]); + assert!(page().contains("application revision `aaa111`")); + assert!(page().contains("This manual is for application revision bbb222")); + assert!(page().contains("✅"), "the result is still the run's"); + // For the same build, and for a part the run did say. + docs( + &suite, + &out, + &[ + "--expect-app-revision", + "aaa111", + "--expect-fixture", + "seed-7", + ], + ); + assert!(page().contains("This is what this manual is for.")); + // With nothing expected, it only says what the run said. + docs(&suite, &out, &[]); + assert!(page().contains("application revision `aaa111`")); + assert!(!page().contains("This manual is for") && !page().contains("what this manual is for")); +} diff --git a/crates/story/src/coverage.rs b/crates/story/src/coverage.rs index 71c717c..f333891 100644 --- a/crates/story/src/coverage.rs +++ b/crates/story/src/coverage.rs @@ -766,6 +766,7 @@ Story: Get a quote Run { id: id.into(), started_ms: 1, + provenance: Default::default(), } } diff --git a/crates/story/src/lib.rs b/crates/story/src/lib.rs index 481dd64..fa57621 100644 --- a/crates/story/src/lib.rs +++ b/crates/story/src/lib.rs @@ -24,9 +24,9 @@ pub use plan::{ reaching_plans, untested, visit_plans, }; pub use report::{ - Attachment, CheckResult, PlanResult, Policy, Report, Run, Status, StepResult, Verdict, Window, - fingerprint, load_results, load_results_for, now_ms, platform_shots, results_file, - screenshots_named, + Agreement, Attachment, CheckResult, PlanResult, Policy, Provenance, Report, Run, Status, + StepResult, Verdict, Window, fingerprint, load_results, load_results_for, now_ms, + platform_shots, results_file, screenshots_named, }; #[cfg(test)] diff --git a/crates/story/src/report.rs b/crates/story/src/report.rs index 2c4a4ce..6e9cf0f 100644 --- a/crates/story/src/report.rs +++ b/crates/story/src/report.rs @@ -66,6 +66,85 @@ pub struct Run { pub id: String, /// Wall-clock milliseconds since the Unix epoch. pub started_ms: u64, + /// What it ran against, as far as whoever started it said. + pub provenance: Provenance, +} + +/// What a run was made against: which build of the program, and what data it had. A test's +/// fingerprint says the test hasn't changed; this says what it was a test of, so a pass saved +/// for one build isn't read as evidence about another. +/// +/// Every part is what the caller said (`RunOptions::provenance`, `expectation-run +/// --app-revision`), or nothing. Nothing is collected from the environment, and the results +/// and the manual publish it, so say only what may be published: a commit, a version, the name +/// of a fixture; not a credential, and not the data itself. A part nobody gave is unknown, +/// and is never guessed. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Provenance { + /// The program's revision: a commit, a version, a build number. + pub app_revision: Option, + /// The kind of data it ran on: `fixtures`, `live`, `recorded`. + pub data_mode: Option, + /// Which fixture or data source that was, by a name that is safe to publish. + pub fixture: Option, +} + +/// How a result's [`Provenance`] compares with the one a manual is written for. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Agreement { + /// Every part that was asked for is what the run said. + Matching, + /// A part differs: the result is about something else. It is still what that run saw, and + /// not a failure. + Mismatched, + /// Nothing differs, but the run didn't say a part that was asked for. + Unknown, +} + +impl Provenance { + /// Each part, as the results files key it and as the manual calls it. + pub fn parts(&self) -> [(&'static str, &'static str, Option<&str>); 3] { + [ + ( + "app-revision", + "application revision", + self.app_revision.as_deref(), + ), + ("data-mode", "data mode", self.data_mode.as_deref()), + ("fixture", "fixture", self.fixture.as_deref()), + ] + } + + /// Nothing was said. + pub fn is_empty(&self) -> bool { + self.parts().iter().all(|(_, _, v)| v.is_none()) + } + + fn set(&mut self, key: &str, value: &str) { + let slot = match key { + "app-revision" => &mut self.app_revision, + "data-mode" => &mut self.data_mode, + "fixture" => &mut self.fixture, + // A part a later version says: not this one's to keep. + _ => return, + }; + *slot = Some(value.to_string()); + } + + /// How this compares with `expected`, over the parts `expected` says. `None` when it says + /// none: there is nothing to compare with. + pub fn agreement(&self, expected: &Provenance) -> Option { + let mut out = None; + for ((_, _, mine), (_, _, want)) in self.parts().into_iter().zip(expected.parts()) { + let Some(want) = want else { continue }; + match mine { + Some(mine) if mine != want => return Some(Agreement::Mismatched), + Some(_) => out = out.or(Some(Agreement::Matching)), + None => out = Some(Agreement::Unknown), + } + } + out + } } impl Run { @@ -78,8 +157,15 @@ impl Run { Run { id: format!("{started_ms:x}-{:x}-{n:x}", std::process::id()), started_ms, + provenance: Provenance::default(), } } + + /// The same run, made against `provenance`. + pub fn against(mut self, provenance: Provenance) -> Run { + self.provenance = provenance; + self + } } /// Wall-clock milliseconds since the Unix epoch: what step windows and runs are measured in, so @@ -482,6 +568,12 @@ impl Report { match run { Some(x) => { let _ = writeln!(out, "run\t{}\t{}", field(&x.id), x.started_ms); + // What the run was against, a line for each part that was said. + for (key, _, value) in x.provenance.parts() { + if let Some(v) = value.map(field).filter(|v| !v.is_empty()) { + let _ = writeln!(out, "provenance\t{key}\t{v}"); + } + } } None => out.push_str("run\t-\t-\n"), } @@ -563,9 +655,16 @@ impl Report { (id, Ok(started_ms)) => Some(Run { id: id.to_string(), started_ms, + provenance: Provenance::default(), }), }; } + // About the run line before it, and so about the plans after. + ["provenance", key, value, ..] => { + if let Some(run) = run.as_mut() { + run.provenance.set(key, value); + } + } ["window", n, start, end] => { let Some((_, r)) = current.as_mut() else { continue;