From a897804549ecab3c9c166051873a21f2887d7528 Mon Sep 17 00:00:00 2001 From: Patrick Tescher Date: Wed, 11 Nov 2026 09:44:26 -0800 Subject: [PATCH 1/3] Picture a start that couldn't be set up When the reset or an arrangement fails, the runner still asks for a screenshot and keeps it as the failed start's, in the docs directory or the store. The setup's error and the failed verdict stand whether or not there is a picture, the later steps are skipped as before, and nothing is asked of the driver when screenshots are off. A failed step's picture says so in the manual: "What it looked like when this failed." Closes #80 Co-Authored-By: Claude --- book/src/drivers.md | 4 + crates/docs/src/html.rs | 3 + crates/docs/src/markdown.rs | 2 + crates/harness/src/run.rs | 7 + crates/harness/tests/setup_failure.rs | 206 ++++++++++++++++++ .../00.png | Bin .../01.png | Bin .../02.png | Bin .../coverage--boil-back-to-warm/00.png | Bin .../coverage--boil-back-to-warm/01.png | Bin .../coverage--boil-back-to-warm/02.png | Bin .../coverage--boil-back-to-warm/03.png | Bin .../coverage--hot-to-boil/00.png | Bin .../coverage--hot-to-boil/01.png | Bin .../coverage--hot-to-boil/02.png | Bin .../00.png | Bin .../01.png | Bin .../02.png | Bin .../00.png | Bin .../01.png | Bin .../02.png | Bin .../00.png | Bin .../01.png | Bin .../02.png | Bin .../00.png | Bin .../01.png | Bin .../02.png | Bin .../03.png | Bin .../04.png | Bin .../00.png | Bin .../01.png | Bin .../02.png | Bin .../coverage--boil-back-to-warm/00.png | Bin .../coverage--boil-back-to-warm/01.png | Bin .../coverage--boil-back-to-warm/02.png | Bin .../coverage--boil-back-to-warm/03.png | Bin .../swiftui}/coverage--hot-to-boil/00.png | Bin .../swiftui}/coverage--hot-to-boil/01.png | Bin .../swiftui}/coverage--hot-to-boil/02.png | Bin .../00.png | Bin .../01.png | Bin .../02.png | Bin .../00.png | Bin .../01.png | Bin .../02.png | Bin .../00.png | Bin .../01.png | Bin .../02.png | Bin .../00.png | Bin .../01.png | Bin .../02.png | Bin .../03.png | Bin .../04.png | Bin 53 files changed, 222 insertions(+) create mode 100644 crates/harness/tests/setup_failure.rs rename demos/kettle/docs/{platforms/react-native/screenshots => screenshots/platforms/react-native}/choosing-a-preset--step-to-hot-and-start/00.png (100%) rename demos/kettle/docs/{platforms/react-native/screenshots => screenshots/platforms/react-native}/choosing-a-preset--step-to-hot-and-start/01.png (100%) rename demos/kettle/docs/{platforms/react-native/screenshots => screenshots/platforms/react-native}/choosing-a-preset--step-to-hot-and-start/02.png (100%) rename demos/kettle/docs/{platforms/react-native/screenshots => screenshots/platforms/react-native}/coverage--boil-back-to-warm/00.png (100%) rename demos/kettle/docs/{platforms/react-native/screenshots => screenshots/platforms/react-native}/coverage--boil-back-to-warm/01.png (100%) rename demos/kettle/docs/{platforms/react-native/screenshots => screenshots/platforms/react-native}/coverage--boil-back-to-warm/02.png (100%) rename demos/kettle/docs/{platforms/react-native/screenshots => screenshots/platforms/react-native}/coverage--boil-back-to-warm/03.png (100%) rename demos/kettle/docs/{platforms/react-native/screenshots => screenshots/platforms/react-native}/coverage--hot-to-boil/00.png (100%) rename demos/kettle/docs/{platforms/react-native/screenshots => screenshots/platforms/react-native}/coverage--hot-to-boil/01.png (100%) rename demos/kettle/docs/{platforms/react-native/screenshots => screenshots/platforms/react-native}/coverage--hot-to-boil/02.png (100%) rename demos/kettle/docs/{platforms/react-native/screenshots => screenshots/platforms/react-native}/coverage--running-dry-stops-the-kettle/00.png (100%) rename demos/kettle/docs/{platforms/react-native/screenshots => screenshots/platforms/react-native}/coverage--running-dry-stops-the-kettle/01.png (100%) rename demos/kettle/docs/{platforms/react-native/screenshots => screenshots/platforms/react-native}/coverage--running-dry-stops-the-kettle/02.png (100%) rename demos/kettle/docs/{platforms/react-native/screenshots => screenshots/platforms/react-native}/coverage--the-child-lock-ignores-the-start-button/00.png (100%) rename demos/kettle/docs/{platforms/react-native/screenshots => screenshots/platforms/react-native}/coverage--the-child-lock-ignores-the-start-button/01.png (100%) rename demos/kettle/docs/{platforms/react-native/screenshots => screenshots/platforms/react-native}/coverage--the-child-lock-ignores-the-start-button/02.png (100%) rename demos/kettle/docs/{platforms/react-native/screenshots => screenshots/platforms/react-native}/first-cup--turn-it-up-then-turn-it-off/00.png (100%) rename demos/kettle/docs/{platforms/react-native/screenshots => screenshots/platforms/react-native}/first-cup--turn-it-up-then-turn-it-off/01.png (100%) rename demos/kettle/docs/{platforms/react-native/screenshots => screenshots/platforms/react-native}/first-cup--turn-it-up-then-turn-it-off/02.png (100%) rename demos/kettle/docs/{platforms/react-native/screenshots => screenshots/platforms/react-native}/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/00.png (100%) rename demos/kettle/docs/{platforms/react-native/screenshots => screenshots/platforms/react-native}/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/01.png (100%) rename demos/kettle/docs/{platforms/react-native/screenshots => screenshots/platforms/react-native}/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/02.png (100%) rename demos/kettle/docs/{platforms/react-native/screenshots => screenshots/platforms/react-native}/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/03.png (100%) rename demos/kettle/docs/{platforms/react-native/screenshots => screenshots/platforms/react-native}/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/04.png (100%) rename demos/kettle/docs/{platforms/swiftui/screenshots => screenshots/platforms/swiftui}/choosing-a-preset--step-to-hot-and-start/00.png (100%) rename demos/kettle/docs/{platforms/swiftui/screenshots => screenshots/platforms/swiftui}/choosing-a-preset--step-to-hot-and-start/01.png (100%) rename demos/kettle/docs/{platforms/swiftui/screenshots => screenshots/platforms/swiftui}/choosing-a-preset--step-to-hot-and-start/02.png (100%) rename demos/kettle/docs/{platforms/swiftui/screenshots => screenshots/platforms/swiftui}/coverage--boil-back-to-warm/00.png (100%) rename demos/kettle/docs/{platforms/swiftui/screenshots => screenshots/platforms/swiftui}/coverage--boil-back-to-warm/01.png (100%) rename demos/kettle/docs/{platforms/swiftui/screenshots => screenshots/platforms/swiftui}/coverage--boil-back-to-warm/02.png (100%) rename demos/kettle/docs/{platforms/swiftui/screenshots => screenshots/platforms/swiftui}/coverage--boil-back-to-warm/03.png (100%) rename demos/kettle/docs/{platforms/swiftui/screenshots => screenshots/platforms/swiftui}/coverage--hot-to-boil/00.png (100%) rename demos/kettle/docs/{platforms/swiftui/screenshots => screenshots/platforms/swiftui}/coverage--hot-to-boil/01.png (100%) rename demos/kettle/docs/{platforms/swiftui/screenshots => screenshots/platforms/swiftui}/coverage--hot-to-boil/02.png (100%) rename demos/kettle/docs/{platforms/swiftui/screenshots => screenshots/platforms/swiftui}/coverage--running-dry-stops-the-kettle/00.png (100%) rename demos/kettle/docs/{platforms/swiftui/screenshots => screenshots/platforms/swiftui}/coverage--running-dry-stops-the-kettle/01.png (100%) rename demos/kettle/docs/{platforms/swiftui/screenshots => screenshots/platforms/swiftui}/coverage--running-dry-stops-the-kettle/02.png (100%) rename demos/kettle/docs/{platforms/swiftui/screenshots => screenshots/platforms/swiftui}/coverage--the-child-lock-ignores-the-start-button/00.png (100%) rename demos/kettle/docs/{platforms/swiftui/screenshots => screenshots/platforms/swiftui}/coverage--the-child-lock-ignores-the-start-button/01.png (100%) rename demos/kettle/docs/{platforms/swiftui/screenshots => screenshots/platforms/swiftui}/coverage--the-child-lock-ignores-the-start-button/02.png (100%) rename demos/kettle/docs/{platforms/swiftui/screenshots => screenshots/platforms/swiftui}/first-cup--turn-it-up-then-turn-it-off/00.png (100%) rename demos/kettle/docs/{platforms/swiftui/screenshots => screenshots/platforms/swiftui}/first-cup--turn-it-up-then-turn-it-off/01.png (100%) rename demos/kettle/docs/{platforms/swiftui/screenshots => screenshots/platforms/swiftui}/first-cup--turn-it-up-then-turn-it-off/02.png (100%) rename demos/kettle/docs/{platforms/swiftui/screenshots => screenshots/platforms/swiftui}/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/00.png (100%) rename demos/kettle/docs/{platforms/swiftui/screenshots => screenshots/platforms/swiftui}/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/01.png (100%) rename demos/kettle/docs/{platforms/swiftui/screenshots => screenshots/platforms/swiftui}/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/02.png (100%) rename demos/kettle/docs/{platforms/swiftui/screenshots => screenshots/platforms/swiftui}/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/03.png (100%) rename demos/kettle/docs/{platforms/swiftui/screenshots => screenshots/platforms/swiftui}/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/04.png (100%) diff --git a/book/src/drivers.md b/book/src/drivers.md index 1210362..7b14211 100644 --- a/book/src/drivers.md +++ b/book/src/drivers.md @@ -158,6 +158,10 @@ acceptance.docs(Some(&report)).write("docs")?; With a `docs_dir`, screenshots go to `/screenshots//.`. +A test whose start can't be set up (the reset or an arrangement fails) fails there, and its +other steps are skipped. The runner still asks for a screenshot, so the report shows what the +system looked like when it went wrong. If there is none, the failure stands as it is. + ### Several runners `Acceptance::publish(dir, runner, &report)` saves a run's results as diff --git a/crates/docs/src/html.rs b/crates/docs/src/html.rs index 6a4f212..95c9ca2 100644 --- a/crates/docs/src/html.rs +++ b/crates/docs/src/html.rs @@ -1024,6 +1024,9 @@ impl Site<'_> { esc(&alt), if stale { "
From before this page changed.
" + } else if status == Status::Failed { + // Evidence of what went wrong, not a picture of what should be there. + "
What it looked like when this failed.
" } else { "" } diff --git a/crates/docs/src/markdown.rs b/crates/docs/src/markdown.rs index 24ec1a1..c6f7291 100644 --- a/crates/docs/src/markdown.rs +++ b/crates/docs/src/markdown.rs @@ -236,6 +236,8 @@ fn image( ); if stale { let _ = writeln!(o, "*From before this page changed.*\n"); + } else if status == Status::Failed { + let _ = writeln!(o, "*What it looked like when this failed.*\n"); } } None if reader => {} diff --git a/crates/harness/src/run.rs b/crates/harness/src/run.rs index 2bac19f..686a8d7 100644 --- a/crates/harness/src/run.rs +++ b/crates/harness/src/run.rs @@ -357,6 +357,13 @@ fn run_steps( start_ms: began, end_ms: now_ms(), }); + // What it looked like when it went wrong, if the driver can still say. The failure + // is the setup's whether or not there is a picture of it. + start.screenshot = shoot(driver, 0); + start.window = Some(Window { + start_ms: began, + end_ms: now_ms(), + }); keep(driver, 0); start }; diff --git a/crates/harness/tests/setup_failure.rs b/crates/harness/tests/setup_failure.rs new file mode 100644 index 0000000..8959d5b --- /dev/null +++ b/crates/harness/tests/setup_failure.rs @@ -0,0 +1,206 @@ +//! A test whose start can't be set up fails, and still shows what the system looked like. + +use expectation_harness::{Acceptance, Capture, Driver, RunOptions, Screenshot}; +use expectation_model::{Event, Value, Variable}; +use expectation_story::Status; +use std::path::{Path, PathBuf}; + +const LAMP: &str = " +Vocabulary: + State: the lamp is off | on + Event: the user flips the switch +Background: + Given the lamp is off +Feature: Lamp + Expectation: Flipping turns it on + Given the lamp is off + When the user flips the switch + Then the lamp is on + Expectation: Flipping turns it off + Given the lamp is on + When the user flips the switch + Then the lamp is off +Story: Off again + Given the lamp is on + When the user flips the switch + Then the lamp is off +"; + +/// A lamp whose reset or arrangement can break, and whose camera can too. +#[derive(Default)] +struct Lamp { + on: bool, + reset_fails: bool, + arrange_fails: bool, + camera_fails: bool, + shots: usize, + performed: usize, + /// One capture waits for whoever asks next. + captured: bool, +} + +impl Driver for Lamp { + fn reset(&mut self) -> Result<(), String> { + self.on = false; + self.captured = true; + if self.reset_fails { + return Err("the lamp is unplugged".into()); + } + Ok(()) + } + fn arrange(&mut self, _: &Variable, _: &Value) -> Result<(), String> { + if self.arrange_fails { + return Err("the switch is stuck".into()); + } + self.on = true; + Ok(()) + } + fn set(&mut self, _: &Variable, _: &Value) -> Result<(), String> { + Ok(()) + } + fn perform(&mut self, _: &Event) -> Result<(), String> { + self.performed += 1; + self.on = !self.on; + Ok(()) + } + fn observe(&mut self, _: &Variable) -> Result { + Ok(Value::Enum(if self.on { "on" } else { "off" }.into())) + } + fn take_attachments(&mut self) -> Vec { + if !std::mem::take(&mut self.captured) { + return vec![]; + } + vec![Capture { + name: "lamp.log".into(), + media_type: "text/plain".into(), + bytes: b"click".to_vec(), + }] + } + fn screenshot(&mut self) -> Result, String> { + self.shots += 1; + if self.camera_fails { + return Err("no camera".into()); + } + Ok(Some(Screenshot::svg(format!( + "broken {}", + self.shots + )))) + } +} + +fn lamp() -> Acceptance { + let loaded = expectation_language::load_sources("lamp", &[("lamp.expect", LAMP)]); + Acceptance::from_loaded(loaded).unwrap_or_else(|e| panic!("{e}")) +} + +fn fresh(name: &str) -> PathBuf { + let dir = Path::new(env!("CARGO_TARGET_TMPDIR")).join(name); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + dir +} + +/// Both ways a start can fail, each with both places a screenshot can go. +#[test] +fn a_start_that_cant_be_set_up_is_pictured_as_it_failed() { + let a = lamp(); + for (reset_fails, error) in [ + (true, "couldn't reset: the lamp is unplugged"), + ( + false, + "couldn't arrange the lamp is on: the switch is stuck", + ), + ] { + for stored in [false, true] { + let root = fresh(&format!("setup-failure-{reset_fails}-{stored}")); + let (docs, store, kept) = (root.join("docs"), root.join("store"), root.join("kept")); + let mut driver = Lamp { + reset_fails, + arrange_fails: !reset_fails, + ..Default::default() + }; + let report = a.run( + &mut driver, + &RunOptions { + docs_dir: (!stored).then(|| docs.clone()), + screenshot_store: stored.then(|| store.clone()), + attachments_dir: Some(kept.clone()), + ..Default::default() + }, + ); + let r = &report.results[0]; + // The failure is the setup's, as it was. + assert_eq!(r.status, Status::Failed); + assert_eq!(r.start.status, Status::Failed); + assert_eq!(r.start.error.as_deref(), Some(error)); + // Nothing after it ran. + assert!(r.steps.iter().all(|s| s.status == Status::Skipped)); + assert!(r.steps.iter().all(|s| s.screenshot.is_none())); + assert_eq!(driver.performed, 0); + // What it looked like is the start's screenshot, where screenshots go. + let shot = r.start.screenshot.as_deref().expect("a screenshot"); + let file = if stored { + store.join(shot.strip_prefix("screenshots/").unwrap()) + } else { + assert_eq!(shot, format!("screenshots/{}/00.svg", a.plans[0].slug)); + docs.join(shot) + }; + assert!(std::fs::read_to_string(file).unwrap().contains("broken")); + assert!(r.start.window.is_some()); + // And what the driver captured is still kept, for the start. + assert_eq!(r.attachments.len(), 1); + assert_eq!(r.attachments[0].step, 0); + assert!(kept.join(&r.attachments[0].path).is_file()); + + // The manual says the picture is of the failure. + if !stored { + a.docs(Some(&report)).write(&docs).unwrap(); + let page = std::fs::read_to_string( + docs.join("tests").join(format!("{}.md", a.plans[0].slug)), + ) + .unwrap(); + assert!(page.contains(shot), "{page}"); + assert!(page.contains("What it looked like when this failed.")); + } + } + } +} + +#[test] +fn a_camera_that_fails_too_leaves_the_setup_failure_as_it_is() { + let a = lamp(); + let docs = fresh("setup-failure-no-camera"); + let mut driver = Lamp { + reset_fails: true, + camera_fails: true, + ..Default::default() + }; + let report = a.run( + &mut driver, + &RunOptions { + docs_dir: Some(docs), + ..Default::default() + }, + ); + let r = &report.results[0]; + assert_eq!(r.status, Status::Failed); + assert_eq!( + r.start.error.as_deref(), + Some("couldn't reset: the lamp is unplugged") + ); + assert_eq!(r.start.screenshot, None); + assert!(driver.shots > 0, "it tried"); +} + +#[test] +fn without_screenshots_a_failed_setup_asks_for_none() { + let a = lamp(); + let mut driver = Lamp { + reset_fails: true, + ..Default::default() + }; + let report = a.run(&mut driver, &RunOptions::default()); + assert_eq!(report.results[0].status, Status::Failed); + assert_eq!(report.results[0].start.screenshot, None); + assert_eq!(driver.shots, 0); +} diff --git a/demos/kettle/docs/platforms/react-native/screenshots/choosing-a-preset--step-to-hot-and-start/00.png b/demos/kettle/docs/screenshots/platforms/react-native/choosing-a-preset--step-to-hot-and-start/00.png similarity index 100% rename from demos/kettle/docs/platforms/react-native/screenshots/choosing-a-preset--step-to-hot-and-start/00.png rename to demos/kettle/docs/screenshots/platforms/react-native/choosing-a-preset--step-to-hot-and-start/00.png diff --git a/demos/kettle/docs/platforms/react-native/screenshots/choosing-a-preset--step-to-hot-and-start/01.png b/demos/kettle/docs/screenshots/platforms/react-native/choosing-a-preset--step-to-hot-and-start/01.png similarity index 100% rename from demos/kettle/docs/platforms/react-native/screenshots/choosing-a-preset--step-to-hot-and-start/01.png rename to demos/kettle/docs/screenshots/platforms/react-native/choosing-a-preset--step-to-hot-and-start/01.png diff --git a/demos/kettle/docs/platforms/react-native/screenshots/choosing-a-preset--step-to-hot-and-start/02.png b/demos/kettle/docs/screenshots/platforms/react-native/choosing-a-preset--step-to-hot-and-start/02.png similarity index 100% rename from demos/kettle/docs/platforms/react-native/screenshots/choosing-a-preset--step-to-hot-and-start/02.png rename to demos/kettle/docs/screenshots/platforms/react-native/choosing-a-preset--step-to-hot-and-start/02.png diff --git a/demos/kettle/docs/platforms/react-native/screenshots/coverage--boil-back-to-warm/00.png b/demos/kettle/docs/screenshots/platforms/react-native/coverage--boil-back-to-warm/00.png similarity index 100% rename from demos/kettle/docs/platforms/react-native/screenshots/coverage--boil-back-to-warm/00.png rename to demos/kettle/docs/screenshots/platforms/react-native/coverage--boil-back-to-warm/00.png diff --git a/demos/kettle/docs/platforms/react-native/screenshots/coverage--boil-back-to-warm/01.png b/demos/kettle/docs/screenshots/platforms/react-native/coverage--boil-back-to-warm/01.png similarity index 100% rename from demos/kettle/docs/platforms/react-native/screenshots/coverage--boil-back-to-warm/01.png rename to demos/kettle/docs/screenshots/platforms/react-native/coverage--boil-back-to-warm/01.png diff --git a/demos/kettle/docs/platforms/react-native/screenshots/coverage--boil-back-to-warm/02.png b/demos/kettle/docs/screenshots/platforms/react-native/coverage--boil-back-to-warm/02.png similarity index 100% rename from demos/kettle/docs/platforms/react-native/screenshots/coverage--boil-back-to-warm/02.png rename to demos/kettle/docs/screenshots/platforms/react-native/coverage--boil-back-to-warm/02.png diff --git a/demos/kettle/docs/platforms/react-native/screenshots/coverage--boil-back-to-warm/03.png b/demos/kettle/docs/screenshots/platforms/react-native/coverage--boil-back-to-warm/03.png similarity index 100% rename from demos/kettle/docs/platforms/react-native/screenshots/coverage--boil-back-to-warm/03.png rename to demos/kettle/docs/screenshots/platforms/react-native/coverage--boil-back-to-warm/03.png diff --git a/demos/kettle/docs/platforms/react-native/screenshots/coverage--hot-to-boil/00.png b/demos/kettle/docs/screenshots/platforms/react-native/coverage--hot-to-boil/00.png similarity index 100% rename from demos/kettle/docs/platforms/react-native/screenshots/coverage--hot-to-boil/00.png rename to demos/kettle/docs/screenshots/platforms/react-native/coverage--hot-to-boil/00.png diff --git a/demos/kettle/docs/platforms/react-native/screenshots/coverage--hot-to-boil/01.png b/demos/kettle/docs/screenshots/platforms/react-native/coverage--hot-to-boil/01.png similarity index 100% rename from demos/kettle/docs/platforms/react-native/screenshots/coverage--hot-to-boil/01.png rename to demos/kettle/docs/screenshots/platforms/react-native/coverage--hot-to-boil/01.png diff --git a/demos/kettle/docs/platforms/react-native/screenshots/coverage--hot-to-boil/02.png b/demos/kettle/docs/screenshots/platforms/react-native/coverage--hot-to-boil/02.png similarity index 100% rename from demos/kettle/docs/platforms/react-native/screenshots/coverage--hot-to-boil/02.png rename to demos/kettle/docs/screenshots/platforms/react-native/coverage--hot-to-boil/02.png diff --git a/demos/kettle/docs/platforms/react-native/screenshots/coverage--running-dry-stops-the-kettle/00.png b/demos/kettle/docs/screenshots/platforms/react-native/coverage--running-dry-stops-the-kettle/00.png similarity index 100% rename from demos/kettle/docs/platforms/react-native/screenshots/coverage--running-dry-stops-the-kettle/00.png rename to demos/kettle/docs/screenshots/platforms/react-native/coverage--running-dry-stops-the-kettle/00.png diff --git a/demos/kettle/docs/platforms/react-native/screenshots/coverage--running-dry-stops-the-kettle/01.png b/demos/kettle/docs/screenshots/platforms/react-native/coverage--running-dry-stops-the-kettle/01.png similarity index 100% rename from demos/kettle/docs/platforms/react-native/screenshots/coverage--running-dry-stops-the-kettle/01.png rename to demos/kettle/docs/screenshots/platforms/react-native/coverage--running-dry-stops-the-kettle/01.png diff --git a/demos/kettle/docs/platforms/react-native/screenshots/coverage--running-dry-stops-the-kettle/02.png b/demos/kettle/docs/screenshots/platforms/react-native/coverage--running-dry-stops-the-kettle/02.png similarity index 100% rename from demos/kettle/docs/platforms/react-native/screenshots/coverage--running-dry-stops-the-kettle/02.png rename to demos/kettle/docs/screenshots/platforms/react-native/coverage--running-dry-stops-the-kettle/02.png diff --git a/demos/kettle/docs/platforms/react-native/screenshots/coverage--the-child-lock-ignores-the-start-button/00.png b/demos/kettle/docs/screenshots/platforms/react-native/coverage--the-child-lock-ignores-the-start-button/00.png similarity index 100% rename from demos/kettle/docs/platforms/react-native/screenshots/coverage--the-child-lock-ignores-the-start-button/00.png rename to demos/kettle/docs/screenshots/platforms/react-native/coverage--the-child-lock-ignores-the-start-button/00.png diff --git a/demos/kettle/docs/platforms/react-native/screenshots/coverage--the-child-lock-ignores-the-start-button/01.png b/demos/kettle/docs/screenshots/platforms/react-native/coverage--the-child-lock-ignores-the-start-button/01.png similarity index 100% rename from demos/kettle/docs/platforms/react-native/screenshots/coverage--the-child-lock-ignores-the-start-button/01.png rename to demos/kettle/docs/screenshots/platforms/react-native/coverage--the-child-lock-ignores-the-start-button/01.png diff --git a/demos/kettle/docs/platforms/react-native/screenshots/coverage--the-child-lock-ignores-the-start-button/02.png b/demos/kettle/docs/screenshots/platforms/react-native/coverage--the-child-lock-ignores-the-start-button/02.png similarity index 100% rename from demos/kettle/docs/platforms/react-native/screenshots/coverage--the-child-lock-ignores-the-start-button/02.png rename to demos/kettle/docs/screenshots/platforms/react-native/coverage--the-child-lock-ignores-the-start-button/02.png diff --git a/demos/kettle/docs/platforms/react-native/screenshots/first-cup--turn-it-up-then-turn-it-off/00.png b/demos/kettle/docs/screenshots/platforms/react-native/first-cup--turn-it-up-then-turn-it-off/00.png similarity index 100% rename from demos/kettle/docs/platforms/react-native/screenshots/first-cup--turn-it-up-then-turn-it-off/00.png rename to demos/kettle/docs/screenshots/platforms/react-native/first-cup--turn-it-up-then-turn-it-off/00.png diff --git a/demos/kettle/docs/platforms/react-native/screenshots/first-cup--turn-it-up-then-turn-it-off/01.png b/demos/kettle/docs/screenshots/platforms/react-native/first-cup--turn-it-up-then-turn-it-off/01.png similarity index 100% rename from demos/kettle/docs/platforms/react-native/screenshots/first-cup--turn-it-up-then-turn-it-off/01.png rename to demos/kettle/docs/screenshots/platforms/react-native/first-cup--turn-it-up-then-turn-it-off/01.png diff --git a/demos/kettle/docs/platforms/react-native/screenshots/first-cup--turn-it-up-then-turn-it-off/02.png b/demos/kettle/docs/screenshots/platforms/react-native/first-cup--turn-it-up-then-turn-it-off/02.png similarity index 100% rename from demos/kettle/docs/platforms/react-native/screenshots/first-cup--turn-it-up-then-turn-it-off/02.png rename to demos/kettle/docs/screenshots/platforms/react-native/first-cup--turn-it-up-then-turn-it-off/02.png diff --git a/demos/kettle/docs/platforms/react-native/screenshots/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/00.png b/demos/kettle/docs/screenshots/platforms/react-native/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/00.png similarity index 100% rename from demos/kettle/docs/platforms/react-native/screenshots/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/00.png rename to demos/kettle/docs/screenshots/platforms/react-native/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/00.png diff --git a/demos/kettle/docs/platforms/react-native/screenshots/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/01.png b/demos/kettle/docs/screenshots/platforms/react-native/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/01.png similarity index 100% rename from demos/kettle/docs/platforms/react-native/screenshots/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/01.png rename to demos/kettle/docs/screenshots/platforms/react-native/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/01.png diff --git a/demos/kettle/docs/platforms/react-native/screenshots/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/02.png b/demos/kettle/docs/screenshots/platforms/react-native/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/02.png similarity index 100% rename from demos/kettle/docs/platforms/react-native/screenshots/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/02.png rename to demos/kettle/docs/screenshots/platforms/react-native/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/02.png diff --git a/demos/kettle/docs/platforms/react-native/screenshots/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/03.png b/demos/kettle/docs/screenshots/platforms/react-native/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/03.png similarity index 100% rename from demos/kettle/docs/platforms/react-native/screenshots/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/03.png rename to demos/kettle/docs/screenshots/platforms/react-native/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/03.png diff --git a/demos/kettle/docs/platforms/react-native/screenshots/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/04.png b/demos/kettle/docs/screenshots/platforms/react-native/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/04.png similarity index 100% rename from demos/kettle/docs/platforms/react-native/screenshots/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/04.png rename to demos/kettle/docs/screenshots/platforms/react-native/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/04.png diff --git a/demos/kettle/docs/platforms/swiftui/screenshots/choosing-a-preset--step-to-hot-and-start/00.png b/demos/kettle/docs/screenshots/platforms/swiftui/choosing-a-preset--step-to-hot-and-start/00.png similarity index 100% rename from demos/kettle/docs/platforms/swiftui/screenshots/choosing-a-preset--step-to-hot-and-start/00.png rename to demos/kettle/docs/screenshots/platforms/swiftui/choosing-a-preset--step-to-hot-and-start/00.png diff --git a/demos/kettle/docs/platforms/swiftui/screenshots/choosing-a-preset--step-to-hot-and-start/01.png b/demos/kettle/docs/screenshots/platforms/swiftui/choosing-a-preset--step-to-hot-and-start/01.png similarity index 100% rename from demos/kettle/docs/platforms/swiftui/screenshots/choosing-a-preset--step-to-hot-and-start/01.png rename to demos/kettle/docs/screenshots/platforms/swiftui/choosing-a-preset--step-to-hot-and-start/01.png diff --git a/demos/kettle/docs/platforms/swiftui/screenshots/choosing-a-preset--step-to-hot-and-start/02.png b/demos/kettle/docs/screenshots/platforms/swiftui/choosing-a-preset--step-to-hot-and-start/02.png similarity index 100% rename from demos/kettle/docs/platforms/swiftui/screenshots/choosing-a-preset--step-to-hot-and-start/02.png rename to demos/kettle/docs/screenshots/platforms/swiftui/choosing-a-preset--step-to-hot-and-start/02.png diff --git a/demos/kettle/docs/platforms/swiftui/screenshots/coverage--boil-back-to-warm/00.png b/demos/kettle/docs/screenshots/platforms/swiftui/coverage--boil-back-to-warm/00.png similarity index 100% rename from demos/kettle/docs/platforms/swiftui/screenshots/coverage--boil-back-to-warm/00.png rename to demos/kettle/docs/screenshots/platforms/swiftui/coverage--boil-back-to-warm/00.png diff --git a/demos/kettle/docs/platforms/swiftui/screenshots/coverage--boil-back-to-warm/01.png b/demos/kettle/docs/screenshots/platforms/swiftui/coverage--boil-back-to-warm/01.png similarity index 100% rename from demos/kettle/docs/platforms/swiftui/screenshots/coverage--boil-back-to-warm/01.png rename to demos/kettle/docs/screenshots/platforms/swiftui/coverage--boil-back-to-warm/01.png diff --git a/demos/kettle/docs/platforms/swiftui/screenshots/coverage--boil-back-to-warm/02.png b/demos/kettle/docs/screenshots/platforms/swiftui/coverage--boil-back-to-warm/02.png similarity index 100% rename from demos/kettle/docs/platforms/swiftui/screenshots/coverage--boil-back-to-warm/02.png rename to demos/kettle/docs/screenshots/platforms/swiftui/coverage--boil-back-to-warm/02.png diff --git a/demos/kettle/docs/platforms/swiftui/screenshots/coverage--boil-back-to-warm/03.png b/demos/kettle/docs/screenshots/platforms/swiftui/coverage--boil-back-to-warm/03.png similarity index 100% rename from demos/kettle/docs/platforms/swiftui/screenshots/coverage--boil-back-to-warm/03.png rename to demos/kettle/docs/screenshots/platforms/swiftui/coverage--boil-back-to-warm/03.png diff --git a/demos/kettle/docs/platforms/swiftui/screenshots/coverage--hot-to-boil/00.png b/demos/kettle/docs/screenshots/platforms/swiftui/coverage--hot-to-boil/00.png similarity index 100% rename from demos/kettle/docs/platforms/swiftui/screenshots/coverage--hot-to-boil/00.png rename to demos/kettle/docs/screenshots/platforms/swiftui/coverage--hot-to-boil/00.png diff --git a/demos/kettle/docs/platforms/swiftui/screenshots/coverage--hot-to-boil/01.png b/demos/kettle/docs/screenshots/platforms/swiftui/coverage--hot-to-boil/01.png similarity index 100% rename from demos/kettle/docs/platforms/swiftui/screenshots/coverage--hot-to-boil/01.png rename to demos/kettle/docs/screenshots/platforms/swiftui/coverage--hot-to-boil/01.png diff --git a/demos/kettle/docs/platforms/swiftui/screenshots/coverage--hot-to-boil/02.png b/demos/kettle/docs/screenshots/platforms/swiftui/coverage--hot-to-boil/02.png similarity index 100% rename from demos/kettle/docs/platforms/swiftui/screenshots/coverage--hot-to-boil/02.png rename to demos/kettle/docs/screenshots/platforms/swiftui/coverage--hot-to-boil/02.png diff --git a/demos/kettle/docs/platforms/swiftui/screenshots/coverage--running-dry-stops-the-kettle/00.png b/demos/kettle/docs/screenshots/platforms/swiftui/coverage--running-dry-stops-the-kettle/00.png similarity index 100% rename from demos/kettle/docs/platforms/swiftui/screenshots/coverage--running-dry-stops-the-kettle/00.png rename to demos/kettle/docs/screenshots/platforms/swiftui/coverage--running-dry-stops-the-kettle/00.png diff --git a/demos/kettle/docs/platforms/swiftui/screenshots/coverage--running-dry-stops-the-kettle/01.png b/demos/kettle/docs/screenshots/platforms/swiftui/coverage--running-dry-stops-the-kettle/01.png similarity index 100% rename from demos/kettle/docs/platforms/swiftui/screenshots/coverage--running-dry-stops-the-kettle/01.png rename to demos/kettle/docs/screenshots/platforms/swiftui/coverage--running-dry-stops-the-kettle/01.png diff --git a/demos/kettle/docs/platforms/swiftui/screenshots/coverage--running-dry-stops-the-kettle/02.png b/demos/kettle/docs/screenshots/platforms/swiftui/coverage--running-dry-stops-the-kettle/02.png similarity index 100% rename from demos/kettle/docs/platforms/swiftui/screenshots/coverage--running-dry-stops-the-kettle/02.png rename to demos/kettle/docs/screenshots/platforms/swiftui/coverage--running-dry-stops-the-kettle/02.png diff --git a/demos/kettle/docs/platforms/swiftui/screenshots/coverage--the-child-lock-ignores-the-start-button/00.png b/demos/kettle/docs/screenshots/platforms/swiftui/coverage--the-child-lock-ignores-the-start-button/00.png similarity index 100% rename from demos/kettle/docs/platforms/swiftui/screenshots/coverage--the-child-lock-ignores-the-start-button/00.png rename to demos/kettle/docs/screenshots/platforms/swiftui/coverage--the-child-lock-ignores-the-start-button/00.png diff --git a/demos/kettle/docs/platforms/swiftui/screenshots/coverage--the-child-lock-ignores-the-start-button/01.png b/demos/kettle/docs/screenshots/platforms/swiftui/coverage--the-child-lock-ignores-the-start-button/01.png similarity index 100% rename from demos/kettle/docs/platforms/swiftui/screenshots/coverage--the-child-lock-ignores-the-start-button/01.png rename to demos/kettle/docs/screenshots/platforms/swiftui/coverage--the-child-lock-ignores-the-start-button/01.png diff --git a/demos/kettle/docs/platforms/swiftui/screenshots/coverage--the-child-lock-ignores-the-start-button/02.png b/demos/kettle/docs/screenshots/platforms/swiftui/coverage--the-child-lock-ignores-the-start-button/02.png similarity index 100% rename from demos/kettle/docs/platforms/swiftui/screenshots/coverage--the-child-lock-ignores-the-start-button/02.png rename to demos/kettle/docs/screenshots/platforms/swiftui/coverage--the-child-lock-ignores-the-start-button/02.png diff --git a/demos/kettle/docs/platforms/swiftui/screenshots/first-cup--turn-it-up-then-turn-it-off/00.png b/demos/kettle/docs/screenshots/platforms/swiftui/first-cup--turn-it-up-then-turn-it-off/00.png similarity index 100% rename from demos/kettle/docs/platforms/swiftui/screenshots/first-cup--turn-it-up-then-turn-it-off/00.png rename to demos/kettle/docs/screenshots/platforms/swiftui/first-cup--turn-it-up-then-turn-it-off/00.png diff --git a/demos/kettle/docs/platforms/swiftui/screenshots/first-cup--turn-it-up-then-turn-it-off/01.png b/demos/kettle/docs/screenshots/platforms/swiftui/first-cup--turn-it-up-then-turn-it-off/01.png similarity index 100% rename from demos/kettle/docs/platforms/swiftui/screenshots/first-cup--turn-it-up-then-turn-it-off/01.png rename to demos/kettle/docs/screenshots/platforms/swiftui/first-cup--turn-it-up-then-turn-it-off/01.png diff --git a/demos/kettle/docs/platforms/swiftui/screenshots/first-cup--turn-it-up-then-turn-it-off/02.png b/demos/kettle/docs/screenshots/platforms/swiftui/first-cup--turn-it-up-then-turn-it-off/02.png similarity index 100% rename from demos/kettle/docs/platforms/swiftui/screenshots/first-cup--turn-it-up-then-turn-it-off/02.png rename to demos/kettle/docs/screenshots/platforms/swiftui/first-cup--turn-it-up-then-turn-it-off/02.png diff --git a/demos/kettle/docs/platforms/swiftui/screenshots/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/00.png b/demos/kettle/docs/screenshots/platforms/swiftui/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/00.png similarity index 100% rename from demos/kettle/docs/platforms/swiftui/screenshots/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/00.png rename to demos/kettle/docs/screenshots/platforms/swiftui/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/00.png diff --git a/demos/kettle/docs/platforms/swiftui/screenshots/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/01.png b/demos/kettle/docs/screenshots/platforms/swiftui/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/01.png similarity index 100% rename from demos/kettle/docs/platforms/swiftui/screenshots/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/01.png rename to demos/kettle/docs/screenshots/platforms/swiftui/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/01.png diff --git a/demos/kettle/docs/platforms/swiftui/screenshots/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/02.png b/demos/kettle/docs/screenshots/platforms/swiftui/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/02.png similarity index 100% rename from demos/kettle/docs/platforms/swiftui/screenshots/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/02.png rename to demos/kettle/docs/screenshots/platforms/swiftui/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/02.png diff --git a/demos/kettle/docs/platforms/swiftui/screenshots/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/03.png b/demos/kettle/docs/screenshots/platforms/swiftui/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/03.png similarity index 100% rename from demos/kettle/docs/platforms/swiftui/screenshots/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/03.png rename to demos/kettle/docs/screenshots/platforms/swiftui/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/03.png diff --git a/demos/kettle/docs/platforms/swiftui/screenshots/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/04.png b/demos/kettle/docs/screenshots/platforms/swiftui/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/04.png similarity index 100% rename from demos/kettle/docs/platforms/swiftui/screenshots/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/04.png rename to demos/kettle/docs/screenshots/platforms/swiftui/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/04.png From a6b548993cda725eebc8c40bf9b9d653b3bb5e73 Mon Sep 17 00:00:00 2001 From: Patrick Tescher Date: Wed, 11 Nov 2026 09:44:33 -0800 Subject: [PATCH 2/3] Each platform's results name its own screenshots expectation-run wrote a later platform's screenshots under its own tree (platforms//screenshots/) while its manual looked in the docs' shared screenshots/, under the same names the default platform uses. The manual showed "No screenshot of this step", or the default platform's picture. A later platform's per-test screenshots now go to screenshots/platforms/// in the shared docs directory, and its results name them so (RunOptions::screenshots_under, which Acceptance::run_options and expectation-run set). A per-test name outside that folder is from before this and is shown as a placeholder, in the platform's own manual and where the default manual borrows its results. Screenshots kept by content are unchanged. The kettle demo's SwiftUI and React Native screenshots move to their folders, and their results name them there. Closes #77 Co-Authored-By: Claude --- book/src/platforms.md | 17 ++ crates/cli/src/lib.rs | 8 +- crates/harness/src/lib.rs | 30 +++- crates/harness/src/run.rs | 20 ++- crates/harness/tests/demo_results.rs | 6 +- crates/remote/src/bin/expectation-run.rs | 6 +- crates/remote/tests/platform_screenshots.rs | 170 ++++++++++++++++++ crates/story/src/lib.rs | 3 +- crates/story/src/report.rs | 27 +++ .../kettle/docs/results/app@React Native.tsv | 48 ++--- demos/kettle/docs/results/app@SwiftUI.tsv | 48 ++--- 11 files changed, 326 insertions(+), 57 deletions(-) create mode 100644 crates/remote/tests/platform_screenshots.rs diff --git a/book/src/platforms.md b/book/src/platforms.md index 9f345a4..afbd2a6 100644 --- a/book/src/platforms.md +++ b/book/src/platforms.md @@ -54,6 +54,23 @@ and merges them with what other runners saved. under `platforms//`. Every HTML page has a Platform menu that opens the same page in the other manual. +The platforms share one docs directory, and so one `screenshots/`. The default platform's are in +`screenshots//`, and each other platform's in a folder of its own, +`screenshots/platforms///`, so the same test on two platforms never shows the other's +picture. `expectation-run --docs` does this. A driver in Rust gets it from +`acceptance.run_options("docs")`, or by setting `RunOptions::screenshots_under` to +`acceptance.screenshots_under()`: + +```rust +let acceptance = Acceptance::load_for("expectations", "iOS")?; +let report = acceptance.run(&mut driver, &acceptance.run_options("docs")); +acceptance.publish("docs".as_ref(), "app", &report)?; +``` + +Results saved before this name a later platform's screenshots as `screenshots//…`. Its +manual shows a placeholder for those until the tests run again. Screenshots kept by content in +a store are the same on every platform. + ## Saying what each platform is The docs have a menu that picks the version of the manual. Say what each platform is with diff --git a/crates/cli/src/lib.rs b/crates/cli/src/lib.rs index 4932260..b29d0e2 100644 --- a/crates/cli/src/lib.rs +++ b/crates/cli/src/lib.rs @@ -303,7 +303,13 @@ fn docs( || Path::new(out).join("results"), |r| Path::new(r).to_path_buf(), ); - let report = expectation_story::load_results_for(&results_dir, &c.plans, name.as_deref()); + let mut report = + expectation_story::load_results_for(&results_dir, &c.plans, name.as_deref()); + // A later platform's pictures are in its own folder of the shared screenshots; a + // per-test name outside it is another platform's picture, or nobody's. + if let Some(p) = platform { + report.only_shots_under(&expectation_story::platform_shots(p)); + } // Coverage files that extensions wrote next to the results, held to these results. let (found, errors) = expectation_story::load_contracts_for(&results_dir, name.as_deref()); for e in errors { diff --git a/crates/harness/src/lib.rs b/crates/harness/src/lib.rs index 23c994b..d61a2c1 100644 --- a/crates/harness/src/lib.rs +++ b/crates/harness/src/lib.rs @@ -89,6 +89,29 @@ impl Acceptance { } } + /// The folder under the docs' `screenshots/` for this platform's per-test screenshots + /// (`platforms/`), so each platform's results name its own pictures. `None` for the + /// default platform and a suite without platforms: theirs are in `screenshots/` itself. + pub fn screenshots_under(&self) -> Option { + match self.suite.platform.as_deref() { + Some(p) if self.suite.platforms.first().map(String::as_str) != Some(p) => { + Some(expectation_story::platform_shots(p)) + } + _ => None, + } + } + + /// What to run with so that [`Acceptance::publish`] to the docs directory `dir` finds the + /// screenshots: they are taken into `dir/screenshots/`, a later platform's in a folder of + /// its own there. + pub fn run_options(&self, dir: impl Into) -> RunOptions { + RunOptions { + docs_dir: Some(dir.into()), + screenshots_under: self.screenshots_under(), + ..Default::default() + } + } + pub fn from_loaded(loaded: Loaded) -> Result { let Loaded { suite, @@ -233,11 +256,16 @@ impl Acceptance { std::fs::create_dir_all(&results)?; let file = expectation_story::results_file(runner, self.platform.as_deref()); std::fs::write(results.join(file), report.save(&self.plans))?; - let merged = expectation_story::load_results_for( + let mut merged = expectation_story::load_results_for( &results, &self.plans, self.suite.platform.as_deref(), ); + // A later platform's pictures are in its own folder of the shared screenshots; a + // per-test name outside it is another platform's picture, or nobody's. + if let Some(under) = self.screenshots_under() { + merged.only_shots_under(&under); + } // What extensions made of this platform's evidence, held to these results. let (found, _) = expectation_story::load_contracts_for(&results, self.suite.platform.as_deref()); diff --git a/crates/harness/src/run.rs b/crates/harness/src/run.rs index 686a8d7..310c3f6 100644 --- a/crates/harness/src/run.rs +++ b/crates/harness/src/run.rs @@ -22,6 +22,13 @@ pub struct RunOptions { /// --screenshots `), or make it the docs' own `screenshots/` directory. Screenshots /// are taken when this or `docs_dir` is set. pub screenshot_store: Option, + /// A folder under `screenshots/` for this run's per-test screenshots, so that runs which + /// share a docs directory don't write over each other's: with `platforms/android`, they go + /// to `/screenshots/platforms/android//.` and the results name + /// them so. [`Acceptance::run_options`](crate::Acceptance::run_options) sets it for a + /// platform other than the suite's first. A store has no use for it: its names are the + /// content's. + pub screenshots_under: Option, /// How many times to [`Driver::advance`] while waiting for a step's checks to hold. pub patience: u32, /// Ask the driver for its controls and which ones each test used (see @@ -39,6 +46,7 @@ impl Default for RunOptions { RunOptions { docs_dir: None, screenshot_store: None, + screenshots_under: None, patience: 100, controls: true, attachments_dir: None, @@ -302,10 +310,18 @@ fn run_steps( } // Per test and step under the docs, replaced each run; or by content in a store. + let named = match opts + .screenshots_under + .as_deref() + .map(|u| u.trim_matches('/')) + { + Some(under) if !under.is_empty() => format!("screenshots/{under}/{}", plan.slug), + _ => format!("screenshots/{}", plan.slug), + }; let shots = match (&opts.screenshot_store, &opts.docs_dir) { (Some(_), _) => None, (None, Some(d)) => { - let dir = d.join("screenshots").join(&plan.slug); + let dir = d.join(&named); let _ = std::fs::remove_dir_all(&dir); Some(dir) } @@ -321,7 +337,7 @@ fn run_steps( let name = format!("{n:02}.{}", shot.extension); std::fs::create_dir_all(dir).ok()?; std::fs::write(dir.join(&name), &shot.bytes).ok()?; - Some(format!("screenshots/{}/{name}", plan.slug)) + Some(format!("{named}/{name}")) }; let mut attachments = vec![]; let mut keep = |driver: &mut dyn Driver, n: usize| { diff --git a/crates/harness/tests/demo_results.rs b/crates/harness/tests/demo_results.rs index ea34772..b834128 100644 --- a/crates/harness/tests/demo_results.rs +++ b/crates/harness/tests/demo_results.rs @@ -42,7 +42,7 @@ fn the_demos_saved_results_still_match_its_tests() { } /// A platform other than the first is written under `platforms//`, but its screenshots -/// are the run's, in the docs' own `screenshots/`: its pages show them from there. +/// are in the docs' own `screenshots/`, in a folder of their own: its pages show them from there. #[test] fn a_later_platforms_manual_shows_the_shared_screenshots() { let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../demos/kettle"); @@ -75,7 +75,9 @@ fn a_later_platforms_manual_shows_the_shared_screenshots() { let mut shown = 0; for entry in std::fs::read_dir(tree.join("site")).unwrap().flatten() { let page = std::fs::read_to_string(entry.path()).unwrap_or_default(); - shown += page.matches("../../../screenshots/").count(); + shown += page + .matches("../../../screenshots/platforms/swiftui/") + .count(); assert!( !page.contains(r#"src="../screenshots/"#), "{}", diff --git a/crates/remote/src/bin/expectation-run.rs b/crates/remote/src/bin/expectation-run.rs index d6080f9..254ad59 100644 --- a/crates/remote/src/bin/expectation-run.rs +++ b/crates/remote/src/bin/expectation-run.rs @@ -163,11 +163,13 @@ fn main() { .unwrap_or_else(|e| fail(&e)); eprintln!("{} ({}) connected", app.name, app.platform); - let tree = docs.as_ref().map(|d| acceptance.tree(d)); let report = acceptance.run( &mut driver, &RunOptions { - docs_dir: tree, + // Every platform's screenshots are in the docs' own screenshots/, a later + // platform's in a folder of its own there, so its results name its own pictures. + docs_dir: docs.clone(), + screenshots_under: acceptance.screenshots_under(), attachments_dir: attachments, screenshot_store: store.map(PathBuf::from), ..Default::default() diff --git a/crates/remote/tests/platform_screenshots.rs b/crates/remote/tests/platform_screenshots.rs new file mode 100644 index 0000000..4115965 --- /dev/null +++ b/crates/remote/tests/platform_screenshots.rs @@ -0,0 +1,170 @@ +//! `expectation-run` on two platforms of one suite, into one docs directory: each platform's +//! manual shows its own screenshots, on the run itself and when `expectation docs` writes the +//! manuals again, and never another platform's. + +use expectation_remote::client::{Client, Handlers, Reply}; +use std::io::{BufRead, BufReader}; +use std::path::Path; +use std::process::{Command, Stdio}; +use std::time::Duration; + +const SUITE: &str = " +Vocabulary: + Platform: iOS | Android + State: the lamp is off | on + Event: the user flips the switch +Background: + Given the lamp is off +Feature: Lamp + Expectation: Flipping turns it on + Given the lamp is off + When the user flips the switch + Then the lamp is on +Story: Light + Given the lamp is off + When the user flips the switch + Then the lamp is on +"; + +/// Run the suite for `platform` with the real runner, against a lamp that draws itself with +/// the platform's name. +fn run(suite: &Path, docs: &Path, platform: &'static str) { + let mut runner = Command::new(env!("CARGO_BIN_EXE_expectation-run")) + .arg(suite) + .args(["--platform", platform, "--addr", "127.0.0.1:0"]) + .args(["--settle", "0", "--wait", "20", "--runner", "app", "--docs"]) + .arg(docs) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + // "waiting for an app at http://127.0.0.1:PORT ..." + let mut said = BufReader::new(runner.stderr.take().unwrap()); + let mut line = String::new(); + said.read_line(&mut line).unwrap(); + let addr = line + .split("http://") + .nth(1) + .and_then(|s| s.split_whitespace().next()) + .unwrap_or_else(|| panic!("no address in {line:?}")) + .to_string(); + let h: Handlers = Handlers::new() + .observe("the lamp", |on: &bool| if *on { "on" } else { "off" }) + .arrange("the lamp", |on, v| *on = v.to_string() == "on") + .on("the user flips the switch", |on| *on = !*on) + .on_reset(|on| *on = false) + .on_screenshot(move |on| { + Ok(Reply::Svg(format!( + "{platform} {on}" + ))) + }); + let client = Client::connect(&addr, h.hello(platform, "Lamp")); + std::thread::spawn(move || { + let mut on = false; + while let Some(r) = client.recv_timeout(Duration::from_secs(20)) { + h.handle(&mut on, r); + } + }); + let mut rest = String::new(); + std::io::Read::read_to_string(&mut said, &mut rest).unwrap(); + assert!(runner.wait().unwrap().success(), "{line}{rest}"); +} + +/// Every generated page under `dir`, as one text. +fn pages(dir: &Path) -> String { + let mut out = String::new(); + for sub in ["site", "tests", "tutorials"] { + for e in std::fs::read_dir(dir.join(sub)) + .into_iter() + .flatten() + .flatten() + { + if e.path().is_file() { + out.push_str(&std::fs::read_to_string(e.path()).unwrap_or_default()); + } + } + } + out +} + +/// Each platform's manual shows its own pictures and none of the other's. +fn each_shows_its_own(docs: &Path) { + let android = pages(&docs.join("platforms/android")); + assert!(android.contains("../../../screenshots/platforms/android/light/")); + assert!(!android.contains("No screenshot"), "a picture is missing"); + assert!(!android.contains("screenshots/light/"), "an iOS picture"); + let ios = pages(docs); + assert!(ios.contains("../screenshots/light/")); + assert!(!ios.contains("No screenshot"), "a picture is missing"); + assert!( + !ios.contains("screenshots/platforms/"), + "an Android picture" + ); +} + +#[test] +fn each_platforms_manual_shows_its_own_screenshots() { + let root = Path::new(env!("CARGO_TARGET_TMPDIR")).join("platform-screenshots"); + let _ = std::fs::remove_dir_all(&root); + let (suite, docs) = (root.join("suite"), root.join("docs")); + std::fs::create_dir_all(&suite).unwrap(); + std::fs::write(suite.join("lamp.expect"), SUITE).unwrap(); + + run(&suite, &docs, "iOS"); + run(&suite, &docs, "Android"); + + // The same test on both, each with its own pictures, in the docs' one screenshots/. + let read = |p: &str| std::fs::read_to_string(docs.join(p)).unwrap(); + assert!(read("screenshots/light/00.svg").contains("iOS false")); + assert!(read("screenshots/light/01.svg").contains("iOS true")); + assert!(read("screenshots/platforms/android/light/00.svg").contains("Android false")); + assert!(read("screenshots/platforms/android/light/01.svg").contains("Android true")); + assert!( + !docs.join("platforms/android/screenshots").exists(), + "a platform's tree has screenshots of its own" + ); + // The results say whose they are. + assert!( + read("results/app@Android.tsv").contains("\tscreenshots/platforms/android/light/00.svg\t") + ); + assert!(read("results/app@iOS.tsv").contains("\tscreenshots/light/00.svg\t")); + each_shows_its_own(&docs); + + // Written again from the saved results, by the command a docs build runs. + for tree in [docs.join("site"), docs.join("platforms")] { + std::fs::remove_dir_all(tree).unwrap(); + } + let manifest = Path::new(env!("CARGO_MANIFEST_DIR")).join("../cli/Cargo.toml"); + let written = Command::new(env!("CARGO")) + .args(["run", "-q", "--manifest-path"]) + .arg(&manifest) + .args(["--bin", "expectation", "--", "docs"]) + .arg(&suite) + .arg(&docs) + .output() + .unwrap(); + assert!( + written.status.success(), + "{}", + String::from_utf8_lossy(&written.stderr) + ); + each_shows_its_own(&docs); + + // Results from before platforms had folders of their own name the default's pictures: + // Android's manual shows a placeholder for them, not an iOS screenshot. + let old = + read("results/app@Android.tsv").replace("screenshots/platforms/android/", "screenshots/"); + std::fs::write(docs.join("results/app@Android.tsv"), old).unwrap(); + let written = Command::new(env!("CARGO")) + .args(["run", "-q", "--manifest-path"]) + .arg(&manifest) + .args(["--bin", "expectation", "--", "docs"]) + .arg(&suite) + .arg(&docs) + .output() + .unwrap(); + assert!(written.status.success()); + let android = pages(&docs.join("platforms/android")); + assert!(android.contains("No screenshot")); + assert!(!android.contains("screenshots/light/"), "an iOS picture"); +} diff --git a/crates/story/src/lib.rs b/crates/story/src/lib.rs index 240f7f1..4686348 100644 --- a/crates/story/src/lib.rs +++ b/crates/story/src/lib.rs @@ -25,7 +25,8 @@ pub use plan::{ }; pub use report::{ Attachment, CheckResult, PlanResult, Report, Run, Status, StepResult, Verdict, Window, - fingerprint, load_results, load_results_for, now_ms, results_file, screenshots_named, + fingerprint, load_results, load_results_for, now_ms, platform_shots, results_file, + screenshots_named, }; #[cfg(test)] diff --git a/crates/story/src/report.rs b/crates/story/src/report.rs index 47e7256..7ae6b35 100644 --- a/crates/story/src/report.rs +++ b/crates/story/src/report.rs @@ -257,6 +257,27 @@ impl Report { } } + /// For the results of a platform whose per-test screenshots are in `screenshots//` + /// (see [`platform_shots`]): forget the ones named anywhere else. Such a name is from + /// before platforms had folders of their own, and what is at it now may be another + /// platform's picture. Screenshots named by content (`screenshots/.png`) stay: the + /// same name is the same picture on every platform. + pub fn only_shots_under(&mut self, under: &str) { + let own = format!("screenshots/{}/", under.trim_matches('/')); + for r in &mut self.results { + for step in std::iter::once(&mut r.start).chain(&mut r.steps) { + let other = step.screenshot.as_deref().is_some_and(|p| { + !p.starts_with(&own) + && p.strip_prefix("screenshots/") + .is_none_or(|rest| rest.contains('/')) + }); + if other { + step.screenshot = None; + } + } + } + } + /// Nothing failed. Tests that aren't supported yet, are pending, or weren't run, don't count /// against it, and none of them counts as verified either. pub fn passed(&self) -> bool { @@ -659,6 +680,12 @@ pub fn fingerprint(plan: &Plan) -> String { format!("{h:016x}") } +/// The folder under `screenshots/` for the per-test screenshots of `platform`, when it isn't +/// the suite's first: `platforms/`. The first platform's are in `screenshots/` itself. +pub fn platform_shots(platform: &str) -> String { + format!("platforms/{}", expectation_model::text::slug(platform)) +} + /// The screenshots a results file names, by file name (`.png` for /// `screenshots/.png`), whatever test or step they are from. pub fn screenshots_named(text: &str) -> Vec { diff --git a/demos/kettle/docs/results/app@React Native.tsv b/demos/kettle/docs/results/app@React Native.tsv index 2ed7425..0a3a57c 100644 --- a/demos/kettle/docs/results/app@React Native.tsv +++ b/demos/kettle/docs/results/app@React Native.tsv @@ -1,77 +1,77 @@ expectation-results 1 plan first-cup--turn-it-up-then-turn-it-off passed 6dc077f1ef5d3850 -step 0 passed 1 screenshots/first-cup--turn-it-up-then-turn-it-off/00.png - +step 0 passed 1 screenshots/platforms/react-native/first-cup--turn-it-up-then-turn-it-off/00.png - check 1 e:off - check 1 n:0 - check 1 e:WARM - check 1 e:off - -step 1 passed 2 screenshots/first-cup--turn-it-up-then-turn-it-off/01.png - +step 1 passed 2 screenshots/platforms/react-native/first-cup--turn-it-up-then-turn-it-off/01.png - check 1 e:heating - -step 2 passed 1 screenshots/first-cup--turn-it-up-then-turn-it-off/02.png - +step 2 passed 1 screenshots/platforms/react-native/first-cup--turn-it-up-then-turn-it-off/02.png - check 1 e:off - plan keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil passed 0869da5fc28933a3 -step 0 passed 1 screenshots/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/00.png - +step 0 passed 1 screenshots/platforms/react-native/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/00.png - check 1 e:off - check 1 n:0 - check 1 e:WARM - check 1 e:off - -step 1 passed 1 screenshots/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/01.png - +step 1 passed 1 screenshots/platforms/react-native/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/01.png - check 1 e:on - -step 2 passed 1 screenshots/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/02.png - +step 2 passed 1 screenshots/platforms/react-native/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/02.png - check 1 e:off - -step 3 passed 1 screenshots/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/03.png - +step 3 passed 1 screenshots/platforms/react-native/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/03.png - check 1 e:off - -step 4 passed 3 screenshots/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/04.png - +step 4 passed 3 screenshots/platforms/react-native/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/04.png - check 1 e:heating - plan choosing-a-preset--step-to-hot-and-start passed 7b544b6f78d36f9b -step 0 passed 1 screenshots/choosing-a-preset--step-to-hot-and-start/00.png - +step 0 passed 1 screenshots/platforms/react-native/choosing-a-preset--step-to-hot-and-start/00.png - check 1 e:off - check 1 n:0 - check 1 e:WARM - check 1 e:off - -step 1 passed 1 screenshots/choosing-a-preset--step-to-hot-and-start/01.png - +step 1 passed 1 screenshots/platforms/react-native/choosing-a-preset--step-to-hot-and-start/01.png - check 1 e:HOT - -step 2 passed 1 screenshots/choosing-a-preset--step-to-hot-and-start/02.png - +step 2 passed 1 screenshots/platforms/react-native/choosing-a-preset--step-to-hot-and-start/02.png - check 1 e:heating - plan coverage--the-child-lock-ignores-the-start-button passed 61683b4489b46f6a -step 0 passed 1 screenshots/coverage--the-child-lock-ignores-the-start-button/00.png - +step 0 passed 1 screenshots/platforms/react-native/coverage--the-child-lock-ignores-the-start-button/00.png - check 1 e:off - check 1 n:0 - check 1 e:WARM - check 1 e:off - -step 1 passed 1 screenshots/coverage--the-child-lock-ignores-the-start-button/01.png - +step 1 passed 1 screenshots/platforms/react-native/coverage--the-child-lock-ignores-the-start-button/01.png - check 1 e:on - -step 2 passed 1 screenshots/coverage--the-child-lock-ignores-the-start-button/02.png - +step 2 passed 1 screenshots/platforms/react-native/coverage--the-child-lock-ignores-the-start-button/02.png - check 1 e:off - plan coverage--running-dry-stops-the-kettle passed 1d4dd4296c369feb -step 0 passed 1 screenshots/coverage--running-dry-stops-the-kettle/00.png - +step 0 passed 1 screenshots/platforms/react-native/coverage--running-dry-stops-the-kettle/00.png - check 1 e:off - check 1 n:0 - check 1 e:WARM - check 1 e:off - -step 1 passed 3 screenshots/coverage--running-dry-stops-the-kettle/01.png - +step 1 passed 3 screenshots/platforms/react-native/coverage--running-dry-stops-the-kettle/01.png - check 1 e:heating - -step 2 passed 1 screenshots/coverage--running-dry-stops-the-kettle/02.png - +step 2 passed 1 screenshots/platforms/react-native/coverage--running-dry-stops-the-kettle/02.png - check 1 e:dry - plan coverage--hot-to-boil passed b10aec935a91fd8f -step 0 passed 1 screenshots/coverage--hot-to-boil/00.png - +step 0 passed 1 screenshots/platforms/react-native/coverage--hot-to-boil/00.png - check 1 e:off - check 1 n:0 - check 1 e:WARM - check 1 e:off - -step 1 passed 1 screenshots/coverage--hot-to-boil/01.png - +step 1 passed 1 screenshots/platforms/react-native/coverage--hot-to-boil/01.png - check 1 e:HOT - -step 2 passed 1 screenshots/coverage--hot-to-boil/02.png - +step 2 passed 1 screenshots/platforms/react-native/coverage--hot-to-boil/02.png - check 1 e:BOIL - plan coverage--boil-back-to-warm passed ea4b006c234a964d -step 0 passed 1 screenshots/coverage--boil-back-to-warm/00.png - +step 0 passed 1 screenshots/platforms/react-native/coverage--boil-back-to-warm/00.png - check 1 e:off - check 1 n:0 - check 1 e:WARM - check 1 e:off - -step 1 passed 1 screenshots/coverage--boil-back-to-warm/01.png - +step 1 passed 1 screenshots/platforms/react-native/coverage--boil-back-to-warm/01.png - check 1 e:HOT - -step 2 passed 1 screenshots/coverage--boil-back-to-warm/02.png - +step 2 passed 1 screenshots/platforms/react-native/coverage--boil-back-to-warm/02.png - check 1 e:BOIL - -step 3 passed 1 screenshots/coverage--boil-back-to-warm/03.png - +step 3 passed 1 screenshots/platforms/react-native/coverage--boil-back-to-warm/03.png - check 1 e:WARM - diff --git a/demos/kettle/docs/results/app@SwiftUI.tsv b/demos/kettle/docs/results/app@SwiftUI.tsv index e9a66f6..739e5b7 100644 --- a/demos/kettle/docs/results/app@SwiftUI.tsv +++ b/demos/kettle/docs/results/app@SwiftUI.tsv @@ -1,77 +1,77 @@ expectation-results 1 plan first-cup--turn-it-up-then-turn-it-off passed 6dc077f1ef5d3850 -step 0 passed 1 screenshots/first-cup--turn-it-up-then-turn-it-off/00.png - +step 0 passed 1 screenshots/platforms/swiftui/first-cup--turn-it-up-then-turn-it-off/00.png - check 1 e:off - check 1 n:0 - check 1 e:WARM - check 1 e:off - -step 1 passed 1 screenshots/first-cup--turn-it-up-then-turn-it-off/01.png - +step 1 passed 1 screenshots/platforms/swiftui/first-cup--turn-it-up-then-turn-it-off/01.png - check 1 e:heating - -step 2 passed 1 screenshots/first-cup--turn-it-up-then-turn-it-off/02.png - +step 2 passed 1 screenshots/platforms/swiftui/first-cup--turn-it-up-then-turn-it-off/02.png - check 1 e:off - plan keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil passed 0869da5fc28933a3 -step 0 passed 1 screenshots/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/00.png - +step 0 passed 1 screenshots/platforms/swiftui/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/00.png - check 1 e:off - check 1 n:0 - check 1 e:WARM - check 1 e:off - -step 1 passed 1 screenshots/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/01.png - +step 1 passed 1 screenshots/platforms/swiftui/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/01.png - check 1 e:on - -step 2 passed 1 screenshots/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/02.png - +step 2 passed 1 screenshots/platforms/swiftui/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/02.png - check 1 e:off - -step 3 passed 1 screenshots/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/03.png - +step 3 passed 1 screenshots/platforms/swiftui/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/03.png - check 1 e:off - -step 4 passed 1 screenshots/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/04.png - +step 4 passed 1 screenshots/platforms/swiftui/keeping-little-hands-off--turn-the-dial-with-the-lock-on-then-unlock-and-boil/04.png - check 1 e:heating - plan choosing-a-preset--step-to-hot-and-start passed 7b544b6f78d36f9b -step 0 passed 1 screenshots/choosing-a-preset--step-to-hot-and-start/00.png - +step 0 passed 1 screenshots/platforms/swiftui/choosing-a-preset--step-to-hot-and-start/00.png - check 1 e:off - check 1 n:0 - check 1 e:WARM - check 1 e:off - -step 1 passed 1 screenshots/choosing-a-preset--step-to-hot-and-start/01.png - +step 1 passed 1 screenshots/platforms/swiftui/choosing-a-preset--step-to-hot-and-start/01.png - check 1 e:HOT - -step 2 passed 1 screenshots/choosing-a-preset--step-to-hot-and-start/02.png - +step 2 passed 1 screenshots/platforms/swiftui/choosing-a-preset--step-to-hot-and-start/02.png - check 1 e:heating - plan coverage--the-child-lock-ignores-the-start-button passed 61683b4489b46f6a -step 0 passed 1 screenshots/coverage--the-child-lock-ignores-the-start-button/00.png - +step 0 passed 1 screenshots/platforms/swiftui/coverage--the-child-lock-ignores-the-start-button/00.png - check 1 e:off - check 1 n:0 - check 1 e:WARM - check 1 e:off - -step 1 passed 1 screenshots/coverage--the-child-lock-ignores-the-start-button/01.png - +step 1 passed 1 screenshots/platforms/swiftui/coverage--the-child-lock-ignores-the-start-button/01.png - check 1 e:on - -step 2 passed 1 screenshots/coverage--the-child-lock-ignores-the-start-button/02.png - +step 2 passed 1 screenshots/platforms/swiftui/coverage--the-child-lock-ignores-the-start-button/02.png - check 1 e:off - plan coverage--running-dry-stops-the-kettle passed 1d4dd4296c369feb -step 0 passed 1 screenshots/coverage--running-dry-stops-the-kettle/00.png - +step 0 passed 1 screenshots/platforms/swiftui/coverage--running-dry-stops-the-kettle/00.png - check 1 e:off - check 1 n:0 - check 1 e:WARM - check 1 e:off - -step 1 passed 1 screenshots/coverage--running-dry-stops-the-kettle/01.png - +step 1 passed 1 screenshots/platforms/swiftui/coverage--running-dry-stops-the-kettle/01.png - check 1 e:heating - -step 2 passed 1 screenshots/coverage--running-dry-stops-the-kettle/02.png - +step 2 passed 1 screenshots/platforms/swiftui/coverage--running-dry-stops-the-kettle/02.png - check 1 e:dry - plan coverage--hot-to-boil passed b10aec935a91fd8f -step 0 passed 1 screenshots/coverage--hot-to-boil/00.png - +step 0 passed 1 screenshots/platforms/swiftui/coverage--hot-to-boil/00.png - check 1 e:off - check 1 n:0 - check 1 e:WARM - check 1 e:off - -step 1 passed 1 screenshots/coverage--hot-to-boil/01.png - +step 1 passed 1 screenshots/platforms/swiftui/coverage--hot-to-boil/01.png - check 1 e:HOT - -step 2 passed 1 screenshots/coverage--hot-to-boil/02.png - +step 2 passed 1 screenshots/platforms/swiftui/coverage--hot-to-boil/02.png - check 1 e:BOIL - plan coverage--boil-back-to-warm passed ea4b006c234a964d -step 0 passed 1 screenshots/coverage--boil-back-to-warm/00.png - +step 0 passed 1 screenshots/platforms/swiftui/coverage--boil-back-to-warm/00.png - check 1 e:off - check 1 n:0 - check 1 e:WARM - check 1 e:off - -step 1 passed 1 screenshots/coverage--boil-back-to-warm/01.png - +step 1 passed 1 screenshots/platforms/swiftui/coverage--boil-back-to-warm/01.png - check 1 e:HOT - -step 2 passed 1 screenshots/coverage--boil-back-to-warm/02.png - +step 2 passed 1 screenshots/platforms/swiftui/coverage--boil-back-to-warm/02.png - check 1 e:BOIL - -step 3 passed 1 screenshots/coverage--boil-back-to-warm/03.png - +step 3 passed 1 screenshots/platforms/swiftui/coverage--boil-back-to-warm/03.png - check 1 e:WARM - From 8bfafc3ffef5a47f25c928c194eef6f976508eb7 Mon Sep 17 00:00:00 2001 From: Patrick Tescher Date: Wed, 11 Nov 2026 20:02:55 -0800 Subject: [PATCH 3/3] The JavaScript client's tests don't pick a port fetch refuses A test took a random port from 5000 to 5999. fetch refuses 5060 and 5061, so on those the app never reached the runner and the test failed after 20 seconds. Co-Authored-By: Claude --- clients/js/test.mjs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/clients/js/test.mjs b/clients/js/test.mjs index cf82aeb..2849da7 100644 --- a/clients/js/test.mjs +++ b/clients/js/test.mjs @@ -55,7 +55,8 @@ test('a suite runs through the JS driver', async () => { /** Runs the turnstile suite through a JS driver that `extra` adds handlers to, keeping attachments in `keep`. */ async function runTurnstile(extra, keep) { - const port = 5000 + Math.floor(Math.random() * 1000); + // Not 5060 or 5061: fetch refuses to connect to them (they are SIP's), and no app would arrive. + const port = 5100 + Math.floor(Math.random() * 900); const docs = fs.mkdtempSync(path.join(os.tmpdir(), 'expectation-js-docs-')); const args = [path.join(here, 'fixture'), '--addr', `127.0.0.1:${port}`, '--settle', '0', '--wait', '20', '--docs', docs]; if (keep) args.push('--attachments', keep);