From 3d11f2701eea76549b64949d9445210327ea2ea3 Mon Sep 17 00:00:00 2001 From: Patrick Tescher Date: Mon, 9 Nov 2026 11:35:18 -0800 Subject: [PATCH 1/2] Associate code coverage with test runs and steps An extension (extensions/code-coverage) reads the LCOV or Istanbul coverage an app handed over as step attachments, works out per-step differences (baseline, restarts, gaps) and writes results/.code.tsv. Expectation reads it (story::codecov) and shows a separate, labelled Code coverage section on the test's page, apart from the result: totals by step, what each step added, the instrumented revision and scope, and a link to a separately hosted report. Captures count only for their own run and test fingerprint, and missing or unreadable captures are shown as that, not as zero. Closes #73 Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/ci.yml | 6 + book/src/evidence.md | 46 +++ crates/cli/src/lib.rs | 15 + crates/docs/src/html.rs | 95 ++++- crates/docs/src/lib.rs | 101 +++++ crates/docs/src/markdown.rs | 77 ++++ crates/harness/src/lib.rs | 4 + crates/story/src/codecov.rs | 384 ++++++++++++++++++ crates/story/src/lib.rs | 5 + crates/story/src/report.rs | 4 +- extensions/code-coverage/README.md | 41 ++ .../bin/expectation-code-coverage.js | 9 + extensions/code-coverage/package.json | 25 ++ extensions/code-coverage/src/analyze.js | 67 +++ extensions/code-coverage/src/cli.js | 145 +++++++ extensions/code-coverage/src/formats.js | 99 +++++ extensions/code-coverage/src/results.js | 75 ++++ extensions/code-coverage/src/tsv.js | 14 + .../code-coverage/test/analyze.test.mjs | 87 ++++ extensions/code-coverage/test/e2e.test.mjs | 67 +++ 20 files changed, 1363 insertions(+), 3 deletions(-) create mode 100644 crates/story/src/codecov.rs create mode 100644 extensions/code-coverage/README.md create mode 100755 extensions/code-coverage/bin/expectation-code-coverage.js create mode 100644 extensions/code-coverage/package.json create mode 100644 extensions/code-coverage/src/analyze.js create mode 100644 extensions/code-coverage/src/cli.js create mode 100644 extensions/code-coverage/src/formats.js create mode 100644 extensions/code-coverage/src/results.js create mode 100644 extensions/code-coverage/src/tsv.js create mode 100644 extensions/code-coverage/test/analyze.test.mjs create mode 100644 extensions/code-coverage/test/e2e.test.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 49287bf..420b947 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,3 +43,9 @@ jobs: working-directory: extensions/har-openapi env: EXPECTATION_E2E: "1" + # Coverage an app hands over as attachments, the extension, and `expectation docs`, end to end. + - run: cargo build -p expectation_cli -p expectation_remote + - run: npm test + working-directory: extensions/code-coverage + env: + EXPECTATION_E2E: "1" diff --git a/book/src/evidence.md b/book/src/evidence.md index 00481a5..82a089b 100644 --- a/book/src/evidence.md +++ b/book/src/evidence.md @@ -88,6 +88,52 @@ branch of a JSON response becomes an outcome. It also reads a HAR made by any ot it leaves unknown. `crates/harness/examples/quote` runs the whole pipeline, with a stand-in quote service. +## Code coverage + +Contract coverage asks which documented outcomes were seen. Code coverage asks which of the +program's own source ran, and links that to the test that ran it. It is a separate thing with a +separate section on the test's page, because it means something different. + +The application collects it (Istanbul, or anything that writes LCOV) and hands it over as step +attachments, in JavaScript with `onAttachments`. Then +[`extensions/code-coverage`](https://github.com/ptescher/expectation/tree/main/extensions/code-coverage) +reads LCOV or Istanbul's `coverage-final.json` and writes `results/code[@platform].code.tsv`: + +```bash +expectation-run expectations --docs docs --attachments captures +npx expectation-code-coverage --results docs/results/app.tsv --attachments captures \ + --revision "$(git rev-parse HEAD)" --scope "src/**, without tests" \ + --report-url "https://coverage.example.com/{run}/{slug}/" +expectation docs expectations docs +``` + +Each test's page then has a "Code coverage" section: the source revision and scope that were +instrumented, the lines, branches and functions hit at the start and after each step, what each +step added, and a link to a full HTML report that is hosted somewhere else (`{run}` and `{slug}` +are filled in for each test). Raw coverage files stay in the attachments directory, outside the +manual. + +What the section claims is limited on purpose: + +- **It is observed execution.** It never changes a test's result or an expectation's verdict, + and it doesn't say anything was asserted. A line that ran in a failed test ran. +- **Missing is not uncovered.** A step with no capture, or with one that can't be read, is shown + as that. Zero lines hit and "no data" are different. +- **A step's part is only claimed when it can be.** Counters count from when the app started, so + a step's part is the difference from the capture before. If a counter goes down, the app + restarted, and nothing is attributed to that step. If a step's capture is missing, the next + difference is labelled as spanning the steps between. Anything else that ran in the same + window, such as background work, is in the numbers, and the page says so. With + `--counters per-step` the app empties its counters each time it hands them over. +- **It counts for its own run and test.** Like contract evidence, a capture is shown only with + the run it came from and a test that hasn't changed since. + +Code coverage files are read the same way as the others: tab-separated, a few hundred characters +a field, 16 MiB at most, and the report link must be an `http` or `https` address. The format is +described at the top of +[`crates/story/src/codecov.rs`](https://github.com/ptescher/expectation/blob/main/crates/story/src/codecov.rs). +Expectation doesn't instrument the app: collecting coverage is the app's and the driver's job. + ## Writing an extension An extension reads: diff --git a/crates/cli/src/lib.rs b/crates/cli/src/lib.rs index 7c2b86b..1b1b516 100644 --- a/crates/cli/src/lib.rs +++ b/crates/cli/src/lib.rs @@ -241,6 +241,7 @@ fn docs( .map_or_else(Default::default, expectation_docs::Screenshots::parse), reader, shared: None, + code: &[], }; if let Some(expectation_docs::Screenshots::Dir(d)) = shots.as_deref().map(expectation_docs::Screenshots::parse) @@ -264,6 +265,7 @@ fn docs( }; // Every platform's tests and results, then the pages: each one's dropdown knows the others' pages. let mut manuals = vec![]; + let mut codes = vec![]; for name in &names { let platform = name .as_deref() @@ -316,6 +318,18 @@ fn docs( expectation_story::Coverage::judge(contract, &c.plans, &report) }) .collect(); + // How much of the program's source ran, from files an extension wrote next to the results. + let (code, errors) = + expectation_story::load_code_coverage_for(&results_dir, name.as_deref()); + for e in errors { + eprintln!("warning: code coverage file {e}"); + } + for c in &code { + for p in &c.problems { + eprintln!("warning: {}: {p}", c.file); + } + } + codes.push(code); // Just some sections, for a preview: resolved in this platform's suite. let partial = (!sections.is_empty()).then(|| { let ids: Vec<_> = sections @@ -370,6 +384,7 @@ fn docs( partial: partial.as_ref(), look: expectation_docs::Look { shared: (i > 0 && names[i].is_some()).then_some((Path::new(out), 2)), + code: &codes[i], ..look }, }; diff --git a/crates/docs/src/html.rs b/crates/docs/src/html.rs index b7fcdcc..f6a6083 100644 --- a/crates/docs/src/html.rs +++ b/crates/docs/src/html.rs @@ -14,12 +14,31 @@ use expectation_graph::Node; use expectation_model::text::paragraphs; use expectation_model::{ExpId, FeatureId, Keyword, PhraseBody, SectionId, StepText, Suite}; use expectation_story::{ - Attribution, Because, Check, CheckResult, Completeness, Credit, Origin, Source, State, Status, - StepResult, Verdict, + Attribution, Basis, Because, CaptureState, Check, CheckResult, Completeness, Credit, Origin, + Source, State, Status, StepResult, Verdict, }; use std::collections::HashMap; use std::fmt::Write; +/// What a capture's "new" numbers say, in words. +fn new_since(c: &expectation_story::Capture) -> String { + let Some([l, b, f]) = c.new else { + return match c.basis { + Basis::Baseline => "the first capture: what ran before the test".into(), + Basis::Restart => "the program restarted, so nothing is attributed".into(), + _ => "can't be told".into(), + }; + }; + let what = format!("{l} lines, {b} branches, {f} functions"); + match c.basis { + Basis::Step => what, + Basis::Spans(from) => { + format!("{what}, since step {from} (the captures between are missing)") + } + _ => what, + } +} + pub fn esc(s: &str) -> String { s.replace('&', "&") .replace('<', "<") @@ -1859,9 +1878,81 @@ impl Site<'_> { o.push_str(&self.step_evidence(i, n + 1)); o.push_str(""); } + o.push_str(&self.code_coverage(i)); self.page(Here::Test(i), &plan.title, &o) } + /// Code coverage of the instrumented program while this test ran, apart from its result. + fn code_coverage(&self, plan: usize) -> String { + let mut o = String::new(); + for (cov, caps) in self.d.code_coverage_of(plan) { + let _ = write!( + o, + r#"

Code coverage

Which of the program's own source ran while this test did. It is observed execution: it doesn't change the result above, and it doesn't show that anything was checked. Counters are the program's totals since it started; "new" is what a step added, when that can be told, and can include activity the test didn't cause.

"# + ); + let mut about = vec![]; + for (label, v) in [ + ("Source", &cov.name), + ("Revision", &cov.revision), + ("Scope", &cov.scope), + ] { + if !v.is_empty() { + about.push(format!("{label}: {}", esc(v))); + } + } + if !about.is_empty() { + let _ = write!(o, "

{}

", about.join(" · ")); + } + o.push_str(""); + for c in &caps { + let at = if c.step == 0 { + r##"Start"##.to_string() + } else { + format!(r##"Step {0}"##, c.step) + }; + match c.state { + CaptureState::Captured => { + let n = |k: expectation_story::Count| format!("{} of {}", k.hit, k.found); + let _ = write!( + o, + "", + n(c.lines), + n(c.branches), + n(c.functions), + esc(&new_since(c)) + ); + } + other => { + let what = if other == CaptureState::Missing { + "No capture at this step" + } else { + "A capture that can't be used" + }; + let note = if c.note.is_empty() { + String::new() + } else { + format!(": {}", esc(&c.note)) + }; + let _ = write!( + o, + r#""# + ); + } + } + } + o.push_str("
CaptureLinesBranchesFunctionsNew since
{at}{}{}{}{}
{at}{what}{note}. That isn't the same as nothing being covered.
"); + if let Some(url) = caps.first().and_then(|c| cov.report_for(&c.plan, &c.run)) { + let _ = write!( + o, + r#"

The full coverage report, hosted separately.

"#, + esc(&url) + ); + } + o.push_str("
"); + } + o + } + /// The contract evidence that counts for a step: what it did, and the contract it was held against. fn step_evidence(&self, plan: usize, step: usize) -> String { let found = self.d.evidence_at(plan, step); diff --git a/crates/docs/src/lib.rs b/crates/docs/src/lib.rs index 9e94f92..b745c03 100644 --- a/crates/docs/src/lib.rs +++ b/crates/docs/src/lib.rs @@ -80,6 +80,9 @@ pub struct Look<'a> { /// directory that holds the shared `screenshots/`, and how many folders up it is from this /// manual. `None` is a manual that has its own. pub shared: Option<(&'a Path, usize)>, + /// Code coverage files (see [`expectation_story::CodeCoverage`]): shown on the test pages + /// as observed execution, apart from the results. + pub code: &'a [expectation_story::CodeCoverage], } /// Where the screenshots are that the results name as `screenshots/`. @@ -329,6 +332,26 @@ impl Docs<'_> { out } + /// The code coverage that counts for test `plan`: per file, its captures of this run of the + /// test as it is now, and how many other captures it holds that were left out. + pub fn code_coverage_of( + &self, + plan: usize, + ) -> Vec<( + &expectation_story::CodeCoverage, + Vec<&expectation_story::Capture>, + )> { + let Some(report) = self.report else { + return vec![]; + }; + self.look + .code + .iter() + .map(|c| (c, c.for_plan(self.plans, report, plan))) + .filter(|(_, found)| !found.is_empty()) + .collect() + } + /// Write the Markdown pages and the HTML site under `dir`. Pages of stories or features /// that no longer exist are removed (only the generated kinds, in the folders this writes /// to); other files in the directory are left alone. @@ -1481,4 +1504,82 @@ Feature: Paying assert!(!ground.contains("Shown for")); assert!(ground.contains("selected>desktop")); } + + #[test] + fn code_coverage_is_shown_as_observed_execution_for_the_run_and_test_it_came_from() { + let src = "Vocabulary:\n State: the lamp is on | off\n Event: the user flips the switch\nBackground:\n Given the lamp is off\nStory: Using the lamp\n As a user\n I want light\n So that I see\n Scenario: x\n When the user flips the switch\n Then the lamp is on\n"; + let loaded = expectation_language::load_sources("S", &[("s.expect", src)]); + assert!(!loaded.has_errors(), "{}", loaded.render_diagnostics()); + let suite = loaded.suite; + let analysis = expectation_graph::analyze(&suite); + let plans = expectation_story::plan_stories(&suite, &analysis).plans; + let mut report = Report::not_run(&plans); + for (r, p) in report.results.iter_mut().zip(&plans) { + *r = expectation_story::PlanResult::with(Status::Passed, p); + r.run = Some(expectation_story::Run { + id: "r1".into(), + started_ms: 1, + }); + } + let (slug, print) = (&plans[0].slug, expectation_story::fingerprint(&plans[0])); + let file = |run: &str, print: &str| { + format!( + "expectation-code-coverage 1\nsource\tapp\tabc123\tsrc/**\thttps://cov.example/r\ncapture\t{run}\t{slug}\t{print}\t0\tcaptured\tbaseline\t-\t5\t10\t1\t4\t2\t3\t-\t-\t-\ncapture\t{run}\t{slug}\t{print}\t1\tcaptured\tstep\t-\t7\t10\t2\t4\t2\t3\t2\t1\t0\ncapture\t{run}\t{slug}\t{print}\t2\tmissing\tnone\tno snapshot\t0\t0\t0\t0\t0\t0\t-\t-\t-\n" + ) + }; + let pages = |code: &[expectation_story::CodeCoverage]| { + Docs { + suite: &suite, + analysis: &analysis, + plans: &plans, + report: Some(&report), + suite_dir: None, + siblings: &[], + outline: None, + inventory: &[], + contracts: &[], + partial: None, + look: Look { + code, + ..Look::default() + }, + } + .pages() + }; + let find = |pages: &[Page], path: &str| { + pages + .iter() + .find(|p| p.path == path) + .unwrap_or_else(|| panic!("{path}")) + .content + .clone() + }; + let good = [ + expectation_story::CodeCoverage::parse("app.code.tsv", &file("r1", &print)).unwrap(), + ]; + let shown = pages(&good); + let (html, md) = ( + find(&shown, &format!("site/test-{slug}.html")), + find(&shown, &format!("tests/{slug}.md")), + ); + for page in [&html, &md] { + assert!(page.contains("Code coverage"), "{page}"); + assert!(page.contains("observed execution")); + assert!(page.contains("abc123") && page.contains("src/")); + assert!(page.contains("7 of 10")); + assert!(page.contains("2 lines, 1 branches, 0 functions")); + assert!(page.contains("No capture at this step")); + assert!(page.contains("isn't the same as nothing being covered")); + assert!(page.contains("https://cov.example/r")); + } + // The result is what it was. + assert!(html.contains("passed")); + // Another run's captures, or a changed test's, are not shown. + for other in [file("r2", &print), file("r1", "0000")] { + let bad = [expectation_story::CodeCoverage::parse("app.code.tsv", &other).unwrap()]; + let hidden = pages(&bad); + assert!(!find(&hidden, &format!("site/test-{slug}.html")).contains("Code coverage")); + assert!(!find(&hidden, &format!("tests/{slug}.md")).contains("Code coverage")); + } + } } diff --git a/crates/docs/src/markdown.rs b/crates/docs/src/markdown.rs index 2f3ec5b..ee93795 100644 --- a/crates/docs/src/markdown.rs +++ b/crates/docs/src/markdown.rs @@ -1086,9 +1086,86 @@ fn test_page(d: &Docs, f: &Facts, i: usize) -> String { ); step_evidence(d, &mut o, i, n + 1); } + code_coverage(d, &mut o, i); o } +/// Code coverage of the instrumented program while a test ran, apart from its result. +fn code_coverage(d: &Docs, o: &mut String, plan: usize) { + use expectation_story::{Basis, CaptureState}; + for (cov, caps) in d.code_coverage_of(plan) { + let _ = writeln!( + o, + "## Code coverage\n\nWhich of the program's own source ran while this test did. It is observed execution: it doesn't change the result above, and it doesn't show that anything was checked. Counters are the program's totals since it started; \"new\" is what a step added, when that can be told, and can include activity the test didn't cause.\n" + ); + let about: Vec = [ + ("Source", &cov.name), + ("Revision", &cov.revision), + ("Scope", &cov.scope), + ] + .iter() + .filter(|(_, v)| !v.is_empty()) + .map(|(k, v)| format!("{k}: {}", cell(v))) + .collect(); + if !about.is_empty() { + let _ = writeln!(o, "{}\n", about.join(" · ")); + } + let _ = writeln!( + o, + "| Capture | Lines | Branches | Functions | New since |\n| --- | --- | --- | --- | --- |" + ); + for c in &caps { + let at = if c.step == 0 { + "Start".to_string() + } else { + format!("Step {}", c.step) + }; + if c.state != CaptureState::Captured { + let what = if c.state == CaptureState::Missing { + "No capture at this step" + } else { + "A capture that can't be used" + }; + let _ = writeln!( + o, + "| {at} | {what}{}. That isn't the same as nothing being covered. | | | |", + if c.note.is_empty() { + String::new() + } else { + format!(": {}", cell(&c.note)) + } + ); + continue; + } + let n = |k: expectation_story::Count| format!("{} of {}", k.hit, k.found); + let new = match (c.new, c.basis) { + (Some([l, b, f]), Basis::Spans(from)) => format!( + "{l} lines, {b} branches, {f} functions, since step {from} (the captures between are missing)" + ), + (Some([l, b, f]), _) => format!("{l} lines, {b} branches, {f} functions"), + (None, Basis::Baseline) => "the first capture: what ran before the test".into(), + (None, Basis::Restart) => "the program restarted, so nothing is attributed".into(), + (None, _) => "can't be told".into(), + }; + let _ = writeln!( + o, + "| {at} | {} | {} | {} | {new} |", + n(c.lines), + n(c.branches), + n(c.functions) + ); + } + o.push('\n'); + if let Some(url) = caps.first().and_then(|c| cov.report_for(&c.plan, &c.run)) { + let _ = writeln!( + o, + "[The full coverage report](<{}>), hosted separately.\n", + url.replace('>', "%3E") + ); + } + } +} + /// The contract evidence that counts for a step, if any: what the step did, as the extension /// labelled it, and the contract it was held against. fn step_evidence(d: &Docs, o: &mut String, plan: usize, step: usize) { diff --git a/crates/harness/src/lib.rs b/crates/harness/src/lib.rs index 52d64ba..23c994b 100644 --- a/crates/harness/src/lib.rs +++ b/crates/harness/src/lib.rs @@ -220,6 +220,7 @@ impl Acceptance { .map_or_else(Default::default, expectation_docs::Screenshots::parse), reader: self.reader, shared: None, + code: &[], }, } } @@ -246,6 +247,9 @@ impl Acceptance { .collect(); let mut docs = self.docs(Some(&merged)); docs.contracts = &judged; + let (code, _) = + expectation_story::load_code_coverage_for(&results, self.suite.platform.as_deref()); + docs.look.code = &code; if self.tree(dir) != dir { docs.look.shared = Some((dir, 2)); } diff --git a/crates/story/src/codecov.rs b/crates/story/src/codecov.rs new file mode 100644 index 0000000..e0846c9 --- /dev/null +++ b/crates/story/src/codecov.rs @@ -0,0 +1,384 @@ +//! Code coverage: how much of the instrumented program's source ran while a test did. +//! +//! This is a different thing from the contract coverage in [`crate::coverage`], which asks which +//! outcomes of a contract were seen. Here the question is which lines, branches and functions of +//! the application's own source were executed. Expectation doesn't instrument the app or read +//! LCOV or Istanbul data itself: the application or its driver collects it, hands it over as +//! step [attachments](crate::Attachment), and an extension (`extensions/code-coverage`) works out +//! the numbers and writes a file next to the results. This module reads it, and holds it to +//! these rules: +//! +//! - It is observed execution. It never changes a test's result or an expectation's verdict, and +//! it says nothing about what a test asserted. +//! - A capture counts only for the run it came from, and for the test as it was fingerprinted +//! then. Captures from another run, or from a test that has changed since, are not shown. +//! - A capture that is missing or unusable is shown as such, never as zero: no data is not +//! uncovered code. +//! - What a step added is only claimed when the counters were cumulative and never went down. +//! After a restart, or when captures skipped steps, it says so instead of guessing, and it +//! can't know what unrelated background activity ran in the same window. +//! +//! The file is `results/[@platform].code.tsv`, tab-separated, `-` for nothing, and lines of +//! a kind not listed here are skipped: +//! +//! ```text +//! expectation-code-coverage 1 +//! source +//! capture +//! +//! +//! ``` +//! +//! (`capture` is one line.) `` is 0 for the test's start. `` is `captured`, +//! `missing` (nothing was captured there) or `incomplete` (something was, but it can't be +//! used). `` says what the `` numbers are: `baseline` (the first capture of the +//! test: the program's state before it), `step` (since the capture of the step before), +//! `spans:` (since the capture of step `n`, which is earlier than the one before), `restart` +//! (the counters went down, so the program restarted and nothing is attributed) or `none`. + +use crate::plan::Plan; +use crate::report::{Report, fingerprint}; + +/// The largest file read. +pub const MAX_BYTES: u64 = 16 * 1024 * 1024; +const MAX_TEXT: usize = 300; +const MAX_ID: usize = 128; + +/// Hit and found counts of one kind of thing in the source. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Count { + pub hit: u64, + pub found: u64, +} + +/// What a capture holds. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CaptureState { + Captured, + /// Nothing was captured at this step. + Missing, + /// A capture was made but can't be read or is empty. + Incomplete, +} + +impl CaptureState { + pub fn name(self) -> &'static str { + match self { + CaptureState::Captured => "captured", + CaptureState::Missing => "missing", + CaptureState::Incomplete => "incomplete", + } + } + + fn parse(s: &str) -> Option { + [ + CaptureState::Captured, + CaptureState::Missing, + CaptureState::Incomplete, + ] + .into_iter() + .find(|x| x.name() == s) + } +} + +/// What a capture's "new" numbers are relative to. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Basis { + /// The first capture of the test: what the program had run before it. + Baseline, + /// Since the capture of the step before. + Step, + /// Since the capture of an earlier step, as captures in between are missing. + Spans(usize), + /// The counters went down: the program restarted, so nothing is attributed. + Restart, + None, +} + +impl Basis { + fn parse(s: &str) -> Option { + match s { + "baseline" => Some(Basis::Baseline), + "step" => Some(Basis::Step), + "restart" => Some(Basis::Restart), + "none" => Some(Basis::None), + _ => s + .strip_prefix("spans:") + .and_then(|n| n.parse().ok()) + .map(Basis::Spans), + } + } +} + +/// The capture taken at one step of one test of one run. +#[derive(Clone, Debug, PartialEq)] +pub struct Capture { + pub run: String, + pub plan: String, + pub fingerprint: String, + /// 0 for the test's start. + pub step: usize, + pub state: CaptureState, + pub basis: Basis, + pub note: String, + /// The program's totals when the capture was taken (cumulative since it started). + pub lines: Count, + pub branches: Count, + pub functions: Count, + /// Lines, branches and functions first run since the capture [`Basis`] names, if it + /// attributes any. + pub new: Option<[u64; 3]>, +} + +/// One code coverage file. +#[derive(Clone, Debug, PartialEq)] +pub struct CodeCoverage { + /// The file's name, for messages. + pub file: String, + /// What was instrumented: a name for the program or package. + pub name: String, + /// The source revision that was instrumented (a commit, say). + pub revision: String, + /// Which source is counted ("src/**, without tests"). + pub scope: String, + /// A separately hosted HTML report, as a web address. + pub report: Option, + pub captures: Vec, + /// Lines that couldn't be read. + pub problems: Vec, +} + +fn clean(s: &str, max: usize) -> String { + let s: String = s.chars().filter(|c| !c.is_control()).take(max).collect(); + if s == "-" { String::new() } else { s } +} + +impl CodeCoverage { + /// A code coverage file's text. A file without the `expectation-code-coverage 1` header is + /// an error; a line that can't be read is left out, with why in [`CodeCoverage::problems`]. + pub fn parse(file: &str, text: &str) -> Result { + let mut lines = text.lines(); + if lines.next().map(str::trim) != Some("expectation-code-coverage 1") { + return Err(format!( + "{file}: not a code coverage file (no \"expectation-code-coverage 1\" line)" + )); + } + let mut out = CodeCoverage { + file: file.to_string(), + name: String::new(), + revision: String::new(), + scope: String::new(), + report: None, + captures: vec![], + problems: vec![], + }; + for (n, line) in lines.enumerate() { + let n = n + 2; + let f: Vec<&str> = line.split('\t').collect(); + match f[0] { + "source" if f.len() >= 5 => { + out.name = clean(f[1], MAX_TEXT); + out.revision = clean(f[2], MAX_TEXT); + out.scope = clean(f[3], MAX_TEXT); + let url = clean(f[4], 2000); + // A link on a published page: only a web address, never a script or a file. + out.report = + (url.starts_with("https://") || url.starts_with("http://")).then_some(url); + if out.report.is_none() && f[4] != "-" { + out.problems + .push(format!("line {n}: the report address isn't http(s)")); + } + } + "capture" => match capture(&f) { + Some(c) => out.captures.push(c), + None => out + .problems + .push(format!("line {n}: can't read this capture")), + }, + _ => {} + } + } + Ok(out) + } + + /// The address of the hosted report for one test of one run: the file's, with `{run}` and + /// `{slug}` in it filled in. + pub fn report_for(&self, plan: &str, run: &str) -> Option { + let enc = |s: &str| -> String { + s.bytes() + .map(|b| match b { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'.' | b'_' | b'~' => { + (b as char).to_string() + } + b => format!("%{b:02X}"), + }) + .collect() + }; + self.report + .as_ref() + .map(|u| u.replace("{run}", &enc(run)).replace("{slug}", &enc(plan))) + } + + /// The captures that count for test `plan` (index `i` in `plans`), whose result is `report`'s: + /// from the run its result came from, of the test as it is now. In step order. + pub fn for_plan(&self, plans: &[Plan], report: &Report, i: usize) -> Vec<&Capture> { + let (Some(plan), Some(result)) = (plans.get(i), report.results.get(i)) else { + return vec![]; + }; + let Some(run) = result.run.as_ref().filter(|_| result.status.ran()) else { + return vec![]; + }; + let print = fingerprint(plan); + let mut found: Vec<&Capture> = self + .captures + .iter() + .filter(|c| c.plan == plan.slug && c.fingerprint == print && c.run == run.id) + .collect(); + found.sort_by_key(|c| c.step); + found + } +} + +fn capture(f: &[&str]) -> Option { + if f.len() < 17 { + return None; + } + let num = |i: usize| -> Option { f[i].parse().ok() }; + let count = |a: usize| -> Option { + Some(Count { + hit: num(a)?, + found: num(a + 1)?, + }) + }; + let state = CaptureState::parse(f[5])?; + let new = match (f[14], f[15], f[16]) { + ("-", "-", "-") => None, + (a, b, c) => Some([a.parse().ok()?, b.parse().ok()?, c.parse().ok()?]), + }; + Some(Capture { + run: clean(f[1], MAX_ID), + plan: clean(f[2], MAX_ID), + fingerprint: clean(f[3], MAX_ID), + step: f[4].parse().ok()?, + state, + basis: Basis::parse(f[6])?, + note: clean(f[7], MAX_TEXT), + lines: count(8)?, + branches: count(10)?, + functions: count(12)?, + new, + }) +} + +/// The code coverage file for `name` on `platform`: `app@iOS.code.tsv`, or `app.code.tsv`. +pub fn code_coverage_file(name: &str, platform: Option<&str>) -> String { + match platform { + Some(p) => format!("{name}@{p}.code.tsv"), + None => format!("{name}.code.tsv"), + } +} + +/// Is this file in a results directory a code coverage file? +pub fn is_code_coverage_file(path: &std::path::Path) -> bool { + path.file_name() + .and_then(|n| n.to_str()) + .is_some_and(|n| n.ends_with(".code.tsv")) +} + +/// Every code coverage file in `dir` for `platform` (and those for every platform), read, with an +/// error for each that couldn't be. +pub fn load_code_coverage_for( + dir: &std::path::Path, + platform: Option<&str>, +) -> (Vec, Vec) { + let mut files: Vec<_> = std::fs::read_dir(dir) + .into_iter() + .flatten() + .flatten() + .map(|e| e.path()) + .filter(|p| is_code_coverage_file(p)) + .filter(|p| { + let name = p.file_name().and_then(|n| n.to_str()).unwrap_or(""); + match name.trim_end_matches(".code.tsv").split_once('@') { + Some((_, on)) => Some(on) == platform, + None => true, + } + }) + .collect(); + files.sort(); + let (mut found, mut errors) = (vec![], vec![]); + for f in files { + let name = f + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or("") + .to_string(); + if std::fs::metadata(&f).is_ok_and(|m| m.len() > MAX_BYTES) { + errors.push(format!("{name}: over {} MiB, so not read", MAX_BYTES >> 20)); + continue; + } + match std::fs::read_to_string(&f) { + Ok(text) => match CodeCoverage::parse(&name, &text) { + Ok(c) => found.push(c), + Err(e) => errors.push(e), + }, + Err(e) => errors.push(format!("{name}: {e}")), + } + } + (found, errors) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn row(run: &str, step: usize, state: &str, basis: &str, tail: &str) -> String { + format!("capture\t{run}\tgo\tfp\t{step}\t{state}\t{basis}\t-\t{tail}\n") + } + + #[test] + fn a_file_is_read_and_unreadable_lines_are_listed() { + let text = format!( + "expectation-code-coverage 1\nsource\tapp\tabc123\tsrc/**\thttps://cov.example/{{run}}\n{}{}capture\tr\tgo\tfp\tx\n{}future\tkind\n", + row("r", 0, "captured", "baseline", "5\t10\t1\t4\t2\t3\t-\t-\t-"), + row("r", 1, "captured", "step", "7\t10\t2\t4\t2\t3\t2\t1\t0"), + row("r", 2, "missing", "none", "0\t0\t0\t0\t0\t0\t-\t-\t-"), + ); + let c = CodeCoverage::parse("app.code.tsv", &text).unwrap(); + assert_eq!((c.name.as_str(), c.revision.as_str()), ("app", "abc123")); + assert_eq!(c.report.as_deref(), Some("https://cov.example/{run}")); + assert_eq!(c.captures.len(), 3); + assert_eq!(c.problems.len(), 1); + assert_eq!(c.captures[1].new, Some([2, 1, 0])); + assert_eq!(c.captures[0].basis, Basis::Baseline); + assert_eq!(c.captures[2].state, CaptureState::Missing); + assert_eq!(c.captures[1].lines, Count { hit: 7, found: 10 }); + } + + #[test] + fn a_report_address_must_be_http() { + let text = "expectation-code-coverage 1\nsource\tapp\t-\t-\tjavascript:alert(1)\n"; + let c = CodeCoverage::parse("a.code.tsv", text).unwrap(); + assert_eq!(c.report, None); + assert_eq!(c.problems.len(), 1); + assert!(CodeCoverage::parse("a", "nope\n").is_err()); + } + + #[test] + fn the_report_address_names_the_run_and_test() { + let text = + "expectation-code-coverage 1\nsource\tapp\t-\t-\thttps://c.example/{run}/{slug}.html\n"; + let c = CodeCoverage::parse("a", text).unwrap(); + assert_eq!( + c.report_for("a-b", "r 1").as_deref(), + Some("https://c.example/r%201/a-b.html") + ); + } + + #[test] + fn spans_and_restarts_are_read() { + assert_eq!(Basis::parse("spans:2"), Some(Basis::Spans(2))); + assert_eq!(Basis::parse("restart"), Some(Basis::Restart)); + assert_eq!(Basis::parse("spans:x"), None); + } +} diff --git a/crates/story/src/lib.rs b/crates/story/src/lib.rs index 8ea4f13..143f3e4 100644 --- a/crates/story/src/lib.rs +++ b/crates/story/src/lib.rs @@ -2,12 +2,17 @@ //! routes) into test plans, [`report`] holds what happened when they ran, and [`coverage`](mod@coverage) //! reads what extensions made of the evidence those runs left, held to the results. +pub mod codecov; pub mod controls; pub mod coverage; pub mod plan; pub mod report; pub mod sha256; +pub use codecov::{ + Basis, Capture, CaptureState, CodeCoverage, Count, code_coverage_file, is_code_coverage_file, + load_code_coverage_for, +}; pub use controls::{Control, ControlRow, Gap, control_coverage, controls_json}; pub use coverage::{ Attribution, Completeness, Contract, Coverage, Credit, Evidence, Item, Judged, Note, Source, diff --git a/crates/story/src/report.rs b/crates/story/src/report.rs index 250bee5..1766fbc 100644 --- a/crates/story/src/report.rs +++ b/crates/story/src/report.rs @@ -701,7 +701,9 @@ pub fn load_results_for(dir: &std::path::Path, plans: &[Plan], platform: Option< .map(|e| e.path()) .filter(|p| p.extension().is_some_and(|x| x == "tsv")) // Coverage files sit next to the results; they aren't results. - .filter(|p| !crate::coverage::is_coverage_file(p)) + .filter(|p| { + !crate::coverage::is_coverage_file(p) && !crate::codecov::is_code_coverage_file(p) + }) .filter(|p| { match p .file_stem() diff --git a/extensions/code-coverage/README.md b/extensions/code-coverage/README.md new file mode 100644 index 0000000..760e4d6 --- /dev/null +++ b/extensions/code-coverage/README.md @@ -0,0 +1,41 @@ +# expectation-code-coverage + +An [Expectation](https://github.com/ptescher/expectation) extension. It associates the code +coverage an instrumented app handed over (LCOV, or Istanbul's `coverage-final.json`, as step +attachments) with the run, test and step that produced it, and writes the file `expectation docs` +turns into a "Code coverage" section on each test's page. + +It reports observed execution. It doesn't pass or fail a test, and a covered line is not an +assertion. + +## Use + +```bash +expectation-run expectations --docs docs --attachments captures +npx expectation-code-coverage --results docs/results/app.tsv --attachments captures \ + --revision "$(git rev-parse HEAD)" --scope "src/**, without tests" \ + --report-url "https://coverage.example.com/{run}/{slug}/" +expectation docs expectations docs +``` + +The app hands over a snapshot after the test starts and after each step (`onAttachments` in +the JavaScript client). Every attachment that looks like coverage (`*.lcov`, `*.info`, +`coverage-final.json`, or an LCOV or Istanbul media type) counts; several at one step are added up. +The file is `docs/results/code[@platform].code.tsv`. + +## What it will and won't say + +- Counters are taken to be cumulative, from when the app started. A step's part is what ran in + its snapshot and not the one before. Use `--counters per-step` if the app empties its counters + when it hands them over. +- A counter that goes down means a restart: that step gets no attribution, and later steps are + compared with the restarted program. +- A step with no capture is `missing`, and an unreadable one is `incomplete`. Neither is + reported as zero coverage. The next difference after a gap is labelled as spanning steps. +- Background activity can't be told from the step's own, so numbers are "new since", not "caused + by". +- Raw files stay in the attachments directory. Nothing in the coverage file holds file paths or + source. + +`npm test` runs the unit tests and an end-to-end test (build `expectation` and +`expectation-run` first). diff --git a/extensions/code-coverage/bin/expectation-code-coverage.js b/extensions/code-coverage/bin/expectation-code-coverage.js new file mode 100755 index 0000000..d44fdff --- /dev/null +++ b/extensions/code-coverage/bin/expectation-code-coverage.js @@ -0,0 +1,9 @@ +#!/usr/bin/env node +import { main, USAGE } from '../src/cli.js'; + +try { + main(process.argv.slice(2)); +} catch (err) { + process.stderr.write(`${err.message}\n\n${USAGE}\n`); + process.exit(2); +} diff --git a/extensions/code-coverage/package.json b/extensions/code-coverage/package.json new file mode 100644 index 0000000..c4b4fd9 --- /dev/null +++ b/extensions/code-coverage/package.json @@ -0,0 +1,25 @@ +{ + "name": "expectation-code-coverage", + "version": "0.1.0", + "description": "An Expectation extension: associate LCOV or Istanbul coverage that an instrumented app handed over with the test run and step that produced it, and write the code coverage the manual shows", + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/ptescher/expectation", + "directory": "extensions/code-coverage" + }, + "type": "module", + "bin": { + "expectation-code-coverage": "bin/expectation-code-coverage.js" + }, + "files": [ + "bin", + "src" + ], + "engines": { + "node": ">=18" + }, + "scripts": { + "test": "node --test test/*.test.mjs" + } +} diff --git a/extensions/code-coverage/src/analyze.js b/extensions/code-coverage/src/analyze.js new file mode 100644 index 0000000..30a084b --- /dev/null +++ b/extensions/code-coverage/src/analyze.js @@ -0,0 +1,67 @@ +// Associating coverage snapshots with the run and step that produced them. +// +// A coverage counter is cumulative: it counts from when the app started, and an app that +// hands it over after each step gives running totals. What a step added is the difference +// between its snapshot and the one before. That is only a claim when: +// - the counters never went down (they did if the app restarted), and +// - the snapshot before is the previous step's (else the difference spans several steps). +// Nothing here can tell activity the step caused from activity that merely happened during it, +// so the page says what the number is. Execution is observed, never judged: a verdict is not +// touched. + +import { totals } from './formats.js'; + +const newByKind = (counts, before) => { + const n = { L: 0, B: 0, F: 0 }; + for (const [k, c] of counts) if (c > 0 && !((before.get(k) ?? 0) > 0)) n[k[0]] += 1; + return [n.L, n.B, n.F]; +}; + +/** + * One row for each step of one test of one run. + * @param {{ snapshots: Map} | {error: string}>, steps: number }} test + * @param {{ counters?: 'cumulative' | 'per-step' }} options + */ +export function rowsFor(test, options = {}) { + const perStep = options.counters === 'per-step'; + if (!test.snapshots.size) { + return [{ step: 0, state: 'missing', basis: 'none', note: 'no coverage was attached to this test' }]; + } + const rows = []; + let prev = null; // { step, counts } of the last usable snapshot + for (let step = 0; step <= test.steps; step++) { + const s = test.snapshots.get(step); + if (!s) { + rows.push({ step, state: 'missing', basis: 'none', note: 'no capture at this step' }); + continue; + } + if ('error' in s) { + rows.push({ step, state: 'incomplete', basis: 'none', note: s.error }); + continue; + } + let counts = s.counts; + let row; + if (perStep) { + // Each capture is what ran since the one before: the totals are the running sum. + const before = prev ? prev.counts : new Map(); + counts = new Map(before); + for (const [k, n] of s.counts) counts.set(k, (counts.get(k) ?? 0) + n); + row = prev + ? { basis: prev.step === step - 1 ? 'step' : `spans:${prev.step}`, new: newByKind(counts, before), note: '' } + : { basis: 'baseline', new: null, note: '' }; + } else if (!prev) { + row = { basis: 'baseline', new: null, note: '' }; + } else if ([...prev.counts].some(([k, n]) => s.counts.has(k) && s.counts.get(k) < n)) { + row = { basis: 'restart', new: null, note: 'a counter went down, so the program restarted' }; + } else { + row = { + basis: prev.step === step - 1 ? 'step' : `spans:${prev.step}`, + new: newByKind(counts, prev.counts), + note: '', + }; + } + prev = { step, counts }; + rows.push({ step, state: 'captured', totals: totals({ counts }), ...row }); + } + return rows; +} diff --git a/extensions/code-coverage/src/cli.js b/extensions/code-coverage/src/cli.js new file mode 100644 index 0000000..d79ae05 --- /dev/null +++ b/extensions/code-coverage/src/cli.js @@ -0,0 +1,145 @@ +// expectation-code-coverage: the command. + +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { dirname, join, resolve, sep } from 'node:path'; +import { rowsFor } from './analyze.js'; +import { coverageKind, merge, parseCoverage } from './formats.js'; +import { readResults, ran } from './results.js'; +import { line } from './tsv.js'; + +export const USAGE = `usage: expectation-code-coverage --results --attachments + [--revision ] [--scope ] [--name ] + [--report-url
] [--counters cumulative|per-step] [--id ] [--out ] + +Associates the coverage an instrumented app handed over (LCOV, or Istanbul's coverage-final.json, +as step attachments) with the run, test and step that produced it, and writes the code coverage +that \`expectation docs\` shows next to the results: +/[@platform].code.tsv. + + --results the run's results, as Expectation saved them + --attachments where the run kept what the driver captured (expectation-run --attachments) + --revision the source revision that was instrumented (say it: coverage is about one) + --scope which source is counted, e.g. "src/**, without tests" + --name the program's name in the manual (default the results' runner) + --report-url a separately hosted HTML report; {run} and {slug} stand for the run and the test + --counters cumulative (default): each capture is the totals since the app started, so a + step's part is the difference. per-step: the app empties its counters when + handing them over, so each capture is that step's own + --id the file's name (default code) + --out write here instead + +Coverage is observed execution. It changes no result and no verdict. A step with no capture, or +with one that can't be read, is shown as that, not as uncovered code.`; + +export function parseArgs(argv) { + const o = { id: 'code', counters: 'cumulative' }; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + const value = () => { + const v = argv[++i]; + if (v === undefined) throw new Error(`${a} needs a value`); + return v; + }; + switch (a) { + case '--results': o.results = value(); break; + case '--attachments': o.attachments = value(); break; + case '--revision': o.revision = value(); break; + case '--scope': o.scope = value(); break; + case '--name': o.name = value(); break; + case '--report-url': o.reportUrl = value(); break; + case '--counters': o.counters = value(); break; + case '--id': o.id = value(); break; + case '--out': o.out = value(); break; + case '-h': + case '--help': o.help = true; break; + default: throw new Error(`unknown option ${a}`); + } + } + if (!o.help) { + if (!o.results || !o.attachments) throw new Error('--results and --attachments are needed'); + if (!['cumulative', 'per-step'].includes(o.counters)) throw new Error('--counters is cumulative or per-step'); + if (!/^[A-Za-z0-9._-]+$/.test(o.id)) throw new Error('--id is letters, digits, ".", "_" and "-"'); + if (o.reportUrl && !/^https?:\/\//.test(o.reportUrl)) throw new Error('--report-url is an http(s) address'); + } + return o; +} + +/** The coverage snapshots a test's steps attached, by step; a file that can't be read is an error entry. */ +export function snapshotsOf(plan, dir) { + const root = resolve(dir); + const snapshots = new Map(); + const files = new Map(); + for (const a of plan.attachments) { + const kind = coverageKind(a.path, a.mediaType); + if (!kind) continue; + files.set(a.step, [...(files.get(a.step) ?? []), { kind, a }]); + } + for (const [step, list] of files) { + let merged = null; + let error = null; + for (const { kind, a } of list) { + const path = resolve(root, a.path); + // A results file is input too: an attachment can't point outside the directory. + if (!path.startsWith(root + sep)) throw new Error(`${a.path}: outside ${dir}`); + try { + const one = parseCoverage(kind, readFileSync(path, 'utf8')); + merged = merged ? merge(merged, one) : one; + } catch (e) { + error = `${a.path.split('/').pop()}: ${e.message}`; + } + } + snapshots.set(step, error ? { error } : merged); + } + const steps = Math.max(0, ...snapshots.keys(), ...plan.windows.keys(), ...plan.attachments.map((a) => a.step)); + return { snapshots, steps }; +} + +const num = (n) => String(n); + +/** The code coverage file's text. */ +export function coverageFile(results, tests, o) { + let out = 'expectation-code-coverage 1\n'; + // The address may have {run} and {slug} in it; the manual fills them in for each test. + out += line('source', o.name ?? results.runner, o.revision ?? '', o.scope ?? '', o.reportUrl ?? ''); + for (const { plan, rows } of tests) { + for (const r of rows) { + const t = r.totals; + out += line( + 'capture', + plan.run, + plan.slug, + plan.fingerprint, + num(r.step), + r.state, + r.basis, + r.note ?? '', + ...(t ? [t.L.hit, t.L.found, t.B.hit, t.B.found, t.F.hit, t.F.found] : [0, 0, 0, 0, 0, 0]).map(num), + ...(r.new ? r.new.map(num) : ['', '', '']), + ); + } + } + return out; +} + +export function main(argv, log = (s) => process.stderr.write(s + '\n')) { + const o = parseArgs(argv); + if (o.help) { + process.stdout.write(USAGE + '\n'); + return null; + } + const results = readResults(o.results); + const tests = []; + for (const plan of results.plans) { + if (!plan.run || !ran(plan)) continue; + tests.push({ plan, rows: rowsFor(snapshotsOf(plan, o.attachments), o) }); + } + const file = `${o.id}${results.platform ? `@${results.platform}` : ''}.code.tsv`; + const out = o.out ?? join(dirname(o.results), file); + mkdirSync(dirname(out), { recursive: true }); + writeFileSync(out, coverageFile(results, tests, o)); + const rows = tests.flatMap((t) => t.rows); + const n = (s) => rows.filter((r) => r.state === s).length; + log(`${tests.length} tests: ${n('captured')} captures, ${n('missing')} missing, ${n('incomplete')} unusable -> ${out}`); + if (!o.revision) log('note: without --revision the manual cannot say which source was instrumented'); + return out; +} diff --git a/extensions/code-coverage/src/formats.js b/extensions/code-coverage/src/formats.js new file mode 100644 index 0000000..8889fb8 --- /dev/null +++ b/extensions/code-coverage/src/formats.js @@ -0,0 +1,99 @@ +// Reading coverage data: LCOV (`.info`, `.lcov`) and Istanbul's JSON (`coverage-final.json`). +// Both become a Map from the key of one thing in the source (a line, a branch, a function) to +// how many times it ran, so two snapshots can be compared. + +/** + * @typedef {{ counts: Map }} Snapshot + * keys: `L|file|line`, `B|file|line|block|branch`, `F|file|name` + */ + +const add = (m, key, n) => m.set(key, (m.get(key) ?? 0) + n); + +/** An LCOV tracefile's text. Throws if it holds nothing of the kind. */ +export function parseLcov(text) { + const counts = new Map(); + let file = null; + for (const raw of text.split(/\r?\n/)) { + const l = raw.trim(); + if (l.startsWith('SF:')) file = l.slice(3); + else if (l === 'end_of_record') file = null; + else if (file === null) continue; + else if (l.startsWith('DA:')) { + const [line, hits] = l.slice(3).split(','); + if (/^\d+$/.test(line) && /^\d+$/.test(hits)) add(counts, `L|${file}|${line}`, Number(hits)); + } else if (l.startsWith('BRDA:')) { + const [line, block, branch, taken] = l.slice(5).split(','); + if (line !== undefined && taken !== undefined) { + add(counts, `B|${file}|${line}|${block}|${branch}`, /^\d+$/.test(taken) ? Number(taken) : 0); + } + } else if (l.startsWith('FN:')) { + const name = l.slice(3).split(',').slice(1).join(','); + if (name) add(counts, `F|${file}|${name}`, 0); + } else if (l.startsWith('FNDA:')) { + const [hits, ...name] = l.slice(5).split(','); + if (/^\d+$/.test(hits) && name.length) add(counts, `F|${file}|${name.join(',')}`, Number(hits)); + } + } + if (!counts.size) throw new Error('no SF/DA records: not an LCOV file, or nothing was instrumented'); + return { counts }; +} + +/** Istanbul's coverage map (`coverage-final.json`) as parsed JSON. */ +export function parseIstanbul(json) { + const counts = new Map(); + if (!json || typeof json !== 'object' || Array.isArray(json)) throw new Error('not an Istanbul coverage map'); + for (const [key, f] of Object.entries(json)) { + if (!f || typeof f !== 'object' || !f.statementMap) continue; + const file = f.path ?? key; + // A line ran as often as its most-run statement that starts there, as Istanbul reports it. + const lines = new Map(); + for (const [id, loc] of Object.entries(f.statementMap)) { + const line = loc?.start?.line; + if (!Number.isInteger(line)) continue; + lines.set(line, Math.max(lines.get(line) ?? 0, Number(f.s?.[id] ?? 0))); + } + for (const [line, n] of lines) add(counts, `L|${file}|${line}`, n); + for (const [id, taken] of Object.entries(f.b ?? {})) { + const line = f.branchMap?.[id]?.line ?? f.branchMap?.[id]?.loc?.start?.line ?? 0; + (Array.isArray(taken) ? taken : []).forEach((n, i) => add(counts, `B|${file}|${line}|${id}|${i}`, Number(n) || 0)); + } + for (const [id, n] of Object.entries(f.f ?? {})) { + add(counts, `F|${file}|${f.fnMap?.[id]?.name ?? id}`, Number(n) || 0); + } + } + if (!counts.size) throw new Error('no statements in the Istanbul coverage map'); + return { counts }; +} + +/** Is this attachment coverage data, going by its name and media type? */ +export function coverageKind(path, mediaType = '') { + const name = path.toLowerCase(); + if (name.endsWith('.lcov') || name.endsWith('.info') || /lcov/.test(mediaType)) return 'lcov'; + if (/istanbul/.test(mediaType) || /(^|[/-])coverage(-final)?\.json$/.test(name) || name.endsWith('.istanbul.json')) { + return 'istanbul'; + } + return null; +} + +/** The snapshot in `text`, whichever format `kind` says it is. */ +export function parseCoverage(kind, text) { + return kind === 'lcov' ? parseLcov(text) : parseIstanbul(JSON.parse(text)); +} + +/** Add `b`'s counts into `a`, for two files captured at the same step. */ +export function merge(a, b) { + const counts = new Map(a.counts); + for (const [k, n] of b.counts) add(counts, k, n); + return { counts }; +} + +/** Hit and found for lines, branches and functions. */ +export function totals(snapshot) { + const t = { L: { hit: 0, found: 0 }, B: { hit: 0, found: 0 }, F: { hit: 0, found: 0 } }; + for (const [k, n] of snapshot.counts) { + const c = t[k[0]]; + c.found += 1; + if (n > 0) c.hit += 1; + } + return t; +} diff --git a/extensions/code-coverage/src/results.js b/extensions/code-coverage/src/results.js new file mode 100644 index 0000000..cf44249 --- /dev/null +++ b/extensions/code-coverage/src/results.js @@ -0,0 +1,75 @@ +// Reading an Expectation results file (`results/[@platform].tsv`): which tests ran in +// which run, their fingerprints, when each step ran, and what the driver attached. + +import { readFileSync } from 'node:fs'; +import { basename } from 'node:path'; + +/** + * @typedef {{ start: number, end: number }} Window + * @typedef {{ step: number, mediaType: string, path: string }} Attachment + * @typedef {{ slug: string, status: string, fingerprint: string, run: string | null, + * windows: Map, attachments: Attachment[] }} PlanResult + * @typedef {{ file: string, runner: string, platform: string | null, + * runs: Map, plans: PlanResult[] }} Results + */ + +/** The results in `text`, saved by Expectation; `file` is its name, for the platform. */ +export function parseResults(text, file) { + const lines = text.split('\n'); + if (lines[0]?.trim() !== 'expectation-results 1') { + throw new Error(`${file}: not an Expectation results file (no "expectation-results 1" line)`); + } + const name = basename(file).replace(/\.tsv$/, ''); + const [runner, platform] = name.includes('@') ? name.split('@', 2) : [name, null]; + /** @type {Results} */ + const out = { file, runner, platform, runs: new Map(), plans: [] }; + let run = null; + let plan = null; + for (const l of lines.slice(1)) { + const f = l.split('\t'); + switch (f[0]) { + case 'run': + if (f.length >= 3 && f[1] !== '-' && /^\d+$/.test(f[2])) { + run = f[1]; + out.runs.set(run, Number(f[2])); + } else { + run = null; + } + break; + case 'plan': + if (f.length < 3) break; + plan = { + slug: f[1], + status: f[2], + fingerprint: f[3] ?? '', + run, + windows: new Map(), + attachments: [], + }; + out.plans.push(plan); + break; + case 'window': + if (plan && f.length === 4 && [1, 2, 3].every((i) => /^\d+$/.test(f[i]))) { + plan.windows.set(Number(f[1]), { start: Number(f[2]), end: Number(f[3]) }); + } + break; + case 'attachment': + if (plan && f.length === 4 && /^\d+$/.test(f[1])) { + plan.attachments.push({ step: Number(f[1]), mediaType: f[2], path: f[3] }); + } + break; + default: + break; + } + } + return out; +} + +export function readResults(path) { + return parseResults(readFileSync(path, 'utf8'), path); +} + +/** Did this result run (pass or fail), so that it could have made traffic? */ +export function ran(plan) { + return plan.status === 'passed' || plan.status === 'failed'; +} diff --git a/extensions/code-coverage/src/tsv.js b/extensions/code-coverage/src/tsv.js new file mode 100644 index 0000000..9a130f0 --- /dev/null +++ b/extensions/code-coverage/src/tsv.js @@ -0,0 +1,14 @@ +// Writing tab-separated records that can't break their own format: no tabs, newlines or other +// control characters in a field, and an empty field written as `-`. + +/** A field as it can be written: control characters dropped, cut to `max` characters. */ +export function field(value, max = 300) { + const s = String(value ?? '').replace(/[\u0000-\u001f\u007f-\u009f]/g, ' ').trim(); + const cut = [...s].slice(0, max).join(''); + return cut === '' ? '-' : cut; +} + +/** One record: a kind and its fields, as a line. */ +export function line(kind, ...fields) { + return [kind, ...fields.map((f) => field(f))].join('\t') + '\n'; +} diff --git a/extensions/code-coverage/test/analyze.test.mjs b/extensions/code-coverage/test/analyze.test.mjs new file mode 100644 index 0000000..644704d --- /dev/null +++ b/extensions/code-coverage/test/analyze.test.mjs @@ -0,0 +1,87 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { parseLcov, parseIstanbul, coverageKind, totals } from '../src/formats.js'; +import { rowsFor } from '../src/analyze.js'; + +const lcov = (hits) => + `SF:src/a.js\n${hits.map((h, i) => `DA:${i + 1},${h}`).join('\n')}\nBRDA:1,0,0,${hits[0] ? 1 : '-'}\nFN:1,main\nFNDA:${hits[0]},main\nend_of_record\n`; +const snap = (hits) => parseLcov(lcov(hits)); + +test('LCOV: lines, branches (a dash is not taken) and functions', () => { + const t = totals(snap([2, 0, 1])); + assert.deepEqual(t.L, { hit: 2, found: 3 }); + assert.deepEqual(t.B, { hit: 1, found: 1 }); + assert.deepEqual(t.F, { hit: 1, found: 1 }); + assert.deepEqual(totals(snap([0, 0])).B, { hit: 0, found: 1 }); + assert.throws(() => parseLcov('nothing here'), /not an LCOV/); +}); + +test('Istanbul: a line ran as often as its most-run statement', () => { + const s = parseIstanbul({ + '/x/a.js': { + path: '/x/a.js', + statementMap: { 0: { start: { line: 1 } }, 1: { start: { line: 1 } }, 2: { start: { line: 2 } } }, + s: { 0: 0, 1: 3, 2: 0 }, + branchMap: { 0: { line: 2 } }, + b: { 0: [1, 0] }, + fnMap: { 0: { name: 'go' } }, + f: { 0: 1 }, + }, + }); + const t = totals(s); + assert.deepEqual(t.L, { hit: 1, found: 2 }); + assert.deepEqual(t.B, { hit: 1, found: 2 }); + assert.deepEqual(t.F, { hit: 1, found: 1 }); + assert.throws(() => parseIstanbul([]), /not an Istanbul/); +}); + +test('attachments are recognised by name or media type', () => { + assert.equal(coverageKind('r/t/00/lcov.info'), 'lcov'); + assert.equal(coverageKind('r/t/00/x', 'text/x-lcov'), 'lcov'); + assert.equal(coverageKind('r/t/00/coverage-final.json'), 'istanbul'); + assert.equal(coverageKind('r/t/00/network.har', 'application/json'), null); +}); + +const test3 = (snapshots, steps = 2) => ({ snapshots: new Map(snapshots), steps }); + +test('cumulative counters: the first capture is a baseline, later ones add what newly ran', () => { + const rows = rowsFor(test3([[0, snap([1, 0, 0])], [1, snap([2, 1, 0])], [2, snap([3, 1, 1])]])); + assert.deepEqual(rows.map((r) => r.basis), ['baseline', 'step', 'step']); + assert.equal(rows[0].new, null); + assert.deepEqual(rows[1].new, [1, 0, 0]); + assert.deepEqual(rows[2].new, [1, 0, 0]); + assert.deepEqual(rows[2].totals.L, { hit: 3, found: 3 }); +}); + +test('a counter that goes down means the program restarted: nothing is attributed to that step', () => { + const rows = rowsFor(test3([[0, snap([5, 5, 0])], [1, snap([1, 0, 0])], [2, snap([2, 1, 0])]])); + assert.equal(rows[1].basis, 'restart'); + assert.equal(rows[1].new, null); + // Later steps are held against the restarted program, not the one before. + assert.equal(rows[2].basis, 'step'); + assert.deepEqual(rows[2].new, [1, 0, 0]); +}); + +test('a missing capture is missing, not zero, and the next difference says it spans steps', () => { + const rows = rowsFor(test3([[0, snap([1, 0, 0])], [2, snap([1, 1, 1])]])); + assert.equal(rows[1].state, 'missing'); + assert.equal(rows[1].totals, undefined); + assert.equal(rows[2].basis, 'spans:0'); + assert.deepEqual(rows[2].new, [2, 0, 0]); +}); + +test('an unreadable capture is incomplete, and a test with none is missing', () => { + const rows = rowsFor(test3([[0, snap([1])], [1, { error: 'lcov.info: not an LCOV file' }]], 1)); + assert.equal(rows[1].state, 'incomplete'); + assert.match(rows[1].note, /not an LCOV/); + const none = rowsFor(test3([])); + assert.deepEqual(none.map((r) => r.state), ['missing']); +}); + +test('per-step counters: each capture is that step\'s own, and totals run on', () => { + const rows = rowsFor(test3([[0, snap([1, 0, 0])], [1, snap([0, 1, 0])], [2, snap([0, 0, 0])]]), { counters: 'per-step' }); + assert.deepEqual(rows.map((r) => r.basis), ['baseline', 'step', 'step']); + assert.deepEqual(rows[1].new, [1, 0, 0]); + assert.deepEqual(rows[2].new, [0, 0, 0]); + assert.deepEqual(rows[2].totals.L, { hit: 2, found: 3 }); +}); diff --git a/extensions/code-coverage/test/e2e.test.mjs b/extensions/code-coverage/test/e2e.test.mjs new file mode 100644 index 0000000..50cc9fc --- /dev/null +++ b/extensions/code-coverage/test/e2e.test.mjs @@ -0,0 +1,67 @@ +// The whole path: a JS app hands over coverage as step attachments, the real runner keeps them, +// the extension associates them with the steps, and the real `expectation docs` shows them. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { spawn, spawnSync } from 'node:child_process'; +import { createRequire } from 'node:module'; +import { fileURLToPath } from 'node:url'; +import path from 'node:path'; +import fs from 'node:fs'; +import os from 'node:os'; +import { main } from '../src/cli.js'; + +const require = createRequire(import.meta.url); +const here = path.dirname(fileURLToPath(import.meta.url)); +const { ExpectationDriver } = require('../../../clients/js/index.js'); +const runner = path.join(here, '../../../target/debug/expectation-run'); +const cli = path.join(here, '../../../target/debug/expectation'); +const fixture = path.join(here, '../../../clients/js/fixture'); + +// It needs the Rust binaries built, so it runs only with EXPECTATION_E2E=1 (as CI does). +test('coverage handed over by an app shows on the test page, apart from its result', { skip: process.env.EXPECTATION_E2E !== '1' && 'set EXPECTATION_E2E=1' }, async () => { + const port = 6000 + Math.floor(Math.random() * 1000); + const work = fs.mkdtempSync(path.join(os.tmpdir(), 'expectation-cov-')); + const docs = path.join(work, 'docs'); + const keep = path.join(work, 'captures'); + const run = spawn(runner, [fixture, '--addr', `127.0.0.1:${port}`, '--settle', '0', '--wait', '20', '--docs', docs, '--runner', 'app', '--attachments', keep]); + let out = ''; + run.stdout.on('data', (d) => (out += d)); + run.stderr.on('data', (d) => (out += d)); + + // A cumulative counter: each step runs a line more of the app's source. + const t = { unlocked: false, coins: 0, ran: 1 }; + const lcov = () => `SF:src/gate.js\n${[1, 2, 3, 4, 5].map((l) => `DA:${l},${l <= t.ran ? 1 : 0}`).join('\n')}\nend_of_record\n`; + const driver = new ExpectationDriver({ platform: 'node', app: 'Turnstile' }) + .observe('the turnstile', () => (t.unlocked ? 'unlocked' : 'locked')) + .observe('the coin box', () => t.coins) + .arrange('the turnstile', (v) => { t.unlocked = v === 'unlocked'; }) + .arrange('the coin box', (v) => { t.coins = v; }) + .on('a visitor inserts a coin', () => { t.coins += 1; t.unlocked = true; t.ran += 1; }) + .on('a visitor pushes the arm', () => { t.unlocked = false; t.ran += 2; }) + .onReset(() => { t.unlocked = false; t.coins = 0; }) + .onAttachments(() => [{ name: 'lcov.info', type: 'text/x-lcov', text: lcov() }]); + driver.start(`http://127.0.0.1:${port}`); + const code = await new Promise((r) => run.on('exit', r)); + driver.stop(); + assert.equal(code, 0, out); + + const results = path.join(docs, 'results/app.tsv'); + const written = main(['--results', results, '--attachments', keep, '--revision', 'abc1234', '--scope', 'src/**', '--report-url', 'https://cov.example/{run}/{slug}/'], () => {}); + assert.equal(written, path.join(docs, 'results/code.code.tsv')); + + const made = spawnSync(cli, ['docs', fixture, docs], { encoding: 'utf8' }); + assert.equal(made.status, 0, made.stderr); + const slug = 'a-day-at-the-zoo'; + const page = fs.readFileSync(path.join(docs, `site/test-${slug}.html`), 'utf8'); + assert.match(page, /Code coverage/); + assert.match(page, /abc1234/); + assert.match(page, /observed execution/); + assert.match(page, /1 of 5/); + assert.match(page, / /lcov\.info$/.test(p)), []); +}); From 1cfb724d24cf0cc3cf77b5e952b26c1a6e501573 Mon Sep 17 00:00:00 2001 From: Patrick Tescher Date: Mon, 9 Nov 2026 11:49:15 -0800 Subject: [PATCH 2/2] Share what extensions have in common, and tighten code coverage Rust: story::sidecar holds what every extension file shares (finding files by kind and platform, the size cap, field cleaning, run/test crediting); contract coverage and code coverage both use it, and results are told from extension files by one registry of kinds. JS: extensions/kit holds the results reader, attachment finder (links can't leave the directory) and record writer; both extensions use it. Code coverage fixes from review: parse errors in the manual are fixed messages, never the file's words; a counter that goes down or something that stops being counted is a restart; the same line in two files is one line; a step must hold the same files as the first; a .info or coverage.json that isn't coverage is ignored; Istanbul functions are keyed by id; 'spans' is now 'since' so it can't be confused with HAR's; the default output is .code.tsv; plurals, unique ids and URL encoding. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/ci.yml | 4 + book/src/evidence.md | 13 +- crates/cli/src/lib.rs | 2 +- crates/docs/src/html.rs | 24 ++- crates/docs/src/lib.rs | 25 ++- crates/docs/src/markdown.rs | 11 +- crates/story/src/codecov.rs | 81 ++------ crates/story/src/coverage.rs | 83 +------- crates/story/src/lib.rs | 1 + crates/story/src/report.rs | 6 +- crates/story/src/sidecar.rs | 186 ++++++++++++++++++ extensions/code-coverage/.gitignore | 1 + extensions/code-coverage/README.md | 4 +- extensions/code-coverage/package-lock.json | 34 ++++ extensions/code-coverage/package.json | 3 + extensions/code-coverage/src/analyze.js | 8 +- extensions/code-coverage/src/cli.js | 78 ++++---- extensions/code-coverage/src/formats.js | 46 +++-- .../code-coverage/test/analyze.test.mjs | 38 +++- extensions/code-coverage/test/e2e.test.mjs | 4 +- extensions/code-coverage/test/files.test.mjs | 66 +++++++ extensions/har-openapi/package-lock.json | 13 ++ extensions/har-openapi/package.json | 3 +- extensions/har-openapi/src/analyze.js | 5 +- extensions/har-openapi/src/cli.js | 21 +- extensions/har-openapi/src/results.js | 75 ------- extensions/har-openapi/src/tsv.js | 14 -- extensions/har-openapi/test/analyze.test.mjs | 2 +- extensions/kit/README.md | 17 ++ extensions/kit/package.json | 25 +++ extensions/kit/src/attachments.js | 38 ++++ extensions/kit/src/index.js | 5 + .../{code-coverage => kit}/src/results.js | 7 +- extensions/{code-coverage => kit}/src/tsv.js | 0 extensions/kit/test/kit.test.mjs | 48 +++++ 35 files changed, 667 insertions(+), 324 deletions(-) create mode 100644 crates/story/src/sidecar.rs create mode 100644 extensions/code-coverage/.gitignore create mode 100644 extensions/code-coverage/package-lock.json create mode 100644 extensions/code-coverage/test/files.test.mjs delete mode 100644 extensions/har-openapi/src/results.js delete mode 100644 extensions/har-openapi/src/tsv.js create mode 100644 extensions/kit/README.md create mode 100644 extensions/kit/package.json create mode 100644 extensions/kit/src/attachments.js create mode 100644 extensions/kit/src/index.js rename extensions/{code-coverage => kit}/src/results.js (88%) rename extensions/{code-coverage => kit}/src/tsv.js (100%) create mode 100644 extensions/kit/test/kit.test.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 420b947..9fa4bb3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,6 +36,8 @@ jobs: - uses: actions/setup-node@v4 with: node-version: 22 + - run: npm test + working-directory: extensions/kit - run: npm ci working-directory: extensions/har-openapi # The unit tests, and the quote example end to end: a run, this extension, `expectation docs`. @@ -45,6 +47,8 @@ jobs: EXPECTATION_E2E: "1" # Coverage an app hands over as attachments, the extension, and `expectation docs`, end to end. - run: cargo build -p expectation_cli -p expectation_remote + - run: npm ci + working-directory: extensions/code-coverage - run: npm test working-directory: extensions/code-coverage env: diff --git a/book/src/evidence.md b/book/src/evidence.md index 82a089b..6dd892b 100644 --- a/book/src/evidence.md +++ b/book/src/evidence.md @@ -97,7 +97,8 @@ separate section on the test's page, because it means something different. The application collects it (Istanbul, or anything that writes LCOV) and hands it over as step attachments, in JavaScript with `onAttachments`. Then [`extensions/code-coverage`](https://github.com/ptescher/expectation/tree/main/extensions/code-coverage) -reads LCOV or Istanbul's `coverage-final.json` and writes `results/code[@platform].code.tsv`: +reads LCOV or Istanbul's `coverage-final.json` and writes `results/[@platform].code.tsv` +(`app.code.tsv` for the default runner): ```bash expectation-run expectations --docs docs --attachments captures @@ -122,7 +123,7 @@ What the section claims is limited on purpose: - **A step's part is only claimed when it can be.** Counters count from when the app started, so a step's part is the difference from the capture before. If a counter goes down, the app restarted, and nothing is attributed to that step. If a step's capture is missing, the next - difference is labelled as spanning the steps between. Anything else that ran in the same + difference is labelled as since the earlier step it is compared with. Anything else that ran in the same window, such as background work, is in the numbers, and the page says so. With `--counters per-step` the app empties its counters each time it hands them over. - **It counts for its own run and test.** Like contract evidence, a capture is shown only with @@ -136,6 +137,14 @@ Expectation doesn't instrument the app: collecting coverage is the app's and the ## Writing an extension +Extensions in JavaScript can start from +[`extensions/kit`](https://github.com/ptescher/expectation/tree/main/extensions/kit): it reads the +results file, finds the attached files (refusing any path that leaves the attachments directory, +links included) and writes records that can't break the format. Expectation reads every +extension's file the same way, whatever kind it is: the file for another platform is left out, a +file over 16 MiB isn't read, fields are cut, and evidence counts only for its own run and test as +it was. + An extension reads: - the results file (`results/[@platform].tsv`), for the run id, each test's slug and diff --git a/crates/cli/src/lib.rs b/crates/cli/src/lib.rs index 1b1b516..4932260 100644 --- a/crates/cli/src/lib.rs +++ b/crates/cli/src/lib.rs @@ -550,7 +550,7 @@ fn prune_screenshots(store: &str, dirs: &[&str], dry_run: bool) { for entry in std::fs::read_dir(d).into_iter().flatten().flatten() { let path = entry.path(); if path.extension().is_none_or(|e| e != "tsv") - || expectation_story::is_coverage_file(&path) + || expectation_story::sidecar::is_sidecar(&path, None) { continue; } diff --git a/crates/docs/src/html.rs b/crates/docs/src/html.rs index f6a6083..6a4f212 100644 --- a/crates/docs/src/html.rs +++ b/crates/docs/src/html.rs @@ -20,6 +20,17 @@ use expectation_story::{ use std::collections::HashMap; use std::fmt::Write; +/// "2 lines, 1 branch, 0 functions". +pub(crate) fn counted(l: u64, b: u64, f: u64) -> String { + let pl = |n: u64, one: &str, many: &str| format!("{n} {}", if n == 1 { one } else { many }); + format!( + "{}, {}, {}", + pl(l, "line", "lines"), + pl(b, "branch", "branches"), + pl(f, "function", "functions") + ) +} + /// What a capture's "new" numbers say, in words. fn new_since(c: &expectation_story::Capture) -> String { let Some([l, b, f]) = c.new else { @@ -29,10 +40,10 @@ fn new_since(c: &expectation_story::Capture) -> String { _ => "can't be told".into(), }; }; - let what = format!("{l} lines, {b} branches, {f} functions"); + let what = counted(l, b, f); match c.basis { Basis::Step => what, - Basis::Spans(from) => { + Basis::Since(from) => { format!("{what}, since step {from} (the captures between are missing)") } _ => what, @@ -1885,10 +1896,15 @@ impl Site<'_> { /// Code coverage of the instrumented program while this test ran, apart from its result. fn code_coverage(&self, plan: usize) -> String { let mut o = String::new(); - for (cov, caps) in self.d.code_coverage_of(plan) { + for (n, (cov, caps)) in self.d.code_coverage_of(plan).into_iter().enumerate() { + let id = if n == 0 { + "code-coverage".to_string() + } else { + format!("code-coverage-{}", n + 1) + }; let _ = write!( o, - r#"

Code coverage

Which of the program's own source ran while this test did. It is observed execution: it doesn't change the result above, and it doesn't show that anything was checked. Counters are the program's totals since it started; "new" is what a step added, when that can be told, and can include activity the test didn't cause.

"# + r#"

Code coverage

Which of the program's own source ran while this test did. It is observed execution: it doesn't change the result above, and it doesn't show that anything was checked. Counts are running totals over the test; "new" is what a step added since the capture before it, when that can be told, and can include activity the test didn't cause.

"# ); let mut about = vec![]; for (label, v) in [ diff --git a/crates/docs/src/lib.rs b/crates/docs/src/lib.rs index b745c03..f99340d 100644 --- a/crates/docs/src/lib.rs +++ b/crates/docs/src/lib.rs @@ -333,7 +333,7 @@ impl Docs<'_> { } /// The code coverage that counts for test `plan`: per file, its captures of this run of the - /// test as it is now, and how many other captures it holds that were left out. + /// test as it is now. A file with none that count isn't listed. pub fn code_coverage_of( &self, plan: usize, @@ -1507,7 +1507,7 @@ Feature: Paying #[test] fn code_coverage_is_shown_as_observed_execution_for_the_run_and_test_it_came_from() { - let src = "Vocabulary:\n State: the lamp is on | off\n Event: the user flips the switch\nBackground:\n Given the lamp is off\nStory: Using the lamp\n As a user\n I want light\n So that I see\n Scenario: x\n When the user flips the switch\n Then the lamp is on\n"; + let src = "Vocabulary:\n State: the lamp is on | off\n Event: the user flips the switch\n Event: the user presses reset\nBackground:\n Given the lamp is off\nFeature: Lamp\n Expectation: Flip\n Given the lamp is off\n When the user flips the switch\n Then the lamp is on\n Expectation: Reset\n Given the lamp is on\n When the user presses reset\n Then the lamp is off\nStory: Using the lamp\n As a user\n I want light\n So that I see\n Scenario: x\n When the user flips the switch\n Then the lamp is on\n When the user presses reset\n Then the lamp is off\n"; let loaded = expectation_language::load_sources("S", &[("s.expect", src)]); assert!(!loaded.has_errors(), "{}", loaded.render_diagnostics()); let suite = loaded.suite; @@ -1567,13 +1567,32 @@ Feature: Paying assert!(page.contains("observed execution")); assert!(page.contains("abc123") && page.contains("src/")); assert!(page.contains("7 of 10")); - assert!(page.contains("2 lines, 1 branches, 0 functions")); + assert!(page.contains("2 lines, 1 branch, 0 functions")); assert!(page.contains("No capture at this step")); assert!(page.contains("isn't the same as nothing being covered")); assert!(page.contains("https://cov.example/r")); } // The result is what it was. assert!(html.contains("passed")); + // Text from the file can't become markup, in either kind of page. + let hostile = file("r1", &print) + .replace( + "source\tapp\tabc123\tsrc/**", + "source\tapp\t\">\t[a](http://evil)", + ) + .replace("no snapshot", ""); + let bad = [expectation_story::CodeCoverage::parse("a.code.tsv", &hostile).unwrap()]; + let shown = pages(&bad); + let html = find(&shown, &format!("site/test-{slug}.html")); + let md = find(&shown, &format!("tests/{slug}.md")); + for page in [&html, &md] { + assert!( + !page.contains("