diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 49287bf..9fa4bb3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,6 +36,8 @@ jobs: - uses: actions/setup-node@v4 with: node-version: 22 + - run: npm test + working-directory: extensions/kit - run: npm ci working-directory: extensions/har-openapi # The unit tests, and the quote example end to end: a run, this extension, `expectation docs`. @@ -43,3 +45,11 @@ jobs: working-directory: extensions/har-openapi env: EXPECTATION_E2E: "1" + # Coverage an app hands over as attachments, the extension, and `expectation docs`, end to end. + - run: cargo build -p expectation_cli -p expectation_remote + - run: npm ci + working-directory: extensions/code-coverage + - run: npm test + working-directory: extensions/code-coverage + env: + EXPECTATION_E2E: "1" diff --git a/book/src/evidence.md b/book/src/evidence.md index 00481a5..6dd892b 100644 --- a/book/src/evidence.md +++ b/book/src/evidence.md @@ -88,8 +88,63 @@ branch of a JSON response becomes an outcome. It also reads a HAR made by any ot it leaves unknown. `crates/harness/examples/quote` runs the whole pipeline, with a stand-in quote service. +## Code coverage + +Contract coverage asks which documented outcomes were seen. Code coverage asks which of the +program's own source ran, and links that to the test that ran it. It is a separate thing with a +separate section on the test's page, because it means something different. + +The application collects it (Istanbul, or anything that writes LCOV) and hands it over as step +attachments, in JavaScript with `onAttachments`. Then +[`extensions/code-coverage`](https://github.com/ptescher/expectation/tree/main/extensions/code-coverage) +reads LCOV or Istanbul's `coverage-final.json` and writes `results/[@platform].code.tsv` +(`app.code.tsv` for the default runner): + +```bash +expectation-run expectations --docs docs --attachments captures +npx expectation-code-coverage --results docs/results/app.tsv --attachments captures \ + --revision "$(git rev-parse HEAD)" --scope "src/**, without tests" \ + --report-url "https://coverage.example.com/{run}/{slug}/" +expectation docs expectations docs +``` + +Each test's page then has a "Code coverage" section: the source revision and scope that were +instrumented, the lines, branches and functions hit at the start and after each step, what each +step added, and a link to a full HTML report that is hosted somewhere else (`{run}` and `{slug}` +are filled in for each test). Raw coverage files stay in the attachments directory, outside the +manual. + +What the section claims is limited on purpose: + +- **It is observed execution.** It never changes a test's result or an expectation's verdict, + and it doesn't say anything was asserted. A line that ran in a failed test ran. +- **Missing is not uncovered.** A step with no capture, or with one that can't be read, is shown + as that. Zero lines hit and "no data" are different. +- **A step's part is only claimed when it can be.** Counters count from when the app started, so + a step's part is the difference from the capture before. If a counter goes down, the app + restarted, and nothing is attributed to that step. If a step's capture is missing, the next + difference is labelled as since the earlier step it is compared with. Anything else that ran in the same + window, such as background work, is in the numbers, and the page says so. With + `--counters per-step` the app empties its counters each time it hands them over. +- **It counts for its own run and test.** Like contract evidence, a capture is shown only with + the run it came from and a test that hasn't changed since. + +Code coverage files are read the same way as the others: tab-separated, a few hundred characters +a field, 16 MiB at most, and the report link must be an `http` or `https` address. The format is +described at the top of +[`crates/story/src/codecov.rs`](https://github.com/ptescher/expectation/blob/main/crates/story/src/codecov.rs). +Expectation doesn't instrument the app: collecting coverage is the app's and the driver's job. + ## Writing an extension +Extensions in JavaScript can start from +[`extensions/kit`](https://github.com/ptescher/expectation/tree/main/extensions/kit): it reads the +results file, finds the attached files (refusing any path that leaves the attachments directory, +links included) and writes records that can't break the format. Expectation reads every +extension's file the same way, whatever kind it is: the file for another platform is left out, a +file over 16 MiB isn't read, fields are cut, and evidence counts only for its own run and test as +it was. + An extension reads: - the results file (`results/[@platform].tsv`), for the run id, each test's slug and diff --git a/crates/cli/src/lib.rs b/crates/cli/src/lib.rs index 7c2b86b..4932260 100644 --- a/crates/cli/src/lib.rs +++ b/crates/cli/src/lib.rs @@ -241,6 +241,7 @@ fn docs( .map_or_else(Default::default, expectation_docs::Screenshots::parse), reader, shared: None, + code: &[], }; if let Some(expectation_docs::Screenshots::Dir(d)) = shots.as_deref().map(expectation_docs::Screenshots::parse) @@ -264,6 +265,7 @@ fn docs( }; // Every platform's tests and results, then the pages: each one's dropdown knows the others' pages. let mut manuals = vec![]; + let mut codes = vec![]; for name in &names { let platform = name .as_deref() @@ -316,6 +318,18 @@ fn docs( expectation_story::Coverage::judge(contract, &c.plans, &report) }) .collect(); + // How much of the program's source ran, from files an extension wrote next to the results. + let (code, errors) = + expectation_story::load_code_coverage_for(&results_dir, name.as_deref()); + for e in errors { + eprintln!("warning: code coverage file {e}"); + } + for c in &code { + for p in &c.problems { + eprintln!("warning: {}: {p}", c.file); + } + } + codes.push(code); // Just some sections, for a preview: resolved in this platform's suite. let partial = (!sections.is_empty()).then(|| { let ids: Vec<_> = sections @@ -370,6 +384,7 @@ fn docs( partial: partial.as_ref(), look: expectation_docs::Look { shared: (i > 0 && names[i].is_some()).then_some((Path::new(out), 2)), + code: &codes[i], ..look }, }; @@ -535,7 +550,7 @@ fn prune_screenshots(store: &str, dirs: &[&str], dry_run: bool) { for entry in std::fs::read_dir(d).into_iter().flatten().flatten() { let path = entry.path(); if path.extension().is_none_or(|e| e != "tsv") - || expectation_story::is_coverage_file(&path) + || expectation_story::sidecar::is_sidecar(&path, None) { continue; } diff --git a/crates/docs/src/html.rs b/crates/docs/src/html.rs index b7fcdcc..6a4f212 100644 --- a/crates/docs/src/html.rs +++ b/crates/docs/src/html.rs @@ -14,12 +14,42 @@ use expectation_graph::Node; use expectation_model::text::paragraphs; use expectation_model::{ExpId, FeatureId, Keyword, PhraseBody, SectionId, StepText, Suite}; use expectation_story::{ - Attribution, Because, Check, CheckResult, Completeness, Credit, Origin, Source, State, Status, - StepResult, Verdict, + Attribution, Basis, Because, CaptureState, Check, CheckResult, Completeness, Credit, Origin, + Source, State, Status, StepResult, Verdict, }; use std::collections::HashMap; use std::fmt::Write; +/// "2 lines, 1 branch, 0 functions". +pub(crate) fn counted(l: u64, b: u64, f: u64) -> String { + let pl = |n: u64, one: &str, many: &str| format!("{n} {}", if n == 1 { one } else { many }); + format!( + "{}, {}, {}", + pl(l, "line", "lines"), + pl(b, "branch", "branches"), + pl(f, "function", "functions") + ) +} + +/// What a capture's "new" numbers say, in words. +fn new_since(c: &expectation_story::Capture) -> String { + let Some([l, b, f]) = c.new else { + return match c.basis { + Basis::Baseline => "the first capture: what ran before the test".into(), + Basis::Restart => "the program restarted, so nothing is attributed".into(), + _ => "can't be told".into(), + }; + }; + let what = counted(l, b, f); + match c.basis { + Basis::Step => what, + Basis::Since(from) => { + format!("{what}, since step {from} (the captures between are missing)") + } + _ => what, + } +} + pub fn esc(s: &str) -> String { s.replace('&', "&") .replace('<', "<") @@ -1859,9 +1889,86 @@ impl Site<'_> { o.push_str(&self.step_evidence(i, n + 1)); o.push_str(""); } + o.push_str(&self.code_coverage(i)); self.page(Here::Test(i), &plan.title, &o) } + /// Code coverage of the instrumented program while this test ran, apart from its result. + fn code_coverage(&self, plan: usize) -> String { + let mut o = String::new(); + for (n, (cov, caps)) in self.d.code_coverage_of(plan).into_iter().enumerate() { + let id = if n == 0 { + "code-coverage".to_string() + } else { + format!("code-coverage-{}", n + 1) + }; + let _ = write!( + o, + r#"

Code coverage

Which of the program's own source ran while this test did. It is observed execution: it doesn't change the result above, and it doesn't show that anything was checked. Counts are running totals over the test; "new" is what a step added since the capture before it, when that can be told, and can include activity the test didn't cause.

"# + ); + let mut about = vec![]; + for (label, v) in [ + ("Source", &cov.name), + ("Revision", &cov.revision), + ("Scope", &cov.scope), + ] { + if !v.is_empty() { + about.push(format!("{label}: {}", esc(v))); + } + } + if !about.is_empty() { + let _ = write!(o, "

{}

", about.join(" · ")); + } + o.push_str(""); + for c in &caps { + let at = if c.step == 0 { + r##"Start"##.to_string() + } else { + format!(r##"Step {0}"##, c.step) + }; + match c.state { + CaptureState::Captured => { + let n = |k: expectation_story::Count| format!("{} of {}", k.hit, k.found); + let _ = write!( + o, + "", + n(c.lines), + n(c.branches), + n(c.functions), + esc(&new_since(c)) + ); + } + other => { + let what = if other == CaptureState::Missing { + "No capture at this step" + } else { + "A capture that can't be used" + }; + let note = if c.note.is_empty() { + String::new() + } else { + format!(": {}", esc(&c.note)) + }; + let _ = write!( + o, + r#""# + ); + } + } + } + o.push_str("
CaptureLinesBranchesFunctionsNew since
{at}{}{}{}{}
{at}{what}{note}. That isn't the same as nothing being covered.
"); + if let Some(url) = caps.first().and_then(|c| cov.report_for(&c.plan, &c.run)) { + let _ = write!( + o, + r#"

The full coverage report, hosted separately.

"#, + esc(&url) + ); + } + o.push_str("
"); + } + o + } + /// The contract evidence that counts for a step: what it did, and the contract it was held against. fn step_evidence(&self, plan: usize, step: usize) -> String { let found = self.d.evidence_at(plan, step); diff --git a/crates/docs/src/lib.rs b/crates/docs/src/lib.rs index 9e94f92..f99340d 100644 --- a/crates/docs/src/lib.rs +++ b/crates/docs/src/lib.rs @@ -80,6 +80,9 @@ pub struct Look<'a> { /// directory that holds the shared `screenshots/`, and how many folders up it is from this /// manual. `None` is a manual that has its own. pub shared: Option<(&'a Path, usize)>, + /// Code coverage files (see [`expectation_story::CodeCoverage`]): shown on the test pages + /// as observed execution, apart from the results. + pub code: &'a [expectation_story::CodeCoverage], } /// Where the screenshots are that the results name as `screenshots/`. @@ -329,6 +332,26 @@ impl Docs<'_> { out } + /// The code coverage that counts for test `plan`: per file, its captures of this run of the + /// test as it is now. A file with none that count isn't listed. + pub fn code_coverage_of( + &self, + plan: usize, + ) -> Vec<( + &expectation_story::CodeCoverage, + Vec<&expectation_story::Capture>, + )> { + let Some(report) = self.report else { + return vec![]; + }; + self.look + .code + .iter() + .map(|c| (c, c.for_plan(self.plans, report, plan))) + .filter(|(_, found)| !found.is_empty()) + .collect() + } + /// Write the Markdown pages and the HTML site under `dir`. Pages of stories or features /// that no longer exist are removed (only the generated kinds, in the folders this writes /// to); other files in the directory are left alone. @@ -1481,4 +1504,101 @@ Feature: Paying assert!(!ground.contains("Shown for")); assert!(ground.contains("selected>desktop")); } + + #[test] + fn code_coverage_is_shown_as_observed_execution_for_the_run_and_test_it_came_from() { + let src = "Vocabulary:\n State: the lamp is on | off\n Event: the user flips the switch\n Event: the user presses reset\nBackground:\n Given the lamp is off\nFeature: Lamp\n Expectation: Flip\n Given the lamp is off\n When the user flips the switch\n Then the lamp is on\n Expectation: Reset\n Given the lamp is on\n When the user presses reset\n Then the lamp is off\nStory: Using the lamp\n As a user\n I want light\n So that I see\n Scenario: x\n When the user flips the switch\n Then the lamp is on\n When the user presses reset\n Then the lamp is off\n"; + let loaded = expectation_language::load_sources("S", &[("s.expect", src)]); + assert!(!loaded.has_errors(), "{}", loaded.render_diagnostics()); + let suite = loaded.suite; + let analysis = expectation_graph::analyze(&suite); + let plans = expectation_story::plan_stories(&suite, &analysis).plans; + let mut report = Report::not_run(&plans); + for (r, p) in report.results.iter_mut().zip(&plans) { + *r = expectation_story::PlanResult::with(Status::Passed, p); + r.run = Some(expectation_story::Run { + id: "r1".into(), + started_ms: 1, + }); + } + let (slug, print) = (&plans[0].slug, expectation_story::fingerprint(&plans[0])); + let file = |run: &str, print: &str| { + format!( + "expectation-code-coverage 1\nsource\tapp\tabc123\tsrc/**\thttps://cov.example/r\ncapture\t{run}\t{slug}\t{print}\t0\tcaptured\tbaseline\t-\t5\t10\t1\t4\t2\t3\t-\t-\t-\ncapture\t{run}\t{slug}\t{print}\t1\tcaptured\tstep\t-\t7\t10\t2\t4\t2\t3\t2\t1\t0\ncapture\t{run}\t{slug}\t{print}\t2\tmissing\tnone\tno snapshot\t0\t0\t0\t0\t0\t0\t-\t-\t-\n" + ) + }; + let pages = |code: &[expectation_story::CodeCoverage]| { + Docs { + suite: &suite, + analysis: &analysis, + plans: &plans, + report: Some(&report), + suite_dir: None, + siblings: &[], + outline: None, + inventory: &[], + contracts: &[], + partial: None, + look: Look { + code, + ..Look::default() + }, + } + .pages() + }; + let find = |pages: &[Page], path: &str| { + pages + .iter() + .find(|p| p.path == path) + .unwrap_or_else(|| panic!("{path}")) + .content + .clone() + }; + let good = [ + expectation_story::CodeCoverage::parse("app.code.tsv", &file("r1", &print)).unwrap(), + ]; + let shown = pages(&good); + let (html, md) = ( + find(&shown, &format!("site/test-{slug}.html")), + find(&shown, &format!("tests/{slug}.md")), + ); + for page in [&html, &md] { + assert!(page.contains("Code coverage"), "{page}"); + assert!(page.contains("observed execution")); + assert!(page.contains("abc123") && page.contains("src/")); + assert!(page.contains("7 of 10")); + assert!(page.contains("2 lines, 1 branch, 0 functions")); + assert!(page.contains("No capture at this step")); + assert!(page.contains("isn't the same as nothing being covered")); + assert!(page.contains("https://cov.example/r")); + } + // The result is what it was. + assert!(html.contains("passed")); + // Text from the file can't become markup, in either kind of page. + let hostile = file("r1", &print) + .replace( + "source\tapp\tabc123\tsrc/**", + "source\tapp\t\">\t[a](http://evil)", + ) + .replace("no snapshot", ""); + let bad = [expectation_story::CodeCoverage::parse("a.code.tsv", &hostile).unwrap()]; + let shown = pages(&bad); + let html = find(&shown, &format!("site/test-{slug}.html")); + let md = find(&shown, &format!("tests/{slug}.md")); + for page in [&html, &md] { + assert!( + !page.contains("