From c00a7abd2e183dfa61fc25a654ab1428599c24ee Mon Sep 17 00:00:00 2001 From: BootIntel Agent Date: Sat, 26 Sep 2026 12:29:44 +0000 Subject: [PATCH] Bring the detector set to parity with the browser library MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The browser detector library at bootintel.com/tools/fingerprint is the source of truth for the detector set — it is what the web tool and the legacy Node analyzer both run, and cli/sync-detectors.mjs already keeps those two in step. This crate shipped 9 of its 14 detectors and folded two of the rest into the wrong label, so the same log produced different JSON depending on which implementation read it. Adds Runtime firmware (OpenSBI), ROM identifier (ESP-ROM), Firmware SDK (ESP-IDF), Userland (BusyBox) and Flash layout (MTD partition map), in the browser's registration order, and moves OpenSBI out of Bootloader — which is an output-schema change for anyone parsing findings by label. On a RISC-V log that prints both, U-Boot won the old precedence chain and the OpenSBI version disappeared from CLI output entirely. Also aligns the machinery the two new detectors depend on: * source/line_number are derived by re-running the detector per line and matching value+detail, the browser's algorithm. The old substring search could attach evidence for a match that exists on no single line, and an aggregate finding (Flash layout counts partitions across the whole table) correctly gets no evidence fields at all. * the two policy detectors read the original lines, not the normalized ones, as the browser does. * partition offsets are de-duplicated: bootintel-9.txt prints its table twice and reported 14 partitions for a 7-partition device. Proven rather than asserted: tests/browser_parity.rs runs the browser library over all 31 corpus logs (twice each — as captured, and behind a terminal timestamp) and diffs {label, value, detail} against this crate's, log for log. It found four further divergences that are fixed here: a Kernel detail stitched across line breaks, an arch claimed from a device-family string, a PID-1 claim made from a BusyBox banner, an init claim made from a distro greeting — plus a partition size that rounded 1.25 MiB down where JavaScript's toFixed rounds it up. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 113 +++++++++ CONTRIBUTING.md | 11 +- Cargo.toml | 2 +- README.md | 6 +- crates/cli/src/analyze/pacer.rs | 2 +- crates/cli/src/cmd/detectors.rs | 17 ++ crates/cli/tests/browser_detectors.mjs | 67 +++++ crates/cli/tests/browser_parity.rs | 249 ++++++++++++++++++ crates/cli/tests/corpus_smoke.rs | 106 +++++++- crates/detectors/README.md | 6 +- crates/detectors/src/lib.rs | 310 ++++++++++++++++++----- crates/detectors/tests/detector_tests.rs | 204 +++++++++++++-- 12 files changed, 998 insertions(+), 95 deletions(-) create mode 100644 crates/cli/tests/browser_detectors.mjs create mode 100644 crates/cli/tests/browser_parity.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index f1a2e84..6d8d391 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,119 @@ All notable changes to bootintel-cli are documented here. Format follows [Keep a ## [Unreleased] +Detector-set parity with the browser detector library at +bootintel.com/tools/fingerprint, which is the source of truth for the +set: it is what the web tool and the legacy Node analyzer both run. This +crate shipped 9 of its 14 detectors and folded two others into the wrong +label, so the same log produced different JSON depending on which +implementation read it. + +**This is an output-schema change, which the policy at the top of this +file calls MAJOR.** One finding moves to a different label, one moves out +of a label it never belonged in, and five new labels appear (all three +below), so anything parsing `.findings[]` by label needs a look before +upgrading. The version is deliberately not bumped here — +releasing is handled separately, and the 0.4.0 entry records how this +project numbers a breaking change while pre-1.0. + +### Added +- Five detectors, bringing the set to 14 and `bootintel version` to + `client-side detectors: 14`: + - **Runtime firmware** — OpenSBI, the RISC-V M-mode runtime. + - **ROM identifier** — the Espressif mask-ROM build stamp + (`ESP-ROM:esp32-20160718`), which is what a ROM-level exploit is + written against. + - **Firmware SDK** — ESP-IDF version off the 2nd-stage bootloader + banner. On an ESP target this is the CVE-relevant identifier; the ROM + stamp rarely moves, the SDK does. + - **Userland** — BusyBox version. It is printed by 7 of the 31 public + corpus logs and this crate reported it zero times, so the offline + story was missing the most common userland component in the category. + - **Flash layout** — the MTD partition map (15 of the 31 corpus logs + register one), as `N partitions` plus the named list with sizes. This + is what a flash-clip read needs in order to know where to read. +- `cargo test -p bootintel --test browser_parity` — runs the browser + library over all 31 corpus logs (transpiled in-process and evaluated in + a `vm`, the same way the website repo's own parity test does) and + asserts the `{label, value, detail}` sequence matches this crate's, log + for log. Each log is compared twice, as captured and with + `[12:34:56.123] ` in front of every line, because line-prefix + normalization is the one place the two could agree on a bare log and + still disagree on a real capture. It skips loudly when `node` or the website checkout is absent, + since the crate is published and `cargo test` has to pass for someone + who only has this repo; `BOOTINTEL_REQUIRE_PARITY=1` turns a skip into + a failure and `BOOTINTEL_BROWSER_DETECTORS` points at `detectors.ts`. +- Corpus coverage guards: BusyBox must fire on every corpus log that + prints its banner (7), the partition map on every log that registers + one (15), and no log may report the same partition twice. + +### Changed +- **`OpenSBI x.y` moves from `Bootloader` to `Runtime firmware`.** It is + the M-mode runtime that hands off to U-Boot, not a bootloader, and the + browser has always reported it under its own label. Folding it into + `Bootloader` also *hid* it: on a RISC-V board that prints both (sample + `bootintel-6.txt`), U-Boot won the precedence chain and the OpenSBI + version disappeared from CLI output entirely. It is now reported + alongside the bootloader. +- **An ESP log gains `ROM identifier` and `Firmware SDK` findings.** The + `Bootloader: Espressif ROM bootloader` finding is unchanged — the ROM + build stamp and the SDK version are additional labels, not a rename. +- Consumers parsing JSON by label: the two bullets above are the whole + behaviour change. `Bootloader` for U-Boot / coreboot / GRUB / ESP, + `Kernel`, `CPU / Arch`, `Init system`, `Device family`, `Network`, + `Web admin`, `Telnet exposure` and `Autoboot interruptable` keep their + labels, values and relative order; the five new labels are interleaved + in the browser's registration order (`Bootloader`, `Runtime firmware`, + `ROM identifier`, `Firmware SDK`, `Kernel`, `CPU / Arch`, `Userland`, + `Flash layout`, `Init system`, …) because order is part of what makes + the two implementations comparable. + +### Fixed +- Four divergences from the browser found by the new parity test, each of + which made the CLI assert something the web tool did not: + - **Garbled `Kernel` detail.** The build-metadata group was mandatory + and `\s`-separated, so on a kernel banner that prints no toolchain + parentheses the match ran across line breaks and swept up unrelated + text: `bootintel-6.txt` reported a detail of + `riscv64-unk2OF: fdt: Ignoring memory range 0x40000000 …` and + `bootintel-17.txt` one of + `gcc version 4.6.3 (Ti2CPU: ARMv7 Processor [412fc09a] revision 10 …`, + each stitched out of two different lines. The group is now optional + and line-bounded, so a banner with no toolchain parentheses yields a + version and no detail. No log loses its `Kernel` finding. + - **`CPU / Arch: Xtensa (ESP)` claimed from `esp32` / `esp8266`.** + Those name a device family, not a CPU, and are reported as such by + `Device family`. Only the literal arch name counts now. + - **`Init system: BusyBox init`.** A BusyBox banner says what the + userland is, not what PID 1 is. The claim is replaced by the new + `Userland` finding — `bootintel-17.txt` reported + `Init system: BusyBox init` and now reports + `Userland: BusyBox 1.20.2`. + - **`Init system: systemd` inferred from a `Welcome to … Linux` + greeting.** A distro greeting is not evidence of PID 1; + `systemd[1]:` is. +- `source` and `line_number` are now derived exactly the way the browser + derives them — the detector is re-run against each line on its own and + the first line reproducing the same `value` + `detail` is the evidence. + The old substring search over the normalized lines could attach a + `source` for a match that exists on no single line. An aggregate + finding (`Flash layout`, whose value counts partitions across the whole + table) correctly carries neither field. +- The two policy detectors (`Telnet exposure`, `Autoboot interruptable`) + read the original lines rather than the normalized ones, as the browser + does. Stripping timestamps and ANSI is a presentation choice for the + inventory detectors; a policy rule should see what the capture + contained. +- Partition offsets are de-duplicated, so a capture that prints the table + twice (a reset loop, or two flash devices registering) does not inflate + the count — `bootintel-9.txt` reports its real 7 partitions rather + than 14. +- Partition sizes round the way JavaScript's `toFixed` does (ties away + from zero) rather than the way Rust's formatter does (ties to even). + Exact ties are common in a partition table: the 1280 KiB `kernel` + region in `bootintel-12.txt` is `1.3M` in the browser and was `1.2M` + here. + ## [0.4.2] — 2026-09-26 — published as `bootintel` ### Changed diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 626b462..b367d13 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -119,13 +119,14 @@ Component names loosely mirror the tree — `cli`, `detectors`, `tui`, `api`, `d ## Adding a detector -New detectors live in `crates/detectors/src/lib.rs`. +New detectors live in `crates/detectors/src/lib.rs`, but the browser detector library at bootintel.com/tools/fingerprint is the source of truth for the set — it is what the web tool and the legacy Node analyzer run. Add the detector there first, then port it here. 1. Add the regex as a `static LazyLock`. -2. Add a `fn detect_thing(&str) -> Option` that runs the regex and returns a `Finding`. -3. Wire it into `analyze()` alongside the existing detectors. -4. Add unit tests: at least one positive case and one negative-case log excerpt that shouldn't match. -5. If the detector should sync with the browser detector library at bootintel.com/tools/fingerprint, coordinate with a maintainer — the browser + CLI detector sets should stay label-aligned. +2. Add a `fn run_thing(&str) -> Option` that runs the regex and returns a `Finding`. Do not set `source` from anything but the matched text, and never set `line_number` — `analyze()` derives both by re-running the detector per line, so a hand-set `source` breaks the original-evidence contract. +3. Wire it into `ALL_DETECTORS` at the same position its browser counterpart occupies. Order and labels are part of the output contract. +4. Add a description to `cmd::detectors::DESCRIPTIONS` (a missing one fails a test) and update `detector_labels_stable`. +5. Add unit tests: at least one positive case and one negative-case log excerpt that shouldn't match. +6. Run `cargo test -p bootintel --test browser_parity` against a checkout of the website repo (`BOOTINTEL_BROWSER_DETECTORS=/path/to/frontend/src/lib/detectors.ts`, `BOOTINTEL_REQUIRE_PARITY=1` so it cannot silently skip). It runs the browser library over all 31 corpus logs and diffs the findings against this crate's. ## Adding a subcommand diff --git a/Cargo.toml b/Cargo.toml index 07763ee..e485b34 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ # `lz-str` (for /decode-compatible share URLs). # # Ships `bootintel scan / share / ports / version / term / analyze` -# with the 9-detector streaming subset + json/text/sarif/junit output. +# with the 14-detector set + json/text/sarif/junit output. # Client-side identification is offline + free; full CVE matching runs # server-side via bootintel.com's API (opt-in via --api). diff --git a/README.md b/README.md index faaca2a..b9ff8a5 100644 --- a/README.md +++ b/README.md @@ -307,7 +307,7 @@ Distribution scope deliberately excludes: no auto-update (attack surface), no te Cargo.toml # workspace manifest crates/ ├── detectors/ # pure client-side detector library -│ ├── src/lib.rs # 9 detectors +│ ├── src/lib.rs # 14 detectors │ └── tests/detector_tests.rs # unit tests └── cli/ # the `bootintel` binary ├── src/ @@ -392,6 +392,8 @@ Two fields were added and no existing key changed name or meaning: | `analysis_status` | envelope | `matched` or `unrecognized` — pairs with exit codes 0 / 3 | | `line_number` | each finding | 1-based line of `source` in the analyzed log; omitted when unknown | +Both are **derived** from the log, not set by the detector: the detector is re-run against each line on its own and the first line that reproduces the same `value` + `detail` is the evidence. An aggregate finding whose value spans many lines — `Flash layout` counts partitions across the whole MTD table — therefore carries neither field, because no single line is its source. + `source` now carries the **original** log line, prefix and all, rather than only the substring the detector regex matched. Prefixed captures (`[12:34:56.789] `, ISO-8601 timestamps, ANSI colour) are normalized @@ -404,7 +406,7 @@ SARIF v2.1.0 and JUnit XML outputs conform to their respective specs and validat ## Detector sync discipline -Nine detectors live at `crates/detectors/src/lib.rs` and mirror the browser detector library at bootintel.com/tools/fingerprint. When adding a detector, keep the label set aligned between the two so JSON output round-trips cleanly. +Fourteen detectors live at `crates/detectors/src/lib.rs` and mirror the browser detector library at bootintel.com/tools/fingerprint, which is the source of truth for the set — labels *and* registration order, so JSON output is comparable between the two. `cargo test -p bootintel --test browser_parity` proves it: it runs the browser library over all 31 corpus logs and asserts the `{label, value, detail}` sequence matches this crate's, log for log. When adding a detector, add it to the browser library first. ## Contributing diff --git a/crates/cli/src/analyze/pacer.rs b/crates/cli/src/analyze/pacer.rs index c6a62ce..1d6d0e1 100644 --- a/crates/cli/src/analyze/pacer.rs +++ b/crates/cli/src/analyze/pacer.rs @@ -6,7 +6,7 @@ //! CPU on high-baud lines; running on a plain timer would delay //! findings on quiet lines that emit one banner and then wait. //! -//! The full re-run of all 9 detectors against a 40 KiB log takes +//! The full re-run of all 14 detectors against a 40 KiB log takes //! <5ms on modern hardware, so the pacer's job is really about //! amortizing display flicker + not blocking the serial reader, //! not about compute cost. diff --git a/crates/cli/src/cmd/detectors.rs b/crates/cli/src/cmd/detectors.rs index 8eb620f..7f5f3e4 100644 --- a/crates/cli/src/cmd/detectors.rs +++ b/crates/cli/src/cmd/detectors.rs @@ -23,11 +23,28 @@ pub struct Args { /// detector without a description here fails the build. const DESCRIPTIONS: &[(&str, &str)] = &[ ("Bootloader", "U-Boot / Barebox / CFE + build timestamp"), + ( + "Runtime firmware", + "OpenSBI — the RISC-V M-mode runtime below the bootloader", + ), + ("ROM identifier", "Mask-ROM build stamp (ESP-ROM)"), + ( + "Firmware SDK", + "Vendor SDK the image was built against (ESP-IDF)", + ), ("Kernel", "Linux version + build metadata"), ( "CPU / Arch", "ARMv7 / ARM64 / MIPS / RISC-V / x86_64 family", ), + ( + "Userland", + "BusyBox version — what the userland is built on", + ), + ( + "Flash layout", + "MTD partition map — offsets, names, sizes for a flash read", + ), ( "Init system", "systemd / procd / OpenRC / SysV — what PID 1 is", diff --git a/crates/cli/tests/browser_detectors.mjs b/crates/cli/tests/browser_detectors.mjs new file mode 100644 index 0000000..8eab4f5 --- /dev/null +++ b/crates/cli/tests/browser_detectors.mjs @@ -0,0 +1,67 @@ +// Browser-parity harness: run the browser detector library (the source of +// truth) over a directory of boot logs and print its findings as JSON. +// +// Usage: node browser_detectors.mjs +// +// Output: { "": { "raw": [ { label, value, detail }, … ], "prefixed": [ … ] }, … } +// +// The `prefixed` variant is every line behind `[12:34:56.123] `. It is not +// padding: line-prefix normalization is the one place the two +// implementations could agree on a bare log and still disagree on a real +// capture, and the browser exempts two policy detectors from it. +// +// `detectors.ts` is TypeScript living in the website repo, so it is +// transpiled in-process with the `typescript` package resolved from that +// repo's own node_modules (found by walking up from the file) and evaluated +// in a fresh vm context — the same trick cli/test-parity.mjs uses, so both +// parity harnesses agree on what "the browser implementation" means. +import fs from 'node:fs' +import path from 'node:path' +import vm from 'node:vm' +import { createRequire } from 'node:module' + +const [detectorsPath, logsDir] = process.argv.slice(2) +if (!detectorsPath || !logsDir) { + console.error('usage: browser_detectors.mjs ') + process.exit(2) +} + +function findTypescript(from) { + let dir = path.resolve(path.dirname(from)) + for (;;) { + const candidate = path.join(dir, 'node_modules', 'typescript', 'package.json') + if (fs.existsSync(candidate)) return createRequire(candidate)('typescript') + const parent = path.dirname(dir) + if (parent === dir) break + dir = parent + } + // Last resort: whatever `typescript` resolves to for this script. + return createRequire(import.meta.url)('typescript') +} + +const ts = findTypescript(detectorsPath) +const source = fs.readFileSync(detectorsPath, 'utf8') +const context = { exports: {} } +vm.runInNewContext( + ts.transpileModule(source, { + compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2022 }, + }).outputText, + context, +) + +const rows = (log) => + context.exports.analyze(log).map((f) => ({ + label: f.label, + value: f.value, + detail: f.detail === undefined ? null : f.detail, + })) + +const out = {} +for (const file of fs.readdirSync(logsDir).filter((f) => f.endsWith('.txt')).sort()) { + const log = fs.readFileSync(path.join(logsDir, file), 'utf8') + out[file] = { + raw: rows(log), + prefixed: rows(log.split(/\r\n|\n|\r/).map((line) => `[12:34:56.123] ${line}`).join('\n')), + } +} +process.stdout.write(JSON.stringify(out, null, 2) + '\n') diff --git a/crates/cli/tests/browser_parity.rs b/crates/cli/tests/browser_parity.rs new file mode 100644 index 0000000..0a751b8 --- /dev/null +++ b/crates/cli/tests/browser_parity.rs @@ -0,0 +1,249 @@ +//! Corpus parity against the browser detector library. +//! +//! The browser library (`frontend/src/lib/detectors.ts` in the website +//! repo) is the source of truth for the detector set: it is what the web +//! tool and the legacy Node analyzer both run, and JSON output is meant +//! to be comparable between implementations. This test proves the Rust +//! crate agrees with it on the whole public corpus rather than trusting +//! that two hand-maintained copies of 14 regex sets stayed in step. +//! +//! How it runs the browser side: `tests/browser_detectors.mjs` transpiles +//! the TypeScript in-process and evaluates it in a `vm` context, the same +//! way the website repo's own `cli/test-parity.mjs` does, then prints +//! every finding as JSON. No network, no build step. +//! +//! Skips (loudly) when the pieces aren't here — `node` missing, or the +//! website checkout not on this machine — because the crate is published +//! and `cargo test` has to pass for someone who only has this repo. Set +//! `BOOTINTEL_REQUIRE_PARITY=1` to turn those skips into failures (CI on +//! a machine that does have both). Point `BOOTINTEL_BROWSER_DETECTORS` at +//! `detectors.ts` if your checkout lives somewhere unusual. + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use bootintel_detectors::analyze; + +/// One finding reduced to the fields both implementations promise to +/// agree on. `source` / `line_number` are derived from the log and are +/// asserted separately (`evidence_is_the_original_line`). +#[derive(Debug, PartialEq, Eq)] +struct Row { + label: String, + value: String, + detail: Option, +} + +fn repo_root() -> Option { + // crates/cli → ../.. + let manifest = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + Some(manifest.parent()?.parent()?.to_path_buf()) +} + +fn samples_dir() -> Option { + let dir = repo_root()?.join("samples"); + dir.is_dir().then_some(dir) +} + +/// Locate the browser detector library: explicit env var first, then the +/// usual places a website checkout sits relative to this repo. +fn browser_detectors() -> Option { + if let Some(p) = std::env::var_os("BOOTINTEL_BROWSER_DETECTORS") { + let p = PathBuf::from(p); + return p.is_file().then_some(p); + } + let rel = Path::new("frontend/src/lib/detectors.ts"); + let mut candidates = vec![PathBuf::from("/opt/bootintel.com").join(rel)]; + if let Some(root) = repo_root() { + if let Some(parent) = root.parent() { + candidates.push(parent.join("bootintel.com").join(rel)); + candidates.push(parent.join("bootintel").join(rel)); + } + } + candidates.into_iter().find(|p| p.is_file()) +} + +fn have_node() -> bool { + Command::new("node") + .arg("--version") + .output() + .is_ok_and(|o| o.status.success()) +} + +/// `[skip]` unless the caller demanded the real thing. +fn skip(why: &str) { + assert!( + std::env::var_os("BOOTINTEL_REQUIRE_PARITY").is_none(), + "BOOTINTEL_REQUIRE_PARITY is set but {why}" + ); + eprintln!("[skip] browser parity: {why}"); +} + +/// Split a log the way `analyze()` does — on CRLF, LF **or** bare CR. +fn split_lines(log: &str) -> Vec<&str> { + let mut out = Vec::new(); + let mut rest = log; + while let Some(i) = rest.find(['\n', '\r']) { + out.push(&rest[..i]); + let skip = if rest[i..].starts_with("\r\n") { 2 } else { 1 }; + rest = &rest[i + skip..]; + } + out.push(rest); + out +} + +/// Both shapes of every corpus log: as captured, and with a terminal +/// timestamp in front of every line. +#[derive(Debug, PartialEq, Eq)] +struct Variants { + raw: Vec, + prefixed: Vec, +} + +/// Run the browser library over `samples/` and return its findings. +fn browser_findings() -> Option> { + let Some(samples) = samples_dir() else { + skip("samples/ is not in this checkout"); + return None; + }; + let Some(detectors) = browser_detectors() else { + skip("detectors.ts not found — set BOOTINTEL_BROWSER_DETECTORS to the website checkout"); + return None; + }; + if !have_node() { + skip("node is not on PATH"); + return None; + } + let harness = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests") + .join("browser_detectors.mjs"); + let out = Command::new("node") + .arg(&harness) + .arg(&detectors) + .arg(&samples) + .output() + .expect("failed to spawn node"); + assert!( + out.status.success(), + "browser harness failed ({}):\n{}", + out.status, + String::from_utf8_lossy(&out.stderr) + ); + let parsed: serde_json::Value = + serde_json::from_slice(&out.stdout).expect("browser harness did not print JSON"); + let rows = |v: &serde_json::Value| -> Vec { + v.as_array() + .expect("each variant maps to an array of findings") + .iter() + .map(|r| Row { + label: r["label"].as_str().expect("label is a string").to_string(), + value: r["value"].as_str().expect("value is a string").to_string(), + detail: r["detail"].as_str().map(str::to_string), + }) + .collect() + }; + let mut findings = BTreeMap::new(); + for (file, variants) in parsed.as_object().expect("harness must print an object") { + findings.insert( + file.clone(), + Variants { + raw: rows(&variants["raw"]), + prefixed: rows(&variants["prefixed"]), + }, + ); + } + Some(findings) +} + +fn rust_findings(log: &str) -> Vec { + analyze(log) + .into_iter() + .map(|f| Row { + label: f.label, + value: f.value, + detail: f.detail, + }) + .collect() +} + +#[test] +fn every_corpus_log_matches_the_browser_implementation() { + let Some(expected) = browser_findings() else { + return; + }; + let samples = samples_dir().expect("samples/ exists if the harness ran"); + assert!( + expected.len() >= 31, + "expected the full public corpus, saw {} logs", + expected.len() + ); + for (file, want) in &expected { + let log = std::fs::read_to_string(samples.join(file)).expect("corpus log is readable"); + assert_eq!( + rust_findings(&log), + want.raw, + "{file}: CLI findings differ from the browser implementation" + ); + // Same log behind a terminal timestamp. Normalization is where the + // two implementations could agree on a bare log and still disagree + // on a real capture — and the browser deliberately exempts the two + // policy detectors from it, so that exemption is under test here. + let prefixed = split_lines(&log) + .iter() + .map(|l| format!("[12:34:56.123] {l}")) + .collect::>() + .join("\n"); + assert_eq!( + rust_findings(&prefixed), + want.prefixed, + "{file}: CLI findings differ from the browser implementation on a timestamped capture" + ); + } +} + +/// The evidence contract: whatever `source` a finding carries has to be +/// the original input line at `line_number`, verbatim — prefix, ANSI and +/// all. `cli/test-parity.mjs` asserts the same thing on the browser side. +#[test] +fn evidence_is_the_original_line() { + let Some(samples) = samples_dir() else { + eprintln!("[skip] samples/ is not in this checkout"); + return; + }; + let mut checked = 0; + for entry in std::fs::read_dir(&samples).expect("samples/ is readable") { + let path = entry.expect("readable dir entry").path(); + if path.extension().is_none_or(|e| e != "txt") { + continue; + } + let log = std::fs::read_to_string(&path).expect("corpus log is readable"); + let lines = split_lines(&log); + for f in analyze(&log) { + let (Some(source), Some(n)) = (f.source.as_deref(), f.line_number) else { + // An aggregate finding (Flash layout) has no single + // originating line, and that is the contract. + assert_eq!( + f.line_number, + None, + "{}: {} has a line number but no source", + path.display(), + f.label + ); + continue; + }; + assert_eq!( + lines[n - 1], + source, + "{}: {} points at line {n} but that is not what the log says", + path.display(), + f.label + ); + checked += 1; + } + } + assert!( + checked > 50, + "expected real evidence to check, saw {checked}" + ); +} diff --git a/crates/cli/tests/corpus_smoke.rs b/crates/cli/tests/corpus_smoke.rs index a29d7e1..8a97040 100644 --- a/crates/cli/tests/corpus_smoke.rs +++ b/crates/cli/tests/corpus_smoke.rs @@ -98,9 +98,11 @@ fn ti_am62a7_hits_uboot_and_arm64() { #[test] fn riscv_vf2_hits_opensbi_and_riscv() { expect_label("bootintel-6.txt", "Bootloader", "U-Boot"); - // OpenSBI is the M-mode runtime; U-Boot proper is what shows up - // last as the "Bootloader" finding (highest-precedence bootloader - // regex hit). The RISC-V arch tag is what confirms the platform. + // OpenSBI is the M-mode runtime that hands off to U-Boot, so this + // log reports both, under separate labels. OpenSBI used to be + // reported as the "Bootloader" — and on this log it lost the + // precedence race against U-Boot and vanished entirely. + expect_label("bootintel-6.txt", "Runtime firmware", "OpenSBI 1.0"); expect_label("bootintel-6.txt", "CPU / Arch", "RISC-V"); } @@ -143,3 +145,101 @@ fn nonsense_log_produces_no_findings() { let findings = analyze("random\ntext\nwith\nno\nboot\ncontent"); assert!(findings.is_empty()); } + +// ── Corpus coverage guards for the aggregate detectors ─────────────── +// +// Mirrors the website repo's `cli/test-parity.mjs` coverage block. A +// count is not the assertion — the assertion is that every log which +// *prints* the evidence produces the finding, so a regex that quietly +// stops matching one vendor's banner shape fails here instead of +// silently shrinking the offline story. + +fn corpus_logs() -> Vec<(String, String)> { + let Some(dir) = samples_dir() else { + return Vec::new(); + }; + let mut out = Vec::new(); + for entry in std::fs::read_dir(dir).expect("samples/ is readable") { + let path = entry.expect("readable dir entry").path(); + if path.extension().is_some_and(|e| e == "txt") { + let name = path.file_name().unwrap().to_string_lossy().into_owned(); + out.push((name, std::fs::read_to_string(&path).expect("readable log"))); + } + } + out.sort(); + out +} + +#[test] +fn busybox_fires_on_every_log_that_prints_its_banner() { + let logs = corpus_logs(); + if logs.is_empty() { + eprintln!("[skip] samples/ not in this checkout — test is a no-op"); + return; + } + let banner = regex::Regex::new(r"(?i)BusyBox\s+v?\d").unwrap(); + let mut hits = 0; + for (name, log) in &logs { + if !banner.is_match(log) { + continue; + } + hits += 1; + assert!( + analyze(log).iter().any(|f| f.label == "Userland"), + "{name} prints a BusyBox banner but Userland did not fire" + ); + } + assert_eq!( + hits, 7, + "7 of the 31 public corpus logs print a BusyBox banner" + ); +} + +#[test] +fn flash_layout_fires_on_every_log_with_a_partition_map() { + let logs = corpus_logs(); + if logs.is_empty() { + eprintln!("[skip] samples/ not in this checkout — test is a no-op"); + return; + } + let table = regex::Regex::new(r#"(?i)0x[0-9a-f]+-0x[0-9a-f]+\s*:\s*""#).unwrap(); + let mut hits = 0; + for (name, log) in &logs { + if !table.is_match(log) { + continue; + } + hits += 1; + assert!( + analyze(log).iter().any(|f| f.label == "Flash layout"), + "{name} has an MTD partition map but Flash layout did not fire" + ); + } + assert_eq!( + hits, 15, + "15 of the 31 public corpus logs register an MTD partition map" + ); +} + +#[test] +fn no_corpus_log_reports_a_duplicate_partition() { + let logs = corpus_logs(); + if logs.is_empty() { + eprintln!("[skip] samples/ not in this checkout — test is a no-op"); + return; + } + for (name, log) in &logs { + let Some(layout) = analyze(log).into_iter().find(|f| f.label == "Flash layout") else { + continue; + }; + let detail = layout.detail.unwrap_or_default(); + let parts: Vec<&str> = detail.split(", ").collect(); + let unique: std::collections::HashSet<&&str> = parts.iter().collect(); + assert_eq!( + unique.len(), + parts.len(), + "{name} lists a partition twice — a capture can print the table more than once \ + (reset loop, or two flash devices registering) and offsets must be de-duplicated: \ + {detail}" + ); + } +} diff --git a/crates/detectors/README.md b/crates/detectors/README.md index cec1713..2a74a21 100644 --- a/crates/detectors/README.md +++ b/crates/detectors/README.md @@ -2,7 +2,7 @@ Client-side boot-log detectors for BootIntel. Mirrors the streaming-identification subset of the browser detector library at bootintel.com/tools/fingerprint. -**Scope:** identification only — bootloader / kernel / CPU / init / device family / network / web admin / telnet / autoboot exposure. CVE matching, exploit paths, and PDF reports run server-side via BootIntel's API and are NOT in this crate. +**Scope:** identification only — bootloader / runtime firmware / ROM identifier / firmware SDK / kernel / CPU / userland / flash layout / init / device family / network / web admin / telnet / autoboot exposure. CVE matching, exploit paths, and PDF reports run server-side via BootIntel's API and are NOT in this crate. ## Usage @@ -17,9 +17,9 @@ for finding in analyze(&log) { ## Sync discipline -Nine detectors, each mirroring one entry in the upstream browser detector library. Keep label sets aligned when adding a new detector so JSON output round-trips between browser + CLI. +Fourteen detectors, each mirroring one entry in the upstream browser detector library, in the same order. That library is the source of truth; keep labels and order aligned when adding a detector so JSON output round-trips between browser + CLI. The `browser_parity` test in the CLI crate runs both implementations over the public corpus and asserts they agree. -Regex patterns are copied verbatim from the browser source where the flavor allows. All 9 use only features supported by both JavaScript regex and Rust `regex` (no lookaround, no backreferences). +Regex patterns are copied verbatim from the browser source where the flavor allows. All 14 use only features supported by both JavaScript regex and Rust `regex` (no lookaround, no backreferences). ## License diff --git a/crates/detectors/src/lib.rs b/crates/detectors/src/lib.rs index d99f234..42312c6 100644 --- a/crates/detectors/src/lib.rs +++ b/crates/detectors/src/lib.rs @@ -1,7 +1,7 @@ //! Client-side boot-log detectors for BootIntel. //! -//! Mirrors the 9-detector streaming subset of the browser detector -//! library at bootintel.com/tools/fingerprint. Kept pure (no I/O, no +//! Mirrors the 14-detector browser detector library at +//! bootintel.com/tools/fingerprint. Kept pure (no I/O, no //! async, no serde) so this crate can be reused in other contexts — //! a future WASM build for the browser tool, a plugin for third-party //! firmware-analysis tooling, etc. @@ -82,19 +82,38 @@ impl Finding { /// (unmodified) line, so `source` always shows the user what their /// capture actually contained, prefix and all. /// -/// Ported from the legacy Node analyzer's `analyze()` so both -/// implementations agree on what a "line" is and what gets stripped. +/// Ported from the browser `analyze()` so both implementations agree +/// on what a "line" is and what gets stripped. +/// +/// Two exceptions, mirroring the browser: the policy observations +/// (`Telnet exposure`, `Autoboot interruptable`) run against the +/// **original** lines. Normalizing is a presentation choice for the +/// inventory detectors; a policy rule should see exactly what the +/// capture contained. pub fn analyze(log: &str) -> Vec { let original = split_lines(log); - let normalized: Vec = original.iter().map(|l| normalize_line(l)).collect(); + let normalized_owned: Vec = original.iter().map(|l| normalize_line(l)).collect(); + let normalized: Vec<&str> = normalized_owned.iter().map(String::as_str).collect(); let norm_log = normalized.join("\n"); + let orig_log = original.join("\n"); ALL_DETECTORS .iter() - .filter_map(|d| (d.run)(&norm_log)) - .map(|f| attach_evidence(f, &original, &normalized)) + .filter_map(|d| { + let (joined, lines) = if ORIGINAL_INPUT_LABELS.contains(&d.label) { + (&orig_log, &original) + } else { + (&norm_log, &normalized) + }; + let finding = (d.run)(joined)?; + Some(attach_evidence(d, finding, &original, lines)) + }) .collect() } +/// Detectors that read the user's original lines instead of the +/// normalized ones. Same two labels the browser exempts. +const ORIGINAL_INPUT_LABELS: &[&str] = &["Telnet exposure", "Autoboot interruptable"]; + /// Split on any of CRLF / LF / CR. `str::lines()` only handles LF and /// CRLF; a bare-CR stream (some bootloaders emit CR-only line endings) /// would otherwise arrive as one giant line and defeat every @@ -169,22 +188,29 @@ fn normalize_line(line: &str) -> String { /// Point a finding's `source` at the original, unmodified line that /// produced it, and record that line's 1-based number. /// -/// Detectors return the matched substring as `source`, taken from the -/// normalized text. Locating it is a plain substring search over the -/// normalized lines — far cheaper than the Node analyzer's re-run of -/// every detector against every line, and exact for the same reason -/// (the needle came out of that text verbatim). -fn attach_evidence(mut f: Finding, original: &[&str], normalized: &[String]) -> Finding { - let Some(src) = f.source.clone() else { - return f; - }; - // A few regexes (`[^)]+`) can span a newline; anchor on the first - // physical line of the match. - let needle = src.split('\n').next().unwrap_or(&src); - if needle.is_empty() { - return f; - } - if let Some(i) = normalized.iter().position(|l| l.contains(needle)) { +/// `source` and `line_number` are **derived**, never set by the +/// detector: the detector is re-run against each line on its own and +/// the first line that reproduces the same `value` + `detail` is the +/// originating evidence. Byte-for-byte the browser's algorithm, which +/// matters for two reasons: +/// +/// * a detector that set its own `source` from a whole-log match +/// could point at text that is not on any single line, breaking +/// the "source is the original line" contract the CSV/JSON +/// consumers rely on; +/// * an **aggregate** detector (`Flash layout`, whose `value` counts +/// partitions across many lines) cannot be reproduced from one +/// line, so no line matches and the finding correctly ends up with +/// no `source` / `line_number` — the evidence is the whole table. +/// +/// When no line reproduces the finding, whatever `source` the detector +/// itself recorded is left in place (also the browser's behavior: it +/// spreads the located evidence over the finding only when found). +fn attach_evidence(d: &Detector, mut f: Finding, original: &[&str], lines: &[&str]) -> Finding { + if let Some(i) = lines + .iter() + .position(|line| (d.run)(line).is_some_and(|c| c.value == f.value && c.detail == f.detail)) + { f.source = Some(original[i].to_string()); f.line_number = Some(i + 1); } @@ -242,6 +268,18 @@ static ALL_DETECTORS: &[Detector] = &[ label: "Bootloader", run: run_bootloader, }, + Detector { + label: "Runtime firmware", + run: run_runtime_firmware, + }, + Detector { + label: "ROM identifier", + run: run_rom_identifier, + }, + Detector { + label: "Firmware SDK", + run: run_firmware_sdk, + }, Detector { label: "Kernel", run: run_kernel, @@ -250,6 +288,14 @@ static ALL_DETECTORS: &[Detector] = &[ label: "CPU / Arch", run: run_cpu_arch, }, + Detector { + label: "Userland", + run: run_userland, + }, + Detector { + label: "Flash layout", + run: run_flash_layout, + }, Detector { label: "Init system", run: run_init_system, @@ -286,8 +332,6 @@ static RE_COREBOOT: LazyLock = LazyLock::new(|| Regex::new(r"(?m)^coreboot-([\w.\-]+)").unwrap()); static RE_GRUB: LazyLock = LazyLock::new(|| Regex::new(r"(?m)^GRUB\s+(?:version\s+)?([\d.]+)").unwrap()); -static RE_OPENSBI: LazyLock = - LazyLock::new(|| Regex::new(r"(?i)OpenSBI\s+v?([\d.]+)").unwrap()); static RE_ESP_ROM: LazyLock = LazyLock::new(|| Regex::new(r"(?i)(rst:0x1\s+\(POWERON_RESET\)|esp_image:|chip is)").unwrap()); @@ -312,11 +356,6 @@ fn run_bootloader(log: &str) -> Option { let source = m.get(0)?.as_str(); return Some(Finding::new("Bootloader", format!("GRUB {ver}")).source(source)); } - if let Some(m) = RE_OPENSBI.captures(log) { - let ver = m.get(1)?.as_str(); - let source = m.get(0)?.as_str(); - return Some(Finding::new("Bootloader", format!("OpenSBI {ver}")).source(source)); - } if let Some(m) = RE_ESP_ROM.captures(log) { let source = m.get(0)?.as_str(); return Some( @@ -328,10 +367,66 @@ fn run_bootloader(log: &str) -> Option { None } -// ── 2. Kernel ──────────────────────────────────────────────────────── +// ── 2. Runtime firmware ────────────────────────────────────────────── +// +// OpenSBI is the RISC-V M-mode runtime that hands off to U-Boot, not a +// bootloader — a RISC-V board reports both, and folding OpenSBI into +// `Bootloader` (as this crate used to) both mislabeled it and hid +// whichever of the two lost the precedence race. + +static RE_OPENSBI: LazyLock = LazyLock::new(|| { + Regex::new(r"(?m)^\s*OpenSBI\s+v?(\d+(?:\.\d+)+(?:[-+][A-Za-z0-9_.+\-]+)?)").unwrap() +}); + +fn run_runtime_firmware(log: &str) -> Option { + let m = RE_OPENSBI.captures(log)?; + let ver = m.get(1)?.as_str(); + Some(Finding::new("Runtime firmware", format!("OpenSBI {ver}"))) +} -static RE_LINUX: LazyLock = - LazyLock::new(|| Regex::new(r"Linux version\s+(\S+)\s+\([^)]+\)\s+\(([^)]+)\)").unwrap()); +// ── 3. ROM identifier ──────────────────────────────────────────────── +// +// The mask-ROM build stamp an ESP prints before anything else +// (`ESP-ROM:esp32s3-20210327`). It identifies the silicon revision's +// ROM image, which is what a ROM-level exploit is written against — +// separate from the `Bootloader` finding the same log also produces. + +static RE_ESP_ROM_ID: LazyLock = + LazyLock::new(|| Regex::new(r"(?m)^\s*ESP-ROM:([A-Za-z0-9._+\-]+)").unwrap()); + +fn run_rom_identifier(log: &str) -> Option { + let m = RE_ESP_ROM_ID.captures(log)?; + let id = m.get(1)?.as_str(); + Some(Finding::new( + "ROM identifier", + format!("Espressif ROM {id}"), + )) +} + +// ── 4. Firmware SDK ────────────────────────────────────────────────── +// +// ESP-IDF version out of the 2nd-stage bootloader banner. The SDK +// version is the CVE-relevant identifier on an ESP target — the ROM +// stamp above rarely moves, the SDK does. + +static RE_ESP_IDF: LazyLock = LazyLock::new(|| { + Regex::new(r"\bboot: ESP-IDF\s+([A-Za-z0-9._+\-]+)\s+2nd stage bootloader\b").unwrap() +}); + +fn run_firmware_sdk(log: &str) -> Option { + let m = RE_ESP_IDF.captures(log)?; + let ver = m.get(1)?.as_str(); + Some(Finding::new("Firmware SDK", format!("ESP-IDF {ver}"))) +} + +// ── 5. Kernel ──────────────────────────────────────────────────────── + +// The build-metadata parentheses are optional: plenty of vendor kernels +// print `Linux version 3.0.15-ts-armv7l` and stop. `[ \t]` rather than +// `\s` so the match cannot run past the end of the banner line. +static RE_LINUX: LazyLock = LazyLock::new(|| { + Regex::new(r"Linux version[ \t]+(\S+)(?:[ \t]+\([^\r\n)]+\)[ \t]+\(([^\r\n)]+)\))?").unwrap() +}); static RE_DARWIN: LazyLock = LazyLock::new(|| Regex::new(r"Darwin Kernel Version\s+([^:]+):").unwrap()); static RE_FREERTOS: LazyLock = @@ -342,16 +437,14 @@ static RE_ZEPHYR: LazyLock = fn run_kernel(log: &str) -> Option { if let Some(m) = RE_LINUX.captures(log) { let ver = m.get(1)?.as_str(); - let toolchain = m.get(2)?.as_str(); let source_full = m.get(0)?.as_str(); - // Match the TS slicing: detail truncated at 80, source at 200. - let detail: String = toolchain.chars().take(80).collect(); + // Match the browser slicing: detail truncated at 80, source at 200. let source: String = source_full.chars().take(200).collect(); - return Some( - Finding::new("Kernel", format!("Linux {ver}")) - .detail(detail) - .source(source), - ); + let mut f = Finding::new("Kernel", format!("Linux {ver}")).source(source); + if let Some(toolchain) = m.get(2) { + f = f.detail(toolchain.as_str().chars().take(80).collect::()); + } + return Some(f); } if let Some(m) = RE_DARWIN.captures(log) { let ver = m.get(1)?.as_str().trim(); @@ -371,7 +464,7 @@ fn run_kernel(log: &str) -> Option { None } -// ── 3. CPU / Arch ──────────────────────────────────────────────────── +// ── 6. CPU / Arch ──────────────────────────────────────────────────── static RE_MIPS: LazyLock = LazyLock::new(|| { Regex::new(r"(?i)CPU\s*\d?\s+revision is:\s+\w+\s+\((MIPS\s+[\w\-]+)\)").unwrap() @@ -385,8 +478,10 @@ static RE_RISCV: LazyLock = LazyLock::new(|| Regex::new(r"(?i)Linux version.*riscv|hart\s+\d+:\s+running").unwrap()); static RE_X86: LazyLock = LazyLock::new(|| Regex::new(r"(?i)Linux version.*x86_64").unwrap()); -static RE_XTENSA: LazyLock = - LazyLock::new(|| Regex::new(r"(?i)(xtensa|esp32|esp8266)").unwrap()); +// Only the literal architecture name. `esp32` / `esp8266` are a device +// family, not a CPU, and are reported as such by `Device family` — +// matching them here made the CLI claim an arch the browser did not. +static RE_XTENSA: LazyLock = LazyLock::new(|| Regex::new(r"(?i)\bxtensa\b").unwrap()); fn run_cpu_arch(log: &str) -> Option { if let Some(m) = RE_MIPS.captures(log) { @@ -410,16 +505,120 @@ fn run_cpu_arch(log: &str) -> Option { None } -// ── 4. Init system ─────────────────────────────────────────────────── +// ── 7. Userland ────────────────────────────────────────────────────── +// +// BusyBox prints its version on 7 of the 31 public corpus captures and +// this crate reported it zero times, so the offline story was missing +// the most common userland component in the category. + +static RE_BUSYBOX: LazyLock = + LazyLock::new(|| Regex::new(r"(?i)BusyBox\s+v?(\d[\d.]*)").unwrap()); + +fn run_userland(log: &str) -> Option { + let m = RE_BUSYBOX.captures(log)?; + let ver = m.get(1)?.as_str(); + Some(Finding::new("Userland", format!("BusyBox {ver}"))) +} + +// ── 8. Flash layout ────────────────────────────────────────────────── +// +// The partition map is what a flash-clip read needs, and it appears in +// roughly half the corpus. Offsets come straight from the kernel's own +// MTD registration lines. +// +// The one **aggregate** detector: it folds many lines into a single +// finding whose `value` counts partitions, so no single line reproduces +// it and `attach_evidence` leaves it without a `source` / `line_number`. + +static RE_MTD_PART: LazyLock = LazyLock::new(|| { + // Character class spelled out rather than `\w` so it means the same + // set as the browser's `[\w/.-]` (ASCII) instead of Rust's + // Unicode-aware `\w`. + Regex::new(r#"(?i)0x0*([0-9a-f]+)-0x0*([0-9a-f]+)\s*:\s*"([A-Za-z0-9_/.\-]+)""#).unwrap() +}); + +fn run_flash_layout(log: &str) -> Option { + let mut seen: std::collections::HashSet = std::collections::HashSet::new(); + let mut named: Vec = Vec::new(); + for m in RE_MTD_PART.captures_iter(log) { + let (Some(start), Some(end), Some(name)) = (m.get(1), m.get(2), m.get(3)) else { + continue; + }; + // A capture can print the partition table more than once (a + // reset loop, or two flash devices registering). Key on the + // exact offsets so the same region is listed once rather than + // inflating the count — bootintel-9.txt reports 14 partitions + // without this for a device that has 7. + if !seen.insert(format!( + "{}-{}-{}", + start.as_str(), + end.as_str(), + name.as_str() + )) { + continue; + } + let size = hex_to_f64(end.as_str()) - hex_to_f64(start.as_str()); + named.push(format!("{} ({})", name.as_str(), human_size(size))); + } + if named.is_empty() { + return None; + } + let plural = if named.len() == 1 { "" } else { "s" }; + Some( + Finding::new("Flash layout", format!("{} partition{plural}", named.len())) + .detail(named.join(", ")), + ) +} + +/// `parseInt(hex, 16)` in f64, so a hostile line with a 40-digit offset +/// widens to infinity the way the browser does instead of overflowing. +fn hex_to_f64(hex: &str) -> f64 { + hex.chars().fold(0.0, |acc, c| { + acc * 16.0 + f64::from(c.to_digit(16).unwrap_or(0)) + }) +} + +/// Render a partition size the way the browser does: whole kibibytes, +/// or mebibytes with one decimal place when it isn't a whole MiB. +/// +/// The decimal is computed in integer tenths rather than with `{:.1}` +/// because the two languages break ties differently: JavaScript's +/// `toFixed` rounds a tie away from zero (1.25 → "1.3") while Rust's +/// formatter rounds to even (1.25 → "1.2"). Partition tables hit exact +/// ties routinely — any whole 1.25 MiB region does it, e.g. the 1280 KiB +/// `kernel` partition in sample bootintel-12.txt — so the naive version +/// diverges from the browser on real corpus logs. +fn human_size(size: f64) -> String { + let kb = (size / 1024.0).round(); + // `-0.0` would print as "-0"; the browser prints "0". + let kb = if kb == 0.0 { 0.0 } else { kb }; + if kb < 1024.0 { + return format!("{kb}K"); + } + let whole = kb as i128; + if whole % 1024 == 0 { + return format!("{}M", whole / 1024); + } + // round((kb / 1024) * 10) with ties away from zero, in integers. + // Falls back to the formatter only for absurd offsets (a hostile + // 40-hex-digit line) where the integer math would overflow. + match whole + .checked_mul(20) + .and_then(|v| v.checked_add(1024)) + .map(|v| v / 2048) + { + Some(tenths) => format!("{}.{}M", tenths / 10, tenths % 10), + None => format!("{:.1}M", kb / 1024.0), + } +} + +// ── 9. Init system ─────────────────────────────────────────────────── static RE_PROCD_START: LazyLock = LazyLock::new(|| Regex::new(r"(?m)^procd:").unwrap()); static RE_PROCD_INIT: LazyLock = LazyLock::new(|| Regex::new(r"procd:\s+-\s+init").unwrap()); -static RE_SYSTEMD: LazyLock = - LazyLock::new(|| Regex::new(r"(?m)systemd\[1\]:|Welcome to \w+ Linux").unwrap()); +static RE_SYSTEMD: LazyLock = LazyLock::new(|| Regex::new(r"(?m)systemd\[1\]:").unwrap()); static RE_SYSV: LazyLock = LazyLock::new(|| Regex::new(r"(?i)INIT:\s+version\s+([\d.]+)").unwrap()); -static RE_BUSYBOX_INIT: LazyLock = - LazyLock::new(|| Regex::new(r"BusyBox v[\d.]+\s+\([^)]+\)\s+built-in shell").unwrap()); static RE_RUNIT: LazyLock = LazyLock::new(|| Regex::new(r"runit:|runit-init").unwrap()); fn run_init_system(log: &str) -> Option { @@ -433,16 +632,13 @@ fn run_init_system(log: &str) -> Option { let ver = m.get(1)?.as_str(); return Some(Finding::new("Init system", format!("SysV init {ver}"))); } - if RE_BUSYBOX_INIT.is_match(log) { - return Some(Finding::new("Init system", "BusyBox init")); - } if RE_RUNIT.is_match(log) { return Some(Finding::new("Init system", "runit")); } None } -// ── 5. Device family ───────────────────────────────────────────────── +// ── 10. Device family ───────────────────────────────────────────────── static RE_OPENWRT: LazyLock = LazyLock::new(|| Regex::new(r"(?i)openwrt").unwrap()); static RE_OPENWRT_GCC: LazyLock = @@ -500,7 +696,7 @@ fn run_device_family(log: &str) -> Option { None } -// ── 6. Network ─────────────────────────────────────────────────────── +// ── 11. Network ─────────────────────────────────────────────────────── static RE_DHCP: LazyLock = LazyLock::new(|| Regex::new(r"(?i)DHCP\s+(?:client\s+)?bound to address\s+([\d.]+)").unwrap()); @@ -529,7 +725,7 @@ fn run_network(log: &str) -> Option { None } -// ── 7. Web admin ───────────────────────────────────────────────────── +// ── 12. Web admin ───────────────────────────────────────────────────── static RE_UHTTPD: LazyLock = LazyLock::new(|| Regex::new(r"(?i)uhttpd\[\d+\]:\s+Listening on\s+([\d.:]+)").unwrap()); @@ -560,7 +756,7 @@ fn run_web_admin(log: &str) -> Option { None } -// ── 8. Telnet exposure ─────────────────────────────────────────────── +// ── 13. Telnet exposure ─────────────────────────────────────────────── static RE_TELNET: LazyLock = LazyLock::new(|| { Regex::new(r"(?i)telnetd?\[\d+\]?[^\n]*?(?:listening|started|on\s+\d)").unwrap() @@ -577,7 +773,7 @@ fn run_telnet_exposure(log: &str) -> Option { None } -// ── 9. Autoboot interruptable ──────────────────────────────────────── +// ── 14. Autoboot interruptable ──────────────────────────────────────── static RE_AUTOBOOT_ANY: LazyLock = LazyLock::new(|| Regex::new(r"Hit any key to stop autoboot:\s*[1-9]").unwrap()); diff --git a/crates/detectors/tests/detector_tests.rs b/crates/detectors/tests/detector_tests.rs index 4b6a261..84860ae 100644 --- a/crates/detectors/tests/detector_tests.rs +++ b/crates/detectors/tests/detector_tests.rs @@ -63,13 +63,6 @@ fn bootloader_coreboot() { assert_eq!(f.value, "coreboot 4.15"); } -#[test] -fn bootloader_opensbi() { - let log = "OpenSBI v1.4\nBoot HART ID : 1\n"; - let f = find(log, "Bootloader").unwrap(); - assert_eq!(f.value, "OpenSBI 1.4"); -} - #[test] fn bootloader_esp_rom() { let log = "rst:0x1 (POWERON_RESET),boot:0x13 (SPI_FAST_FLASH_BOOT)\n"; @@ -84,7 +77,66 @@ fn bootloader_none_on_gibberish() { assert!(find(log, "Bootloader").is_none()); } -// ── 2. Kernel ──────────────────────────────────────────────────────── +// ── 2. Runtime firmware ────────────────────────────────────────────── + +#[test] +fn runtime_firmware_opensbi() { + let log = "OpenSBI v1.4\nBoot HART ID : 1\n"; + let f = find(log, "Runtime firmware").unwrap(); + assert_eq!(f.value, "OpenSBI 1.4"); + // OpenSBI is a runtime, not a bootloader. It used to be reported + // under `Bootloader`, which disagreed with the browser tool on the + // same input. + assert!(find(log, "Bootloader").is_none()); +} + +#[test] +fn runtime_firmware_and_bootloader_coexist() { + // A RISC-V board hands off OpenSBI → U-Boot → Linux, and reports + // both. Folding them into one detector hid whichever lost. + let log = "OpenSBI v1.0\nU-Boot 2021.10 (Nov 10 2022 - 13:29:36 +0800)\n"; + assert_eq!(find(log, "Runtime firmware").unwrap().value, "OpenSBI 1.0"); + assert_eq!(find(log, "Bootloader").unwrap().value, "U-Boot 2021.10"); +} + +#[test] +fn runtime_firmware_needs_a_dotted_version() { + // `\d+(?:\.\d+)+` — a bare major refuses rather than reporting + // "OpenSBI 1". + assert!(find("OpenSBI v1\n", "Runtime firmware").is_none()); +} + +// ── 3. ROM identifier ──────────────────────────────────────────────── + +#[test] +fn rom_identifier_esp_rom() { + let log = "ESP-ROM:esp32s3-20210327\nBuild:Mar 27 2021\n"; + let f = find(log, "ROM identifier").unwrap(); + assert_eq!(f.value, "Espressif ROM esp32s3-20210327"); +} + +#[test] +fn rom_identifier_absent_on_non_esp_log() { + assert!(find("U-Boot 2020.10 (Sep 17 2023)\n", "ROM identifier").is_none()); +} + +// ── 4. Firmware SDK ────────────────────────────────────────────────── + +#[test] +fn firmware_sdk_esp_idf() { + let log = "I (27) boot: ESP-IDF v5.1.2 2nd stage bootloader\n"; + let f = find(log, "Firmware SDK").unwrap(); + assert_eq!(f.value, "ESP-IDF v5.1.2"); +} + +#[test] +fn firmware_sdk_absent_without_the_bootloader_banner() { + // The version has to come off the 2nd-stage bootloader line; a + // passing mention of ESP-IDF is not a version claim. + assert!(find("built with ESP-IDF v5.1.2\n", "Firmware SDK").is_none()); +} + +// ── 5. Kernel ──────────────────────────────────────────────────────── #[test] fn kernel_linux() { @@ -94,6 +146,16 @@ fn kernel_linux() { assert!(f.detail.unwrap().starts_with("gcc-11.2.0")); } +#[test] +fn kernel_linux_without_build_metadata() { + // Plenty of vendor kernels print the version and stop. The build + // parentheses are optional, and `detail` is then absent. + let log = "Linux version 3.0.15-ts-armv7l\n"; + let f = find(log, "Kernel").unwrap(); + assert_eq!(f.value, "Linux 3.0.15-ts-armv7l"); + assert_eq!(f.detail, None); +} + #[test] fn kernel_freertos() { let log = "FreeRTOS Kernel V10.4.3 starting on core 0\n"; @@ -108,7 +170,7 @@ fn kernel_zephyr() { assert_eq!(f.value, "Zephyr v3.5.0-2547-g1234567"); } -// ── 3. CPU / Arch ──────────────────────────────────────────────────── +// ── 6. CPU / Arch ──────────────────────────────────────────────────── #[test] fn cpu_mips() { @@ -131,7 +193,104 @@ fn cpu_riscv_from_hart() { assert_eq!(f.value, "RISC-V"); } -// ── 4. Init system ─────────────────────────────────────────────────── +#[test] +fn cpu_xtensa_needs_the_arch_name() { + // `esp32` alone is a device family, not a CPU — it is reported by + // `Device family`. Claiming an arch from it disagreed with the + // browser tool. + assert_eq!( + find("xtensa: booting\n", "CPU / Arch").unwrap().value, + "Xtensa (ESP)" + ); + assert!(find("esp_image: segment 0\n", "CPU / Arch").is_none()); +} + +// ── 7. Userland ────────────────────────────────────────────────────── + +#[test] +fn userland_busybox() { + let log = "BusyBox v1.35.0 (2022-03-01) built-in shell (ash)\n"; + let f = find(log, "Userland").unwrap(); + assert_eq!(f.value, "BusyBox 1.35.0"); +} + +#[test] +fn userland_busybox_is_not_an_init_claim() { + // A BusyBox banner says what the userland is, not what PID 1 is. + // This crate used to report "BusyBox init" for it, which the + // browser tool never did. + let log = "BusyBox v1.35.0 (2022-03-01) built-in shell (ash)\n"; + assert!(find(log, "Init system").is_none()); +} + +#[test] +fn userland_absent_without_a_version() { + assert!(find("starting busybox applets\n", "Userland").is_none()); +} + +// ── 8. Flash layout ────────────────────────────────────────────────── + +const MTD_TABLE: &str = concat!( + "0x000000000000-0x000000020000 : \"u-boot\"\n", + "0x000000020000-0x000000030000 : \"u-boot-env\"\n", + "0x000000030000-0x000000200000 : \"kernel\"\n", +); + +#[test] +fn flash_layout_lists_partitions_with_sizes() { + let f = find(MTD_TABLE, "Flash layout").unwrap(); + assert_eq!(f.value, "3 partitions"); + assert_eq!( + f.detail.unwrap(), + "u-boot (128K), u-boot-env (64K), kernel (1.8M)" + ); +} + +#[test] +fn flash_layout_is_an_aggregate_so_it_has_no_single_line_evidence() { + // The value counts partitions across many lines, so no single line + // reproduces it and the evidence fields stay empty. Pointing at one + // arbitrary MTD line would be a lie about where the value came from. + let f = find(MTD_TABLE, "Flash layout").unwrap(); + assert_eq!(f.source, None); + assert_eq!(f.line_number, None); +} + +#[test] +fn flash_layout_deduplicates_a_repeated_table() { + // A reset loop (or two flash devices registering) prints the table + // twice. The same offsets must be listed once. + let log = format!("{MTD_TABLE}reset\n{MTD_TABLE}"); + let f = find(&log, "Flash layout").unwrap(); + assert_eq!(f.value, "3 partitions"); + let names: Vec<&str> = f.detail.as_deref().unwrap().split(", ").collect(); + let unique: std::collections::HashSet<&&str> = names.iter().collect(); + assert_eq!( + unique.len(), + names.len(), + "no partition may be listed twice" + ); +} + +#[test] +fn flash_layout_whole_megabytes_drop_the_decimal() { + let log = "0x000000000000-0x000000100000 : \"one\"\n0x000000100000-0x000000280000 : \"onepointfive\"\n"; + let f = find(log, "Flash layout").unwrap(); + assert_eq!(f.detail.unwrap(), "one (1M), onepointfive (1.5M)"); +} + +#[test] +fn flash_layout_singular_for_one_partition() { + let log = "0x000000000000-0x000000020000 : \"u-boot\"\n"; + assert_eq!(find(log, "Flash layout").unwrap().value, "1 partition"); +} + +#[test] +fn flash_layout_absent_without_a_partition_map() { + assert!(find("mtd: device 0 (boot)\n", "Flash layout").is_none()); +} + +// ── 9. Init system ─────────────────────────────────────────────────── #[test] fn init_procd() { @@ -148,14 +307,7 @@ fn init_systemd() { assert_eq!(f.value, "systemd"); } -#[test] -fn init_busybox() { - let log = "BusyBox v1.35.0 (2022-03-01) built-in shell (ash)\n"; - let f = find(log, "Init system").unwrap(); - assert_eq!(f.value, "BusyBox init"); -} - -// ── 5. Device family ───────────────────────────────────────────────── +// ── 10. Device family ───────────────────────────────────────────────── #[test] fn family_openwrt() { @@ -178,7 +330,7 @@ fn family_qualcomm() { assert_eq!(f.value, "Qualcomm IPQ"); } -// ── 6. Network ─────────────────────────────────────────────────────── +// ── 11. Network ─────────────────────────────────────────────────────── #[test] fn network_dhcp() { @@ -195,7 +347,7 @@ fn network_dnsmasq() { assert_eq!(f.value, "dnsmasq 2.86 active"); } -// ── 7. Web admin ───────────────────────────────────────────────────── +// ── 12. Web admin ───────────────────────────────────────────────────── #[test] fn web_uhttpd() { @@ -212,7 +364,7 @@ fn web_nginx() { assert_eq!(f.value, "nginx 1.20.1"); } -// ── 8. Telnet exposure ─────────────────────────────────────────────── +// ── 13. Telnet exposure ─────────────────────────────────────────────── #[test] fn telnet_started() { @@ -228,7 +380,7 @@ fn telnet_absent_on_clean_log() { assert!(find(log, "Telnet exposure").is_none()); } -// ── 9. Autoboot interruptable ──────────────────────────────────────── +// ── 14. Autoboot interruptable ──────────────────────────────────────── #[test] fn autoboot_3_second_countdown() { @@ -256,15 +408,21 @@ fn detector_labels_stable() { labels, vec![ "Bootloader", + "Runtime firmware", + "ROM identifier", + "Firmware SDK", "Kernel", "CPU / Arch", + "Userland", + "Flash layout", "Init system", "Device family", "Network", "Web admin", "Telnet exposure", "Autoboot interruptable", - ] + ], + "order and labels must match the browser detector library exactly" ); }