diff --git a/crates/fbuild-header-scan/src/lib.rs b/crates/fbuild-header-scan/src/lib.rs index 38d5aca2..0ad0c888 100644 --- a/crates/fbuild-header-scan/src/lib.rs +++ b/crates/fbuild-header-scan/src/lib.rs @@ -9,9 +9,13 @@ mod scanner; mod walker; -pub use scanner::{IncludeKind, IncludeRef, Span, active_defines, scan, scan_active}; +pub use scanner::{ + IncludeKind, IncludeRef, Span, active_defines, defined_macro_names, scan, scan_active, + scan_active_with_known, +}; pub use walker::{ - WalkResult, WalkState, walk, walk_active, walk_with_state, walk_with_state_active, + WalkResult, WalkState, collect_defined_macro_names, walk, walk_active, walk_with_state, + walk_with_state_active, walk_with_state_active_known, }; /// Bumped whenever the scanner output shape changes. Mixed into cache keys so a diff --git a/crates/fbuild-header-scan/src/scanner.rs b/crates/fbuild-header-scan/src/scanner.rs index 2c42cb68..00da80cc 100644 --- a/crates/fbuild-header-scan/src/scanner.rs +++ b/crates/fbuild-header-scan/src/scanner.rs @@ -9,7 +9,7 @@ //! preprocessor conditionals — false positives are acceptable, false negatives //! are not) when using `scan`. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; /// Whether an include used `<...>` (system / search-path) or `"..."` (quoted / /// same-directory-first). @@ -194,8 +194,45 @@ pub fn scan(src: &str) -> Vec { /// active `#define` in the same file apply to subsequent lines, matching the /// part of preprocessing relevant to library discovery. pub fn scan_active(src: &str, defines: &HashMap) -> Vec { + scan_active_with_known(src, defines, &HashSet::new()) +} + +/// [`scan_active`] told which macro names the wider corpus defines. +/// +/// `defined_somewhere` is the union of every `#define`d name reachable from +/// the seeds. A guard on a name in that set cannot be decided from the +/// compiler command line alone, so every arm is scanned; a guard on a name +/// nobody defines is honestly false and stays pruned (FastLED/fbuild#1371). +pub fn scan_active_with_known( + src: &str, + defines: &HashMap, + defined_somewhere: &HashSet, +) -> Vec { let mut macros = defines.clone(); - scan(&active_source(src, &mut macros)) + scan(&active_source(src, &mut macros, defined_somewhere)) +} + +/// Every macro name this source `#define`s, in any branch. +/// +/// Deliberately textual: the point is to know what the corpus *could* define, +/// so conditionals must not filter it. +pub fn defined_macro_names(src: &str) -> Vec { + let mut names = Vec::new(); + for line in src.lines() { + let Some(directive) = line.trim_start().strip_prefix('#').map(str::trim_start) else { + continue; + }; + let (name, rest) = split_directive(directive); + if name != "define" { + continue; + } + let (macro_name, _) = split_directive(rest); + let macro_name = macro_name.split('(').next().unwrap_or(""); + if !macro_name.is_empty() { + names.push(macro_name.to_string()); + } + } + names } /// Return `defines` plus macros declared in active branches of `src`. @@ -204,72 +241,170 @@ pub fn scan_active(src: &str, defines: &HashMap) -> Vec) -> HashMap { let mut macros = defines.clone(); - let _ = active_source(src, &mut macros); + let _ = active_source(src, &mut macros, &HashSet::new()); macros } #[derive(Clone, Copy)] struct Conditional { - parent_active: bool, + /// Whether lines were being kept when this conditional opened. + parent_scan: bool, + /// Whether `#define`s were being applied when this conditional opened. + parent_define: bool, + /// A branch of this group was decidably taken, so later `#elif`/`#else` + /// arms are dead. Never set for an undecidable group. branch_taken: bool, + /// The group's condition could not be decided, so every arm is scanned. + unknown: bool, +} + +/// Apply one branch decision to the scan/define state. +/// +/// The two states are deliberately separate. Scanning is generous — the +/// scanner's contract is that false positives are acceptable and false +/// negatives are not — while `#define` application stays strict, because a +/// macro picked up from a branch that may not be compiled would go on to +/// decide *other* conditions wrongly. +fn apply_decision(decision: Decision, parent_scan: bool, parent_define: bool) -> (bool, bool) { + match decision { + Decision::True => (parent_scan, parent_define), + // The LDF `#if 0` hint idiom: never compiled, so an include here is a + // dependency declaration. Scanned, but its defines are not real. + Decision::LiteralFalse => (parent_scan, false), + Decision::False => (false, false), + Decision::Unknown => (parent_scan, false), + } } -fn active_source(src: &str, macros: &mut HashMap) -> String { - let mut stack = Vec::new(); +/// The name of this file's own include guard, if it has the standard shape. +/// +/// A header that opens `#ifndef FOO_H` / `#define FOO_H` defines its own guard +/// macro, so `FOO_H` lands in the corpus-wide name set and the guard would +/// read as *undecidable* — which would switch off `#define` application for +/// the entire body of nearly every header in the project, and cascade into +/// every later guard in the same file. The guard is not really undecidable: +/// on the inclusion that matters it is not yet defined, so the body is taken. +fn self_include_guard(src: &str) -> Option { + let mut directives = src.lines().filter_map(|line| { + let directive = line.trim_start().strip_prefix('#')?.trim_start(); + let (name, rest) = split_directive(directive); + if name.is_empty() { + None + } else { + Some((name, rest)) + } + }); + let (first_name, first_rest) = directives.next()?; + if first_name != "ifndef" { + return None; + } + let guard = first_token(first_rest); + if guard.is_empty() { + return None; + } + let (second_name, second_rest) = directives.next()?; + if second_name == "define" && first_token(second_rest) == guard { + Some(guard.to_string()) + } else { + None + } +} + +fn active_source( + src: &str, + macros: &mut HashMap, + defined_somewhere: &HashSet, +) -> String { + let self_guard = self_include_guard(src); + let mut stack: Vec = Vec::new(); + // `scan` keeps lines for the include scan; `active` gates `#define`. + let mut scan = true; let mut active = true; let mut output = String::with_capacity(src.len()); for line in src.split_inclusive('\n') { let directive = line.trim_start().strip_prefix('#').map(str::trim_start); - let mut keep = active; + let mut keep = scan; if let Some(directive) = directive { let (name, rest) = split_directive(directive); match name { - "if" => { - let current = active && eval_condition(rest, macros); - stack.push(Conditional { - parent_active: active, - branch_taken: current, - }); - active = current; - keep = false; - } - "ifdef" => { - let current = active && macros.contains_key(first_token(rest)); - stack.push(Conditional { - parent_active: active, - branch_taken: current, - }); - active = current; - keep = false; - } - "ifndef" => { - let current = active && !macros.contains_key(first_token(rest)); + "if" | "ifdef" | "ifndef" => { + let decision = match name { + "if" => eval_decision(rest, macros, defined_somewhere), + // A bare `#ifdef X` is undecidable for the same reason + // `defined(X)` is: the macro set is the command line, + // not the preprocessor's running state. + "ifdef" => { + decide_defined(macros, defined_somewhere, first_token(rest), false) + } + _ => decide_defined(macros, defined_somewhere, first_token(rest), true), + }; + // A file's own `#ifndef FOO_H` is taken on the inclusion + // that matters, whatever the corpus says about `FOO_H`. + let decision = if name == "ifndef" + && self_guard.as_deref() == Some(first_token(rest)) + && !macros.contains_key(first_token(rest)) + { + Decision::True + } else { + decision + }; + let (next_scan, next_active) = if scan { + apply_decision(decision, scan, active) + } else { + (false, false) + }; stack.push(Conditional { - parent_active: active, - branch_taken: current, + parent_scan: scan, + parent_define: active, + branch_taken: scan && decision == Decision::True, + unknown: scan && decision == Decision::Unknown, }); - active = current; + scan = next_scan; + active = next_active; keep = false; } "elif" => { if let Some(current) = stack.last_mut() { - active = current.parent_active - && !current.branch_taken - && eval_condition(rest, macros); - current.branch_taken |= active; + if current.unknown { + // Undecidable group: every arm is scanned, none + // contributes defines. + scan = current.parent_scan; + active = false; + } else if current.branch_taken { + scan = false; + active = false; + } else { + let decision = eval_decision(rest, macros, defined_somewhere); + let (next_scan, next_active) = apply_decision( + decision, + current.parent_scan, + current.parent_define, + ); + scan = next_scan; + active = next_active; + current.branch_taken |= decision == Decision::True; + current.unknown |= decision == Decision::Unknown; + } } keep = false; } "else" => { if let Some(current) = stack.last_mut() { - active = current.parent_active && !current.branch_taken; - current.branch_taken = true; + if current.unknown { + scan = current.parent_scan; + active = false; + } else { + scan = current.parent_scan && !current.branch_taken; + active = current.parent_define && !current.branch_taken; + current.branch_taken = true; + } } keep = false; } "endif" => { if let Some(current) = stack.pop() { - active = current.parent_active; + scan = current.parent_scan; + active = current.parent_define; } keep = false; } @@ -284,6 +419,12 @@ fn active_source(src: &str, macros: &mut HashMap) -> String { macros.remove(first_token(rest)); keep = false; } + // A `#define`/`#undef` inside a branch that is only being + // scanned speculatively must not reach the macro set. The + // directive line itself is never include-bearing either. + "define" | "undef" => { + keep = false; + } _ => {} } } @@ -296,6 +437,34 @@ fn active_source(src: &str, macros: &mut HashMap) -> String { output } +/// Decide an `#ifdef` / `#ifndef` against the available macro set. +/// +/// Present means decidable. Absent means *unknown*, not false — see +/// [`Decision`]. +fn decide_defined( + macros: &HashMap, + defined_somewhere: &HashSet, + name: &str, + negated: bool, +) -> Decision { + if name.is_empty() { + return Decision::LiteralFalse; + } + if macros.contains_key(name) { + if negated { + Decision::False + } else { + Decision::True + } + } else if defined_somewhere.contains(name) { + Decision::Unknown + } else if negated { + Decision::True + } else { + Decision::False + } +} + fn split_directive(input: &str) -> (&str, &str) { let trimmed = input.trim_start(); let end = trimmed.find(char::is_whitespace).unwrap_or(trimmed.len()); @@ -310,19 +479,76 @@ fn first_token(input: &str) -> &str { .unwrap_or("") } -fn eval_condition(input: &str, macros: &HashMap) -> bool { +/// What a preprocessor condition evaluates to, given an incomplete macro set. +/// +/// The third state is the point. `scan_active` is handed the *compiler +/// command line* only — macros a header defines are not threaded through the +/// walk, because the walker visits each file once, in BFS order, with a +/// shared cache, and that is not preprocessor order. So a guard like +/// `#if defined(FL_IS_SAMD21)` is not false; it is *unknown*, and the +/// difference matters: FastLED derives `FL_IS_SAMD21` several headers deep +/// from `-D__SAMD21G18A__`, and treating it as false made an include that is +/// genuinely compiled invisible to library selection +/// (FastLED/fbuild#1371). +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum Decision { + True, + /// Decidably false from the macros actually available. + False, + /// False, and reached without consulting a single macro — `#if 0`. + /// + /// Distinguished from [`Decision::False`] because an include inside a + /// literal-false block cannot be there to be compiled. It is a dependency + /// declaration: the PlatformIO LDF `#if 0` hint idiom. + LiteralFalse, + /// Referenced a macro that is not in the available set, so the branch + /// cannot be decided. + Unknown, +} + +fn eval_decision( + input: &str, + macros: &HashMap, + defined_somewhere: &HashSet, +) -> Decision { let mut parser = ConditionParser { input: input.as_bytes(), index: 0, macros, + defined_somewhere, + saw_unknown_macro: false, + saw_any_macro: false, }; - parser.parse_or() != 0 + let value = parser.parse_or(); + if parser.saw_unknown_macro { + Decision::Unknown + } else if value != 0 { + Decision::True + } else if parser.saw_any_macro { + Decision::False + } else { + Decision::LiteralFalse + } } struct ConditionParser<'a> { input: &'a [u8], index: usize, macros: &'a HashMap, + /// Every macro name `#define`d anywhere in the reachable source corpus. + /// + /// This is what separates "the project never defines this" from "the + /// project defines this somewhere the walk could not thread to us". Only + /// the second is undecidable; the first is honestly false, and treating + /// it as unknown would select libraries behind branches that genuinely + /// never compile. + defined_somewhere: &'a HashSet, + /// Set when the expression consulted a macro that is not available, which + /// makes the whole condition undecidable rather than false. + saw_unknown_macro: bool, + /// Set when the expression consulted any macro at all, which separates + /// `#if 0` from a guard that happened to evaluate to zero. + saw_any_macro: bool, } impl<'a> ConditionParser<'a> { @@ -388,15 +614,28 @@ impl<'a> ConditionParser<'a> { self.consume(b"("); let name = self.token(); self.consume(b")"); + self.saw_any_macro = true; + if !self.macros.contains_key(name) && self.defined_somewhere.contains(name) { + self.saw_unknown_macro = true; + } return i64::from(self.macros.contains_key(name)); } if let Some(value) = parse_number(token) { return value; } - self.macros - .get(token) - .and_then(|value| parse_number(value)) - .unwrap_or(0) + if token.is_empty() { + return 0; + } + self.saw_any_macro = true; + match self.macros.get(token).and_then(|value| parse_number(value)) { + Some(value) => value, + None => { + if self.defined_somewhere.contains(token) { + self.saw_unknown_macro = true; + } + 0 + } + } } fn consume(&mut self, expected: &[u8]) -> bool { @@ -521,353 +760,5 @@ fn try_parse_include( } #[cfg(test)] -mod tests { - use super::*; - - fn first(refs: &[IncludeRef]) -> &IncludeRef { - refs.first().expect("expected at least one include ref") - } - - #[test] - fn s01_angled() { - let refs = scan("#include "); - assert_eq!(refs.len(), 1); - assert_eq!(first(&refs).path, "stdio.h"); - assert_eq!(first(&refs).kind, IncludeKind::Angled); - } - - #[test] - fn s02_quoted() { - let refs = scan("#include \"foo.h\""); - assert_eq!(refs.len(), 1); - assert_eq!(first(&refs).path, "foo.h"); - assert_eq!(first(&refs).kind, IncludeKind::Quoted); - } - - #[test] - fn s03_leading_ws() { - let refs = scan(" #include "); - assert_eq!(refs.len(), 1); - assert_eq!(first(&refs).path, "a.h"); - } - - #[test] - fn s04_ws_after_hash() { - let refs = scan("# include "); - assert_eq!(refs.len(), 1); - assert_eq!(first(&refs).path, "a.h"); - } - - #[test] - fn s05_path_with_slashes() { - let refs = scan("#include "); - assert_eq!(refs.len(), 1); - assert_eq!(first(&refs).path, "a/b/c.h"); - } - - #[test] - fn s06_trailing_comment_ignored() { - let refs = scan("#include // trailing\n"); - assert_eq!(refs.len(), 1); - assert_eq!(first(&refs).path, "a.h"); - } - - #[test] - fn s07_garbage_after_first_include_does_not_crash() { - let refs = scan("#include \"a.h\" \"b.h\"\n"); - assert_eq!(refs.len(), 1); - assert_eq!(first(&refs).path, "a.h"); - } - - #[test] - fn s10_line_comment_blocks_include() { - let refs = scan("// #include \n"); - assert!(refs.is_empty(), "got {refs:?}"); - } - - #[test] - fn s11_block_comment_blocks_include() { - let refs = scan("/* #include */\n"); - assert!(refs.is_empty(), "got {refs:?}"); - } - - #[test] - fn s12_multiline_block_comment_blocks_include() { - let refs = scan("/*\n#include \n*/\n"); - assert!(refs.is_empty(), "got {refs:?}"); - } - - #[test] - fn s13_string_literal_blocks_include() { - let refs = scan("const char* s = \"#include \";\n"); - assert!(refs.is_empty(), "got {refs:?}"); - } - - #[test] - fn s14_escaped_quotes_in_string_blocks_include() { - let refs = scan("const char* s = \"\\\"#include \\\"\";\n"); - assert!(refs.is_empty(), "got {refs:?}"); - } - - #[test] - fn s15_raw_string_blocks_include() { - let refs = scan("const char* s = R\"(#include )\";\n"); - assert!(refs.is_empty(), "got {refs:?}"); - } - - #[test] - fn s15_raw_string_with_delim_blocks_include() { - let refs = scan("const char* s = R\"DELIM(#include )DELIM\";\n"); - assert!(refs.is_empty(), "got {refs:?}"); - } - - #[test] - fn s16_char_literal_does_not_swallow() { - let refs = scan("char c = '#';\n#include \n"); - assert_eq!(refs.len(), 1); - assert_eq!(first(&refs).path, "a.h"); - } - - #[test] - fn s17_line_comment_then_include() { - let refs = scan("//#include \n#include \n"); - assert_eq!(refs.len(), 1); - assert_eq!(first(&refs).path, "b.h"); - } - - #[test] - fn s20_span_line_after_blank_lines() { - let refs = scan("\n\n#include "); - assert_eq!(first(&refs).span.line, 3); - assert_eq!(first(&refs).span.col, 1); - } - - #[test] - fn s21_span_col_with_indent() { - let refs = scan(" #include "); - assert_eq!(first(&refs).span.line, 1); - assert_eq!(first(&refs).span.col, 3); - } - - #[test] - fn s30_if_zero_branch_still_scanned() { - let refs = scan("#if 0\n#include \n#endif\n"); - assert_eq!(refs.len(), 1); - assert_eq!(first(&refs).path, "a.h"); - } - - #[test] - fn s31_has_include_branch_still_scanned() { - let refs = scan("#ifdef __has_include\n#include \n#endif\n"); - assert_eq!(refs.len(), 1); - } - - #[test] - fn s32_both_branches_scanned() { - let refs = scan("#if defined(X)\n#include \n#else\n#include \n#endif\n"); - assert_eq!(refs.len(), 2); - assert_eq!(refs[0].path, "a.h"); - assert_eq!(refs[1].path, "b.h"); - } - - #[test] - fn ignores_other_directives() { - let refs = scan("#define FOO 1\n#pragma once\n"); - assert!(refs.is_empty()); - } - - #[test] - fn handles_crlf_line_endings() { - let refs = scan("#include \r\n#include \r\n"); - assert_eq!(refs.len(), 2); - assert_eq!(refs[0].span.line, 1); - assert_eq!(refs[1].span.line, 2); - } - - #[test] - fn does_not_panic_on_unterminated_block_comment() { - let _ = scan("/* unterminated"); - } - - #[test] - fn does_not_panic_on_unterminated_string() { - let _ = scan("const char* s = \"unterminated"); - } - - #[test] - fn does_not_panic_on_unterminated_raw_string() { - let _ = scan("const char* s = R\"DELIM(unterminated"); - } - - #[test] - fn identifier_ending_in_r_does_not_start_raw_string() { - // `FooR` ends in `R` but is an identifier — the next `R"(` must NOT - // be treated as the opener of a raw string. If it were, the scanner - // would consume into RawString state and silently swallow the - // `#include` on the following line — a false negative the module - // contract forbids. - let refs = scan("auto FooR = 0;\n#include \n"); - assert_eq!(refs.len(), 1); - assert_eq!(refs[0].path, "a.h"); - } - - #[test] - fn identifier_ending_in_lr_does_not_start_wide_raw_string() { - // `FooL` precedes `R"(` — the `L` is part of the identifier, not the - // wide-string prefix. Must NOT enter RawString state. - let refs = scan("auto FooL = 0;\n#include \n"); - assert_eq!(refs.len(), 1); - assert_eq!(refs[0].path, "a.h"); - } - - #[test] - fn identifier_ending_in_lower_u_r_does_not_start_raw_string() { - let refs = scan("auto Foou = 0;\n#include \n"); - assert_eq!(refs.len(), 1); - assert_eq!(refs[0].path, "a.h"); - } - - #[test] - fn identifier_ending_in_upper_u_r_does_not_start_raw_string() { - let refs = scan("auto FooU = 0;\n#include \n"); - assert_eq!(refs.len(), 1); - assert_eq!(refs[0].path, "a.h"); - } - - #[test] - fn underscore_before_raw_prefix_blocks_detection() { - // `_R"(...)"` is identifier-continuation; must not start a raw - // string. Critical for code that uses `_R` as a translation macro - // name (common in i18n shims). - let refs = scan("foo_R = 0;\n#include \n"); - assert_eq!(refs.len(), 1); - } - - #[test] - fn digit_before_raw_prefix_blocks_detection() { - // Numbers can appear in identifiers; `foo1R` must not start a raw - // string. - let refs = scan("foo1R = 0;\n#include \n"); - assert_eq!(refs.len(), 1); - } - - #[test] - fn whitespace_before_raw_prefix_starts_raw_string() { - // Positive control — make sure we didn't break legitimate raw - // strings preceded by whitespace. - let refs = scan("auto x = R\"(#include )\";\n#include \n"); - assert_eq!(refs.len(), 1); - assert_eq!(refs[0].path, "a.h"); - } - - #[test] - fn start_of_file_raw_string_still_detected() { - // Boundary case: `R"(...)"` at byte 0 has no previous byte; - // `i > 0` clause must short-circuit and allow detection. - let refs = scan("R\"(#include )\"\n#include \n"); - assert_eq!(refs.len(), 1); - assert_eq!(refs[0].path, "a.h"); - } - - #[test] - fn punctuation_before_raw_prefix_starts_raw_string() { - // `=R"(...)"` — `=` is non-identifier; must enter raw-string state - // and swallow the embedded `#include`. - let refs = scan("auto x =R\"(#include )\";\n#include \n"); - assert_eq!(refs.len(), 1); - assert_eq!(refs[0].path, "a.h"); - } - - #[test] - fn paren_before_raw_prefix_starts_raw_string() { - // `(R"(...)"` — `(` is non-identifier. - let refs = scan("foo(R\"(#include )\");\n#include \n"); - assert_eq!(refs.len(), 1); - assert_eq!(refs[0].path, "a.h"); - } - - #[test] - fn many_includes_in_one_file() { - // Adversary: pile of includes interspersed with comments and - // strings. Confirm count + order are stable. - let src = "// header\n\ - #include \n\ - const char* s = \"#include \";\n\ - #include \"b.h\"\n\ - /* block\n\ - #include \n\ - */\n\ - #include \n"; - let refs = scan(src); - assert_eq!(refs.len(), 3); - assert_eq!(refs[0].path, "a.h"); - assert_eq!(refs[1].path, "b.h"); - assert_eq!(refs[2].path, "c.h"); - } - - #[test] - fn empty_input_returns_empty() { - assert!(scan("").is_empty()); - } - - #[test] - fn lone_hash_does_not_panic() { - let _ = scan("#"); - } - - #[test] - fn hash_then_eof_does_not_panic() { - let _ = scan("#include"); - } - - #[test] - fn null_bytes_do_not_panic() { - // Adversary: embedded NUL inside source. Real toolchains reject - // these but the scanner must not crash. - let _ = scan("foo\0bar\n#include \n"); - } - - #[test] - fn very_long_line_does_not_panic() { - // 64 KB single line. - let mut s = String::from("// "); - s.push_str(&"x".repeat(64 * 1024)); - s.push('\n'); - s.push_str("#include \n"); - let refs = scan(&s); - assert_eq!(refs.len(), 1); - } - - #[test] - fn deeply_nested_block_comments_do_not_panic() { - // C/C++ block comments don't nest, but we still shouldn't choke on - // pathological input. - let s = "/* /* /* */\n#include \n"; - let refs = scan(s); - // After the first `*/`, we're back in code state, so the include - // must be picked up. - assert_eq!(refs.len(), 1); - } - - #[test] - fn active_scan_ignores_disabled_branch() { - let refs = scan_active( - "#if 0\n#include \n#else\n#include \n#endif\n", - &HashMap::new(), - ); - assert_eq!(refs.len(), 1); - assert_eq!(refs[0].path, "SPI.h"); - } - - #[test] - fn active_scan_uses_compiler_and_local_defines() { - let mut defines = HashMap::new(); - defines.insert("ARDUINO".to_string(), "10819".to_string()); - let refs = scan_active( - "#if defined(ARDUINO) && ARDUINO >= 100\n#define USE_SPI 1\n#endif\n#ifdef USE_SPI\n#include \n#endif\n", - &defines, - ); - assert_eq!(refs.len(), 1); - assert_eq!(refs[0].path, "SPI.h"); - } -} +#[path = "scanner_tests.rs"] +mod tests; diff --git a/crates/fbuild-header-scan/src/scanner_tests.rs b/crates/fbuild-header-scan/src/scanner_tests.rs new file mode 100644 index 00000000..a582a6bd --- /dev/null +++ b/crates/fbuild-header-scan/src/scanner_tests.rs @@ -0,0 +1,485 @@ +//! Tests for [`super`]. +//! +//! Split out to keep the implementation file under the workspace 1000-LOC +//! limit; `compiler_tests.rs` is the same pattern. + +use super::*; + +fn first(refs: &[IncludeRef]) -> &IncludeRef { + refs.first().expect("expected at least one include ref") +} + +#[test] +fn s01_angled() { + let refs = scan("#include "); + assert_eq!(refs.len(), 1); + assert_eq!(first(&refs).path, "stdio.h"); + assert_eq!(first(&refs).kind, IncludeKind::Angled); +} + +#[test] +fn s02_quoted() { + let refs = scan("#include \"foo.h\""); + assert_eq!(refs.len(), 1); + assert_eq!(first(&refs).path, "foo.h"); + assert_eq!(first(&refs).kind, IncludeKind::Quoted); +} + +#[test] +fn s03_leading_ws() { + let refs = scan(" #include "); + assert_eq!(refs.len(), 1); + assert_eq!(first(&refs).path, "a.h"); +} + +#[test] +fn s04_ws_after_hash() { + let refs = scan("# include "); + assert_eq!(refs.len(), 1); + assert_eq!(first(&refs).path, "a.h"); +} + +#[test] +fn s05_path_with_slashes() { + let refs = scan("#include "); + assert_eq!(refs.len(), 1); + assert_eq!(first(&refs).path, "a/b/c.h"); +} + +#[test] +fn s06_trailing_comment_ignored() { + let refs = scan("#include // trailing\n"); + assert_eq!(refs.len(), 1); + assert_eq!(first(&refs).path, "a.h"); +} + +#[test] +fn s07_garbage_after_first_include_does_not_crash() { + let refs = scan("#include \"a.h\" \"b.h\"\n"); + assert_eq!(refs.len(), 1); + assert_eq!(first(&refs).path, "a.h"); +} + +#[test] +fn s10_line_comment_blocks_include() { + let refs = scan("// #include \n"); + assert!(refs.is_empty(), "got {refs:?}"); +} + +#[test] +fn s11_block_comment_blocks_include() { + let refs = scan("/* #include */\n"); + assert!(refs.is_empty(), "got {refs:?}"); +} + +#[test] +fn s12_multiline_block_comment_blocks_include() { + let refs = scan("/*\n#include \n*/\n"); + assert!(refs.is_empty(), "got {refs:?}"); +} + +#[test] +fn s13_string_literal_blocks_include() { + let refs = scan("const char* s = \"#include \";\n"); + assert!(refs.is_empty(), "got {refs:?}"); +} + +#[test] +fn s14_escaped_quotes_in_string_blocks_include() { + let refs = scan("const char* s = \"\\\"#include \\\"\";\n"); + assert!(refs.is_empty(), "got {refs:?}"); +} + +#[test] +fn s15_raw_string_blocks_include() { + let refs = scan("const char* s = R\"(#include )\";\n"); + assert!(refs.is_empty(), "got {refs:?}"); +} + +#[test] +fn s15_raw_string_with_delim_blocks_include() { + let refs = scan("const char* s = R\"DELIM(#include )DELIM\";\n"); + assert!(refs.is_empty(), "got {refs:?}"); +} + +#[test] +fn s16_char_literal_does_not_swallow() { + let refs = scan("char c = '#';\n#include \n"); + assert_eq!(refs.len(), 1); + assert_eq!(first(&refs).path, "a.h"); +} + +#[test] +fn s17_line_comment_then_include() { + let refs = scan("//#include \n#include \n"); + assert_eq!(refs.len(), 1); + assert_eq!(first(&refs).path, "b.h"); +} + +#[test] +fn s20_span_line_after_blank_lines() { + let refs = scan("\n\n#include "); + assert_eq!(first(&refs).span.line, 3); + assert_eq!(first(&refs).span.col, 1); +} + +#[test] +fn s21_span_col_with_indent() { + let refs = scan(" #include "); + assert_eq!(first(&refs).span.line, 1); + assert_eq!(first(&refs).span.col, 3); +} + +#[test] +fn s30_if_zero_branch_still_scanned() { + let refs = scan("#if 0\n#include \n#endif\n"); + assert_eq!(refs.len(), 1); + assert_eq!(first(&refs).path, "a.h"); +} + +#[test] +fn s31_has_include_branch_still_scanned() { + let refs = scan("#ifdef __has_include\n#include \n#endif\n"); + assert_eq!(refs.len(), 1); +} + +#[test] +fn s32_both_branches_scanned() { + let refs = scan("#if defined(X)\n#include \n#else\n#include \n#endif\n"); + assert_eq!(refs.len(), 2); + assert_eq!(refs[0].path, "a.h"); + assert_eq!(refs[1].path, "b.h"); +} + +#[test] +fn ignores_other_directives() { + let refs = scan("#define FOO 1\n#pragma once\n"); + assert!(refs.is_empty()); +} + +#[test] +fn handles_crlf_line_endings() { + let refs = scan("#include \r\n#include \r\n"); + assert_eq!(refs.len(), 2); + assert_eq!(refs[0].span.line, 1); + assert_eq!(refs[1].span.line, 2); +} + +#[test] +fn does_not_panic_on_unterminated_block_comment() { + let _ = scan("/* unterminated"); +} + +#[test] +fn does_not_panic_on_unterminated_string() { + let _ = scan("const char* s = \"unterminated"); +} + +#[test] +fn does_not_panic_on_unterminated_raw_string() { + let _ = scan("const char* s = R\"DELIM(unterminated"); +} + +#[test] +fn identifier_ending_in_r_does_not_start_raw_string() { + // `FooR` ends in `R` but is an identifier — the next `R"(` must NOT + // be treated as the opener of a raw string. If it were, the scanner + // would consume into RawString state and silently swallow the + // `#include` on the following line — a false negative the module + // contract forbids. + let refs = scan("auto FooR = 0;\n#include \n"); + assert_eq!(refs.len(), 1); + assert_eq!(refs[0].path, "a.h"); +} + +#[test] +fn identifier_ending_in_lr_does_not_start_wide_raw_string() { + // `FooL` precedes `R"(` — the `L` is part of the identifier, not the + // wide-string prefix. Must NOT enter RawString state. + let refs = scan("auto FooL = 0;\n#include \n"); + assert_eq!(refs.len(), 1); + assert_eq!(refs[0].path, "a.h"); +} + +#[test] +fn identifier_ending_in_lower_u_r_does_not_start_raw_string() { + let refs = scan("auto Foou = 0;\n#include \n"); + assert_eq!(refs.len(), 1); + assert_eq!(refs[0].path, "a.h"); +} + +#[test] +fn identifier_ending_in_upper_u_r_does_not_start_raw_string() { + let refs = scan("auto FooU = 0;\n#include \n"); + assert_eq!(refs.len(), 1); + assert_eq!(refs[0].path, "a.h"); +} + +#[test] +fn underscore_before_raw_prefix_blocks_detection() { + // `_R"(...)"` is identifier-continuation; must not start a raw + // string. Critical for code that uses `_R` as a translation macro + // name (common in i18n shims). + let refs = scan("foo_R = 0;\n#include \n"); + assert_eq!(refs.len(), 1); +} + +#[test] +fn digit_before_raw_prefix_blocks_detection() { + // Numbers can appear in identifiers; `foo1R` must not start a raw + // string. + let refs = scan("foo1R = 0;\n#include \n"); + assert_eq!(refs.len(), 1); +} + +#[test] +fn whitespace_before_raw_prefix_starts_raw_string() { + // Positive control — make sure we didn't break legitimate raw + // strings preceded by whitespace. + let refs = scan("auto x = R\"(#include )\";\n#include \n"); + assert_eq!(refs.len(), 1); + assert_eq!(refs[0].path, "a.h"); +} + +#[test] +fn start_of_file_raw_string_still_detected() { + // Boundary case: `R"(...)"` at byte 0 has no previous byte; + // `i > 0` clause must short-circuit and allow detection. + let refs = scan("R\"(#include )\"\n#include \n"); + assert_eq!(refs.len(), 1); + assert_eq!(refs[0].path, "a.h"); +} + +#[test] +fn punctuation_before_raw_prefix_starts_raw_string() { + // `=R"(...)"` — `=` is non-identifier; must enter raw-string state + // and swallow the embedded `#include`. + let refs = scan("auto x =R\"(#include )\";\n#include \n"); + assert_eq!(refs.len(), 1); + assert_eq!(refs[0].path, "a.h"); +} + +#[test] +fn paren_before_raw_prefix_starts_raw_string() { + // `(R"(...)"` — `(` is non-identifier. + let refs = scan("foo(R\"(#include )\");\n#include \n"); + assert_eq!(refs.len(), 1); + assert_eq!(refs[0].path, "a.h"); +} + +#[test] +fn many_includes_in_one_file() { + // Adversary: pile of includes interspersed with comments and + // strings. Confirm count + order are stable. + let src = "// header\n\ + #include \n\ + const char* s = \"#include \";\n\ + #include \"b.h\"\n\ + /* block\n\ + #include \n\ + */\n\ + #include \n"; + let refs = scan(src); + assert_eq!(refs.len(), 3); + assert_eq!(refs[0].path, "a.h"); + assert_eq!(refs[1].path, "b.h"); + assert_eq!(refs[2].path, "c.h"); +} + +#[test] +fn empty_input_returns_empty() { + assert!(scan("").is_empty()); +} + +#[test] +fn lone_hash_does_not_panic() { + let _ = scan("#"); +} + +#[test] +fn hash_then_eof_does_not_panic() { + let _ = scan("#include"); +} + +#[test] +fn null_bytes_do_not_panic() { + // Adversary: embedded NUL inside source. Real toolchains reject + // these but the scanner must not crash. + let _ = scan("foo\0bar\n#include \n"); +} + +#[test] +fn very_long_line_does_not_panic() { + // 64 KB single line. + let mut s = String::from("// "); + s.push_str(&"x".repeat(64 * 1024)); + s.push('\n'); + s.push_str("#include \n"); + let refs = scan(&s); + assert_eq!(refs.len(), 1); +} + +#[test] +fn deeply_nested_block_comments_do_not_panic() { + // C/C++ block comments don't nest, but we still shouldn't choke on + // pathological input. + let s = "/* /* /* */\n#include \n"; + let refs = scan(s); + // After the first `*/`, we're back in code state, so the include + // must be picked up. + assert_eq!(refs.len(), 1); +} + +#[test] +/// `#if 0` is a dependency declaration, not dead code. +/// +/// This test previously asserted that `Audio.h` was ignored. It is not, +/// and deliberately so: an include that can never be compiled is only +/// there to be *seen* — the PlatformIO LDF hint idiom, which FastLED uses +/// in `platforms/*/ldf_headers.h` to declare dependencies its conditional +/// includes would otherwise hide (FastLED/fbuild#1371). PlatformIO's own +/// `chain` mode honors it by not evaluating conditionals at all. +/// +/// The `#else` arm is scanned too, because that is the arm which actually +/// compiles. Both are dependencies. +fn literal_false_branches_are_scanned_as_ldf_hints() { + let refs = scan_active( + "#if 0\n#include \n#else\n#include \n#endif\n", + &HashMap::new(), + ); + let paths: Vec<&str> = refs.iter().map(|r| r.path.as_str()).collect(); + assert!( + paths.contains(&"Audio.h"), + "the #if 0 hint must be seen: {paths:?}" + ); + assert!( + paths.contains(&"SPI.h"), + "the compiled arm must be seen: {paths:?}" + ); +} + +/// A branch that is decidably false from the *known* macros stays pruned. +/// +/// This is what keeps the change from collapsing into a plain textual +/// scan: when the command line actually settles a guard, it is settled. +#[test] +fn decidably_false_branches_are_still_pruned() { + let mut defines = HashMap::new(); + defines.insert("USE_AUDIO".to_string(), "0".to_string()); + let refs = scan_active( + "#if USE_AUDIO\n#include \n#else\n#include \n#endif\n", + &defines, + ); + let paths: Vec<&str> = refs.iter().map(|r| r.path.as_str()).collect(); + assert_eq!( + paths, + vec!["SPI.h"], + "a known-false guard must still prune: {paths:?}" + ); +} + +/// The FastLED/fbuild#1371 case: a guard on a macro the scan cannot see. +/// +/// `FL_IS_SAMD21` is derived several headers deep from `-D__SAMD21G18A__`, +/// and header-defined macros are not threaded through the walk. Treating +/// that as *false* made an include that is genuinely compiled invisible to +/// library selection; treating it as *unknown* finds it. +#[test] +fn guards_on_unknown_macros_scan_every_arm() { + // The corpus defines these somewhere (FastLED's `is_platform.h`), so + // the guard is undecidable rather than false. + let known: HashSet = ["FL_IS_SAMD21".to_string(), "FL_IS_SAMD51".to_string()].into(); + let refs = scan_active_with_known( + "#if defined(FL_IS_SAMD21) || defined(FL_IS_SAMD51)\n#include \n#endif\n", + &HashMap::new(), + &known, + ); + let paths: Vec<&str> = refs.iter().map(|r| r.path.as_str()).collect(); + assert_eq!( + paths, + vec!["SPI.h"], + "an undecidable guard must not hide its include" + ); +} + +/// `#ifdef` on an unseen macro is undecidable for the same reason. +#[test] +fn ifdef_on_an_unknown_macro_is_undecidable() { + let known: HashSet = ["FL_IS_ARM".to_string()].into(); + let refs = scan_active_with_known( + "#ifdef FL_IS_ARM\n#include \n#else\n#include \n#endif\n", + &HashMap::new(), + &known, + ); + let paths: Vec<&str> = refs.iter().map(|r| r.path.as_str()).collect(); + assert!(paths.contains(&"arm.h"), "{paths:?}"); + assert!(paths.contains(&"other.h"), "{paths:?}"); +} + +/// Defines from a speculatively-scanned branch must not settle later +/// guards — that would let an arm which may never compile prune a real one. +#[test] +fn defines_inside_undecidable_branches_do_not_leak() { + let known: HashSet = ["UNKNOWN_MACRO".to_string(), "PICKED".to_string()].into(); + let refs = scan_active_with_known( + "#ifdef UNKNOWN_MACRO\n#define PICKED 1\n#endif\n#if PICKED\n#include \n#else\n#include \n#endif\n", + &HashMap::new(), + &known, + ); + let paths: Vec<&str> = refs.iter().map(|r| r.path.as_str()).collect(); + // `PICKED` never became known, so the second guard is undecidable too + // and both arms are scanned — rather than `PICKED` being trusted. + assert!(paths.contains(&"picked.h"), "{paths:?}"); + assert!(paths.contains(&"other.h"), "{paths:?}"); +} + +#[test] +fn active_scan_uses_compiler_and_local_defines() { + let mut defines = HashMap::new(); + defines.insert("ARDUINO".to_string(), "10819".to_string()); + let refs = scan_active( + "#if defined(ARDUINO) && ARDUINO >= 100\n#define USE_SPI 1\n#endif\n#ifdef USE_SPI\n#include \n#endif\n", + &defines, + ); + assert_eq!(refs.len(), 1); + assert_eq!(refs[0].path, "SPI.h"); +} + +/// A file's own include guard must not make the rest of the file undecidable. +/// +/// `#ifndef FOO_H` / `#define FOO_H` puts `FOO_H` into the corpus-wide name +/// set — the file defines it, after all. Reading that guard as *undecidable* +/// would switch off `#define` application for the entire body, so +/// `LOCAL_FEATURE` below would never be learned and the `#ifdef` on it would +/// scan both arms. Since nearly every header is guarded this way, that would +/// quietly degrade the whole scan toward textual. +#[test] +fn a_files_own_include_guard_does_not_poison_the_rest_of_it() { + let known: HashSet = ["FOO_H".to_string(), "LOCAL_FEATURE".to_string()].into(); + let refs = scan_active_with_known( + "#ifndef FOO_H\n#define FOO_H\n#define LOCAL_FEATURE 1\n#ifdef LOCAL_FEATURE\n#include \n#else\n#include \n#endif\n#endif\n", + &HashMap::new(), + &known, + ); + let paths: Vec<&str> = refs.iter().map(|r| r.path.as_str()).collect(); + assert_eq!( + paths, + vec!["wanted.h"], + "the guard must stay decidable so the file's own defines still apply: {paths:?}" + ); +} + +/// An `#ifndef` that is *not* the file's self-guard keeps the conservative +/// treatment — it is a real feature test, not a header-reentry check. +#[test] +fn a_non_guard_ifndef_is_still_undecidable() { + let known: HashSet = ["SOME_FEATURE".to_string()].into(); + let refs = scan_active_with_known( + "#include \n#ifndef SOME_FEATURE\n#include \n#else\n#include \n#endif\n", + &HashMap::new(), + &known, + ); + let paths: Vec<&str> = refs.iter().map(|r| r.path.as_str()).collect(); + assert!(paths.contains(&"fallback.h"), "{paths:?}"); + assert!(paths.contains(&"feature.h"), "{paths:?}"); +} diff --git a/crates/fbuild-header-scan/src/walker.rs b/crates/fbuild-header-scan/src/walker.rs index 4fab519d..efc907de 100644 --- a/crates/fbuild-header-scan/src/walker.rs +++ b/crates/fbuild-header-scan/src/walker.rs @@ -21,7 +21,9 @@ use std::path::{Path, PathBuf}; use rayon::prelude::*; -use crate::scanner::{IncludeKind, IncludeRef, scan, scan_active}; +use crate::scanner::{ + IncludeKind, IncludeRef, defined_macro_names, scan, scan_active, scan_active_with_known, +}; /// Result of a walk. `reached` and `unresolved` are sorted for deterministic /// cache keys. @@ -133,6 +135,40 @@ pub fn walk_with_state_active( walk_with_state_scanner(seeds, search_paths, state, &|src| scan_active(src, defines)) } +/// [`walk_with_state_active`] told which macro names the corpus defines. +/// +/// See [`crate::scanner::scan_active_with_known`]: a guard on a macro the +/// project defines somewhere is undecidable from the command line alone, and +/// pruning it hid includes that genuinely compile (FastLED/fbuild#1371). +pub fn walk_with_state_active_known( + seeds: &[PathBuf], + search_paths: &[PathBuf], + defines: &HashMap, + defined_somewhere: &HashSet, + state: &mut WalkState, +) -> WalkResult { + walk_with_state_scanner(seeds, search_paths, state, &|src| { + scan_active_with_known(src, defines, defined_somewhere) + }) +} + +/// Collect every macro name `#define`d in any file reachable from `seeds`. +/// +/// Walks textually (all branches), because the question is what the corpus +/// *could* define — a conditional must not filter the answer. Uses its own +/// [`WalkState`] so the active passes keep their own scan cache semantics. +pub fn collect_defined_macro_names(seeds: &[PathBuf], search_paths: &[PathBuf]) -> HashSet { + let mut state = WalkState::new(); + let result = walk_with_state(seeds, search_paths, &mut state); + let mut names = HashSet::new(); + for path in &result.reached { + if let Ok(src) = std::fs::read_to_string(path) { + names.extend(defined_macro_names(&src)); + } + } + names +} + fn walk_with_state_scanner( seeds: &[PathBuf], search_paths: &[PathBuf], diff --git a/crates/fbuild-library-select/src/cache.rs b/crates/fbuild-library-select/src/cache.rs index c4d148ee..92d451b2 100644 --- a/crates/fbuild-library-select/src/cache.rs +++ b/crates/fbuild-library-select/src/cache.rs @@ -35,11 +35,11 @@ use crate::{Selection, canon}; /// Bump when the scanner's lexical grammar changes in a way that could change /// which `#include` directives it emits for the same source. -pub const SCANNER_VERSION: u32 = 2; +pub const SCANNER_VERSION: u32 = 3; /// Bump when the resolver's 2-pass LDF semantics change (seed expansion, /// attribution, convergence rule, etc.). -pub const LDF_MODE_VERSION: u32 = 4; +pub const LDF_MODE_VERSION: u32 = 5; /// Namespace for the library-selection file cache. pub const NAMESPACE: &str = "library-selection"; diff --git a/crates/fbuild-library-select/src/lib.rs b/crates/fbuild-library-select/src/lib.rs index 38b90648..b2eb1269 100644 --- a/crates/fbuild-library-select/src/lib.rs +++ b/crates/fbuild-library-select/src/lib.rs @@ -20,7 +20,10 @@ use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::path::{Path, PathBuf}; -use fbuild_header_scan::{WalkState, active_defines, walk_with_state, walk_with_state_active}; +use fbuild_header_scan::{ + WalkState, active_defines, collect_defined_macro_names, walk_with_state, + walk_with_state_active_known, +}; use fbuild_packages::library::FrameworkLibrary; use serde::{Deserialize, Serialize}; @@ -284,13 +287,38 @@ fn resolve_with_stats_impl_declared( } } + // Which macro names the reachable corpus defines anywhere. + // + // This is what lets branch evaluation stay honest without hiding real + // dependencies. `#if defined(FL_IS_SAMD21)` cannot be decided from the + // compiler command line, because FastLED derives that macro several + // headers deep and header-defined macros are not threaded through a BFS + // walk — so guards on names the project *does* define are undecidable and + // every arm gets scanned. Guards on names nobody defines stay honestly + // false, which is what keeps a library behind a genuinely dead branch + // from being selected (FastLED/fbuild#1094, #1371). + // + // Only computed when branch evaluation is on; the textual mode already + // scans every arm. + let defined_somewhere = if defines.is_some() { + collect_defined_macro_names(seeds, &full_search_paths) + } else { + Default::default() + }; + // Pass 1: BFS from project seeds. { let _span = tracing::info_span!("ldf_pass", pass = 1u32).entered(); pass_count += 1; tracing::info!(pass = 1u32, "ldf_pass"); let res = match defines { - Some(defines) => walk_with_state_active(seeds, &full_search_paths, defines, &mut state), + Some(defines) => walk_with_state_active_known( + seeds, + &full_search_paths, + defines, + &defined_somewhere, + &mut state, + ), None => walk_with_state(seeds, &full_search_paths, &mut state), }; for p in &res.reached { @@ -325,9 +353,13 @@ fn resolve_with_stats_impl_declared( } } let res = match defines { - Some(defines) => { - walk_with_state_active(&recon_seeds, &full_search_paths, defines, &mut state) - } + Some(defines) => walk_with_state_active_known( + &recon_seeds, + &full_search_paths, + defines, + &defined_somewhere, + &mut state, + ), None => walk_with_state(&recon_seeds, &full_search_paths, &mut state), }; for p in &res.reached { @@ -415,508 +447,5 @@ fn path_in_any(path: &Path, dirs: &[PathBuf]) -> bool { } #[cfg(test)] -mod tests { - use super::*; - use tempfile::TempDir; - - fn write(path: &Path, contents: &str) { - if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent).unwrap(); - } - std::fs::write(path, contents).unwrap(); - } - - fn lib(tmp: &Path, name: &str) -> FrameworkLibrary { - let dir = tmp.join("libraries").join(name); - let src = dir.join("src"); - std::fs::create_dir_all(&src).unwrap(); - FrameworkLibrary { - name: name.to_string(), - dir: dir.clone(), - include_dirs: vec![src.clone()], - source_files: Vec::new(), - } - } - - fn tempdir() -> TempDir { - TempDir::new_in(fbuild_paths::temp_subdir("fbuild-library-select-tests")).unwrap() - } - - #[test] - fn r01_direct_include_selects_library() { - let tmp = tempdir(); - let project_src = tmp.path().join("project").join("src"); - write(&project_src.join("main.cpp"), "#include \n"); - let mut spi = lib(tmp.path(), "SPI"); - write(&spi.include_dirs[0].join("SPI.h"), ""); - let spi_cpp = spi.include_dirs[0].join("SPI.cpp"); - write(&spi_cpp, ""); - spi.source_files.push(spi_cpp.clone()); - - let seeds = vec![project_src.join("main.cpp")]; - let sel = resolve(&seeds, &[project_src], &[spi]); - assert_eq!(sel.required_libraries, vec!["SPI".to_string()]); - assert!(sel.source_files.contains(&canon(&spi_cpp)) || sel.source_files.contains(&spi_cpp)); - } - - #[test] - fn active_resolution_skips_library_in_disabled_branch() { - let tmp = tempdir(); - let project_src = tmp.path().join("project").join("src"); - write( - &project_src.join("main.cpp"), - "#define USE_SPI 1\n#include \n", - ); - write( - &project_src.join("FastLED.h"), - "#if defined(USE_AUDIO)\n#include \n#elif USE_SPI\n#include \n#endif\n", - ); - - let mut audio = lib(tmp.path(), "Audio"); - write(&audio.include_dirs[0].join("Audio.h"), ""); - let audio_cpp = audio.include_dirs[0].join("Audio.cpp"); - write(&audio_cpp, ""); - audio.source_files.push(audio_cpp); - - let mut spi = lib(tmp.path(), "SPI"); - write(&spi.include_dirs[0].join("SPI.h"), ""); - let spi_cpp = spi.include_dirs[0].join("SPI.cpp"); - write(&spi_cpp, ""); - spi.source_files.push(spi_cpp); - - let seeds = vec![project_src.join("main.cpp")]; - let selection = resolve_active(&seeds, &[project_src], &[audio, spi], &HashMap::new()); - assert_eq!(selection.required_libraries, vec!["SPI".to_string()]); - } - - #[test] - fn r02_transitive_library_selection() { - let tmp = tempdir(); - let project_src = tmp.path().join("project").join("src"); - write(&project_src.join("main.cpp"), "#include \n"); - - let mut spi = lib(tmp.path(), "SPI"); - write(&spi.include_dirs[0].join("SPI.h"), "#include \n"); - let spi_cpp = spi.include_dirs[0].join("SPI.cpp"); - write(&spi_cpp, ""); - spi.source_files.push(spi_cpp); - - let mut wire = lib(tmp.path(), "Wire"); - write(&wire.include_dirs[0].join("Wire.h"), ""); - let wire_cpp = wire.include_dirs[0].join("Wire.cpp"); - write(&wire_cpp, ""); - wire.source_files.push(wire_cpp); - - let seeds = vec![project_src.join("main.cpp")]; - let sel = resolve(&seeds, &[project_src], &[spi, wire]); - assert_eq!( - sel.required_libraries, - vec!["SPI".to_string(), "Wire".to_string()] - ); - } - - #[test] - fn r04_pass2_reconciliation_catches_cpp_only_dependency() { - // The whole reason the LDF resolver is 2-pass instead of single-pass - // BFS: a lib's `.cpp` may pull in a second lib that the first lib's - // `.h` does NOT mention. Pass 1 (BFS from project seeds + reached - // headers) cannot see that edge; pass 2 re-seeds with each selected - // lib's full source set and catches it. - // - // Setup: project includes . SPI.h is silent. SPI.cpp includes - // . Wire is only reachable through SPI.cpp. - // - // Expected: pass 1 selects {SPI}; pass 2 (with SPI.cpp as a seed) - // selects {SPI, Wire}. A regression that drops the second pass would - // produce {SPI} only and silently miss Wire at link time. - let tmp = tempdir(); - let project_src = tmp.path().join("project").join("src"); - write(&project_src.join("main.cpp"), "#include \n"); - - let mut spi = lib(tmp.path(), "SPI"); - write( - &spi.include_dirs[0].join("SPI.h"), - "// no transitive includes\n", - ); - let spi_cpp = spi.include_dirs[0].join("SPI.cpp"); - write(&spi_cpp, "#include \n"); - spi.source_files.push(spi_cpp); - - let mut wire = lib(tmp.path(), "Wire"); - write(&wire.include_dirs[0].join("Wire.h"), ""); - let wire_cpp = wire.include_dirs[0].join("Wire.cpp"); - write(&wire_cpp, ""); - wire.source_files.push(wire_cpp); - - let seeds = vec![project_src.join("main.cpp")]; - let sel = resolve(&seeds, &[project_src], &[spi, wire]); - assert_eq!( - sel.required_libraries, - vec!["SPI".to_string(), "Wire".to_string()], - "pass 2 reconciliation must catch Wire reached only via SPI.cpp" - ); - } - - #[test] - fn r03_no_includes_selects_nothing() { - let tmp = tempdir(); - let project_src = tmp.path().join("project").join("src"); - write(&project_src.join("main.cpp"), "int main() { return 0; }\n"); - let spi = lib(tmp.path(), "SPI"); - write(&spi.include_dirs[0].join("SPI.h"), ""); - - let seeds = vec![project_src.join("main.cpp")]; - let sel = resolve(&seeds, &[project_src], &[spi]); - assert!(sel.required_libraries.is_empty()); - } - - #[test] - fn r13_unrelated_library_not_selected() { - let tmp = tempdir(); - let project_src = tmp.path().join("project").join("src"); - write(&project_src.join("main.cpp"), "#include \n"); - - let mut spi = lib(tmp.path(), "SPI"); - write(&spi.include_dirs[0].join("SPI.h"), ""); - let spi_cpp = spi.include_dirs[0].join("SPI.cpp"); - write(&spi_cpp, ""); - spi.source_files.push(spi_cpp); - - let mut fnet = lib(tmp.path(), "FNET"); - write(&fnet.include_dirs[0].join("fnet.h"), ""); - let fnet_cpp = fnet.include_dirs[0].join("fnet.cpp"); - write(&fnet_cpp, ""); - fnet.source_files.push(fnet_cpp); - - let seeds = vec![project_src.join("main.cpp")]; - let sel = resolve(&seeds, &[project_src], &[spi, fnet]); - assert_eq!(sel.required_libraries, vec!["SPI".to_string()]); - } - - #[test] - fn path_prefix_attribution_distinguishes_same_basename() { - let tmp = tempdir(); - let project_src = tmp.path().join("project").join("src"); - write(&project_src.join("main.cpp"), "#include \"foo/config.h\"\n"); - - let mut foo = lib(tmp.path(), "Foo"); - write(&foo.include_dirs[0].join("foo").join("config.h"), ""); - let foo_cpp = foo.include_dirs[0].join("Foo.cpp"); - write(&foo_cpp, ""); - foo.source_files.push(foo_cpp); - - let mut bar = lib(tmp.path(), "Bar"); - // Bar also has a config.h but at its own path — must NOT be selected - // when the project only includes "foo/config.h". - write(&bar.include_dirs[0].join("bar").join("config.h"), ""); - let bar_cpp = bar.include_dirs[0].join("Bar.cpp"); - write(&bar_cpp, ""); - bar.source_files.push(bar_cpp); - - let seeds = vec![project_src.join("main.cpp")]; - let sel = resolve( - &seeds, - &[ - project_src, - foo.include_dirs[0].clone(), - bar.include_dirs[0].clone(), - ], - &[foo, bar], - ); - assert_eq!(sel.required_libraries, vec!["Foo".to_string()]); - } - - #[test] - fn empty_libraries_yields_empty_selection() { - // Adversary: no libraries at all. resolve must terminate cleanly with - // no required_libraries, no panics, and any reached files limited to - // what the walker found from seeds alone. - let tmp = tempdir(); - let project_src = tmp.path().join("project").join("src"); - write(&project_src.join("main.cpp"), "int main() { return 0; }\n"); - let seeds = vec![project_src.join("main.cpp")]; - let sel = resolve(&seeds, &[project_src], &[]); - assert!(sel.required_libraries.is_empty()); - assert!(sel.source_files.is_empty()); - } - - #[test] - fn missing_library_include_dir_does_not_panic() { - // Adversary: a FrameworkLibrary whose include_dirs point at a path - // that doesn't exist on disk (broken framework install, lib not yet - // downloaded). canon() falls back and emits a tracing::warn; the - // resolver must not panic and must return a sensible empty - // selection. - let tmp = tempdir(); - let project_src = tmp.path().join("project").join("src"); - write(&project_src.join("main.cpp"), "int main() { return 0; }\n"); - let phantom = FrameworkLibrary { - name: "Phantom".to_string(), - dir: tmp.path().join("nonexistent").join("Phantom"), - include_dirs: vec![tmp.path().join("nonexistent").join("Phantom").join("src")], - source_files: Vec::new(), - }; - let seeds = vec![project_src.join("main.cpp")]; - let sel = resolve(&seeds, &[project_src], &[phantom]); - assert!(sel.required_libraries.is_empty()); - } - - #[test] - fn many_libraries_in_random_order_returns_sorted() { - // Adversary: 6 libs in deliberately scrambled input order. The - // output must be sorted lexicographically, independent of input - // order — required for stable cache keys (#205 Phase 4). - let tmp = tempdir(); - let project_src = tmp.path().join("project").join("src"); - write( - &project_src.join("main.cpp"), - "#include \n#include \n#include \n\ - #include \n#include \n#include \n", - ); - - let mut libs = Vec::new(); - for name in ["Z", "A", "M", "B", "Y", "K"] { - let mut l = lib(tmp.path(), name); - write(&l.include_dirs[0].join(format!("{name}.h")), ""); - let cpp = l.include_dirs[0].join(format!("{name}.cpp")); - write(&cpp, ""); - l.source_files.push(cpp); - libs.push(l); - } - - let seeds = vec![project_src.join("main.cpp")]; - let sel = resolve(&seeds, &[project_src], &libs); - assert_eq!( - sel.required_libraries, - ["A", "B", "K", "M", "Y", "Z"] - .iter() - .map(|s| s.to_string()) - .collect::>() - ); - } - - #[test] - fn required_libraries_returned_sorted_by_name_not_input_order() { - // Regression guard: pass the libraries in REVERSE name order (Wire - // before SPI) and confirm the output is sorted lexicographically. - // The doc on `Selection::required_libraries` and the cache-key story - // in #205 Phase 4 both depend on this being a pure function of the - // selected *set* of libraries, not their input position. - let tmp = tempdir(); - let project_src = tmp.path().join("project").join("src"); - write( - &project_src.join("main.cpp"), - "#include \n#include \n", - ); - - let mut spi = lib(tmp.path(), "SPI"); - write(&spi.include_dirs[0].join("SPI.h"), ""); - let spi_cpp = spi.include_dirs[0].join("SPI.cpp"); - write(&spi_cpp, ""); - spi.source_files.push(spi_cpp); - - let mut wire = lib(tmp.path(), "Wire"); - write(&wire.include_dirs[0].join("Wire.h"), ""); - let wire_cpp = wire.include_dirs[0].join("Wire.cpp"); - write(&wire_cpp, ""); - wire.source_files.push(wire_cpp); - - let seeds = vec![project_src.join("main.cpp")]; - // Wire is passed BEFORE SPI in the input slice. - let sel = resolve(&seeds, &[project_src], &[wire, spi]); - assert_eq!( - sel.required_libraries, - vec!["SPI".to_string(), "Wire".to_string()] - ); - } - - // ---- lib_deps declarations (FastLED/fbuild#1214) ------------------------ - - /// Build the exact shape from the issue: a sketch that reaches SPI only - /// through a *library* header, which the shallow scan deliberately does - /// not follow. Returns (seeds, search paths, libs, SPI's .cpp). - fn unreachable_spi_fixture( - tmp: &Path, - ) -> (Vec, Vec, Vec, PathBuf) { - let project_src = tmp.join("project").join("src"); - write(&project_src.join("main.cpp"), "// no includes at all\n"); - - let mut spi = lib(tmp, "SPI"); - write(&spi.include_dirs[0].join("SPI.h"), ""); - let spi_cpp = spi.include_dirs[0].join("SPI.cpp"); - write(&spi_cpp, ""); - spi.source_files.push(spi_cpp.clone()); - - ( - vec![project_src.join("main.cpp")], - vec![project_src], - vec![spi], - spi_cpp, - ) - } - - /// Baseline: without a declaration the library is NOT selected. This is - /// the shallow-LDF default (#1094) and must stay that way — the fix adds - /// an opt-in, it does not deepen the scan. - #[test] - fn unreached_library_is_not_selected_without_a_declaration() { - let tmp = tempdir(); - let (seeds, paths, libs, _) = unreachable_spi_fixture(tmp.path()); - - let sel = resolve_with_stats_active_declared(&seeds, &paths, &libs, &HashMap::new(), &[]).0; - - assert!(sel.required_libraries.is_empty(), "{sel:?}"); - } - - #[test] - fn lib_deps_declaration_selects_an_unreached_library() { - let tmp = tempdir(); - let (seeds, paths, libs, spi_cpp) = unreachable_spi_fixture(tmp.path()); - - let sel = resolve_with_stats_active_declared( - &seeds, - &paths, - &libs, - &HashMap::new(), - &["SPI".to_string()], - ) - .0; - - assert_eq!(sel.required_libraries, vec!["SPI".to_string()]); - assert!( - sel.source_files.contains(&canon(&spi_cpp)) || sel.source_files.contains(&spi_cpp), - "declared library's sources must reach the link line: {sel:?}" - ); - } - - /// PlatformIO matches `lib_deps` entries case-insensitively and tolerates - /// owner prefixes and version specs. A declaration that doesn't match - /// because of a `@^1.0` suffix would look like the feature is broken. - #[test] - fn lib_deps_matching_ignores_case_owner_and_version() { - for entry in ["spi", "SPI@^1.0", "arduino/SPI", "arduino/SPI@1.2.3"] { - let tmp = tempdir(); - let (seeds, paths, libs, _) = unreachable_spi_fixture(tmp.path()); - - let sel = resolve_with_stats_active_declared( - &seeds, - &paths, - &libs, - &HashMap::new(), - &[entry.to_string()], - ) - .0; - - assert_eq!( - sel.required_libraries, - vec!["SPI".to_string()], - "entry {entry:?} should match the SPI framework library" - ); - } - } - - /// URLs and local paths name something to be *fetched*; they must not be - /// mangled into a bare name that accidentally matches a framework library. - #[test] - fn lib_deps_urls_and_paths_never_match_a_framework_library() { - for entry in [ - "https://github.com/example/SPI.git", - "file:///opt/SPI", - "./vendor/SPI", - ] { - let tmp = tempdir(); - let (seeds, paths, libs, _) = unreachable_spi_fixture(tmp.path()); - - let sel = resolve_with_stats_active_declared( - &seeds, - &paths, - &libs, - &HashMap::new(), - &[entry.to_string()], - ) - .0; - - assert!( - sel.required_libraries.is_empty(), - "entry {entry:?} must be left to the installer path, got {sel:?}" - ); - } - } - - /// An explicit declaration gets its own dependency chain resolved, exactly - /// as PlatformIO does — the declared library participates in the - /// reconciliation passes rather than being bolted on at the end. - #[test] - fn declared_library_pulls_in_its_own_transitive_dependency() { - let tmp = tempdir(); - let project_src = tmp.path().join("project").join("src"); - write(&project_src.join("main.cpp"), "// no includes\n"); - - let mut wire = lib(tmp.path(), "Wire"); - write(&wire.include_dirs[0].join("Wire.h"), ""); - let wire_cpp = wire.include_dirs[0].join("Wire.cpp"); - write(&wire_cpp, ""); - wire.source_files.push(wire_cpp); - - // SPI.cpp — not its header — is what reaches Wire, so only the - // reconciliation pass can find it. - let mut spi = lib(tmp.path(), "SPI"); - write(&spi.include_dirs[0].join("SPI.h"), ""); - let spi_cpp = spi.include_dirs[0].join("SPI.cpp"); - write(&spi_cpp, "#include \n"); - spi.source_files.push(spi_cpp); - - let seeds = vec![project_src.join("main.cpp")]; - let sel = resolve_with_stats_active_declared( - &seeds, - &[project_src], - &[spi, wire], - &HashMap::new(), - &["SPI".to_string()], - ) - .0; - - assert_eq!( - sel.required_libraries, - vec!["SPI".to_string(), "Wire".to_string()], - "declaring SPI must also bring in the Wire it depends on" - ); - } - - #[test] - fn bundled_framework_lib_deps_are_not_external() { - let tmp = tempdir(); - let libraries = vec![lib(tmp.path(), "BTstackLib"), lib(tmp.path(), "HTTPUpdate")]; - let declared = vec![ - "btSTACKlib".to_string(), - "vendor/HTTPUpdate@^1.3".to_string(), - "ExternalRegistryLib@^2.0".to_string(), - "https://example.com/vendor/local-lib.git".to_string(), - "https://example.com/vendor/HTTPUpdate".to_string(), - "file://vendor/BTstackLib".to_string(), - ]; - - assert_eq!( - external_declared_deps(&declared, &libraries), - vec![ - "ExternalRegistryLib@^2.0".to_string(), - "https://example.com/vendor/local-lib.git".to_string(), - "https://example.com/vendor/HTTPUpdate".to_string(), - "file://vendor/BTstackLib".to_string(), - ] - ); - } - - #[test] - fn declared_dep_name_normalization() { - assert_eq!(declared_dep_name("SPI").as_deref(), Some("spi")); - assert_eq!(declared_dep_name(" Wire@^1.0 ").as_deref(), Some("wire")); - assert_eq!( - declared_dep_name("adafruit/Adafruit GFX").as_deref(), - Some("adafruit gfx") - ); - assert_eq!(declared_dep_name(""), None); - assert_eq!(declared_dep_name("https://example.com/x.git"), None); - assert_eq!(declared_dep_name("./local"), None); - } -} +#[path = "lib_tests.rs"] +mod tests; diff --git a/crates/fbuild-library-select/src/lib_tests.rs b/crates/fbuild-library-select/src/lib_tests.rs new file mode 100644 index 00000000..0c3dced7 --- /dev/null +++ b/crates/fbuild-library-select/src/lib_tests.rs @@ -0,0 +1,623 @@ +//! Tests for [`super`]. +//! +//! Split out to keep the implementation file under the workspace 1000-LOC +//! limit; `compiler_tests.rs` is the same pattern. + +use super::*; +use tempfile::TempDir; + +fn write(path: &Path, contents: &str) { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(path, contents).unwrap(); +} + +fn lib(tmp: &Path, name: &str) -> FrameworkLibrary { + let dir = tmp.join("libraries").join(name); + let src = dir.join("src"); + std::fs::create_dir_all(&src).unwrap(); + FrameworkLibrary { + name: name.to_string(), + dir: dir.clone(), + include_dirs: vec![src.clone()], + source_files: Vec::new(), + } +} + +fn tempdir() -> TempDir { + TempDir::new_in(fbuild_paths::temp_subdir("fbuild-library-select-tests")).unwrap() +} + +#[test] +fn r01_direct_include_selects_library() { + let tmp = tempdir(); + let project_src = tmp.path().join("project").join("src"); + write(&project_src.join("main.cpp"), "#include \n"); + let mut spi = lib(tmp.path(), "SPI"); + write(&spi.include_dirs[0].join("SPI.h"), ""); + let spi_cpp = spi.include_dirs[0].join("SPI.cpp"); + write(&spi_cpp, ""); + spi.source_files.push(spi_cpp.clone()); + + let seeds = vec![project_src.join("main.cpp")]; + let sel = resolve(&seeds, &[project_src], &[spi]); + assert_eq!(sel.required_libraries, vec!["SPI".to_string()]); + assert!(sel.source_files.contains(&canon(&spi_cpp)) || sel.source_files.contains(&spi_cpp)); +} + +/// FastLED/fbuild#1371, end to end: a macro derived inside a header must +/// not hide the include it guards. +/// +/// This is the SAMD shape verbatim. The compiler command line carries +/// `__SAMD21G18A__`; `FL_IS_SAMD21` is derived from it several headers +/// deep, and header-defined macros are not threaded through a BFS walk. +/// Evaluating `#if defined(FL_IS_SAMD21)` as *false* made `` +/// invisible to selection even though the include is genuinely compiled — +/// the build then failed on a missing header that nothing had requested. +#[test] +fn include_guarded_by_a_header_derived_macro_still_selects_its_library() { + let tmp = tempdir(); + let project_src = tmp.path().join("project").join("src"); + write(&project_src.join("main.cpp"), "#include \n"); + write( + &project_src.join("FastLED.h"), + "#include \n#include \n", + ); + // The derivation the walk cannot see: command-line macro in, FastLED + // macro out. + write( + &project_src.join("is_platform.h"), + "#if defined(__SAMD21G18A__)\n#define FL_IS_SAMD21 1\n#endif\n", + ); + write( + &project_src.join("fastspi_arm_sam.h"), + "#if defined(FL_IS_SAMD21) || defined(FL_IS_SAMD51)\n#include \n#endif\n", + ); + + let mut spi = lib(tmp.path(), "SPI"); + write(&spi.include_dirs[0].join("SPI.h"), ""); + let spi_cpp = spi.include_dirs[0].join("SPI.cpp"); + write(&spi_cpp, ""); + spi.source_files.push(spi_cpp); + + let mut defines = HashMap::new(); + defines.insert("__SAMD21G18A__".to_string(), "1".to_string()); + + let seeds = vec![project_src.join("main.cpp")]; + let selection = resolve_active(&seeds, &[project_src], &[spi], &defines); + assert_eq!( + selection.required_libraries, + vec!["SPI".to_string()], + "an include behind a header-derived guard must still select its library" + ); +} + +/// The `#if 0` LDF hint idiom, end to end. +/// +/// `platforms/ldf_headers.h` in FastLED declares dependencies inside +/// `#if 0` blocks precisely because PlatformIO's `chain` LDF scans +/// includes without evaluating conditionals. The block never compiles, so +/// an include there exists only to be seen. +#[test] +fn if_zero_hint_headers_declare_a_dependency() { + let tmp = tempdir(); + let project_src = tmp.path().join("project").join("src"); + write(&project_src.join("main.cpp"), "#include \n"); + write(&project_src.join("FastLED.h"), "#include \n"); + write( + &project_src.join("ldf_headers.h"), + "#if 0\n#include \n#endif\n", + ); + + let mut spi = lib(tmp.path(), "SPI"); + write(&spi.include_dirs[0].join("SPI.h"), ""); + let spi_cpp = spi.include_dirs[0].join("SPI.cpp"); + write(&spi_cpp, ""); + spi.source_files.push(spi_cpp); + + let seeds = vec![project_src.join("main.cpp")]; + let selection = resolve_active(&seeds, &[project_src], &[spi], &HashMap::new()); + assert_eq!( + selection.required_libraries, + vec!["SPI".to_string()], + "an #if 0 hint must declare the dependency" + ); +} + +/// A guard on a macro *nothing* defines stays honestly false. +/// +/// The counterweight to the two tests above, and the reason this is not +/// simply a textual scan: without it, every unresolved guard would select +/// its library and #1094's over-selection would come straight back. +#[test] +fn guard_on_a_macro_no_file_defines_does_not_select() { + let tmp = tempdir(); + let project_src = tmp.path().join("project").join("src"); + write(&project_src.join("main.cpp"), "#include \n"); + write( + &project_src.join("FastLED.h"), + "#if defined(NOBODY_DEFINES_THIS)\n#include \n#endif\n#include \n", + ); + + let mut audio = lib(tmp.path(), "Audio"); + write(&audio.include_dirs[0].join("Audio.h"), ""); + let audio_cpp = audio.include_dirs[0].join("Audio.cpp"); + write(&audio_cpp, ""); + audio.source_files.push(audio_cpp); + + let mut spi = lib(tmp.path(), "SPI"); + write(&spi.include_dirs[0].join("SPI.h"), ""); + let spi_cpp = spi.include_dirs[0].join("SPI.cpp"); + write(&spi_cpp, ""); + spi.source_files.push(spi_cpp); + + let seeds = vec![project_src.join("main.cpp")]; + let selection = resolve_active(&seeds, &[project_src], &[audio, spi], &HashMap::new()); + assert_eq!( + selection.required_libraries, + vec!["SPI".to_string()], + "a guard nothing can satisfy must not pull in a library" + ); +} + +#[test] +fn active_resolution_skips_library_in_disabled_branch() { + let tmp = tempdir(); + let project_src = tmp.path().join("project").join("src"); + write( + &project_src.join("main.cpp"), + "#define USE_SPI 1\n#include \n", + ); + write( + &project_src.join("FastLED.h"), + "#if defined(USE_AUDIO)\n#include \n#elif USE_SPI\n#include \n#endif\n", + ); + + let mut audio = lib(tmp.path(), "Audio"); + write(&audio.include_dirs[0].join("Audio.h"), ""); + let audio_cpp = audio.include_dirs[0].join("Audio.cpp"); + write(&audio_cpp, ""); + audio.source_files.push(audio_cpp); + + let mut spi = lib(tmp.path(), "SPI"); + write(&spi.include_dirs[0].join("SPI.h"), ""); + let spi_cpp = spi.include_dirs[0].join("SPI.cpp"); + write(&spi_cpp, ""); + spi.source_files.push(spi_cpp); + + let seeds = vec![project_src.join("main.cpp")]; + let selection = resolve_active(&seeds, &[project_src], &[audio, spi], &HashMap::new()); + assert_eq!(selection.required_libraries, vec!["SPI".to_string()]); +} + +#[test] +fn r02_transitive_library_selection() { + let tmp = tempdir(); + let project_src = tmp.path().join("project").join("src"); + write(&project_src.join("main.cpp"), "#include \n"); + + let mut spi = lib(tmp.path(), "SPI"); + write(&spi.include_dirs[0].join("SPI.h"), "#include \n"); + let spi_cpp = spi.include_dirs[0].join("SPI.cpp"); + write(&spi_cpp, ""); + spi.source_files.push(spi_cpp); + + let mut wire = lib(tmp.path(), "Wire"); + write(&wire.include_dirs[0].join("Wire.h"), ""); + let wire_cpp = wire.include_dirs[0].join("Wire.cpp"); + write(&wire_cpp, ""); + wire.source_files.push(wire_cpp); + + let seeds = vec![project_src.join("main.cpp")]; + let sel = resolve(&seeds, &[project_src], &[spi, wire]); + assert_eq!( + sel.required_libraries, + vec!["SPI".to_string(), "Wire".to_string()] + ); +} + +#[test] +fn r04_pass2_reconciliation_catches_cpp_only_dependency() { + // The whole reason the LDF resolver is 2-pass instead of single-pass + // BFS: a lib's `.cpp` may pull in a second lib that the first lib's + // `.h` does NOT mention. Pass 1 (BFS from project seeds + reached + // headers) cannot see that edge; pass 2 re-seeds with each selected + // lib's full source set and catches it. + // + // Setup: project includes . SPI.h is silent. SPI.cpp includes + // . Wire is only reachable through SPI.cpp. + // + // Expected: pass 1 selects {SPI}; pass 2 (with SPI.cpp as a seed) + // selects {SPI, Wire}. A regression that drops the second pass would + // produce {SPI} only and silently miss Wire at link time. + let tmp = tempdir(); + let project_src = tmp.path().join("project").join("src"); + write(&project_src.join("main.cpp"), "#include \n"); + + let mut spi = lib(tmp.path(), "SPI"); + write( + &spi.include_dirs[0].join("SPI.h"), + "// no transitive includes\n", + ); + let spi_cpp = spi.include_dirs[0].join("SPI.cpp"); + write(&spi_cpp, "#include \n"); + spi.source_files.push(spi_cpp); + + let mut wire = lib(tmp.path(), "Wire"); + write(&wire.include_dirs[0].join("Wire.h"), ""); + let wire_cpp = wire.include_dirs[0].join("Wire.cpp"); + write(&wire_cpp, ""); + wire.source_files.push(wire_cpp); + + let seeds = vec![project_src.join("main.cpp")]; + let sel = resolve(&seeds, &[project_src], &[spi, wire]); + assert_eq!( + sel.required_libraries, + vec!["SPI".to_string(), "Wire".to_string()], + "pass 2 reconciliation must catch Wire reached only via SPI.cpp" + ); +} + +#[test] +fn r03_no_includes_selects_nothing() { + let tmp = tempdir(); + let project_src = tmp.path().join("project").join("src"); + write(&project_src.join("main.cpp"), "int main() { return 0; }\n"); + let spi = lib(tmp.path(), "SPI"); + write(&spi.include_dirs[0].join("SPI.h"), ""); + + let seeds = vec![project_src.join("main.cpp")]; + let sel = resolve(&seeds, &[project_src], &[spi]); + assert!(sel.required_libraries.is_empty()); +} + +#[test] +fn r13_unrelated_library_not_selected() { + let tmp = tempdir(); + let project_src = tmp.path().join("project").join("src"); + write(&project_src.join("main.cpp"), "#include \n"); + + let mut spi = lib(tmp.path(), "SPI"); + write(&spi.include_dirs[0].join("SPI.h"), ""); + let spi_cpp = spi.include_dirs[0].join("SPI.cpp"); + write(&spi_cpp, ""); + spi.source_files.push(spi_cpp); + + let mut fnet = lib(tmp.path(), "FNET"); + write(&fnet.include_dirs[0].join("fnet.h"), ""); + let fnet_cpp = fnet.include_dirs[0].join("fnet.cpp"); + write(&fnet_cpp, ""); + fnet.source_files.push(fnet_cpp); + + let seeds = vec![project_src.join("main.cpp")]; + let sel = resolve(&seeds, &[project_src], &[spi, fnet]); + assert_eq!(sel.required_libraries, vec!["SPI".to_string()]); +} + +#[test] +fn path_prefix_attribution_distinguishes_same_basename() { + let tmp = tempdir(); + let project_src = tmp.path().join("project").join("src"); + write(&project_src.join("main.cpp"), "#include \"foo/config.h\"\n"); + + let mut foo = lib(tmp.path(), "Foo"); + write(&foo.include_dirs[0].join("foo").join("config.h"), ""); + let foo_cpp = foo.include_dirs[0].join("Foo.cpp"); + write(&foo_cpp, ""); + foo.source_files.push(foo_cpp); + + let mut bar = lib(tmp.path(), "Bar"); + // Bar also has a config.h but at its own path — must NOT be selected + // when the project only includes "foo/config.h". + write(&bar.include_dirs[0].join("bar").join("config.h"), ""); + let bar_cpp = bar.include_dirs[0].join("Bar.cpp"); + write(&bar_cpp, ""); + bar.source_files.push(bar_cpp); + + let seeds = vec![project_src.join("main.cpp")]; + let sel = resolve( + &seeds, + &[ + project_src, + foo.include_dirs[0].clone(), + bar.include_dirs[0].clone(), + ], + &[foo, bar], + ); + assert_eq!(sel.required_libraries, vec!["Foo".to_string()]); +} + +#[test] +fn empty_libraries_yields_empty_selection() { + // Adversary: no libraries at all. resolve must terminate cleanly with + // no required_libraries, no panics, and any reached files limited to + // what the walker found from seeds alone. + let tmp = tempdir(); + let project_src = tmp.path().join("project").join("src"); + write(&project_src.join("main.cpp"), "int main() { return 0; }\n"); + let seeds = vec![project_src.join("main.cpp")]; + let sel = resolve(&seeds, &[project_src], &[]); + assert!(sel.required_libraries.is_empty()); + assert!(sel.source_files.is_empty()); +} + +#[test] +fn missing_library_include_dir_does_not_panic() { + // Adversary: a FrameworkLibrary whose include_dirs point at a path + // that doesn't exist on disk (broken framework install, lib not yet + // downloaded). canon() falls back and emits a tracing::warn; the + // resolver must not panic and must return a sensible empty + // selection. + let tmp = tempdir(); + let project_src = tmp.path().join("project").join("src"); + write(&project_src.join("main.cpp"), "int main() { return 0; }\n"); + let phantom = FrameworkLibrary { + name: "Phantom".to_string(), + dir: tmp.path().join("nonexistent").join("Phantom"), + include_dirs: vec![tmp.path().join("nonexistent").join("Phantom").join("src")], + source_files: Vec::new(), + }; + let seeds = vec![project_src.join("main.cpp")]; + let sel = resolve(&seeds, &[project_src], &[phantom]); + assert!(sel.required_libraries.is_empty()); +} + +#[test] +fn many_libraries_in_random_order_returns_sorted() { + // Adversary: 6 libs in deliberately scrambled input order. The + // output must be sorted lexicographically, independent of input + // order — required for stable cache keys (#205 Phase 4). + let tmp = tempdir(); + let project_src = tmp.path().join("project").join("src"); + write( + &project_src.join("main.cpp"), + "#include \n#include \n#include \n\ + #include \n#include \n#include \n", + ); + + let mut libs = Vec::new(); + for name in ["Z", "A", "M", "B", "Y", "K"] { + let mut l = lib(tmp.path(), name); + write(&l.include_dirs[0].join(format!("{name}.h")), ""); + let cpp = l.include_dirs[0].join(format!("{name}.cpp")); + write(&cpp, ""); + l.source_files.push(cpp); + libs.push(l); + } + + let seeds = vec![project_src.join("main.cpp")]; + let sel = resolve(&seeds, &[project_src], &libs); + assert_eq!( + sel.required_libraries, + ["A", "B", "K", "M", "Y", "Z"] + .iter() + .map(|s| s.to_string()) + .collect::>() + ); +} + +#[test] +fn required_libraries_returned_sorted_by_name_not_input_order() { + // Regression guard: pass the libraries in REVERSE name order (Wire + // before SPI) and confirm the output is sorted lexicographically. + // The doc on `Selection::required_libraries` and the cache-key story + // in #205 Phase 4 both depend on this being a pure function of the + // selected *set* of libraries, not their input position. + let tmp = tempdir(); + let project_src = tmp.path().join("project").join("src"); + write( + &project_src.join("main.cpp"), + "#include \n#include \n", + ); + + let mut spi = lib(tmp.path(), "SPI"); + write(&spi.include_dirs[0].join("SPI.h"), ""); + let spi_cpp = spi.include_dirs[0].join("SPI.cpp"); + write(&spi_cpp, ""); + spi.source_files.push(spi_cpp); + + let mut wire = lib(tmp.path(), "Wire"); + write(&wire.include_dirs[0].join("Wire.h"), ""); + let wire_cpp = wire.include_dirs[0].join("Wire.cpp"); + write(&wire_cpp, ""); + wire.source_files.push(wire_cpp); + + let seeds = vec![project_src.join("main.cpp")]; + // Wire is passed BEFORE SPI in the input slice. + let sel = resolve(&seeds, &[project_src], &[wire, spi]); + assert_eq!( + sel.required_libraries, + vec!["SPI".to_string(), "Wire".to_string()] + ); +} + +// ---- lib_deps declarations (FastLED/fbuild#1214) ------------------------ + +/// Build the exact shape from the issue: a sketch that reaches SPI only +/// through a *library* header, which the shallow scan deliberately does +/// not follow. Returns (seeds, search paths, libs, SPI's .cpp). +fn unreachable_spi_fixture( + tmp: &Path, +) -> (Vec, Vec, Vec, PathBuf) { + let project_src = tmp.join("project").join("src"); + write(&project_src.join("main.cpp"), "// no includes at all\n"); + + let mut spi = lib(tmp, "SPI"); + write(&spi.include_dirs[0].join("SPI.h"), ""); + let spi_cpp = spi.include_dirs[0].join("SPI.cpp"); + write(&spi_cpp, ""); + spi.source_files.push(spi_cpp.clone()); + + ( + vec![project_src.join("main.cpp")], + vec![project_src], + vec![spi], + spi_cpp, + ) +} + +/// Baseline: without a declaration the library is NOT selected. This is +/// the shallow-LDF default (#1094) and must stay that way — the fix adds +/// an opt-in, it does not deepen the scan. +#[test] +fn unreached_library_is_not_selected_without_a_declaration() { + let tmp = tempdir(); + let (seeds, paths, libs, _) = unreachable_spi_fixture(tmp.path()); + + let sel = resolve_with_stats_active_declared(&seeds, &paths, &libs, &HashMap::new(), &[]).0; + + assert!(sel.required_libraries.is_empty(), "{sel:?}"); +} + +#[test] +fn lib_deps_declaration_selects_an_unreached_library() { + let tmp = tempdir(); + let (seeds, paths, libs, spi_cpp) = unreachable_spi_fixture(tmp.path()); + + let sel = resolve_with_stats_active_declared( + &seeds, + &paths, + &libs, + &HashMap::new(), + &["SPI".to_string()], + ) + .0; + + assert_eq!(sel.required_libraries, vec!["SPI".to_string()]); + assert!( + sel.source_files.contains(&canon(&spi_cpp)) || sel.source_files.contains(&spi_cpp), + "declared library's sources must reach the link line: {sel:?}" + ); +} + +/// PlatformIO matches `lib_deps` entries case-insensitively and tolerates +/// owner prefixes and version specs. A declaration that doesn't match +/// because of a `@^1.0` suffix would look like the feature is broken. +#[test] +fn lib_deps_matching_ignores_case_owner_and_version() { + for entry in ["spi", "SPI@^1.0", "arduino/SPI", "arduino/SPI@1.2.3"] { + let tmp = tempdir(); + let (seeds, paths, libs, _) = unreachable_spi_fixture(tmp.path()); + + let sel = resolve_with_stats_active_declared( + &seeds, + &paths, + &libs, + &HashMap::new(), + &[entry.to_string()], + ) + .0; + + assert_eq!( + sel.required_libraries, + vec!["SPI".to_string()], + "entry {entry:?} should match the SPI framework library" + ); + } +} + +/// URLs and local paths name something to be *fetched*; they must not be +/// mangled into a bare name that accidentally matches a framework library. +#[test] +fn lib_deps_urls_and_paths_never_match_a_framework_library() { + for entry in [ + "https://github.com/example/SPI.git", + "file:///opt/SPI", + "./vendor/SPI", + ] { + let tmp = tempdir(); + let (seeds, paths, libs, _) = unreachable_spi_fixture(tmp.path()); + + let sel = resolve_with_stats_active_declared( + &seeds, + &paths, + &libs, + &HashMap::new(), + &[entry.to_string()], + ) + .0; + + assert!( + sel.required_libraries.is_empty(), + "entry {entry:?} must be left to the installer path, got {sel:?}" + ); + } +} + +/// An explicit declaration gets its own dependency chain resolved, exactly +/// as PlatformIO does — the declared library participates in the +/// reconciliation passes rather than being bolted on at the end. +#[test] +fn declared_library_pulls_in_its_own_transitive_dependency() { + let tmp = tempdir(); + let project_src = tmp.path().join("project").join("src"); + write(&project_src.join("main.cpp"), "// no includes\n"); + + let mut wire = lib(tmp.path(), "Wire"); + write(&wire.include_dirs[0].join("Wire.h"), ""); + let wire_cpp = wire.include_dirs[0].join("Wire.cpp"); + write(&wire_cpp, ""); + wire.source_files.push(wire_cpp); + + // SPI.cpp — not its header — is what reaches Wire, so only the + // reconciliation pass can find it. + let mut spi = lib(tmp.path(), "SPI"); + write(&spi.include_dirs[0].join("SPI.h"), ""); + let spi_cpp = spi.include_dirs[0].join("SPI.cpp"); + write(&spi_cpp, "#include \n"); + spi.source_files.push(spi_cpp); + + let seeds = vec![project_src.join("main.cpp")]; + let sel = resolve_with_stats_active_declared( + &seeds, + &[project_src], + &[spi, wire], + &HashMap::new(), + &["SPI".to_string()], + ) + .0; + + assert_eq!( + sel.required_libraries, + vec!["SPI".to_string(), "Wire".to_string()], + "declaring SPI must also bring in the Wire it depends on" + ); +} + +#[test] +fn bundled_framework_lib_deps_are_not_external() { + let tmp = tempdir(); + let libraries = vec![lib(tmp.path(), "BTstackLib"), lib(tmp.path(), "HTTPUpdate")]; + let declared = vec![ + "btSTACKlib".to_string(), + "vendor/HTTPUpdate@^1.3".to_string(), + "ExternalRegistryLib@^2.0".to_string(), + "https://example.com/vendor/local-lib.git".to_string(), + "https://example.com/vendor/HTTPUpdate".to_string(), + "file://vendor/BTstackLib".to_string(), + ]; + + assert_eq!( + external_declared_deps(&declared, &libraries), + vec![ + "ExternalRegistryLib@^2.0".to_string(), + "https://example.com/vendor/local-lib.git".to_string(), + "https://example.com/vendor/HTTPUpdate".to_string(), + "file://vendor/BTstackLib".to_string(), + ] + ); +} + +#[test] +fn declared_dep_name_normalization() { + assert_eq!(declared_dep_name("SPI").as_deref(), Some("spi")); + assert_eq!(declared_dep_name(" Wire@^1.0 ").as_deref(), Some("wire")); + assert_eq!( + declared_dep_name("adafruit/Adafruit GFX").as_deref(), + Some("adafruit gfx") + ); + assert_eq!(declared_dep_name(""), None); + assert_eq!(declared_dep_name("https://example.com/x.git"), None); + assert_eq!(declared_dep_name("./local"), None); +} diff --git a/docs/architecture/library-selection.md b/docs/architecture/library-selection.md index 148ae023..dd1c9083 100644 --- a/docs/architecture/library-selection.md +++ b/docs/architecture/library-selection.md @@ -94,6 +94,45 @@ before its framework dependencies can be selected. This prevents an inactive header anywhere in a large library from self-selecting an unrelated framework library (FastLED/fbuild#1094). +## Conditional includes: decided, undecidable, or a hint + +The walk evaluates preprocessor branches, but only where it honestly can. The +macro set it is handed is the *compiler command line* — macros a header +`#define`s are not threaded through, because the walker visits each file once +in BFS order with a shared cache, and that is not preprocessor order. + +So a guard is resolved three ways: + +| guard | treatment | +|---|---| +| decidable from the command-line macros | evaluated; the dead arm is pruned | +| references a macro the reachable corpus `#define`s somewhere | **undecidable** — every arm is scanned | +| references a macro *nothing* defines | honestly false; the arm is pruned | + +The middle row is FastLED/fbuild#1371: FastLED derives `FL_IS_SAMD21` several +headers deep from `-D__SAMD21G18A__`, so `#if defined(FL_IS_SAMD21)` is not +false — it is unknowable from the command line, and treating it as false hid +an `#include ` that genuinely compiles. The third row is what keeps +this from collapsing into a plain textual scan and reviving the +over-selection of #1094. + +A literal `#if 0` block is scanned as well. An include that can never compile +is there only to be seen: the PlatformIO LDF hint idiom, which FastLED uses in +`platforms/*/ldf_headers.h`. Its `#define`s are *not* applied, since that code +does not run. + +Two separate comparisons with PlatformIO, since it is easy to conflate them: + +- **Against `chain`**, which evaluates no conditionals at all, fbuild is + *stricter*: a guard the command line settles is settled here, and its dead + arm is pruned. +- **Against `chain+`**, which does evaluate conditionals, fbuild is *more + permissive*: `chain+` has no undecidable case, so a guard on a macro it + cannot see reads as false — the exact failure this rule exists to avoid. + +The `#if 0` hint works under `chain` only as a side effect of it evaluating +nothing. `chain+` does not honor it. fbuild honors it deliberately. + ## Why two-pass (not fixed-point) PlatformIO `chain` mode runs BFS from project sources, then ONE diff --git a/dylints/ban_std_pathbuf/Cargo.lock b/dylints/ban_std_pathbuf/Cargo.lock index 0fc7ed4b..c2c604a0 100644 --- a/dylints/ban_std_pathbuf/Cargo.lock +++ b/dylints/ban_std_pathbuf/Cargo.lock @@ -69,7 +69,7 @@ checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" [[package]] name = "ban_std_pathbuf" -version = "0.2.0" +version = "0.2.1" dependencies = [ "dylint_linting", "dylint_testing", diff --git a/dylints/ban_std_pathbuf/Cargo.toml b/dylints/ban_std_pathbuf/Cargo.toml index c76fe4a8..136e9aa8 100644 --- a/dylints/ban_std_pathbuf/Cargo.toml +++ b/dylints/ban_std_pathbuf/Cargo.toml @@ -5,7 +5,7 @@ name = "ban_std_pathbuf" # src/allowlist.txt, so a content-only change to the allowlist does # not invalidate the cached compiled plugin and the OLD allowlist # stays in effect. -version = "0.2.0" +version = "0.2.1" description = "Ban std::path::PathBuf outside the legacy allowlist" edition = "2021" publish = false diff --git a/dylints/ban_std_pathbuf/src/allowlist.txt b/dylints/ban_std_pathbuf/src/allowlist.txt index e197b1cf..1a468a0a 100644 --- a/dylints/ban_std_pathbuf/src/allowlist.txt +++ b/dylints/ban_std_pathbuf/src/allowlist.txt @@ -151,6 +151,11 @@ crates/fbuild-library-select/benches/resolve_cold.rs crates/fbuild-library-select/benches/resolve_warm.rs crates/fbuild-library-select/src/cache.rs crates/fbuild-library-select/src/lib.rs +# Same code as `lib.rs` above, just relocated: the tests moved into +# their own file to keep `lib.rs` under the 1000-LOC gate +# (FastLED/fbuild#1371). A file move must not silently drop the +# exception the moved code already had. +crates/fbuild-library-select/src/lib_tests.rs crates/fbuild-library-select/tests/teensy41_ldf_diag.rs crates/fbuild-packages-fetch/src/cache.rs crates/fbuild-packages-fetch/src/disk_cache/gc.rs