diff --git a/Cargo.lock b/Cargo.lock index 1923b57..2aa86dc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -204,6 +204,7 @@ dependencies = [ "memmap2", "rayon", "regex", + "regex-syntax", "serde", "serde_json", "tempfile", diff --git a/Cargo.toml b/Cargo.toml index a4af229..b10196a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,6 +30,7 @@ ignore = "0.4" memmap2 = "0.9" rayon = "1.10" regex = { version = "1.11", default-features = false, features = ["std", "unicode-perl"] } +regex-syntax = { version = "0.8", default-features = false, features = ["std", "unicode-perl"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" toml = "1.1" diff --git a/src/patterns.rs b/src/patterns.rs index 9601017..b96ee48 100644 --- a/src/patterns.rs +++ b/src/patterns.rs @@ -1,5 +1,6 @@ use anyhow::{Context, Result, bail}; use regex::{Regex, RegexSet}; +use regex_syntax::hir::{Hir, HirKind}; use serde::Deserialize; use std::collections::{HashMap, HashSet}; @@ -127,6 +128,7 @@ pub struct ConstantPatterns { #[derive(Debug, Clone)] pub struct PatternSet { pub libraries: Vec, + /// Conservative whole-file hints; authoritative include regexes live in the owner sets. pub include_sets: HashMap, pub api_sets: HashMap, /// Pre-compiled include patterns per language with library ownership for find_library_anchors @@ -246,7 +248,16 @@ impl PatternSet { for (lang, patterns) in include_patterns { if !patterns.is_empty() { let regex_set = RegexSet::new(&patterns)?; - include_sets.insert(lang, regex_set.clone()); + let hints = patterns + .iter() + .map(|pattern| { + regex_syntax::Parser::new() + .parse(pattern) + .map(|hir| without_assertions(hir).to_string()) + .context("compile include hint") + }) + .collect::, _>>()?; + include_sets.insert(lang, RegexSet::new(hints)?); include_sets_with_owners.insert( lang, IncludeSetWithOwners { @@ -360,6 +371,29 @@ impl PatternSet { } } +// Include regexes run on individual AST nodes. Their start/end and boundary +// assertions do not necessarily hold in the enclosing file. Removing zero-width +// assertions produces a conservative hint without changing authoritative matching. +fn without_assertions(hir: Hir) -> Hir { + match hir.into_kind() { + HirKind::Empty | HirKind::Look(_) => Hir::empty(), + HirKind::Literal(literal) => Hir::literal(literal.0), + HirKind::Class(class) => Hir::class(class), + HirKind::Repetition(mut repetition) => { + repetition.sub = Box::new(without_assertions(*repetition.sub)); + Hir::repetition(repetition) + } + HirKind::Capture(mut capture) => { + capture.sub = Box::new(without_assertions(*capture.sub)); + Hir::capture(capture) + } + HirKind::Concat(parts) => Hir::concat(parts.into_iter().map(without_assertions).collect()), + HirKind::Alternation(parts) => { + Hir::alternation(parts.into_iter().map(without_assertions).collect()) + } + } +} + fn toml_value_to_json(v: toml::Value) -> serde_json::Value { match v { toml::Value::String(s) => serde_json::Value::String(s), diff --git a/src/scan.rs b/src/scan.rs index cfe2b41..7974413 100644 --- a/src/scan.rs +++ b/src/scan.rs @@ -1,5 +1,6 @@ use ahash::{AHashMap as HashMap, AHashSet as HashSet}; use anyhow::{Context, Result}; +use std::borrow::Cow; use std::ops::ControlFlow; use std::time::{Duration, Instant}; use tree_sitter::{Language as TsLanguage, Node, ParseOptions, Parser, Point, Tree}; @@ -137,6 +138,7 @@ pub fn find_library_anchors<'a>( patterns: &'a PatternSet, ) -> Vec> { let mut hits = Vec::new(); + let code = without_comments(content, tree); // Handle libraries without include patterns (fallback to api_regexes) for lib in &patterns.libraries { @@ -145,7 +147,7 @@ pub fn find_library_anchors<'a>( } if lib.include_regexes.is_empty() { // Fallback: scan entire content with api_regexes as a coarse anchor (no AST import nodes) - if lib.api_regexes.iter().any(|re| re.is_match(content)) { + if lib.api_regexes.iter().any(|re| re.is_match(&code)) { hits.push(LibraryHit { library_name: &lib.name, line: 1, @@ -222,6 +224,8 @@ pub fn find_algorithms<'a>( primitive_by_alg.insert(alg.name.clone(), primitive.clone()); } } + let code = without_comments(content, tree); + let content = code.as_ref(); let constants = collect_constants(lang, content, patterns); // Build line cache for fast line/column lookups (O(n) once, O(log n) per lookup) let line_cache = LineCache::new(content); @@ -607,6 +611,36 @@ fn replace_constants_with_map( (resolved, map) } +// Replace AST comment bytes with spaces while retaining newlines and byte +// offsets. String contents (including URLs and comment-like text) stay intact. +fn without_comments<'a>(content: &'a str, tree: &Tree) -> Cow<'a, str> { + let mut masked: Option> = None; + let mut cursor = tree.walk(); + loop { + let node = cursor.node(); + if node.kind() == "comment" || node.kind().ends_with("_comment") { + let bytes = masked.get_or_insert_with(|| content.as_bytes().to_vec()); + for byte in &mut bytes[node.byte_range()] { + if !matches!(*byte, b'\n' | b'\r') { + *byte = b' '; + } + } + } else if cursor.goto_first_child() { + continue; + } + loop { + if cursor.goto_next_sibling() { + break; + } + if !cursor.goto_parent() { + return masked.map_or(Cow::Borrowed(content), |bytes| { + Cow::Owned(String::from_utf8(bytes).expect("mask preserves UTF-8")) + }); + } + } + } +} + fn import_like_nodes<'a>(lang: Language, root: Node<'a>, content: &[u8]) -> Vec> { let mut nodes = Vec::new(); let mut stack = vec![root]; diff --git a/tests/detection_accuracy.rs b/tests/detection_accuracy.rs new file mode 100644 index 0000000..b1c14e5 --- /dev/null +++ b/tests/detection_accuracy.rs @@ -0,0 +1,134 @@ +#[cfg(any(feature = "lang-c", feature = "lang-python"))] +use cipherscope::{patterns::Language, scan_snippet}; + +#[cfg(feature = "lang-c")] +#[test] +fn include_after_copyright_is_not_skipped_by_hint() { + let findings = scan_snippet( + "// Copyright\n#include \n", + Language::C, + "source.c", + ) + .unwrap(); + assert_eq!(findings.len(), 1); + assert_eq!(findings[0].identifier, "OpenSSL"); + assert_eq!(findings[0].evidence.line, 2); +} + +#[cfg(feature = "lang-c")] +#[test] +fn comments_do_not_create_algorithm_findings() { + let findings = scan_snippet( + "#include \n// EVP_aes_256_gcm();\n/* EVP_aes_128_cbc(); */\n", + Language::C, + "source.c", + ) + .unwrap(); + assert_eq!(findings.len(), 1); + assert_eq!(findings[0].identifier, "OpenSSL"); +} + +#[cfg(feature = "lang-c")] +#[test] +fn comments_inside_calls_cannot_trigger_symbol_matches() { + let findings = scan_snippet( + "#include \nvoid f() { unrelated(/* EVP_aes_256_gcm() */); }\n", + Language::C, + "source.c", + ) + .unwrap(); + assert_eq!(findings.len(), 1); +} + +#[cfg(feature = "lang-python")] +#[test] +fn api_only_library_cannot_be_anchored_by_a_comment() { + let patterns = cipherscope::patterns::PatternSet::from_toml( + r#" +[[library]] +name = "TestLib" +languages = ["Python"] +[library.patterns] +apis = ["CryptoLib"] +"#, + ) + .unwrap(); + let findings = cipherscope::scan_with_patterns( + "# CryptoLib.new()\npass\n", + Language::Python, + "source.py", + &patterns, + ) + .unwrap(); + assert!(findings.is_empty()); +} + +#[cfg(feature = "lang-python")] +#[test] +fn node_anchors_work_inside_indented_code_with_absolute_regex_anchors() { + let patterns = cipherscope::patterns::PatternSet::from_toml( + r#" +[[library]] +name = "TestLib" +languages = ["Python"] +[library.patterns] +include = ['\Aimport testlib\z'] +"#, + ) + .unwrap(); + let findings = cipherscope::scan_with_patterns( + "def f():\n import testlib\n pass\n", + Language::Python, + "source.py", + &patterns, + ) + .unwrap(); + assert_eq!(findings.len(), 1); + assert_eq!(findings[0].evidence.line, 2); + assert_eq!(findings[0].evidence.column, 5); + assert!(!cipherscope::scan::has_anchor_hint( + Language::Python, + "print('hello')", + &patterns, + )); +} + +#[cfg(feature = "lang-c")] +#[test] +fn comment_masking_preserves_unicode_byte_columns_and_real_calls() { + let source = "#include \nvoid f() { /* café */ EVP_aes_256_gcm(); }\n"; + let findings = scan_snippet(source, Language::C, "source.c").unwrap(); + let hit = findings.iter().find(|f| f.identifier == "AES-GCM").unwrap(); + assert_eq!(hit.evidence.line, 2); + assert_eq!( + hit.evidence.column, + source.lines().nth(1).unwrap().find("EVP").unwrap() + 1 + ); + assert_eq!(hit.metadata["keySize"], 256); +} + +#[cfg(feature = "lang-c")] +#[test] +fn string_contents_are_preserved_when_masking_comments() { + let patterns = cipherscope::patterns::PatternSet::from_toml( + r#" +[[library]] +name = "TestLib" +languages = ["C"] +[library.patterns] +include = ['testlib.h'] +[[library.algorithms]] +name = "TestAlgorithm" +symbol_patterns = ['https://example.test/crypto'] +"#, + ) + .unwrap(); + let findings = cipherscope::scan_with_patterns( + "#include \nvoid f() { call(\"https://example.test/crypto\"); }\n", + Language::C, + "source.c", + &patterns, + ) + .unwrap(); + assert!(findings.iter().any(|hit| hit.identifier == "TestAlgorithm")); +}