From 9f32f4699897a997e20f20dfc5dd3a7f86069bdf Mon Sep 17 00:00:00 2001 From: liquidsec Date: Mon, 26 Oct 2026 14:57:11 -0400 Subject: [PATCH 1/3] Don't count Imperva's loader script as a challenge --- CHANGELOG.md | 4 +++ src/antibot.rs | 7 +++++- tests/antibot_classify.rs | 51 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fa54503..aeee662 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,10 @@ the ladder falling back to `compatibility` on the second handshake. JavaScript challenge, which no HTTP client passes, so the URL should go to a real browser rather than be retried - **`Response.attempts`** lists what the ladder tried, in order +- Imperva's `_Incapsula_Resource` loader script no longer counts as a + challenge. Imperva appends it to pages it let through, so ordinary pages and + the origin's own error pages were judged `needs_browser`. Imperva's incident + page still counts - **TLS failures now say why.** The reason used to be destroyed in transit: hyper-util's error displays as the literal "client error (Connect)" for every connect-time failure, so a cipher mismatch, a rejected certificate, a DNS diff --git a/src/antibot.rs b/src/antibot.rs index 5c859bd..3e91110 100644 --- a/src/antibot.rs +++ b/src/antibot.rs @@ -215,7 +215,12 @@ pub fn classify(facts: &ResponseFacts) -> Outcome { } // ── Imperva / Incapsula ─────────────────────────────────────── - if facts.body_contains("_Incapsula_Resource") { + // The `&ns=1&cb=` script is the loader Imperva adds to pages it let + // through, the origin's own error pages included, so on its own it's not + // an interception. Imperva's incident page frames itself as `CWUDNSAI`. + if facts.body_contains("_Incapsula_Resource?CWUDNSAI=") + || (facts.body_contains("_Incapsula_Resource") && !facts.body_contains("&ns=1&cb=")) + { return Outcome::Challenge(Vendor::Imperva); } diff --git a/tests/antibot_classify.rs b/tests/antibot_classify.rs index 891f7b3..5a2eb8e 100644 --- a/tests/antibot_classify.rs +++ b/tests/antibot_classify.rs @@ -167,6 +167,57 @@ fn test_akamai_sec_cpt_428_is_its_own_outcome() { ); } +/// The shape of Imperva's incident page, with its identifiers replaced. +const IMPERVA_INCIDENT: &str = concat!( + r#""#, + r#""#, + r#""# +); + +/// The loader Imperva adds to the end of pages it let through. +const IMPERVA_LOADER: &str = r#""#; + +#[test] +fn test_imperva_incident_page_is_a_challenge() { + let h = hdrs(&[("x-cdn", "Imperva")]); + for status in [200, 403, 503] { + assert_eq!( + classify(&facts(status, &h, IMPERVA_INCIDENT)), + Outcome::Challenge(Vendor::Imperva) + ); + } +} + +#[test] +fn test_imperva_loader_on_a_page_is_not_a_challenge() { + // The origin's own page, a 200 or its own 403, with Imperva's loader + // appended. + let h = hdrs(&[("x-cdn", "Imperva")]); + let page = format!( + "IIS Windows Serverwelcome{IMPERVA_LOADER}" + ); + let refusal = format!("

403 Forbidden

{IMPERVA_LOADER}"); + for (status, body) in [(200, &page), (403, &refusal)] { + assert_ne!( + classify(&facts(status, &h, body)), + Outcome::Challenge(Vendor::Imperva) + ); + } +} + +#[test] +fn test_imperva_script_without_the_loader_form_is_a_challenge() { + // Anything else naming the resource still counts, as before. + let h = hdrs(&[]); + let body = r#""#; + assert_eq!( + classify(&facts(200, &h, body)), + Outcome::Challenge(Vendor::Imperva) + ); +} + // ── The false positives that caught the first pass ──────────────── #[test] From 841f3b0da0a8c2def12cad79cb06712a90f889ae Mon Sep 17 00:00:00 2001 From: liquidsec Date: Mon, 26 Oct 2026 15:22:12 -0400 Subject: [PATCH 2/3] Move the anti-bot rules out of code into antibot_rules.json --- CHANGELOG.md | 5 + README.md | 2 +- antibot_rules.json | 126 ++++++++++++ src/antibot.rs | 311 +++++++++++++++-------------- src/python.rs | 22 ++ src/report.rs | 22 +- src/response.rs | 7 +- tests/antibot_classify.rs | 34 ++-- tests/antibot_rules.rs | 115 +++++++++++ tests/antibot_set_rules.rs | 31 +++ tests/python/test_antibot_rules.py | 51 +++++ 11 files changed, 547 insertions(+), 179 deletions(-) create mode 100644 antibot_rules.json create mode 100644 tests/antibot_rules.rs create mode 100644 tests/antibot_set_rules.rs create mode 100644 tests/python/test_antibot_rules.py diff --git a/CHANGELOG.md b/CHANGELOG.md index aeee662..8e5d803 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,11 @@ the ladder falling back to `compatibility` on the second handshake. challenge. Imperva appends it to pages it let through, so ordinary pages and the origin's own error pages were judged `needs_browser`. Imperva's incident page still counts +- **The anti-bot rules are data.** What `protection` recognises now lives in + `antibot_rules.json` instead of code, and `blasthttp.set_antibot_rules(path)` + (`antibot::set_rules` in Rust) replaces them at runtime, so a rule can change + without a rebuild. `Vendor` is a name rather than a fixed list, so a rules + file can add a product - **TLS failures now say why.** The reason used to be destroyed in transit: hyper-util's error displays as the literal "client error (Connect)" for every connect-time failure, so a cipher mismatch, a rejected certificate, a DNS diff --git a/README.md b/README.md index 9eb29d8..28352b6 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ cargo add blasthttp - **Rust performance** — async I/O, zero-copy where possible, and native concurrency give significant speed improvements over pure Python HTTP clients - **SSL cert info on every request** — extracts CN, SANs, emails, issuer, validity dates, and fingerprint during the TLS handshake that's already happening, eliminating the need for a separate connection to gather this information - **Connection profiles, and automatic shifting between them**: `modern` by default (11 cipher suites, TLS 1.2+), `compatibility` for servers that speak only RC4, 3DES, SEED, Camellia, anonymous DH or SSLv3, and `chrome` to imitate a browser's TLS, HTTP/2 and headers together. A handshake the peer cannot negotiate widens the offer; a refusal from a recognised protection product changes what the client claims to be. What works is remembered per host -- **Says why it failed**: a blocked request reports which product blocked it, and a JavaScript challenge reports `needs_browser`, because no HTTP client passes that tier and retrying will not change it +- **Says why it failed**: a blocked request reports which product blocked it, and a JavaScript challenge reports `needs_browser`, because no HTTP client passes that tier and retrying will not change it. The products and how to recognise them are rules in [`antibot_rules.json`](antibot_rules.json), and `blasthttp.set_antibot_rules(path)` swaps in your own at runtime - **No cert validation by default** — offensive-first: connects to self-signed, expired, and misconfigured TLS without extra config - **HTTP/2 support** — automatic via ALPN negotiation, falls back to HTTP/1.1 - **Low-level primitives** — `RawConnection` for byte-level TCP/TLS I/O (bypasses HTTP framing entirely) and `blasthttp.h2` for manual H2 frame construction (includes a permissive HPACK encoder that emits bytes a strict encoder refuses, the building block for H2 smuggling / CRLF-injection tooling) diff --git a/antibot_rules.json b/antibot_rules.json new file mode 100644 index 0000000..b3bc56e --- /dev/null +++ b/antibot_rules.json @@ -0,0 +1,126 @@ +{ + "refusal_statuses": [401, 403, 405, 406, 429, 503], + "rules": [ + { + "vendor": "cloudflare", + "outcome": "challenge", + "when": {"header_contains": {"name": "cf-mitigated", "value": "challenge"}}, + "why": "The unambiguous one, and the only header Cloudflare sets specifically to say it intervened." + }, + { + "vendor": "cloudflare", + "outcome": "challenge", + "when": {"any": [ + {"body_contains": "/cdn-cgi/challenge-platform/"}, + {"body_contains": "window._cf_chl_opt"} + ]}, + "why": "The challenge page's own markers, for when cf-mitigated is missing." + }, + { + "vendor": "akamai", + "outcome": "blocked", + "when": {"any": [ + {"body_contains": "edgesuite"}, + {"all": [ + {"any": [ + {"header_contains": {"name": "server", "value": "akamaighost"}}, + {"header_present": "akamai-grn"}, + {"header_present": "x-akam-sw-version"}, + {"header_present": "x-akamai-transformed"}, + {"header_contains": {"name": "server-timing", "value": "ak_p"}} + ]}, + {"body_contains": "Access Denied"} + ]} + ]}, + "why": "The edge error page. It HTML-entity-encodes its own punctuation, so the body carries errors.edgesuite.net and a search for errors.edgesuite.net never fires; the bare label edgesuite survives that. server: AkamaiGHost is not reliably present either: www.akamai.com's own 403 sends no server header and identifies itself through akamai-grn, x-akam-sw-version and an ak_p server-timing entry instead, so any one of the edge headers will do. Reference # on its own stays insufficient, because plenty of unrelated error pages carry one." + }, + { + "vendor": "akamai", + "outcome": "challenge", + "when": {"all": [ + {"status": [428]}, + {"any": [{"cookie_set": "sec_cpt"}, {"body_contains": "sec-cpt"}]} + ]}, + "why": "sec-cpt is an adaptive challenge that fires after _abck already passed, and it answers 428 rather than 403. Its own outcome, not a block." + }, + { + "vendor": "datadome", + "outcome": "challenge", + "when": {"body_contains": "geo.captcha-delivery.com"} + }, + { + "vendor": "perimeterx", + "outcome": "challenge", + "when": {"any": [{"body_contains": "/px/captcha"}, {"body_contains": "px-captcha"}]} + }, + { + "vendor": "imperva", + "outcome": "challenge", + "when": {"any": [ + {"body_contains": "_Incapsula_Resource?CWUDNSAI="}, + {"all": [ + {"body_contains": "_Incapsula_Resource"}, + {"not": {"body_contains": "&ns=1&cb="}} + ]} + ]}, + "why": "The &ns=1&cb= script is the loader Imperva adds to pages it let through, the origin's own error pages included, so on its own it's not an interception. Imperva's incident page frames itself as CWUDNSAI." + }, + { + "vendor": "awswaf", + "outcome": "challenge", + "when": {"header_present": "x-amzn-waf-action"}, + "why": "The header carries the action, and there is more than one: challenge for the silent JavaScript kind, captcha for the kind wanting a person. Matching only challenge missed wizzair.com, which answers 405 with captcha, so the ladder read a genuine interception as an ordinary refusal and declined to try anything else. Match the header's presence and let the value be detail." + }, + { + "vendor": "kasada", + "outcome": "blocked", + "when": {"all": [ + {"any": [{"header_present": "x-kpsdk-ct"}, {"body_contains": "KPSDK"}]}, + {"status": [429]} + ]}, + "why": "Kasada answers 429 rather than 403 for a refusal." + }, + { + "vendor": "kasada", + "outcome": "present", + "when": {"any": [{"header_present": "x-kpsdk-ct"}, {"body_contains": "KPSDK"}]} + }, + { + "vendor": "cloudflare", + "outcome": "present", + "on_refusal": "blocked", + "when": {"any": [{"cookie_set": "__cf_bm"}, {"cookie_set": "cf_clearance"}]}, + "why": "From here on, sightings of a product on a response nothing above matched. One that set its cookie while refusing us is still a block." + }, + { + "vendor": "akamai", + "outcome": "present", + "on_refusal": "blocked", + "when": {"any": [{"cookie_set": "_abck"}, {"cookie_set": "ak_bmsc"}, {"cookie_set": "bm_sz"}]} + }, + { + "vendor": "datadome", + "outcome": "present", + "on_refusal": "blocked", + "when": {"cookie_set": "datadome"} + }, + { + "vendor": "perimeterx", + "outcome": "present", + "on_refusal": "blocked", + "when": {"cookie_prefix": "_px"} + }, + { + "vendor": "imperva", + "outcome": "present", + "on_refusal": "blocked", + "when": {"any": [{"cookie_prefix": "visid_incap_"}, {"cookie_prefix": "incap_ses_"}]} + }, + { + "vendor": "f5", + "outcome": "present", + "on_refusal": "blocked", + "when": {"header_contains": {"name": "server", "value": "volt-adc"}} + } + ] +} diff --git a/src/antibot.rs b/src/antibot.rs index 3e91110..8e7e7e6 100644 --- a/src/antibot.rs +++ b/src/antibot.rs @@ -16,38 +16,39 @@ //! 2. A response that merely *references* a vendor's script is not a challenge //! page. Every page on a protected site carries the loader; only a page //! shaped like an interception is a block. +//! +//! The rules themselves are data, in `antibot_rules.json`: the copy built in +//! is the default, and [`set_rules`] swaps in another at runtime, so a rule +//! can change without a rebuild. use std::fmt; +use std::sync::{Arc, LazyLock, RwLock}; + +use serde::Deserialize; + +/// Which product answered, named by the rule that matched. Deliberately +/// conservative: guessing a vendor wrong is worse than reporting unknown. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct Vendor(String); + +impl Vendor { + pub fn new(name: impl Into) -> Self { + Vendor(name.into()) + } -/// Which product answered. Not exhaustive, and deliberately conservative: -/// guessing a vendor wrong is worse than reporting `Unknown`. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum Vendor { - Cloudflare, - Akamai, - DataDome, - PerimeterX, - Imperva, - F5, - Kasada, - AwsWaf, - Unknown, + /// A refusal no rule could put a name to. + pub fn unknown() -> Self { + Vendor::new("unknown") + } + + pub fn is_unknown(&self) -> bool { + self.0 == "unknown" + } } impl fmt::Display for Vendor { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let s = match self { - Vendor::Cloudflare => "cloudflare", - Vendor::Akamai => "akamai", - Vendor::DataDome => "datadome", - Vendor::PerimeterX => "perimeterx", - Vendor::Imperva => "imperva", - Vendor::F5 => "f5", - Vendor::Kasada => "kasada", - Vendor::AwsWaf => "awswaf", - Vendor::Unknown => "unknown", - }; - f.write_str(s) + f.write_str(&self.0) } } @@ -92,12 +93,12 @@ impl Outcome { /// fingerprint. The cost of the alternative is doubling the request count /// of every scan that touches a 403, which is most of them. pub fn indicates_protection(&self) -> bool { - !matches!(self.vendor(), None | Some(Vendor::Unknown)) + self.vendor().is_some_and(|v| !v.is_unknown()) } - pub fn vendor(&self) -> Option { + pub fn vendor(&self) -> Option<&Vendor> { match self { - Outcome::Present(v) | Outcome::Challenge(v) | Outcome::Blocked(v) => Some(*v), + Outcome::Present(v) | Outcome::Challenge(v) | Outcome::Blocked(v) => Some(v), _ => None, } } @@ -161,146 +162,154 @@ impl ResponseFacts<'_> { } } -/// Classify one response. +/// The rules a classification runs, as loaded from `antibot_rules.json`. /// -/// Order matters: a challenge outranks a mere sighting of the vendor, and a -/// block outranks both, because a host that challenges us has still told us -/// more than one that refuses outright. -pub fn classify(facts: &ResponseFacts) -> Outcome { - // ── Cloudflare ──────────────────────────────────────────────── - // `cf-mitigated: challenge` is the unambiguous one and the only header - // Cloudflare sets specifically to say it intervened. - if facts.header_matches("cf-mitigated", "challenge") { - return Outcome::Challenge(Vendor::Cloudflare); - } - if facts.body_contains("/cdn-cgi/challenge-platform/") - || facts.body_contains("window._cf_chl_opt") - { - return Outcome::Challenge(Vendor::Cloudflare); - } +/// Order matters, and the first rule to match decides: a challenge outranks a +/// mere sighting of the vendor, and a block outranks both, because a host that +/// challenges us has still told us more than one that refuses outright. +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Rules { + /// Statuses a protection product uses to refuse. 429 belongs because a rate + /// limit and a bot block are the same outcome for a scan, whoever sent it. + refusal_statuses: Vec, + rules: Vec, +} - // ── Akamai ──────────────────────────────────────────────────── - // The edge error page, which is fussier to spot than it looks. - // - // It HTML-entity-encodes its own punctuation, so the body carries - // `errors.edgesuite.net` and a search for `errors.edgesuite.net` - // never fires. Matching the bare label `edgesuite` survives that, since - // only the dots and slashes get encoded. - // - // `server: AkamaiGHost` is also not reliably present: www.akamai.com's own - // 403 sends no `server` header at all and identifies itself through - // `akamai-grn`, `x-akam-sw-version` and an `ak_p` server-timing entry - // instead. So the server header is one signal among several rather than a - // requirement. `Reference #` on its own stays insufficient, because plenty - // of unrelated error pages carry one. - if facts.body_contains("edgesuite") - || (is_akamai_edge(facts) && facts.body_contains("Access Denied")) - { - return Outcome::Blocked(Vendor::Akamai); - } - // sec-cpt is an adaptive challenge that fires after _abck already passed, - // and it answers 428 rather than 403. Its own outcome, not a block. - if facts.status == 428 && (facts.sets_cookie("sec_cpt") || facts.body_contains("sec-cpt")) { - return Outcome::Challenge(Vendor::Akamai); - } +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct Rule { + vendor: String, + outcome: RuleOutcome, + /// What the outcome becomes when the status is a refusal. + on_refusal: Option, + when: Condition, + /// Why the rule is shaped the way it is. Not used, but worth keeping. + #[allow(dead_code)] + why: Option, +} - // ── DataDome ────────────────────────────────────────────────── - if facts.body_contains("geo.captcha-delivery.com") { - return Outcome::Challenge(Vendor::DataDome); - } +#[derive(Debug, Clone, Copy, Deserialize)] +#[serde(rename_all = "snake_case")] +enum RuleOutcome { + Present, + Challenge, + Blocked, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "snake_case", deny_unknown_fields)] +enum Condition { + All(Vec), + Any(Vec), + Not(Box), + Status(Statuses), + HeaderPresent(String), + /// Case-insensitive, in both the name and the value. + HeaderContains { + name: String, + value: String, + }, + CookieSet(String), + CookiePrefix(String), + BodyContains(String), +} - // ── PerimeterX / HUMAN ──────────────────────────────────────── - if facts.body_contains("/px/captcha") || facts.body_contains("px-captcha") { - return Outcome::Challenge(Vendor::PerimeterX); +#[derive(Debug, Deserialize)] +#[serde(untagged)] +enum Statuses { + Codes(Vec), + Named(NamedStatuses), +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "snake_case")] +enum NamedStatuses { + Refusal, +} + +impl Rules { + pub fn from_json(json: &str) -> Result { + let rules: Rules = serde_json::from_str(json)?; + if rules.rules.iter().any(|r| r.vendor.is_empty()) { + return Err(serde::de::Error::custom("every rule needs a vendor")); + } + Ok(rules) } - // ── Imperva / Incapsula ─────────────────────────────────────── - // The `&ns=1&cb=` script is the loader Imperva adds to pages it let - // through, the origin's own error pages included, so on its own it's not - // an interception. Imperva's incident page frames itself as `CWUDNSAI`. - if facts.body_contains("_Incapsula_Resource?CWUDNSAI=") - || (facts.body_contains("_Incapsula_Resource") && !facts.body_contains("&ns=1&cb=")) - { - return Outcome::Challenge(Vendor::Imperva); + /// Classify one response. + pub fn classify(&self, facts: &ResponseFacts) -> Outcome { + let refusal = self.is_refusal(facts.status); + if let Some(rule) = self.rules.iter().find(|r| self.matches(&r.when, facts)) { + let vendor = Vendor::new(&rule.vendor); + return match (refusal, rule.on_refusal) { + (true, Some(outcome)) => outcome, + _ => rule.outcome, + } + .with(vendor); + } + // Nothing identifiable + if (200..400).contains(&facts.status) { + Outcome::Ok + } else if refusal { + Outcome::Blocked(Vendor::unknown()) + } else { + Outcome::Error + } } - // ── AWS WAF ─────────────────────────────────────────────────── - // The header carries the action, and there is more than one: `challenge` - // for the silent JavaScript kind, `captcha` for the kind wanting a person. - // Matching only `challenge` missed wizzair.com, which answers 405 with - // `captcha`, so the ladder read a genuine interception as an ordinary - // refusal and declined to try anything else. Match the header's presence - // and let the value be detail. - if facts.has_header("x-amzn-waf-action") { - return Outcome::Challenge(Vendor::AwsWaf); + fn is_refusal(&self, status: u16) -> bool { + self.refusal_statuses.contains(&status) } - // ── Kasada ──────────────────────────────────────────────────── - // Kasada answers 429 rather than 403 for a refusal. - if facts.has_header("x-kpsdk-ct") || facts.body_contains("KPSDK") { - return if facts.status == 429 { - Outcome::Blocked(Vendor::Kasada) - } else { - Outcome::Present(Vendor::Kasada) - }; + fn matches(&self, condition: &Condition, facts: &ResponseFacts) -> bool { + match condition { + Condition::All(conditions) => conditions.iter().all(|c| self.matches(c, facts)), + Condition::Any(conditions) => conditions.iter().any(|c| self.matches(c, facts)), + Condition::Not(condition) => !self.matches(condition, facts), + Condition::Status(Statuses::Codes(codes)) => codes.contains(&facts.status), + Condition::Status(Statuses::Named(NamedStatuses::Refusal)) => { + self.is_refusal(facts.status) + } + Condition::HeaderPresent(name) => facts.has_header(name), + Condition::HeaderContains { name, value } => { + facts.header_matches(name, &value.to_ascii_lowercase()) + } + Condition::CookieSet(name) => facts.sets_cookie(name), + Condition::CookiePrefix(prefix) => facts.sets_cookie_prefixed(prefix), + Condition::BodyContains(needle) => facts.body_contains(needle), + } } +} - // ── Product present, and we got through ─────────────────────── - // Reached only when nothing above fired, so these are sightings on a - // response that actually carried content. - let present = if facts.sets_cookie("__cf_bm") || facts.sets_cookie("cf_clearance") { - Some(Vendor::Cloudflare) - } else if facts.sets_cookie("_abck") - || facts.sets_cookie("ak_bmsc") - || facts.sets_cookie("bm_sz") - { - Some(Vendor::Akamai) - } else if facts.sets_cookie("datadome") { - Some(Vendor::DataDome) - } else if facts.sets_cookie_prefixed("_px") { - Some(Vendor::PerimeterX) - } else if facts.sets_cookie_prefixed("visid_incap_") || facts.sets_cookie_prefixed("incap_ses_") - { - Some(Vendor::Imperva) - } else if facts.header_matches("server", "volt-adc") { - Some(Vendor::F5) - } else { - None - }; - - if let Some(vendor) = present { - // A product that set its cookie while refusing us is still a block. - return if is_refusal(facts.status) { - Outcome::Blocked(vendor) - } else { - Outcome::Present(vendor) - }; +impl RuleOutcome { + fn with(self, vendor: Vendor) -> Outcome { + match self { + RuleOutcome::Present => Outcome::Present(vendor), + RuleOutcome::Challenge => Outcome::Challenge(vendor), + RuleOutcome::Blocked => Outcome::Blocked(vendor), + } } +} - // ── Nothing identifiable ────────────────────────────────────── - if (200..400).contains(&facts.status) { - Outcome::Ok - } else if is_refusal(facts.status) { - Outcome::Blocked(Vendor::Unknown) - } else { - Outcome::Error +impl Default for Rules { + /// The rules built in from `antibot_rules.json`. + fn default() -> Self { + Rules::from_json(include_str!("../antibot_rules.json")) + .expect("the built-in antibot_rules.json is valid") } } -/// Statuses a protection product uses to refuse. 429 is in here because a rate -/// limit and a bot block are the same outcome for a scan, whoever sent it. -fn is_refusal(status: u16) -> bool { - matches!(status, 401 | 403 | 405 | 406 | 429 | 503) +static RULES: LazyLock>> = + LazyLock::new(|| RwLock::new(Arc::new(Rules::default()))); + +/// Replace the rules every later classification runs, in this process. +pub fn set_rules(rules: Rules) { + *RULES.write().unwrap_or_else(|e| e.into_inner()) = Arc::new(rules); } -/// Is this response coming off Akamai's edge at all? -/// -/// Any one of these is enough. They are the headers the edge attaches to its -/// own responses regardless of whether it is serving content or refusing. -fn is_akamai_edge(facts: &ResponseFacts) -> bool { - facts.header_matches("server", "akamaighost") - || facts.has_header("akamai-grn") - || facts.has_header("x-akam-sw-version") - || facts.has_header("x-akamai-transformed") - || facts.header_matches("server-timing", "ak_p") +/// Classify one response with the current rules. +pub fn classify(facts: &ResponseFacts) -> Outcome { + let rules = RULES.read().unwrap_or_else(|e| e.into_inner()).clone(); + rules.classify(facts) } diff --git a/src/python.rs b/src/python.rs index af39244..00b5466 100644 --- a/src/python.rs +++ b/src/python.rs @@ -2193,6 +2193,27 @@ fn register_h2_submodule<'py>(parent: &Bound<'py, PyModule>) -> PyResult<()> { Ok(()) } +/// Replace the anti-bot rules every response in this process is classified +/// with, from a JSON file shaped like the built-in `antibot_rules.json`. +/// `None` restores the built-in rules. Raises ValueError on a bad file. +#[pyfunction] +#[pyo3(signature = (path=None))] +fn set_antibot_rules(path: Option) -> PyResult<()> { + let rules = match path { + None => crate::antibot::Rules::default(), + Some(path) => { + let json = std::fs::read_to_string(&path).map_err(|e| { + pyo3::exceptions::PyValueError::new_err(format!("{}: {e}", path.display())) + })?; + crate::antibot::Rules::from_json(&json).map_err(|e| { + pyo3::exceptions::PyValueError::new_err(format!("{}: {e}", path.display())) + })? + } + }; + crate::antibot::set_rules(rules); + Ok(()) +} + // ── Module registration ─────────────────────────────────────────── #[pymodule] @@ -2211,6 +2232,7 @@ fn blasthttp(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_function(wrap_pyfunction!(set_antibot_rules, m)?)?; m.add("HTTPStatusError", m.py().get_type::())?; m.add("TransportError", m.py().get_type::())?; register_h2_submodule(m)?; diff --git a/src/report.rs b/src/report.rs index bb7ed42..673ea63 100644 --- a/src/report.rs +++ b/src/report.rs @@ -20,7 +20,7 @@ use serde::{Deserialize, Serialize}; -use crate::antibot::{Outcome, Vendor}; +use crate::antibot::Outcome; /// What one rung of the ladder produced. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -62,7 +62,7 @@ impl Attempt { pub fn from_outcome(profile: &str, status: u16, outcome: &Outcome) -> Self { let vendor = outcome .vendor() - .filter(|v| !matches!(v, Vendor::Unknown)) + .filter(|v| !v.is_unknown()) .map(|v| v.to_string()); let o = match outcome { Outcome::Ok | Outcome::Present(_) => AttemptOutcome::Reached { status }, @@ -111,7 +111,7 @@ impl Conclusion { pub fn from_attempts(last: &Outcome, attempts: &[Attempt]) -> Self { let vendor = last .vendor() - .filter(|v| !matches!(v, Vendor::Unknown)) + .filter(|v| !v.is_unknown()) .map(|v| v.to_string()); match last { Outcome::Ok | Outcome::Present(_) => Conclusion::Reached, @@ -132,13 +132,15 @@ impl Conclusion { #[cfg(test)] mod tests { use super::*; + use crate::antibot::Vendor; #[test] fn test_a_challenge_asks_for_a_browser_and_a_block_does_not() { // The distinction the whole escalation path rests on. A block might // yield to different settings; a challenge needs JavaScript and never // will, so only one of these justifies starting a browser. - let challenge = Conclusion::from_attempts(&Outcome::Challenge(Vendor::Cloudflare), &[]); + let challenge = + Conclusion::from_attempts(&Outcome::Challenge(Vendor::new("cloudflare")), &[]); assert!(challenge.needs_browser()); assert_eq!( challenge, @@ -147,7 +149,7 @@ mod tests { } ); - let blocked = Conclusion::from_attempts(&Outcome::Blocked(Vendor::Akamai), &[]); + let blocked = Conclusion::from_attempts(&Outcome::Blocked(Vendor::new("akamai")), &[]); assert!(!blocked.needs_browser()); } @@ -155,13 +157,13 @@ mod tests { fn test_an_unnamed_vendor_is_left_unnamed() { // Reporting "unknown" as though it were a product would invite a // consumer to branch on it. An absent vendor is the honest answer. - let blocked = Conclusion::from_attempts(&Outcome::Blocked(Vendor::Unknown), &[]); + let blocked = Conclusion::from_attempts(&Outcome::Blocked(Vendor::unknown()), &[]); assert_eq!(blocked, Conclusion::Blocked { vendor: None }); } #[test] fn test_reaching_it_is_not_an_escalation() { - for outcome in [Outcome::Ok, Outcome::Present(Vendor::Akamai)] { + for outcome in [Outcome::Ok, Outcome::Present(Vendor::new("akamai"))] { let c = Conclusion::from_attempts(&outcome, &[]); assert_eq!(c, Conclusion::Reached); assert!(!c.needs_browser()); @@ -171,7 +173,11 @@ mod tests { #[test] fn test_attempts_record_the_profile_that_produced_them() { // The log is only useful if it says which rung did what. - let a = Attempt::from_outcome("modern", 403, &Outcome::Challenge(Vendor::PerimeterX)); + let a = Attempt::from_outcome( + "modern", + 403, + &Outcome::Challenge(Vendor::new("perimeterx")), + ); assert_eq!(a.profile, "modern"); assert_eq!( a.outcome, diff --git a/src/response.rs b/src/response.rs index 5f25b5c..c4b7820 100644 --- a/src/response.rs +++ b/src/response.rs @@ -534,7 +534,10 @@ mod protection_tests { ], "\nAccess Denied\n", ); - assert_eq!(blocked.protection(), Outcome::Blocked(Vendor::Akamai)); + assert_eq!( + blocked.protection(), + Outcome::Blocked(Vendor::new("akamai")) + ); } #[test] @@ -548,7 +551,7 @@ mod protection_tests { ); assert_eq!( challenge.protection(), - Outcome::Challenge(Vendor::Cloudflare) + Outcome::Challenge(Vendor::new("cloudflare")) ); let ordinary = response_with(200, &[("server", "nginx")], "hello"); diff --git a/tests/antibot_classify.rs b/tests/antibot_classify.rs index 5a2eb8e..6c3c030 100644 --- a/tests/antibot_classify.rs +++ b/tests/antibot_classify.rs @@ -66,7 +66,7 @@ fn test_akamai_access_denied_is_a_block() { ]); assert_eq!( classify(&facts(403, &h, AKAMAI_DENIED)), - Outcome::Blocked(Vendor::Akamai) + Outcome::Blocked(Vendor::new("akamai")) ); } @@ -80,7 +80,7 @@ fn test_akamai_block_detected_without_a_server_header() { &h, "\nAccess Denied\n" )), - Outcome::Blocked(Vendor::Akamai) + Outcome::Blocked(Vendor::new("akamai")) ); } @@ -90,7 +90,7 @@ fn test_entity_encoded_edgesuite_still_matches() { let h = hdrs(&[("content-type", "text/html")]); assert_eq!( classify(&facts(403, &h, AKAMAI_DENIED)), - Outcome::Blocked(Vendor::Akamai) + Outcome::Blocked(Vendor::new("akamai")) ); } @@ -106,7 +106,7 @@ fn test_akamai_session_cookie_on_success_is_present() { ("set-cookie", "bm_sz=8C2...; Path=/"), ]); let outcome = classify(&facts(200, &h, "real content")); - assert_eq!(outcome, Outcome::Present(Vendor::Akamai)); + assert_eq!(outcome, Outcome::Present(Vendor::new("akamai"))); assert!(outcome.got_through()); } @@ -122,7 +122,7 @@ fn test_cloudflare_managed_challenge() { ]); assert_eq!( classify(&facts(403, &h, "Just a moment...")), - Outcome::Challenge(Vendor::Cloudflare) + Outcome::Challenge(Vendor::new("cloudflare")) ); } @@ -137,7 +137,7 @@ fn test_cloudflare_waf_block_is_not_a_challenge() { "expected a block, got {:?}", outcome ); - assert_ne!(outcome, Outcome::Challenge(Vendor::Cloudflare)); + assert_ne!(outcome, Outcome::Challenge(Vendor::new("cloudflare"))); } #[test] @@ -149,7 +149,7 @@ fn test_cloudflare_bm_cookie_on_success_is_present() { ]); assert_eq!( classify(&facts(200, &h, "content")), - Outcome::Present(Vendor::Cloudflare) + Outcome::Present(Vendor::new("cloudflare")) ); } @@ -163,7 +163,7 @@ fn test_akamai_sec_cpt_428_is_its_own_outcome() { ]); assert_eq!( classify(&facts(428, &h, r#"{"chlg_duration":30,"difficulty":5}"#)), - Outcome::Challenge(Vendor::Akamai) + Outcome::Challenge(Vendor::new("akamai")) ); } @@ -185,7 +185,7 @@ fn test_imperva_incident_page_is_a_challenge() { for status in [200, 403, 503] { assert_eq!( classify(&facts(status, &h, IMPERVA_INCIDENT)), - Outcome::Challenge(Vendor::Imperva) + Outcome::Challenge(Vendor::new("imperva")) ); } } @@ -202,7 +202,7 @@ fn test_imperva_loader_on_a_page_is_not_a_challenge() { for (status, body) in [(200, &page), (403, &refusal)] { assert_ne!( classify(&facts(status, &h, body)), - Outcome::Challenge(Vendor::Imperva) + Outcome::Challenge(Vendor::new("imperva")) ); } } @@ -214,7 +214,7 @@ fn test_imperva_script_without_the_loader_form_is_a_challenge() { let body = r#""#; assert_eq!( classify(&facts(200, &h, body)), - Outcome::Challenge(Vendor::Imperva) + Outcome::Challenge(Vendor::new("imperva")) ); } @@ -266,7 +266,7 @@ fn test_reference_number_alone_is_not_akamai() { &h, "

Access Denied

Reference #12345

", )); - assert_eq!(outcome, Outcome::Blocked(Vendor::Unknown)); + assert_eq!(outcome, Outcome::Blocked(Vendor::unknown())); } // ── Ordinary responses ──────────────────────────────────────────── @@ -293,7 +293,7 @@ fn test_unidentified_refusal_reports_unknown_vendor() { // to guess a vendor. let h = hdrs(&[("server", "nginx")]); let outcome = classify(&facts(403, &h, "forbidden")); - assert_eq!(outcome, Outcome::Blocked(Vendor::Unknown)); + assert_eq!(outcome, Outcome::Blocked(Vendor::unknown())); assert!(!outcome.got_through()); } @@ -305,7 +305,7 @@ fn test_product_cookie_with_refusal_still_counts_as_blocked() { ("set-cookie", "__cf_bm=xyz; Path=/"), ]); let outcome = classify(&facts(403, &h, "denied")); - assert_eq!(outcome, Outcome::Blocked(Vendor::Cloudflare)); + assert_eq!(outcome, Outcome::Blocked(Vendor::new("cloudflare"))); assert!(!outcome.got_through()); } @@ -321,7 +321,7 @@ fn test_aws_waf_captcha_is_a_challenge() { ("x-cache", "Error from cloudfront"), ]); let outcome = classify(&facts(405, &h, "")); - assert_eq!(outcome, Outcome::Challenge(Vendor::AwsWaf)); + assert_eq!(outcome, Outcome::Challenge(Vendor::new("awswaf"))); assert!( outcome.indicates_protection(), "the ladder only moves on positive evidence of a product" @@ -336,7 +336,7 @@ fn test_a_bare_403_is_not_evidence_of_protection() { // that touches one. let h = hdrs(&[("server", "nginx")]); let outcome = classify(&facts(403, &h, "forbidden")); - assert_eq!(outcome, Outcome::Blocked(Vendor::Unknown)); + assert_eq!(outcome, Outcome::Blocked(Vendor::unknown())); assert!(!outcome.indicates_protection()); } @@ -366,6 +366,6 @@ fn test_a_marker_in_a_multibyte_body_is_still_found() { let body = format!("{}window._cf_chl_opt", "\u{5ae}".repeat(10)); assert_eq!( classify(&facts(403, &h, &body)), - Outcome::Challenge(Vendor::Cloudflare) + Outcome::Challenge(Vendor::new("cloudflare")) ); } diff --git a/tests/antibot_rules.rs b/tests/antibot_rules.rs new file mode 100644 index 0000000..ae14509 --- /dev/null +++ b/tests/antibot_rules.rs @@ -0,0 +1,115 @@ +// Loading anti-bot rules from JSON: what a rules file can say, and how a bad +// one fails. + +use blasthttp::antibot::{Outcome, ResponseFacts, Rules, Vendor}; + +fn facts<'a>(status: u16, headers: &'a [(String, String)], body: &'a str) -> ResponseFacts<'a> { + ResponseFacts { + status, + headers, + body, + } +} + +fn header(name: &str, value: &str) -> Vec<(String, String)> { + vec![(name.to_string(), value.to_string())] +} + +const CUSTOM: &str = r#"{ + "refusal_statuses": [403], + "rules": [ + { + "vendor": "examplewaf", + "outcome": "challenge", + "when": {"all": [{"status": "refusal"}, {"body_contains": "example-challenge"}]} + }, + { + "vendor": "examplewaf", + "outcome": "present", + "on_refusal": "blocked", + "when": {"cookie_prefix": "exwaf_"} + }, + { + "vendor": "otherwaf", + "outcome": "blocked", + "when": {"all": [ + {"header_contains": {"name": "X-Edge", "value": "OtherWAF"}}, + {"not": {"status": [200]}} + ]} + } + ] +}"#; + +#[test] +fn test_a_rules_file_can_name_a_new_vendor() { + let rules = Rules::from_json(CUSTOM).unwrap(); + let none = vec![]; + assert_eq!( + rules.classify(&facts(403, &none, "example-challenge")), + Outcome::Challenge(Vendor::new("examplewaf")) + ); + // the same body on a status the file doesn't call a refusal + assert_eq!( + rules.classify(&facts(503, &none, "example-challenge")), + Outcome::Error + ); +} + +#[test] +fn test_on_refusal_changes_the_outcome_on_a_refusal_only() { + let rules = Rules::from_json(CUSTOM).unwrap(); + let cookie = header("set-cookie", "exwaf_session=abc; Path=/"); + assert_eq!( + rules.classify(&facts(200, &cookie, "welcome")), + Outcome::Present(Vendor::new("examplewaf")) + ); + assert_eq!( + rules.classify(&facts(403, &cookie, "go away")), + Outcome::Blocked(Vendor::new("examplewaf")) + ); +} + +#[test] +fn test_header_contains_ignores_case() { + let rules = Rules::from_json(CUSTOM).unwrap(); + let edge = header("x-edge", "otherwaf-7"); + assert_eq!( + rules.classify(&facts(404, &edge, "")), + Outcome::Blocked(Vendor::new("otherwaf")) + ); + assert_eq!(rules.classify(&facts(200, &edge, "")), Outcome::Ok); +} + +#[test] +fn test_a_rules_file_replaces_the_built_in_rules() { + let rules = Rules::from_json(CUSTOM).unwrap(); + let mitigated = header("cf-mitigated", "challenge"); + assert_eq!( + Rules::default().classify(&facts(403, &mitigated, "")), + Outcome::Challenge(Vendor::new("cloudflare")) + ); + assert_eq!( + rules.classify(&facts(403, &mitigated, "")), + Outcome::Blocked(Vendor::unknown()) + ); +} + +#[test] +fn test_a_bad_rules_file_is_refused() { + for bad in [ + // not JSON + "{", + // a condition that doesn't exist + r#"{"refusal_statuses": [], "rules": [{"vendor": "x", "outcome": "challenge", "when": {"body_matches": "x"}}]}"#, + // an outcome that doesn't exist + r#"{"refusal_statuses": [], "rules": [{"vendor": "x", "outcome": "maybe", "when": {"body_contains": "x"}}]}"#, + // a misspelled field + r#"{"refusal_statuses": [], "rules": [{"vendor": "x", "outcome": "challenge", "wen": {"body_contains": "x"}}]}"#, + // no vendor + r#"{"refusal_statuses": [], "rules": [{"vendor": "", "outcome": "challenge", "when": {"body_contains": "x"}}]}"#, + // no rules at all + r#"{"refusal_statuses": []}"#, + ] { + assert!(Rules::from_json(bad).is_err(), "accepted: {bad}"); + } +} diff --git a/tests/antibot_set_rules.rs b/tests/antibot_set_rules.rs new file mode 100644 index 0000000..9f6d2b5 --- /dev/null +++ b/tests/antibot_set_rules.rs @@ -0,0 +1,31 @@ +// `set_rules` swaps the rules every later classification in the process +// runs. Its own test binary, since the rules are process-wide. + +use blasthttp::antibot::{Outcome, ResponseFacts, Rules, Vendor, classify, set_rules}; + +#[test] +fn test_set_rules_replaces_the_rules_classify_runs() { + let none = vec![]; + let page = ResponseFacts { + status: 200, + headers: &none, + body: "example-challenge", + }; + assert_eq!(classify(&page), Outcome::Ok); + + set_rules( + Rules::from_json( + r#"{"refusal_statuses": [403], "rules": [ + {"vendor": "examplewaf", "outcome": "challenge", "when": {"body_contains": "example-challenge"}} + ]}"#, + ) + .unwrap(), + ); + assert_eq!( + classify(&page), + Outcome::Challenge(Vendor::new("examplewaf")) + ); + + set_rules(Rules::default()); + assert_eq!(classify(&page), Outcome::Ok); +} diff --git a/tests/python/test_antibot_rules.py b/tests/python/test_antibot_rules.py new file mode 100644 index 0000000..5d046a8 --- /dev/null +++ b/tests/python/test_antibot_rules.py @@ -0,0 +1,51 @@ +"""Replacing the anti-bot rules from Python, with a JSON file shaped like the built-in one.""" + +import json + +import pytest + +import blasthttp + +CHALLENGE_PAGE = b"example-challenge" + + +def protection(body): + return blasthttp.Response("https://example.test/", 200, headers=[], body=body).protection + + +@pytest.fixture +def restore_rules(): + yield + blasthttp.set_antibot_rules() + + +def test_a_rules_file_replaces_the_rules(tmp_path, restore_rules): + rules = tmp_path / "rules.json" + rules.write_text( + json.dumps( + { + "refusal_statuses": [403], + "rules": [ + {"vendor": "examplewaf", "outcome": "challenge", "when": {"body_contains": "example-challenge"}} + ], + } + ) + ) + assert protection(CHALLENGE_PAGE) == ("ok", "-") + blasthttp.set_antibot_rules(str(rules)) + assert protection(CHALLENGE_PAGE) == ("challenge", "examplewaf") + blasthttp.set_antibot_rules() + assert protection(CHALLENGE_PAGE) == ("ok", "-") + + +def test_a_bad_rules_file_raises_and_keeps_the_rules(tmp_path, restore_rules): + rules = tmp_path / "rules.json" + rules.write_text('{"refusal_statuses": [], "rules": [{"vendor": "x", "outcome": "maybe", "when": {}}]}') + with pytest.raises(ValueError, match="rules.json"): + blasthttp.set_antibot_rules(str(rules)) + with pytest.raises(ValueError, match="missing.json"): + blasthttp.set_antibot_rules(str(tmp_path / "missing.json")) + assert protection(b'') == ( + "challenge", + "imperva", + ) From c496315fddedb5a1aa3d306fa298f61e469567cb Mon Sep 17 00:00:00 2001 From: liquidsec Date: Mon, 26 Oct 2026 15:41:42 -0400 Subject: [PATCH 3/3] Format test_profiles_and_report.py --- tests/python/test_profiles_and_report.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/python/test_profiles_and_report.py b/tests/python/test_profiles_and_report.py index 6f491d6..a36fd9a 100644 --- a/tests/python/test_profiles_and_report.py +++ b/tests/python/test_profiles_and_report.py @@ -59,9 +59,7 @@ def respond(status, headers, body): if path == "/echo-headers": body = "\n".join(received).encode() respond(200, [("Content-Type", "text/plain")], body) - elif path == "/cf-challenge" and not any( - h.lower().startswith("cookie:") and "pass=1" in h for h in received - ): + elif path == "/cf-challenge" and not any(h.lower().startswith("cookie:") and "pass=1" in h for h in received): # cf-mitigated is the header Cloudflare sets specifically to say it # intervened, and is what separates a challenge from a WAF block. respond(