From 9231501a4a46d485b410946049b9cea91ec952e4 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 5 Aug 2026 22:20:32 +0000 Subject: [PATCH] =?UTF-8?q?docs:=20complete=20rustdoc=20pass=20=E2=80=94?= =?UTF-8?q?=20crate=20docs,=20all=20public=20items,=20#[must=5Fuse]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Crate-level overview in lib.rs covering the three layers (parser/fetchers/spiders) with a runnable Selector quickstart doctest plus no_run fetch and minimal-Spider examples, and a feature-highlight list (::text/::attr via css_get/css_getall, adaptive relocation, stealth headers, robots.txt, AutoThrottle, checkpoints, dev cache, CSV/JSON/JSONL/XML export). - Enabled #![warn(missing_docs)] and documented every public item across all 36 source files: modules, structs, enums (incl. error variants), traits and their provided methods, public fields, and functions — with real semantics (Selector::find_by_text deepest-match rules, Scheduler enqueue-time dedup, TextHandler::clean whitespace/entity behavior, AttributesHandler ordering, AutoThrottle reserve/record, fingerprint options, robots.txt RFC 9309 matching, etc.). - #[must_use] on builder types (FetcherConfigBuilder, SpiderRequestBuilder, CrawlSpider/SitemapSpider/ShopifySpider builders, LinkExtractor, CrawlRule) and their build() methods, and on pure query/accessor methods whose dropped results are almost certainly bugs (Selector css/css_get/css_getall/text/get_all_text/ find_by_* etc., TextHandler/TextHandlers/AttributesHandler/Selectors accessors, Response accessors, stats getters). - No runtime behavior changes: docs, attributes, and doc examples only. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01KDFsMaKk764vogjUW3nqpk --- src/core/attributes_handler.rs | 30 ++++++- src/core/mod.rs | 5 ++ src/core/storage.rs | 25 +++++- src/core/text_handler.rs | 63 ++++++++++++- src/core/text_handlers.rs | 23 ++++- src/fetchers/client.rs | 25 ++++++ src/fetchers/config.rs | 43 ++++++++- src/fetchers/constants.rs | 24 ++++- src/fetchers/encoding.rs | 2 + src/fetchers/mod.rs | 5 ++ src/fetchers/proxy.rs | 12 ++- src/fetchers/response.rs | 39 +++++++- src/fetchers/search.rs | 2 + src/lib.rs | 144 +++++++++++++++++++++++++++++- src/parser/adaptive.rs | 16 +++- src/parser/mod.rs | 4 + src/parser/selector.rs | 130 ++++++++++++++++++++++----- src/parser/selector_generation.rs | 23 ++++- src/parser/selectors.rs | 42 +++++++-- src/parser/translator.rs | 13 ++- src/spiders/cache.rs | 19 ++++ src/spiders/checkpoint.rs | 21 +++++ src/spiders/engine.rs | 40 +++++++++ src/spiders/links.rs | 19 ++++ src/spiders/mod.rs | 6 ++ src/spiders/request.rs | 81 ++++++++++++++++- src/spiders/response.rs | 31 +++++++ src/spiders/result.rs | 61 +++++++++++-- src/spiders/robots.rs | 16 ++++ src/spiders/scheduler.rs | 31 ++++++- src/spiders/session.rs | 16 ++++ src/spiders/spider.rs | 66 +++++++++++++- src/spiders/templates/crawler.rs | 24 +++++ src/spiders/templates/shopify.rs | 15 ++++ src/spiders/templates/sitemap.rs | 18 ++++ src/spiders/throttle.rs | 13 ++- 36 files changed, 1077 insertions(+), 70 deletions(-) diff --git a/src/core/attributes_handler.rs b/src/core/attributes_handler.rs index d3789a2..dcada6d 100644 --- a/src/core/attributes_handler.rs +++ b/src/core/attributes_handler.rs @@ -1,14 +1,23 @@ +//! The [`AttributesHandler`] map returned by +//! [`Selector::attrib`](crate::parser::Selector::attrib): an element's +//! attributes in document order. + use crate::core::TextHandler; use indexmap::IndexMap; /// A read-only mapping of HTML element attributes. -/// All values are wrapped in TextHandler for regex/json capabilities. +/// +/// Attributes keep their document order (backed by an `IndexMap`), and all +/// values are wrapped in [`TextHandler`] for regex/JSON helpers. Indexing +/// by key (`attrs["href"]`) panics on a missing key — use +/// [`AttributesHandler::get`] for a fallible lookup. #[derive(Debug, Clone)] pub struct AttributesHandler { inner: IndexMap, } impl AttributesHandler { + /// Build a handler from `(name, value)` pairs, preserving their order. pub fn new(map: impl IntoIterator) -> Self { let inner: IndexMap = map .into_iter() @@ -17,30 +26,43 @@ impl AttributesHandler { Self { inner } } + /// Look up an attribute value by name (case-sensitive). + #[must_use] pub fn get(&self, key: &str) -> Option<&TextHandler> { self.inner.get(key) } + /// Whether an attribute with this name exists. + #[must_use] pub fn contains_key(&self, key: &str) -> bool { self.inner.contains_key(key) } + /// Number of attributes. + #[must_use] pub fn len(&self) -> usize { self.inner.len() } + /// Whether the element has no attributes. + #[must_use] pub fn is_empty(&self) -> bool { self.inner.is_empty() } + /// Iterate over attribute names, in document order. pub fn keys(&self) -> impl Iterator { self.inner.keys().map(|k| k.as_str()) } + /// Iterate over attribute values, in document order. pub fn values(&self) -> impl Iterator { self.inner.values() } + /// Iterate over `(name, value)` pairs, in document order. pub fn iter(&self) -> impl Iterator { self.inner.iter().map(|(k, v)| (k.as_str(), v)) } - /// Search for attributes whose values match a keyword (exact or partial). + /// Iterate over the attributes whose value equals `keyword` — or + /// merely contains it when `partial` is set — yielding the matching + /// `(name, value)` pairs. pub fn search_values<'a>( &'a self, keyword: &'a str, @@ -60,7 +82,9 @@ impl AttributesHandler { }) } - /// Serialize attributes to JSON string. + /// Serialize the attributes to a JSON object string, keys in document + /// order (empty string on the unlikely event of a serialization error). + #[must_use] pub fn json_string(&self) -> String { let map: IndexMap<&str, &str> = self .inner diff --git a/src/core/mod.rs b/src/core/mod.rs index d9008b1..f19628c 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -1,3 +1,8 @@ +//! Shared value types used across the parser and spiders: string wrappers +//! with scraping helpers ([`TextHandler`], [`TextHandlers`]), a read-only +//! attribute map ([`AttributesHandler`]), and the SQLite storage backing +//! adaptive element relocation ([`storage::SqliteStorage`]). + pub mod attributes_handler; pub mod storage; pub mod text_handler; diff --git a/src/core/storage.rs b/src/core/storage.rs index b58d770..03c4545 100644 --- a/src/core/storage.rs +++ b/src/core/storage.rs @@ -1,9 +1,19 @@ +//! SQLite persistence for adaptive element relocation: element snapshots +//! saved by [`Selector::save`](crate::parser::Selector::save) live here, +//! keyed by page URL plus a hashed identifier. + use rusqlite::{params, Connection, OptionalExtension}; use sha2::{Digest, Sha256}; use std::collections::HashMap; use std::sync::{Mutex, MutexGuard}; /// SQLite-backed storage for adaptive element relocation. +/// +/// One `SqliteStorage` is scoped to a single page URL (lowercased at +/// construction): rows are keyed by `(url, identifier)`, so the same +/// identifier can be reused across different pages without collisions. The +/// connection is guarded by a mutex, making the storage safe to share +/// across threads. pub struct SqliteStorage { conn: Mutex, url: String, @@ -11,7 +21,9 @@ pub struct SqliteStorage { } impl SqliteStorage { - /// Create storage backed by SQLite file. URL is normalized to lowercase. + /// Open (creating if needed) the SQLite database at `db_path`, scoped + /// to `url`. The URL is normalized to lowercase; the database uses WAL + /// journaling and gets its schema created on first use. pub fn new(db_path: &str, url: &str) -> Result { let conn = Self::open(db_path)?; Ok(Self { @@ -56,7 +68,8 @@ impl SqliteStorage { } } - /// Save element data. Uses INSERT OR REPLACE for upsert. + /// Save element data under `identifier` for this storage's URL, + /// replacing any previous entry (INSERT OR REPLACE upsert). pub fn save( &self, identifier: &str, @@ -72,7 +85,9 @@ impl SqliteStorage { Ok(()) } - /// Retrieve stored element data. + /// Retrieve the element data stored under `identifier` for this + /// storage's URL. Returns `Ok(None)` when nothing was saved yet; real + /// database errors propagate as `Err`. pub fn retrieve( &self, identifier: &str, @@ -103,10 +118,14 @@ impl SqliteStorage { } } +/// Errors from [`SqliteStorage`] operations. #[derive(Debug, thiserror::Error)] pub enum StorageError { + /// The underlying SQLite operation failed (I/O, locking, schema, …). #[error("SQLite error: {0}")] Sqlite(#[from] rusqlite::Error), + /// Element data could not be serialized to, or deserialized from, its + /// stored JSON form. #[error("JSON error: {0}")] Json(#[from] serde_json::Error), } diff --git a/src/core/text_handler.rs b/src/core/text_handler.rs index 242e44b..137e08a 100644 --- a/src/core/text_handler.rs +++ b/src/core/text_handler.rs @@ -1,68 +1,109 @@ +//! The [`TextHandler`] string wrapper returned by every text-extraction +//! method in the crate, with regex, JSON, and cleaning helpers. + use regex::Regex; use std::fmt; /// A string wrapper with scraping-specific methods (regex, JSON, cleaning). +/// +/// All transforming methods ([`TextHandler::strip`], [`TextHandler::clean`], +/// …) return a new `TextHandler` and leave the original untouched. Use +/// [`TextHandler::as_str`] / [`TextHandler::into_string`] (or the +/// `Display` / `AsRef` impls) to get at the plain string. #[derive(Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] pub struct TextHandler { value: String, } impl TextHandler { + /// Wrap a string value. pub fn new(value: impl Into) -> Self { Self { value: value.into(), } } + /// The wrapped text as a string slice. + #[must_use] pub fn as_str(&self) -> &str { &self.value } + /// Consume the handler and return the owned `String`. + #[must_use] pub fn into_string(self) -> String { self.value } + /// Whether the wrapped text is empty. + #[must_use] pub fn is_empty(&self) -> bool { self.value.is_empty() } + /// Length of the wrapped text in *bytes* (like [`str::len`]), not + /// characters. + #[must_use] pub fn len(&self) -> usize { self.value.len() } + /// Return a copy with leading and trailing whitespace removed + /// (Python-style name for [`str::trim`]). + #[must_use] pub fn strip(&self) -> TextHandler { TextHandler::new(self.value.trim()) } + /// Return a lowercased copy. + #[must_use] pub fn to_lowercase(&self) -> TextHandler { TextHandler::new(self.value.to_lowercase()) } + /// Return an uppercased copy. + #[must_use] pub fn to_uppercase(&self) -> TextHandler { TextHandler::new(self.value.to_uppercase()) } + /// Whether the text contains the given substring. + #[must_use] pub fn contains_str(&self, pattern: &str) -> bool { self.value.contains(pattern) } + /// Return a copy with every occurrence of `from` replaced by `to`. + #[must_use] pub fn replace_str(&self, from: &str, to: &str) -> TextHandler { TextHandler::new(self.value.replace(from, to)) } + /// Whether the text starts with the given prefix. + #[must_use] pub fn starts_with_str(&self, prefix: &str) -> bool { self.value.starts_with(prefix) } + /// Whether the text ends with the given suffix. + #[must_use] pub fn ends_with_str(&self, suffix: &str) -> bool { self.value.ends_with(suffix) } + /// Split the text on a delimiter, wrapping each piece (empty pieces + /// included, like [`str::split`]). + #[must_use] pub fn split_str(&self, delimiter: &str) -> Vec { self.value.split(delimiter).map(TextHandler::new).collect() } - /// Clean whitespace: remove tabs, CR, LF, collapse spaces. Optionally decode HTML entities. + /// Normalize whitespace: tabs and line feeds become spaces, carriage + /// returns are removed, runs of spaces collapse to one, and the result + /// is trimmed. With `remove_entities`, a fixed set of common HTML + /// entities (`<` `>` `"` `'` `'` ` ` `&`) + /// is decoded afterwards. + #[must_use] pub fn clean(&self, remove_entities: bool) -> TextHandler { let mut s = self.value.replace('\t', " "); s = s.replace('\r', ""); @@ -77,12 +118,20 @@ impl TextHandler { TextHandler::new(s) } - /// Parse text as JSON. + /// Parse the text as JSON. pub fn json(&self) -> Result { serde_json::from_str(&self.value) } - /// Apply regex. Returns captured group 1 if present, else group 0. + /// Apply a regex and return all matches. + /// + /// For each match, capture group 1 is returned when the pattern + /// defines capture groups, otherwise the whole match. Flags: + /// `replace_entities` decodes common HTML entities *before* matching, + /// `clean_match` trims each returned match, and `case_sensitive = + /// false` prepends `(?i)` to the pattern. An invalid pattern yields an + /// empty `Vec`. + #[must_use] pub fn re( &self, pattern: &str, @@ -119,6 +168,9 @@ impl TextHandler { results } + /// Apply a regex and return only the first match (see + /// [`TextHandler::re`] for the flag semantics). + #[must_use] pub fn re_first( &self, pattern: &str, @@ -131,10 +183,15 @@ impl TextHandler { .next() } + /// Parsel-parity accessor: returns `self` (a single value's `.get()` + /// is itself). + #[must_use] pub fn get(&self) -> &TextHandler { self } + /// Parsel-parity accessor: returns the value as a one-element `Vec`. + #[must_use] pub fn getall(&self) -> Vec { vec![self.clone()] } diff --git a/src/core/text_handlers.rs b/src/core/text_handlers.rs index 5af6260..f59494c 100644 --- a/src/core/text_handlers.rs +++ b/src/core/text_handlers.rs @@ -1,32 +1,49 @@ +//! The [`TextHandlers`] collection: a list of [`TextHandler`] values with +//! batch regex operations, mirroring Parsel's `SelectorList` text API. + use crate::core::TextHandler; -/// A list of TextHandler values with batch operations. +/// A list of [`TextHandler`] values with batch operations. +/// +/// Supports indexing (`values[0]`) and iteration by value or reference. #[derive(Debug, Clone, Default)] pub struct TextHandlers { items: Vec, } impl TextHandlers { + /// Create a new `TextHandlers` from a `Vec` of [`TextHandler`]. pub fn new(items: Vec) -> Self { Self { items } } + /// Number of values in the collection. + #[must_use] pub fn len(&self) -> usize { self.items.len() } + /// Whether the collection is empty. + #[must_use] pub fn is_empty(&self) -> bool { self.items.is_empty() } + /// The first value (cloned), or `default` when the collection is empty. + #[must_use] pub fn get(&self, default: Option) -> Option { self.items.first().cloned().or(default) } + /// All values as a slice. + #[must_use] pub fn getall(&self) -> &[TextHandler] { &self.items } + /// Apply a regex to every value and flatten all matches into a new + /// collection (see [`TextHandler::re`] for the flag semantics). + #[must_use] pub fn re( &self, pattern: &str, @@ -42,6 +59,10 @@ impl TextHandlers { TextHandlers::new(items) } + /// Apply a regex across the values in order and return the first match + /// found in any of them (see [`TextHandler::re`] for the flag + /// semantics). + #[must_use] pub fn re_first( &self, pattern: &str, diff --git a/src/fetchers/client.rs b/src/fetchers/client.rs index 794694e..2377d47 100644 --- a/src/fetchers/client.rs +++ b/src/fetchers/client.rs @@ -1,9 +1,20 @@ +//! The [`Fetcher`] HTTP client: retries, stealth headers, proxy rotation, +//! redirect policy, and response-size protection on top of reqwest. + use crate::fetchers::config::FetcherConfig; use crate::fetchers::proxy::ProxyRotator; use crate::fetchers::response::Response; use std::collections::HashMap; use std::time::Duration; +/// Async HTTP client configured through [`FetcherConfig`]. +/// +/// Every request applies the config's headers (with browser-like stealth +/// headers when enabled), retries failed sends up to `config.retries` +/// times with a fixed delay between attempts, enforces the configured +/// response-size cap while streaming the body, and decodes the body using +/// the charset advertised in `Content-Type`. With `proxy_list` set, one +/// client is built per proxy and requests rotate through them round-robin. pub struct Fetcher { config: FetcherConfig, /// One client per rotating proxy when rotation is enabled, otherwise a @@ -12,10 +23,16 @@ pub struct Fetcher { rotator: Option, } +/// Errors from building a [`Fetcher`] or performing a request. #[derive(Debug, thiserror::Error)] pub enum FetcherError { + /// The request could not be completed: every retry attempt failed, the + /// response body exceeded the configured size cap, or a body chunk + /// could not be read. The message carries the underlying cause. #[error("Request failed after retries: {0}")] RequestFailed(String), + /// The underlying reqwest client failed to build (e.g. TLS backend + /// initialization). #[error("HTTP error: {0}")] Http(#[from] reqwest::Error), } @@ -127,11 +144,15 @@ impl Fetcher { } } + /// Send a GET request. pub async fn get(&self, url: &str) -> Result { self.request(reqwest::Method::GET, url, None, None, None) .await } + /// Send a POST request with an optional plain-text `body` or a `json` + /// payload (which also sets `Content-Type: application/json`). When + /// both are given, `json` wins. pub async fn post( &self, url: &str, @@ -142,6 +163,9 @@ impl Fetcher { .await } + /// Send a PUT request with an optional plain-text `body` or a `json` + /// payload (which also sets `Content-Type: application/json`). When + /// both are given, `json` wins. pub async fn put( &self, url: &str, @@ -152,6 +176,7 @@ impl Fetcher { .await } + /// Send a DELETE request. pub async fn delete(&self, url: &str) -> Result { self.request(reqwest::Method::DELETE, url, None, None, None) .await diff --git a/src/fetchers/config.rs b/src/fetchers/config.rs index 3fd63dc..81a0b76 100644 --- a/src/fetchers/config.rs +++ b/src/fetchers/config.rs @@ -1,23 +1,49 @@ +//! [`FetcherConfig`] and its builder: everything tunable about how a +//! [`Fetcher`](crate::fetchers::client::Fetcher) makes requests. + use crate::fetchers::constants; use std::collections::HashMap; /// Configuration for HTTP fetchers. +/// +/// Build one with [`FetcherConfig::builder`] or start from +/// [`FetcherConfig::default`] (30s timeout, 3 retries with a 1s delay, +/// redirects followed up to 10 hops, SSL verified, stealth headers on, +/// 50 MiB body cap, no proxies). #[derive(Debug, Clone)] pub struct FetcherConfig { + /// Total per-request timeout in seconds (connect through body read). pub timeout_secs: u64, + /// How many times a failed request is retried (in addition to the + /// initial attempt). pub retries: u32, + /// Fixed delay in seconds between retry attempts. pub retry_delay_secs: u64, + /// Whether HTTP redirects are followed at all. pub follow_redirects: bool, + /// Maximum number of redirect hops when `follow_redirects` is on. pub max_redirects: u32, + /// Whether TLS certificates are verified. Disabling this is insecure + /// and logs a warning when the fetcher is built. pub verify_ssl: bool, + /// A single proxy URL applied to all protocols, used as a fallback + /// after the per-protocol `proxies` map. pub proxy: Option, + /// Per-protocol proxy map: keys `"http"` / `"https"` bind a proxy to + /// that scheme, any other key acts as a wildcard for all protocols. pub proxies: HashMap, /// Proxy URLs to rotate through, one HTTP client is built per entry and /// selected round-robin per request. Takes precedence over `proxy` / /// `proxies` when non-empty. pub proxy_list: Vec, + /// Extra request headers (lowercase names). These win over the + /// generated stealth headers. pub headers: HashMap, + /// Whether browser-like stealth headers are generated for each request + /// (see [`FetcherConfig::build_headers`]). pub stealthy_headers: bool, + /// Fixed User-Agent. When `None`, a user agent is picked + /// deterministically per URL from a small pool of real browser strings. pub user_agent: Option, /// Maximum response body size in bytes. Responses larger than this are /// aborted to protect against OOM. Defaults to 50 MiB. @@ -140,29 +166,34 @@ impl FetcherConfig { // Builder // --------------------------------------------------------------------------- -/// Builder for [`FetcherConfig`]. +/// Builder for [`FetcherConfig`], starting from the default values. #[derive(Debug, Default)] +#[must_use = "builders do nothing until `.build()` is called"] pub struct FetcherConfigBuilder { inner: FetcherConfig, } impl FetcherConfigBuilder { + /// Create a builder initialised with [`FetcherConfig::default`] values. pub fn new() -> Self { Self { inner: FetcherConfig::default(), } } + /// Set the total per-request timeout, in seconds. pub fn timeout(mut self, secs: u64) -> Self { self.inner.timeout_secs = secs; self } + /// Set how many times a failed request is retried. pub fn retries(mut self, retries: u32) -> Self { self.inner.retries = retries; self } + /// Set the fixed delay between retry attempts, in seconds. pub fn retry_delay(mut self, secs: u64) -> Self { self.inner.retry_delay_secs = secs; self @@ -207,22 +238,28 @@ impl FetcherConfigBuilder { self } + /// Pin a fixed User-Agent string instead of the per-URL pick from the + /// built-in browser pool. pub fn user_agent(mut self, ua: impl Into) -> Self { self.inner.user_agent = Some(ua.into()); self } - /// Enable or disable stealth header generation. + /// Enable or disable stealth header generation (on by default). pub fn stealth(mut self, enabled: bool) -> Self { self.inner.stealthy_headers = enabled; self } + /// Enable or disable following HTTP redirects (on by default, up to + /// 10 hops). pub fn follow_redirects(mut self, follow: bool) -> Self { self.inner.follow_redirects = follow; self } + /// Enable or disable TLS certificate verification. Disabling is + /// insecure and logs a warning when the fetcher is built. pub fn verify_ssl(mut self, verify: bool) -> Self { self.inner.verify_ssl = verify; self @@ -235,6 +272,8 @@ impl FetcherConfigBuilder { self } + /// Finish the builder and return the configured [`FetcherConfig`]. + #[must_use] pub fn build(self) -> FetcherConfig { self.inner } diff --git a/src/fetchers/constants.rs b/src/fetchers/constants.rs index a5e6814..2a3f7f7 100644 --- a/src/fetchers/constants.rs +++ b/src/fetchers/constants.rs @@ -1,3 +1,10 @@ +//! Shared constants for the fetchers: the stealth user-agent pool, default +//! header values, and the status codes treated as bot blocks. + +/// Browser resource types typically blocked by scraping-oriented browser +/// automation to speed up page loads (fonts, images, media, stylesheets, +/// …). Not used by the plain HTTP fetcher itself; provided for browser +/// integrations. pub const BLOCKED_RESOURCE_TYPES: &[&str] = &[ "font", "image", @@ -11,6 +18,9 @@ pub const BLOCKED_RESOURCE_TYPES: &[&str] = &[ "stylesheet", ]; +/// Pool of real browser User-Agent strings (current Chrome and Firefox on +/// Windows/macOS/Linux). When no fixed user agent is configured, one is +/// picked deterministically per URL. pub const USER_AGENTS: &[&str] = &[ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", @@ -19,15 +29,21 @@ pub const USER_AGENTS: &[&str] = &[ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:133.0) Gecko/20100101 Firefox/133.0", ]; -// Bot-blocking / anti-automation status codes only. 5xx server errors are -// deliberately excluded: they are genuine server failures, not bot blocks, and -// classifying them as blocks would wrongly suppress the on_error hook and burn -// the blocked-retry budget (444 is Nginx's non-standard "connection closed"). +/// Bot-blocking / anti-automation status codes only +/// (401/403/407/429/444). 5xx server errors are deliberately excluded: +/// they are genuine server failures, not bot blocks, and classifying them +/// as blocks would wrongly suppress the on_error hook and burn the +/// blocked-retry budget (444 is Nginx's non-standard "connection closed"). pub const BLOCKED_STATUS_CODES: &[u16] = &[401, 403, 407, 429, 444]; +/// Default `Accept` header sent in stealth mode (a real browser's +/// navigation Accept value). pub const ACCEPT_HEADER: &str = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8"; +/// Default `Accept-Language` header sent in stealth mode. pub const ACCEPT_LANGUAGE: &str = "en-US,en;q=0.9"; +/// Default `Accept-Encoding` header sent in stealth mode. All three +/// codings are actually supported by the client. pub const ACCEPT_ENCODING: &str = "gzip, deflate, br"; diff --git a/src/fetchers/encoding.rs b/src/fetchers/encoding.rs index 89356f0..19bf19a 100644 --- a/src/fetchers/encoding.rs +++ b/src/fetchers/encoding.rs @@ -24,6 +24,7 @@ static CHARSET_RE: LazyLock = LazyLock::new(|| { /// /// Handles both bare (`charset=utf-8`) and quoted (`charset="ISO-8859-1"`) /// parameter forms; the quotes are not part of the returned label. +#[must_use] pub fn charset_from_content_type(content_type: &str) -> Option<&str> { CHARSET_RE .captures(content_type) @@ -41,6 +42,7 @@ pub fn charset_from_content_type(content_type: &str) -> Option<&str> { /// legacy labels (`hz-gb-2312`, `iso-2022-kr`, …) to the *replacement* /// encoding, which decodes the entire body to a single U+FFFD — for a /// scraper, falling back to lossy UTF-8 preserves far more of the content. +#[must_use] pub fn decode_body(bytes: &[u8], content_type: &str) -> String { let encoding = charset_from_content_type(content_type) .and_then(|label| encoding_rs::Encoding::for_label_no_replacement(label.as_bytes())) diff --git a/src/fetchers/mod.rs b/src/fetchers/mod.rs index b87f712..d4da0b1 100644 --- a/src/fetchers/mod.rs +++ b/src/fetchers/mod.rs @@ -1,3 +1,8 @@ +//! Async HTTP fetching: a reqwest-backed client ([`client::Fetcher`]) with +//! stealth headers, retries, proxy support (single, per-protocol, and +//! rotating), charset-aware body decoding, and response wrappers that plug +//! straight into the parser. + pub mod client; pub mod config; pub mod constants; diff --git a/src/fetchers/proxy.rs b/src/fetchers/proxy.rs index 01ed94a..5f8cc2b 100644 --- a/src/fetchers/proxy.rs +++ b/src/fetchers/proxy.rs @@ -1,3 +1,6 @@ +//! Round-robin proxy rotation shared by the +//! [`Fetcher`](crate::fetchers::client::Fetcher)'s per-proxy client pool. + use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; @@ -26,7 +29,8 @@ impl ProxyRotator { }) } - /// Return the next proxy in round-robin order. + /// Return the next proxy in round-robin order, advancing the cursor. + #[must_use] pub fn next(&self) -> &str { &self.proxies[self.next_index()] } @@ -34,12 +38,14 @@ impl ProxyRotator { /// Advance the cursor and return the index of the next proxy in /// round-robin order. Useful for indexing a parallel collection (e.g. a /// pool of pre-built HTTP clients) that shares the rotator's ordering. + #[must_use] pub fn next_index(&self) -> usize { self.cursor.fetch_add(1, Ordering::Relaxed) % self.proxies.len() } /// Return a pseudo-random proxy, advancing the cursor so consecutive /// calls don't keep returning the same proxy. + #[must_use] pub fn random(&self) -> &str { let pos = self.cursor.fetch_add(1, Ordering::Relaxed); // Simple pseudo-random selection: mix the position with a constant. @@ -48,10 +54,14 @@ impl ProxyRotator { } /// Number of proxies in the rotator. + #[must_use] pub fn len(&self) -> usize { self.proxies.len() } + /// Whether the rotator holds no proxies (never true in practice, since + /// [`ProxyRotator::new`] rejects empty lists). + #[must_use] pub fn is_empty(&self) -> bool { self.proxies.is_empty() } diff --git a/src/fetchers/response.rs b/src/fetchers/response.rs index 32798f0..dfd4d29 100644 --- a/src/fetchers/response.rs +++ b/src/fetchers/response.rs @@ -1,7 +1,15 @@ +//! The [`Response`] wrapper returned by the +//! [`Fetcher`](crate::fetchers::client::Fetcher): status, headers, decoded +//! body, and one-call access to the parser. + use crate::parser::Selector; use std::collections::HashMap; /// HTTP response wrapper with parser integration. +/// +/// The body is already fully read and decoded to a `String` (using the +/// charset from the `Content-Type` header), so all accessors are cheap and +/// synchronous. #[derive(Debug, Clone)] pub struct Response { status_code: u16, @@ -12,6 +20,8 @@ pub struct Response { } impl Response { + /// Assemble a response from its parts (used by the fetcher and by the + /// dev-mode cache when replaying stored responses). pub fn new( status_code: u16, content_type: String, @@ -28,44 +38,69 @@ impl Response { } } + /// The HTTP status code (e.g. `200`). + #[must_use] pub fn status(&self) -> u16 { self.status_code } + /// The decoded response body. + #[must_use] pub fn text(&self) -> &str { &self.body } + /// The final URL of the response, after any redirects. + #[must_use] pub fn url(&self) -> &str { &self.url } + /// The raw `Content-Type` header value (empty when the server sent + /// none). + #[must_use] pub fn content_type(&self) -> &str { &self.content_type } + /// All response headers, with lowercase names. Headers whose values are + /// not valid UTF-8 are omitted. + #[must_use] pub fn headers(&self) -> &HashMap { &self.headers } + /// Size of the *decoded* body in bytes (not the on-wire + /// `Content-Length`, which may differ after decompression and charset + /// decoding). + #[must_use] pub fn content_length(&self) -> usize { self.body.len() } - /// Parse response body as Selector for HTML extraction. + /// Parse the body as HTML and return a [`Selector`] rooted at the + /// document, with the response URL as base for + /// [`Selector::urljoin`]. Each call re-parses the body. + #[must_use] pub fn selector(&self) -> Selector { Selector::from_html_with_url(&self.body, &self.url) } - /// Parse body as JSON. + /// Parse the body as JSON. pub fn json(&self) -> Result { serde_json::from_str(&self.body) } + /// Whether the status code is 2xx. + #[must_use] pub fn is_success(&self) -> bool { (200..300).contains(&self.status_code) } + /// Whether the status code is one of the bot-blocking codes + /// (401/403/407/429/444 — see + /// [`BLOCKED_STATUS_CODES`](crate::fetchers::constants::BLOCKED_STATUS_CODES)). + #[must_use] pub fn is_blocked(&self) -> bool { crate::fetchers::constants::BLOCKED_STATUS_CODES.contains(&self.status_code) } diff --git a/src/fetchers/search.rs b/src/fetchers/search.rs index 9fd67da..886e0e2 100644 --- a/src/fetchers/search.rs +++ b/src/fetchers/search.rs @@ -24,6 +24,7 @@ use crate::fetchers::response::Response; /// /// Returns `false` for non-DuckDuckGo responses, so it is safe to call on any /// response. +#[must_use] pub fn is_duckduckgo_blocked(response: &Response) -> bool { if !response.url().contains("duckduckgo.com") { return false; @@ -43,6 +44,7 @@ pub fn is_duckduckgo_blocked(response: &Response) -> bool { /// This prepends a scheme to protocol-relative hrefs, then extracts and /// percent-decodes the `uddg` parameter. Any href that is not a DuckDuckGo /// redirect is returned unchanged. +#[must_use] pub fn decode_duckduckgo_href(href: &str) -> String { let full = if let Some(stripped) = href.strip_prefix("//") { format!("https://{}", stripped) diff --git a/src/lib.rs b/src/lib.rs index 238ef53..167ea9c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,146 @@ -//! RUSTScrapling - A Rust port of the Scrapling web scraping framework. +//! RUSTScrapling — a Rust port of the [Scrapling](https://github.com/D4Vinci/Scrapling) +//! web-scraping framework. +//! +//! The crate is organized in three layers, each usable on its own: +//! +//! - **[`parser`]** — HTML parsing and data extraction. [`Selector`] wraps a +//! parsed DOM node and offers CSS selection (including Parsel-style +//! `::text` / `::attr(name)` pseudo-elements via [`Selector::css_get`] / +//! [`Selector::css_getall`]), text extraction, DOM navigation, regex +//! helpers, JSON parsing, and *adaptive element relocation* — a saved +//! element's "unique properties" can be used to find it again by +//! similarity after the page layout changes. +//! - **[`fetchers`]** — async HTTP. [`Fetcher`] is a reqwest-backed client +//! configured through [`FetcherConfig`] (timeouts, retries, redirects, +//! single/per-protocol/rotating proxies, response-size caps) that sends +//! browser-like *stealth headers* by default, and returns a [`Response`] +//! with charset-aware body decoding and direct parser integration. +//! - **[`spiders`]** — crawling. Implement the [`Spider`] trait (or use a +//! ready-made template: [`CrawlSpider`], [`SitemapSpider`], +//! [`ShopifySpider`]) and run it with [`CrawlerEngine`]. The engine +//! handles concurrency limits (global and per-domain), request +//! deduplication, robots.txt compliance (RFC 9309 matching plus +//! `Crawl-delay`), AutoThrottle adaptive per-domain delays, +//! blocked-response retries, checkpoints for pause/resume, and a +//! development-mode response cache for offline iteration. Results come +//! back as a [`CrawlResult`] with live stats and an [`ItemList`] +//! exportable to CSV, JSON, JSON Lines, or XML. +//! +//! # Quickstart: parsing HTML +//! +//! ``` +//! use rust_scrapling::Selector; +//! +//! let html = r#" +//! +//! "#; +//! +//! let page = Selector::from_html(html); +//! +//! // Parsel-style `::text` / `::attr()` pseudo-elements: +//! let names: Vec = page +//! .css_getall("li.item a::text") +//! .into_iter() +//! .map(|t| t.into_string()) +//! .collect(); +//! assert_eq!(names, ["Widget", "Gadget"]); +//! +//! let first_href = page.css_get("li.item a::attr(href)").unwrap(); +//! assert_eq!(first_href.as_str(), "/p/1"); +//! +//! // Plain CSS queries return `Selectors` for further navigation: +//! let prices = page.css(".price"); +//! assert_eq!(prices.len(), 2); +//! assert_eq!(prices[0].text().as_str(), "9.99"); +//! ``` +//! +//! # Fetching a page +//! +//! ```no_run +//! use rust_scrapling::{Fetcher, FetcherConfig}; +//! +//! #[tokio::main] +//! async fn main() -> Result<(), Box> { +//! let config = FetcherConfig::builder() +//! .timeout(20) +//! .stealth(true) // browser-like headers (the default) +//! .build(); +//! let fetcher = Fetcher::new(config)?; +//! +//! let response = fetcher.get("https://example.com").await?; +//! println!("status: {}", response.status()); +//! if let Some(title) = response.selector().css_get("title::text") { +//! println!("title: {}", title); +//! } +//! Ok(()) +//! } +//! ``` +//! +//! # A minimal spider +//! +//! ```no_run +//! use async_trait::async_trait; +//! use rust_scrapling::spiders::response::SpiderResponse; +//! use rust_scrapling::spiders::session::SessionManager; +//! use rust_scrapling::{CrawlerEngine, FetcherConfig, Spider, SpiderRequest}; +//! use std::sync::Arc; +//! +//! struct QuotesSpider; +//! +//! #[async_trait] +//! impl Spider for QuotesSpider { +//! fn name(&self) -> &str { +//! "quotes" +//! } +//! +//! fn start_urls(&self) -> Vec { +//! vec!["https://quotes.toscrape.com/".to_string()] +//! } +//! +//! async fn parse( +//! &self, +//! response: SpiderResponse, +//! ) -> (Vec, Vec) { +//! let items = response +//! .css_getall(".quote .text::text") +//! .into_iter() +//! .map(|text| serde_json::json!({ "quote": text.as_str() })) +//! .collect(); +//! (items, Vec::new()) // no follow-up requests +//! } +//! } +//! +//! #[tokio::main] +//! async fn main() { +//! let sessions = SessionManager::new(FetcherConfig::default()); +//! let engine = CrawlerEngine::new(Arc::new(QuotesSpider), sessions, None) +//! .expect("HTTP client should build"); +//! let result = engine.crawl().await; +//! println!("scraped {} items", result.items.len()); +//! } +//! ``` +//! +//! # Feature highlights +//! +//! - CSS selectors with `::text` / `::attr(name)` extraction +//! ([`Selector::css_get`] / [`Selector::css_getall`]) +//! - Adaptive element relocation that survives page-layout changes +//! ([`parser::adaptive`]) +//! - Stealth (browser-like) request headers, on by default +//! ([`FetcherConfig::build_headers`]) +//! - robots.txt compliance with RFC 9309 longest-match rules and +//! `Crawl-delay` support ([`spiders::robots`]) +//! - AutoThrottle: adaptive per-domain crawl delays ([`spiders::throttle`]) +//! - Checkpoints with pause/resume ([`spiders::checkpoint`], +//! [`CrawlerEngine::request_pause`]) +//! - Development-mode response cache for offline replays +//! ([`spiders::cache`]) +//! - Item export to CSV / JSON / JSON Lines / XML ([`ItemList`]) + +#![warn(missing_docs)] pub mod core; pub mod fetchers; diff --git a/src/parser/adaptive.rs b/src/parser/adaptive.rs index 27b3620..6233d0e 100644 --- a/src/parser/adaptive.rs +++ b/src/parser/adaptive.rs @@ -35,19 +35,24 @@ pub const DEFAULT_RELOCATION_PERCENTAGE: f64 = 40.0; /// page structure changes. Mirrors upstream `_StorageTools.element_to_dict`. #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] pub struct ElementData { + /// The element's tag name. pub tag: String, /// Attributes with whitespace-stripped values; empty values are dropped. #[serde(default)] pub attributes: IndexMap, + /// Trimmed direct text content, `None` when empty. #[serde(default)] pub text: Option, /// Tag names from the root element down to this element. #[serde(default)] pub path: Vec, + /// The parent element's tag name, if the parent is an element. #[serde(default)] pub parent_name: Option, + /// The parent element's attributes (values unstripped). #[serde(default)] pub parent_attribs: IndexMap, + /// The parent's trimmed direct text content, `None` when empty. #[serde(default)] pub parent_text: Option, /// Tag names of the parent's other element children. @@ -60,6 +65,7 @@ pub struct ElementData { impl Selector { /// Capture this element's unique properties for adaptive relocation. + #[must_use] pub fn element_data(&self) -> ElementData { let mut data = ElementData { tag: self.tag().to_string(), @@ -143,6 +149,7 @@ impl Selector { /// is scored against `original`; the group with the highest score is /// returned if it reaches `percentage` (see /// [`DEFAULT_RELOCATION_PERCENTAGE`]), otherwise an empty collection. + #[must_use] pub fn relocate(&self, original: &ElementData, percentage: f64) -> Selectors { let mut best_score = f64::MIN; let mut best: Vec = Vec::new(); @@ -224,8 +231,13 @@ impl Selector { } } -/// Percentage similarity between a stored element and a candidate. Port of -/// upstream `__calculate_similarity_score`, rounded to two decimals. +/// Percentage similarity (`0.0..=100.0`) between a stored element and a +/// candidate. Port of upstream `__calculate_similarity_score`, rounded to +/// two decimals: the mean of per-feature similarity checks (tag, text, +/// attributes with extra weight on `class`/`id`/`href`/`src`, tree path, +/// parent context, siblings), where only features present in `original` +/// are checked. +#[must_use] pub fn similarity_score(original: &ElementData, candidate: &ElementData) -> f64 { let mut score = 0.0; let mut checks = 0u32; diff --git a/src/parser/mod.rs b/src/parser/mod.rs index c630a23..db7cc39 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -1,3 +1,7 @@ +//! HTML parsing and data extraction: CSS selection with `::text`/`::attr()` +//! pseudo-elements, DOM navigation, selector generation, and adaptive +//! element relocation. + pub mod adaptive; pub mod selector; pub mod selector_generation; diff --git a/src/parser/selector.rs b/src/parser/selector.rs index 50e0fae..935a5c5 100644 --- a/src/parser/selector.rs +++ b/src/parser/selector.rs @@ -1,3 +1,6 @@ +//! The [`Selector`] type: a handle to one node of a parsed HTML tree with +//! CSS querying, text extraction, DOM navigation, regex, and JSON helpers. + use crate::core::{AttributesHandler, TextHandler}; use crate::parser::selectors::Selectors; use ego_tree::NodeId; @@ -9,6 +12,11 @@ use url::Url; /// A wrapper around a node in an HTML tree, providing CSS selection, /// text extraction, DOM navigation, regex, and JSON parsing. +/// +/// Cloning is cheap: all `Selector`s produced from the same document share +/// one reference-counted tree, and a clone only copies the node handle. +/// Query methods ([`Selector::css`], [`Selector::children`], …) return new +/// `Selector`s pointing into that same shared tree. #[derive(Debug, Clone)] pub struct Selector { tree: Rc, @@ -28,7 +36,8 @@ impl Selector { } } - /// Parse an HTML document string with a base URL for link resolution. + /// Parse an HTML document string with a base URL used by + /// [`Selector::urljoin`] to resolve relative links. pub fn from_html_with_url(html: &str, url: &str) -> Self { let parsed = Html::parse_document(html); let root_id = parsed.tree.root().id(); @@ -75,12 +84,17 @@ impl Selector { /// Stable identity of the underlying DOM node. Useful for comparing two /// `Selector`s that reference the same node without resorting to HTML /// string equality (which is ambiguous for identical siblings). + #[must_use] pub fn node_id(&self) -> NodeId { self.node_id } /// Return the tag name of this element. - /// Returns "html" for the document root, "#text" for text nodes. + /// + /// Non-element nodes get placeholder names: `"html"` for the document + /// root, `"#text"` for text nodes, `"#comment"` for comments, + /// `"#doctype"` for doctypes, and `"#pi"` for processing instructions. + #[must_use] pub fn tag(&self) -> &str { let node = self.node_ref(); match node.value() { @@ -94,7 +108,10 @@ impl Selector { } /// Return the direct (non-recursive) text content of this element. - /// Only collects immediate text children, not text inside child elements. + /// Only collects immediate text children, not text inside child + /// elements — for the full recursive text use + /// [`Selector::get_all_text`]. + #[must_use] pub fn text(&self) -> TextHandler { let node = self.node_ref(); let mut text = String::new(); @@ -106,7 +123,9 @@ impl Selector { TextHandler::new(text) } - /// Return the element's attributes as an AttributesHandler. + /// Return the element's attributes as an [`AttributesHandler`] + /// (empty for non-element nodes). + #[must_use] pub fn attrib(&self) -> AttributesHandler { if let Some(el) = self.element_ref() { let attrs = el @@ -119,7 +138,10 @@ impl Selector { } } - /// Return the inner HTML of this element as a TextHandler. + /// Return the inner HTML of this element (its children serialized, + /// without the element's own tag). For the document root this is the + /// whole serialized document; for other non-element nodes it is empty. + #[must_use] pub fn html_content(&self) -> TextHandler { if let Some(el) = self.element_ref() { TextHandler::new(el.inner_html()) @@ -130,7 +152,10 @@ impl Selector { } } - /// Return the outer HTML of this element as a TextHandler. + /// Return the outer HTML of this element (the element itself including + /// its tag and children). For the document root this is the whole + /// serialized document; for other non-element nodes it is empty. + #[must_use] pub fn outer_html(&self) -> TextHandler { if let Some(el) = self.element_ref() { TextHandler::new(el.html()) @@ -141,8 +166,15 @@ impl Selector { } } - /// Recursively extract text, skipping tags in `ignore_tags`. - /// If `valid_values` is Some, only include text nodes whose trimmed content is in the set. + /// Recursively extract all text under this node, joined with + /// `separator`. + /// + /// Elements whose tag is in `ignore_tags` are skipped along with their + /// entire subtree (e.g. pass `&["script", "style"]` to drop scripts). + /// With `strip`, each text node is trimmed and empty ones are dropped + /// before joining. If `valid_values` is `Some`, only text nodes whose + /// trimmed content appears in the slice are included. + #[must_use] pub fn get_all_text( &self, separator: &str, @@ -195,7 +227,13 @@ impl Selector { } } - /// Run a CSS selector query and return matching elements as Selectors. + /// Run a CSS selector query and return matching descendant elements. + /// + /// Matches are returned in document order. An invalid selector yields + /// an empty collection rather than an error. Only plain CSS is accepted + /// here — for `::text` / `::attr(name)` extraction use + /// [`Selector::css_get`] / [`Selector::css_getall`]. + #[must_use] pub fn css(&self, selector: &str) -> Selectors { let css_sel = match scraper::Selector::parse(selector) { Ok(s) => s, @@ -233,6 +271,7 @@ impl Selector { /// that has the attribute (elements without it are skipped, matching /// Parsel), /// - a plain selector — each match's outer HTML. + #[must_use] pub fn css_getall(&self, query: &str) -> Vec { let q = crate::parser::translator::parse_css_query(query); self.css(&q.selector) @@ -263,11 +302,13 @@ impl Selector { /// Like [`Self::css_getall`], but returns only the first extracted /// value (Parsel's `.get()`): for `::attr`/`::text` queries this is the /// first element that actually has a value, not merely the first match. + #[must_use] pub fn css_get(&self, query: &str) -> Option { self.css_getall(query).into_iter().next() } - /// Return direct element children. + /// Return direct element children (text and comment nodes are skipped). + #[must_use] pub fn children(&self) -> Selectors { let node = self.node_ref(); let items: Vec = node @@ -278,13 +319,17 @@ impl Selector { Selectors::new(items) } - /// Return the parent element, if any. + /// Return the parent node, if any. Note the parent of a top-level + /// element is the document root, not an element. + #[must_use] pub fn parent(&self) -> Option { let node = self.node_ref(); node.parent().map(|p| self.child_selector(p.id())) } - /// Return sibling elements (excluding self). + /// Return sibling elements (all of the parent's element children + /// except this node), in document order. + #[must_use] pub fn siblings(&self) -> Selectors { let node = self.node_ref(); let self_id = self.node_id; @@ -302,7 +347,9 @@ impl Selector { Selectors::new(items) } - /// Return the next sibling element. + /// Return the next sibling *element*, skipping intervening text and + /// comment nodes. + #[must_use] pub fn next(&self) -> Option { let mut node = self.node_ref(); while let Some(sibling) = node.next_sibling() { @@ -314,7 +361,9 @@ impl Selector { None } - /// Return the previous sibling element. + /// Return the previous sibling *element*, skipping intervening text and + /// comment nodes. + #[must_use] pub fn previous(&self) -> Option { let mut node = self.node_ref(); while let Some(sibling) = node.prev_sibling() { @@ -326,7 +375,9 @@ impl Selector { None } - /// Check whether this element has a given CSS class. + /// Check whether this element has a given CSS class + /// (case-sensitive; `false` for non-element nodes). + #[must_use] pub fn has_class(&self, class_name: &str) -> bool { if let Some(el) = self.element_ref() { el.value() @@ -336,7 +387,11 @@ impl Selector { } } - /// Join a relative URL with the base URL of this selector. + /// Join a relative URL with the base URL of this selector (set via + /// [`Selector::from_html_with_url`]). Falls back to returning + /// `relative_url` unchanged when there is no base URL or either URL + /// fails to parse. + #[must_use] pub fn urljoin(&self, relative_url: &str) -> String { if self.url.is_empty() { return relative_url.to_string(); @@ -350,7 +405,11 @@ impl Selector { } } - /// Apply a regex pattern against the full recursive text of this element. + /// Apply a regex pattern against the full recursive text of this + /// element, returning all matches. When the pattern has capture + /// groups, group 1 is returned for each match, otherwise the whole + /// match. See [`TextHandler::re`] for the flag semantics. + #[must_use] pub fn re( &self, pattern: &str, @@ -362,7 +421,9 @@ impl Selector { text_handler.re(pattern, replace_entities, clean_match, case_sensitive) } - /// Apply a regex and return only the first match. + /// Apply a regex against the full recursive text of this element and + /// return only the first match (see [`Selector::re`]). + #[must_use] pub fn re_first( &self, pattern: &str, @@ -374,13 +435,25 @@ impl Selector { text_handler.re_first(pattern, replace_entities, clean_match, case_sensitive) } - /// Parse the text content of this element as JSON. + /// Parse the *direct* text content of this element (see + /// [`Selector::text`]) as JSON — useful for `