Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 49 additions & 0 deletions src/parser/selector.rs
Original file line number Diff line number Diff line change
Expand Up @@ -218,6 +218,55 @@ impl Selector {
Selectors::new(items)
}

/// Parsel-inspired CSS query with `::text` and `::attr(name)`
/// pseudo-element support, returning all extracted values:
///
/// - `"li::text"` — the full recursive text of each matching element
/// (including `<script>`/`<style>` content, like Parsel), joined
/// without stripping so word spacing inside markup survives, then
/// end-trimmed. Elements with no text yield nothing. Note this
/// diverges from Parsel, which returns one string **per text node**
/// and distinguishes `p::text` (direct children) from `p ::text`
/// (descendants) — here `::text` is always one joined string per
/// matched element, fully recursive.
/// - `"a::attr(href)"` — the attribute value of each matching element
/// that has the attribute (elements without it are skipped, matching
/// Parsel),
/// - a plain selector — each match's outer HTML.
pub fn css_getall(&self, query: &str) -> Vec<TextHandler> {
let q = crate::parser::translator::parse_css_query(query);
self.css(&q.selector)
.into_iter()
.filter_map(|el| {
if q.extract_text {
// Join text nodes without stripping (so word spacing
// inside markup like "a <b>bold</b> word" survives),
// then trim the ends. Text-less elements yield None so
// css_get skips to the first element with actual text,
// symmetric with the ::attr path.
let text = el.get_all_text("", false, &[], None);
let trimmed = text.as_str().trim();
if trimmed.is_empty() {
None
} else {
Some(TextHandler::from(trimmed))
}
} else if let Some(ref attr) = q.extract_attr {
el.get_attribute(attr)
} else {
Some(el.outer_html())
}
})
.collect()
}

/// Like [`Self::css_getall`], but returns only the first extracted
/// value (Parsel's `.get()`): for `::attr`/`::text` queries this is the
/// first element that actually has a value, not merely the first match.
pub fn css_get(&self, query: &str) -> Option<TextHandler> {
self.css_getall(query).into_iter().next()
}

/// Return direct element children.
pub fn children(&self) -> Selectors {
let node = self.node_ref();
Expand Down
12 changes: 12 additions & 0 deletions src/spiders/response.rs
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,18 @@ impl SpiderResponse {
self.selector().css(selector)
}

/// Parsel-inspired CSS query with `::text` / `::attr(name)` support; all
/// extracted values. See [`Selector::css_getall`].
pub fn css_getall(&self, query: &str) -> Vec<crate::core::text_handler::TextHandler> {
self.selector().css_getall(query)
}

/// Parsel-inspired CSS query with `::text` / `::attr(name)` support; first
/// extracted value. See [`Selector::css_get`].
pub fn css_get(&self, query: &str) -> Option<crate::core::text_handler::TextHandler> {
self.selector().css_get(query)
}

pub fn json(&self) -> Result<serde_json::Value, serde_json::Error> {
self.inner.json()
}
Expand Down
78 changes: 78 additions & 0 deletions tests/parser_selector.rs
Original file line number Diff line number Diff line change
Expand Up @@ -313,6 +313,65 @@ fn test_find_all_by_text() {
assert!(found.len() >= 3);
}

#[test]
fn test_css_getall_text_pseudo_element() {
let sel = Selector::from_html(HTML);
let texts: Vec<String> = sel
.css_getall("li.item::text")
.into_iter()
.map(|t| t.as_str().to_string())
.collect();
assert_eq!(texts, vec!["Item 1", "Item 2", "Item 3"]);
}

#[test]
fn test_css_getall_attr_pseudo_element() {
let sel = Selector::from_html(HTML);
let prices: Vec<String> = sel
.css_getall("li.item::attr(data-price)")
.into_iter()
.map(|t| t.as_str().to_string())
.collect();
assert_eq!(prices, vec!["19.99", "29.99", "39.99"]);
// Elements without the attribute are skipped, not empty.
assert!(sel.css_getall("h1::attr(data-missing)").is_empty());
}

#[test]
fn test_css_get_returns_first_value() {
let sel = Selector::from_html(HTML);
assert_eq!(
sel.css_get("li.item::text").map(|t| t.as_str().to_string()),
Some("Item 1".to_string())
);
assert_eq!(
sel.css_get("a.nav-link::attr(href)")
.map(|t| t.as_str().to_string()),
Some("/next".to_string())
);
assert!(sel.css_get(".does-not-exist::text").is_none());
}

#[test]
fn test_css_get_plain_selector_yields_outer_html() {
let sel = Selector::from_html(HTML);
let html = sel.css_get("h1.title").unwrap();
assert!(html.as_str().contains("<h1"));
assert!(html.as_str().contains("Hello World"));
}

#[test]
fn test_css_getall_recursive_text() {
// ::text extracts the full recursive text of each match.
let sel = Selector::from_html(HTML);
let texts: Vec<String> = sel
.css_getall("p.description::text")
.into_iter()
.map(|t| t.as_str().to_string())
.collect();
assert_eq!(texts, vec!["This is a test page."]);
}

#[test]
fn test_find_by_text_first_and_last_match_agree_on_nested_text() {
// Regression: find_by_text(first_match=true) used to match only an
Expand All @@ -333,3 +392,22 @@ fn test_find_by_text_first_and_last_match_agree_on_nested_text() {
.expect("first_match=false must find a match too");
assert_eq!(last.tag(), "span");
}

#[test]
fn test_css_text_skips_text_less_elements() {
// Parity with Parsel and symmetry with ::attr: an element with no text
// nodes yields nothing, so css_get returns the first element that has
// actual text instead of Some("").
let html = r#"<html><body><ul><li></li><li>real</li></ul></body></html>"#;
let sel = Selector::from_html(html);
let texts: Vec<String> = sel
.css_getall("li::text")
.into_iter()
.map(|t| t.as_str().to_string())
.collect();
assert_eq!(texts, vec!["real"]);
assert_eq!(
sel.css_get("li::text").map(|t| t.as_str().to_string()),
Some("real".to_string())
);
}