From aca85f570ea554830519361c1eda81492ae2fadc Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 5 Aug 2026 21:52:28 +0000 Subject: [PATCH 1/2] feat(parser): wire ::text and ::attr() pseudo-elements into css_get/css_getall MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit translator.rs (parse_css_query) existed but was dead code, while the README advertised ::text/::attr() support. New Parsel-style methods on Selector and SpiderResponse: - css_getall("li::text") — full recursive text of each match, with original word spacing preserved (joined unstripped, ends trimmed) - css_getall("a::attr(href)") — attribute values; elements without the attribute are skipped, matching Parsel - css_getall("div.card") — outer HTML of each match - css_get(...) — first value only (Parsel's .get()) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01KDFsMaKk764vogjUW3nqpk --- src/parser/selector.rs | 34 +++++++++++++++++++++++ src/spiders/response.rs | 12 ++++++++ tests/parser_selector.rs | 59 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 105 insertions(+) diff --git a/src/parser/selector.rs b/src/parser/selector.rs index 49030b3..d4c52a9 100644 --- a/src/parser/selector.rs +++ b/src/parser/selector.rs @@ -218,6 +218,40 @@ impl Selector { Selectors::new(items) } + /// Parsel-style CSS query with `::text` and `::attr(name)` pseudo-element + /// support, returning all extracted values: + /// + /// - `"li::text"` — the full (recursive, whitespace-trimmed) text of + /// each matching element, + /// - `"a::attr(href)"` — the attribute value of each matching element + /// that has the attribute (elements without it are skipped), + /// - a plain selector — each match's outer HTML. + pub fn css_getall(&self, query: &str) -> Vec { + let q = crate::parser::translator::parse_css_query(query); + self.css(&q.selector) + .into_iter() + .filter_map(|el| { + if q.extract_text { + // Join text nodes without stripping (so word spacing + // inside markup like "a bold word" survives), + // then trim the ends. + let text = el.get_all_text("", false, &[], None); + Some(TextHandler::from(text.as_str().trim())) + } else if let Some(ref attr) = q.extract_attr { + el.get_attribute(attr) + } else { + Some(el.outer_html()) + } + }) + .collect() + } + + /// Like [`Self::css_getall`], but returns only the first extracted + /// value (Parsel's `.get()`). + pub fn css_get(&self, query: &str) -> Option { + self.css_getall(query).into_iter().next() + } + /// Return direct element children. pub fn children(&self) -> Selectors { let node = self.node_ref(); diff --git a/src/spiders/response.rs b/src/spiders/response.rs index 6b9b8dd..391f110 100644 --- a/src/spiders/response.rs +++ b/src/spiders/response.rs @@ -35,6 +35,18 @@ impl SpiderResponse { self.selector().css(selector) } + /// Parsel-style CSS query with `::text` / `::attr(name)` support; all + /// extracted values. See [`Selector::css_getall`]. + pub fn css_getall(&self, query: &str) -> Vec { + self.selector().css_getall(query) + } + + /// Parsel-style CSS query with `::text` / `::attr(name)` support; first + /// extracted value. See [`Selector::css_get`]. + pub fn css_get(&self, query: &str) -> Option { + self.selector().css_get(query) + } + pub fn json(&self) -> Result { self.inner.json() } diff --git a/tests/parser_selector.rs b/tests/parser_selector.rs index 48a1c4f..5864061 100644 --- a/tests/parser_selector.rs +++ b/tests/parser_selector.rs @@ -313,6 +313,65 @@ fn test_find_all_by_text() { assert!(found.len() >= 3); } +#[test] +fn test_css_getall_text_pseudo_element() { + let sel = Selector::from_html(HTML); + let texts: Vec = sel + .css_getall("li.item::text") + .into_iter() + .map(|t| t.as_str().to_string()) + .collect(); + assert_eq!(texts, vec!["Item 1", "Item 2", "Item 3"]); +} + +#[test] +fn test_css_getall_attr_pseudo_element() { + let sel = Selector::from_html(HTML); + let prices: Vec = sel + .css_getall("li.item::attr(data-price)") + .into_iter() + .map(|t| t.as_str().to_string()) + .collect(); + assert_eq!(prices, vec!["19.99", "29.99", "39.99"]); + // Elements without the attribute are skipped, not empty. + assert!(sel.css_getall("h1::attr(data-missing)").is_empty()); +} + +#[test] +fn test_css_get_returns_first_value() { + let sel = Selector::from_html(HTML); + assert_eq!( + sel.css_get("li.item::text").map(|t| t.as_str().to_string()), + Some("Item 1".to_string()) + ); + assert_eq!( + sel.css_get("a.nav-link::attr(href)") + .map(|t| t.as_str().to_string()), + Some("/next".to_string()) + ); + assert!(sel.css_get(".does-not-exist::text").is_none()); +} + +#[test] +fn test_css_get_plain_selector_yields_outer_html() { + let sel = Selector::from_html(HTML); + let html = sel.css_get("h1.title").unwrap(); + assert!(html.as_str().contains(" = sel + .css_getall("p.description::text") + .into_iter() + .map(|t| t.as_str().to_string()) + .collect(); + assert_eq!(texts, vec!["This is a test page."]); +} + #[test] fn test_find_by_text_first_and_last_match_agree_on_nested_text() { // Regression: find_by_text(first_match=true) used to match only an From bfbe57ef5da281a1769f0cff67ec0f12ef65a84b Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 5 Aug 2026 21:59:42 +0000 Subject: [PATCH 2/2] fix(parser): skip text-less elements in ::text; honest Parsel-divergence docs Review follow-ups: an element with no text nodes now yields nothing from ::text queries (Parsel parity, symmetric with the ::attr path), so css_get returns the first element with actual text instead of Some(""). Doc comments now say Parsel-inspired and spell out the divergences: one joined string per matched element (not per text node), always recursive, end-trimmed, script/style text included. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01KDFsMaKk764vogjUW3nqpk --- src/parser/selector.rs | 31 +++++++++++++++++++++++-------- src/spiders/response.rs | 4 ++-- tests/parser_selector.rs | 19 +++++++++++++++++++ 3 files changed, 44 insertions(+), 10 deletions(-) diff --git a/src/parser/selector.rs b/src/parser/selector.rs index d4c52a9..50e0fae 100644 --- a/src/parser/selector.rs +++ b/src/parser/selector.rs @@ -218,13 +218,20 @@ impl Selector { Selectors::new(items) } - /// Parsel-style CSS query with `::text` and `::attr(name)` pseudo-element - /// support, returning all extracted values: + /// Parsel-inspired CSS query with `::text` and `::attr(name)` + /// pseudo-element support, returning all extracted values: /// - /// - `"li::text"` — the full (recursive, whitespace-trimmed) text of - /// each matching element, + /// - `"li::text"` — the full recursive text of each matching element + /// (including `