From 54e9f712894f346ff4e5610c963bf40a9a901638 Mon Sep 17 00:00:00 2001 From: mkrastev Date: Wed, 15 Jul 2026 13:18:03 +0300 Subject: [PATCH] add cps statement scraper --- datasets/amfv_datasets/scraping/__init__.py | 18 ++ datasets/amfv_datasets/scraping/cli.py | 38 ++- datasets/amfv_datasets/scraping/cps.py | 285 ++++++++++++++++++++ datasets/amfv_datasets/scraping/html.py | 2 + datasets/test/test_scraping_cli.py | 45 ++++ datasets/test/test_scraping_cps.py | 159 +++++++++++ datasets/test/test_scraping_html.py | 9 + 7 files changed, 548 insertions(+), 8 deletions(-) create mode 100644 datasets/amfv_datasets/scraping/cps.py create mode 100644 datasets/test/test_scraping_cps.py diff --git a/datasets/amfv_datasets/scraping/__init__.py b/datasets/amfv_datasets/scraping/__init__.py index cd7cdb0..827edb3 100644 --- a/datasets/amfv_datasets/scraping/__init__.py +++ b/datasets/amfv_datasets/scraping/__init__.py @@ -9,6 +9,16 @@ scrape_listing_documents, ) from amfv_datasets.scraping.cli import OutputFormat, ScraperSource +from amfv_datasets.scraping.cps import ( + CpsFetchError, + CpsStatementRef, + build_statement_text, + list_statements, + listing_page_url, + scrape_cps, + scrape_statement, + statement_ref_from_url, +) from amfv_datasets.scraping.html import ( LinkMode, absolute_unique_urls, @@ -31,6 +41,8 @@ __all__ = [ "GuidanceRef", "GuidanceListingPage", + "CpsFetchError", + "CpsStatementRef", "LinkMode", "NiceFetchError", "OutputFormat", @@ -41,6 +53,7 @@ "USER_AGENT", "absolute_unique_urls", "build_guideline_text", + "build_statement_text", "clean_text", "document_title", "default_client", @@ -48,7 +61,12 @@ "guidance_ref_from_url", "html_to_markdown", "list_published_guidance", + "list_statements", + "listing_page_url", "scrape_guideline", "scrape_listing_documents", "scrape_nice", + "scrape_cps", + "scrape_statement", + "statement_ref_from_url", ] diff --git a/datasets/amfv_datasets/scraping/cli.py b/datasets/amfv_datasets/scraping/cli.py index e8aece3..bb46d84 100644 --- a/datasets/amfv_datasets/scraping/cli.py +++ b/datasets/amfv_datasets/scraping/cli.py @@ -25,6 +25,7 @@ ) from amfv_datasets.scraping.base import ScrapedDocument, ScrapeRun +from amfv_datasets.scraping.cps import scrape_cps from amfv_datasets.scraping.html import LinkMode from amfv_datasets.scraping.nice import scrape_nice @@ -34,6 +35,7 @@ class ScraperSource(StrEnum): ALL = "all" NICE = "nice" + CPS = "cps" class OutputFormat(StrEnum): @@ -68,13 +70,33 @@ def scrape_documents( if documents is not None and documents < 1: raise ValueError(f"documents must be at least 1; got {documents}") - for selected_source in _expand_source(source): - match selected_source: - case ScraperSource.NICE: - return scrape_nice(documents=documents, link_mode=link_mode, url=url) - case ScraperSource.ALL: - raise AssertionError("expanded source cannot be all") - raise AssertionError(f"unsupported source: {source}") + selected_sources = _expand_source(source) + if url is not None and len(selected_sources) != 1: + raise ValueError("--url requires one specific scraper source, not 'all'") + runs = tuple( + _scrape_source(selected_source, documents=documents, link_mode=link_mode, url=url) + for selected_source in selected_sources + ) + if len(runs) == 1: + return runs[0] + total = sum(run.total for run in runs) if all(run.total is not None for run in runs) else None + return ScrapeRun(documents=(document for run in runs for document in run), total=total) + + +def _scrape_source( + source: ScraperSource, + *, + documents: int | None, + link_mode: LinkMode, + url: str | None, +) -> ScrapeRun: + match source: + case ScraperSource.NICE: + return scrape_nice(documents=documents, link_mode=link_mode, url=url) + case ScraperSource.CPS: + return scrape_cps(documents=documents, link_mode=link_mode, url=url) + case ScraperSource.ALL: + raise AssertionError("expanded source cannot be all") def write_jsonl(documents: Iterable[ScrapedDocument], output: TextIO) -> int: @@ -125,7 +147,7 @@ def write_markdown_files(documents: Iterable[ScrapedDocument], output_path: Path def _expand_source(source: ScraperSource) -> tuple[ScraperSource, ...]: if source is ScraperSource.ALL: - return (ScraperSource.NICE,) + return (ScraperSource.NICE, ScraperSource.CPS) return (source,) diff --git a/datasets/amfv_datasets/scraping/cps.py b/datasets/amfv_datasets/scraping/cps.py new file mode 100644 index 0000000..4b9ba1e --- /dev/null +++ b/datasets/amfv_datasets/scraping/cps.py @@ -0,0 +1,285 @@ +"""Scrape Canadian Paediatric Society statements into normalized markdown. + +CPS retains copyright in its position statements and practice points. This +module supports a permission-gated ingestion workflow; obtain written CPS +permission before running a corpus scrape or redistributing its output. +""" + +from __future__ import annotations + +import re +from collections.abc import Iterable +from dataclasses import dataclass +from urllib.parse import unquote, urljoin, urlparse + +import httpx +from lxml import html as lxml_html +from markdownify import MarkdownConverter + +from amfv_datasets.scraping.base import ( + ScrapedDocument, + ScrapeError, + ScrapeRun, + default_client, + scrape_listing_documents, +) +from amfv_datasets.scraping.html import LinkMode, clean_text + +BASE_URL = "https://cps.ca" +STATEMENTS_URL = f"{BASE_URL}/en/documents/statements-by-date" +DOCUMENT_DELAY_SECONDS = 10.0 +LISTING_PAGE_SIZE = 10 + +_POSITION_PATH_RE = re.compile(r"^/(?:en/)?documents/position/(?P[^/]+)/?$", re.IGNORECASE) +_DATE_PATTERNS = { + "posted": re.compile(r"\bPosted:\s*([A-Za-z]+\s+\d{1,2},\s+\d{4})", re.IGNORECASE), + "reaffirmed": re.compile(r"\bReaffirmed:\s*([A-Za-z]+\s+\d{1,2},\s+\d{4})", re.IGNORECASE), + "updated": re.compile(r"\bUpdated:\s*([A-Za-z]+\s+\d{1,2},\s+\d{4})", re.IGNORECASE), +} +_BLANK_LINES_RE = re.compile(r"\n{3,}") +_EMPTY_MARKDOWN_LINK_RE = re.compile(r"(? str: # noqa: ANN001 + classes = element.get("class") or () + href = element.get("href") + if "reference" in classes and href and "#ref" in href: + marker = element.get_text(strip=True).strip("[]") + if "a" in (self.options.get("strip") or ()): + return f"[{marker}]" + return f"[{marker}]({href})" + return super().convert_a(element, text, parent_tags) + + def convert_sup(self, element, text: str, parent_tags: set[str]) -> str: # noqa: ANN001 + if not text.strip(): + return "" + citation = element.find("a", href=lambda href: href and "#ref" in href) + if citation: + marker = element.get_text(strip=True).strip("[]") + if "a" in (self.options.get("strip") or ()): + return f"[{marker}]" + return f"[{marker}]({citation.get('href')})" + return f"{text}" + + +_CPS_MARKDOWN_CONVERTERS = { + LinkMode.KEEP: _CpsMarkdownConverter(bullets="-", heading_style="ATX"), + LinkMode.STRIP: _CpsMarkdownConverter(bullets="-", heading_style="ATX", strip=("a",)), +} + + +class CpsFetchError(ScrapeError): + """Raised when a CPS statement cannot be discovered or parsed.""" + + +@dataclass(frozen=True) +class CpsStatementRef: + """A CPS statement or practice point discovered from the date index.""" + + slug: str + title: str + page_url: str + + +def statement_ref_from_url(url: str, *, title: str | None = None) -> CpsStatementRef: + """Parse a CPS position-statement URL into a canonical reference.""" + parsed = urlparse(url.strip()) + if parsed.scheme not in {"http", "https"} or parsed.netloc.lower() not in {"cps.ca", "www.cps.ca"}: + raise CpsFetchError(f"Enter a CPS statement URL from cps.ca; got {url!r}") + + match = _POSITION_PATH_RE.match(unquote(parsed.path)) + if not match: + raise CpsFetchError(f"Enter a URL like https://cps.ca/en/documents/position/example-statement; got {url!r}") + slug = match.group("slug") + return CpsStatementRef( + slug=slug, + title=title or slug.replace("-", " "), + page_url=f"{BASE_URL}/en/documents/position/{slug}", + ) + + +def listing_page_url(page: int) -> str: + """Return the CPS date-index URL for a one-based page number.""" + if page < 1: + raise ValueError(f"page must be at least 1; got {page}") + if page == 1: + return STATEMENTS_URL + return f"{STATEMENTS_URL}/P{(page - 1) * LISTING_PAGE_SIZE}" + + +def list_statements(client: httpx.Client, page: int) -> list[CpsStatementRef]: + """Return unique statement references from one CPS date-index page.""" + response = client.get(listing_page_url(page)) + response.raise_for_status() + doc = lxml_html.fromstring(response.text) + anchors = doc.xpath("//div[contains(concat(' ', normalize-space(@class), ' '), ' stmt-title ')]//a[@href]") + + refs: list[CpsStatementRef] = [] + seen_urls: set[str] = set() + for anchor in anchors: + href = anchor.get("href") + title = clean_text(anchor.text_content(), drop_numeric_citations=False) + if not href or not title: + continue + try: + ref = statement_ref_from_url(urljoin(BASE_URL, href), title=title) + except CpsFetchError: + continue + if ref.page_url in seen_urls: + continue + seen_urls.add(ref.page_url) + refs.append(ref) + return refs + + +def _content_root(html_text: str) -> lxml_html.HtmlElement: + doc = lxml_html.fromstring(html_text) + selectors = ( + "//div[contains(concat(' ', normalize-space(@class), ' '), ' statement-wrapper ')][1]", + "//main//article[1]", + "//*[@id='main-content']//article[1]", + "//*[@id='main-content'][1]", + "//main[1]", + ) + for selector in selectors: + matches = doc.xpath(selector) + if matches: + return matches[0] + raise CpsFetchError("No readable statement content found on the CPS page") + + +def _normalize_content(root: lxml_html.HtmlElement, *, base_url: str) -> None: + removable = root.xpath( + ".//script | .//style | .//nav | .//form | .//button | .//noscript | " + ".//*[contains(concat(' ', normalize-space(@class), ' '), ' breadcrumb ')] | " + ".//*[contains(@class, 'share')] | " + ".//*[contains(@class, 'social')] | " + ".//*[contains(concat(' ', normalize-space(@class), ' '), ' related-content ')] | " + ".//*[contains(concat(' ', normalize-space(@class), ' '), ' sidebar ')] | " + ".//*[contains(concat(' ', normalize-space(@class), ' '), ' hide-for-print ')] | " + ".//*[contains(concat(' ', normalize-space(@class), ' '), ' show-for-print ')] | " + ".//*[contains(@class, 'print:tw-hidden')] | " + ".//*[contains(concat(' ', normalize-space(@class), ' '), ' --podcast ')]" + ) + for element in removable: + element.drop_tree() + + for link in root.xpath(".//a[contains(@href, '/en/education/test-your-knowledge')]"): + link.drop_tree() + + for image in root.xpath(".//img[@src]"): + filename = urlparse(image.get("src")).path.rsplit("/", 1)[-1].lower() + if filename in _DECORATIVE_IMAGE_FILENAMES: + image.drop_tree() + + for link in root.xpath(".//a[@href]"): + link.set("href", urljoin(base_url, link.get("href"))) + for image in root.xpath(".//img[@src]"): + image.set("src", urljoin(base_url, image.get("src"))) + + for nested in root.xpath(".//strong//strong | .//em//em"): + nested.drop_tag() + for title in root.xpath(".//h1"): + title.drop_tree() + for heading in root.xpath(".//*[self::h1 or self::h2 or self::h3 or self::h4 or self::h5 or self::h6]"): + for emphasis in heading.xpath(".//strong | .//em | .//b | .//i"): + emphasis.drop_tag() + if not clean_text(heading.text_content(), drop_numeric_citations=False) and not heading.xpath(".//img"): + heading.drop_tree() + + +def _content_to_markdown(root: lxml_html.HtmlElement, *, link_mode: LinkMode) -> str: + source = lxml_html.tostring(root, encoding="unicode") + markdown = _CPS_MARKDOWN_CONVERTERS[link_mode].convert(source) + markdown = _EMPTY_MARKDOWN_LINK_RE.sub("", markdown) + lines = [line.rstrip() for line in markdown.splitlines()] + return _BLANK_LINES_RE.sub("\n\n", "\n".join(lines)).strip() + + +def build_statement_text( + html_text: str, + *, + link_mode: LinkMode = LinkMode.KEEP, + base_url: str = BASE_URL, +) -> tuple[str, int]: + """Extract one CPS statement as markdown and return its section count.""" + root = _content_root(html_text) + _normalize_content(root, base_url=base_url) + section_count = max(1, len(root.xpath(".//*[self::h2 or self::h3 or self::h4 or self::h5 or self::h6]"))) + content = _content_to_markdown(root, link_mode=link_mode) + if not content: + raise CpsFetchError("No readable statement content found on the CPS page") + return content, section_count + + +def scrape_statement( + client: httpx.Client, + ref: CpsStatementRef, + *, + link_mode: LinkMode = LinkMode.KEEP, +) -> ScrapedDocument: + """Scrape one CPS statement into the shared document schema.""" + response = client.get(ref.page_url) + response.raise_for_status() + content, section_count = build_statement_text(response.text, link_mode=link_mode, base_url=ref.page_url) + raw_root = _content_root(response.text) + headings = [clean_text(value) for value in raw_root.xpath(".//h1[1]//text()")] + title = " ".join(value for value in headings if value) or ref.title + metadata: dict[str, str] = {"slug": ref.slug} + visible_text = clean_text(raw_root.text_content(), drop_numeric_citations=False) + for key, pattern in _DATE_PATTERNS.items(): + if match := pattern.search(visible_text): + metadata[key] = match.group(1).strip() + return ScrapedDocument( + source="cps", + external_id=f"cps-{ref.slug.lower()}", + title=title, + url=ref.page_url, + content=content, + section_count=section_count, + metadata=metadata, + ) + + +def scrape_cps( + *, + documents: int | None, + link_mode: LinkMode = LinkMode.KEEP, + url: str | None = None, +) -> ScrapeRun: + """Configure a CPS scrape from one URL or the current-statements index.""" + if url is not None: + + def scrape_url() -> Iterable[ScrapedDocument]: + with default_client() as client: + yield scrape_statement(client, statement_ref_from_url(url), link_mode=link_mode) + + return ScrapeRun(documents=scrape_url(), total=1) + + return ScrapeRun( + documents=scrape_listing_documents( + documents=documents, + client_factory=default_client, + list_page=list_statements, + scrape_item=lambda client, ref: scrape_statement(client, ref, link_mode=link_mode), + document_delay_seconds=DOCUMENT_DELAY_SECONDS, + ) + ) + + +__all__ = [ + "BASE_URL", + "CpsFetchError", + "CpsStatementRef", + "STATEMENTS_URL", + "build_statement_text", + "list_statements", + "listing_page_url", + "scrape_cps", + "scrape_statement", + "statement_ref_from_url", +] diff --git a/datasets/amfv_datasets/scraping/html.py b/datasets/amfv_datasets/scraping/html.py index 2bd1c96..1580528 100644 --- a/datasets/amfv_datasets/scraping/html.py +++ b/datasets/amfv_datasets/scraping/html.py @@ -119,6 +119,8 @@ def _absolutize_links(html_text: str, *, base_url: str) -> str: root = lxml_html.fragment_fromstring(html_text, create_parent="div") for link in root.xpath(".//a[@href]"): link.set("href", urljoin(base_url, link.get("href"))) + for image in root.xpath(".//img[@src]"): + image.set("src", urljoin(base_url, image.get("src"))) return "".join(lxml_html.tostring(child, encoding="unicode") for child in root) diff --git a/datasets/test/test_scraping_cli.py b/datasets/test/test_scraping_cli.py index 44f714f..75239b6 100644 --- a/datasets/test/test_scraping_cli.py +++ b/datasets/test/test_scraping_cli.py @@ -11,6 +11,7 @@ from amfv_datasets.scraping.cli import ( ScraperSource, app, + scrape_documents, write_huggingface_dataset, write_jsonl, write_markdown_files, @@ -219,6 +220,50 @@ def fake_scrape_documents( assert json.loads(result.stdout.splitlines()[0])["external_id"] == "nice-ng1" +def test_scrape_documents_dispatches_cps(monkeypatch: pytest.MonkeyPatch) -> None: + """The CPS source delegates to its source-specific scraper.""" + calls = 0 + + def fake_scrape_cps(*, documents: int | None, link_mode: LinkMode, url: str | None) -> ScrapeRun: + nonlocal calls + calls += 1 + assert documents == 2 + assert link_mode is LinkMode.KEEP + assert url == "https://cps.ca/en/documents/position/example" + return ScrapeRun([_document()], total=1) + + monkeypatch.setattr("amfv_datasets.scraping.cli.scrape_cps", fake_scrape_cps) + + run = scrape_documents( + ScraperSource.CPS, + documents=2, + link_mode=LinkMode.KEEP, + url="https://cps.ca/en/documents/position/example", + ) + + assert calls == 1 + assert run.total == 1 + + +def test_scrape_documents_combines_all_sources(monkeypatch: pytest.MonkeyPatch) -> None: + """The all source streams NICE and CPS documents with a combined total.""" + nice = _document() + cps = ScrapedDocument("cps", "cps-example", "CPS example", "https://cps.ca/example", "content") + monkeypatch.setattr( + "amfv_datasets.scraping.cli.scrape_nice", + lambda **_kwargs: ScrapeRun([nice], total=1), + ) + monkeypatch.setattr( + "amfv_datasets.scraping.cli.scrape_cps", + lambda **_kwargs: ScrapeRun([cps], total=1), + ) + + run = scrape_documents(ScraperSource.ALL, documents=1, link_mode=LinkMode.KEEP) + + assert run.total == 2 + assert [document.source for document in run] == ["nice", "cps"] + + def _document() -> ScrapedDocument: return ScrapedDocument( source="nice", diff --git a/datasets/test/test_scraping_cps.py b/datasets/test/test_scraping_cps.py new file mode 100644 index 0000000..26d3bd9 --- /dev/null +++ b/datasets/test/test_scraping_cps.py @@ -0,0 +1,159 @@ +"""Tests for Canadian Paediatric Society statement scraping helpers.""" + +import httpx +import pytest + +from amfv_datasets.scraping.cps import ( + BASE_URL, + STATEMENTS_URL, + CpsFetchError, + CpsStatementRef, + list_statements, + listing_page_url, + scrape_statement, + statement_ref_from_url, +) + + +def test_listing_page_url_uses_cps_offsets() -> None: + """One-based pages map to the CPS P30 offset convention.""" + assert listing_page_url(1) == STATEMENTS_URL + assert listing_page_url(2) == f"{STATEMENTS_URL}/P10" + assert listing_page_url(4) == f"{STATEMENTS_URL}/P30" + + +def test_list_statements_discovers_unique_position_pages() -> None: + """The index yields unique position statements and ignores unrelated links.""" + html_text = """ + + """ + + def handler(request: httpx.Request) -> httpx.Response: + assert str(request.url) == STATEMENTS_URL + return httpx.Response(200, text=html_text) + + with httpx.Client(transport=httpx.MockTransport(handler)) as client: + assert list_statements(client, 1) == [ + CpsStatementRef("acute-asthma", "Acute asthma", f"{BASE_URL}/en/documents/position/acute-asthma"), + CpsStatementRef( + "newborn-glucose", + "Newborn glucose", + f"{BASE_URL}/en/documents/position/newborn-glucose", + ), + ] + + +def test_statement_ref_from_url_normalizes_supported_routes() -> None: + """English and language-neutral CPS routes become canonical HTTPS URLs.""" + assert statement_ref_from_url("http://www.cps.ca/documents/position/Acute-Asthma?print=1#dose") == ( + CpsStatementRef( + "Acute-Asthma", + "Acute Asthma", + "https://cps.ca/en/documents/position/Acute-Asthma", + ) + ) + + +def test_statement_ref_from_url_rejects_non_statement_pages() -> None: + """CPS navigation pages cannot be mistaken for clinical statements.""" + with pytest.raises(CpsFetchError): + statement_ref_from_url("https://cps.ca/en/documents") + + +def test_scrape_statement_preserves_clinical_structure_and_metadata() -> None: + """Statement content is normalized while navigation and sharing chrome are removed.""" + html_text = """ + + Fallback | Canadian Paediatric Society + +

Site heading

+
+
+ +

Position statement

+

Management of well-appearing febrile young infants

+

Posted: Oct 27, 2023 | Reaffirmed: Jan 12, 2026 | Updated: May 27, 2026

+

Recommendations

+
  • Assess the infant:
    • Check vital signs.
+

Use the risk calculator.

+

Current citation[1].

+

Legacy citation[2].

+

Maternal fever above 38oC and counts of 109/L require attention.

+

Full statement PDF + PDF icon

+

+
AgeAction
0-28 daysInvestigate
+

Algorithm

+

+

References

+
    +
  1. Reference one.
  2. +
  3. Reference two.
  4. +
+ Quiz + + + +
+
+
Contact CPS
+ + + """ + ref = CpsStatementRef( + "febrile-young-infants", + "Listing title", + f"{BASE_URL}/en/documents/position/febrile-young-infants", + ) + + def handler(request: httpx.Request) -> httpx.Response: + assert str(request.url) == ref.page_url + return httpx.Response(200, text=html_text) + + with httpx.Client(transport=httpx.MockTransport(handler)) as client: + document = scrape_statement(client, ref) + + assert document.source == "cps" + assert document.external_id == "cps-febrile-young-infants" + assert document.title == "Management of well-appearing febrile young infants" + assert document.section_count == 3 + assert document.metadata == { + "slug": "febrile-young-infants", + "posted": "Oct 27, 2023", + "reaffirmed": "Jan 12, 2026", + "updated": "May 27, 2026", + } + assert "- Assess the infant:\n - Check vital signs." in document.content + assert "[risk calculator](https://cps.ca/en/tools/risk-calculator)" in document.content + assert ( + "Current citation[1](https://cps.ca/en/documents/position/febrile-young-infants#ref1)." + in document.content + ) + assert ( + "Legacy citation[2](https://cps.ca/en/documents/position/febrile-young-infants#ref2)." + in document.content + ) + assert "38oC" in document.content + assert "109/L" in document.content + assert "# Management of well-appearing" not in document.content + assert "## Algorithm" in document.content + assert "## **Algorithm**" not in document.content + assert "| Age | Action |" in document.content + assert "![](https://cps.ca/uploads/flowcharts/febrile-infant.png)" in document.content + assert "## References" in document.content + assert "1. Reference one." in document.content + assert "2. Reference two." in document.content + assert "[Full statement PDF](https://cps.ca/documents/full-statement.pdf)" in document.content + assert "file-pdf.svg" not in document.content + assert "empty-target" not in document.content + assert "Share this statement" not in document.content + assert "Related news" not in document.content + assert "test-your-knowledge" not in document.content + assert "assets/img/test.png" not in document.content + assert "window.track" not in document.content + assert "Contact CPS" not in document.content diff --git a/datasets/test/test_scraping_html.py b/datasets/test/test_scraping_html.py index d34a8fa..6e82ae3 100644 --- a/datasets/test/test_scraping_html.py +++ b/datasets/test/test_scraping_html.py @@ -68,3 +68,12 @@ def test_html_to_markdown_can_strip_links() -> None: html_text = '

Offer treatment.

' assert html_to_markdown(html_text, link_mode=LinkMode.STRIP) == "Offer treatment." + + +def test_html_to_markdown_absolutizes_remote_image_references() -> None: + """Relative image references remain usable outside the source website.""" + html_text = '

Treatment flowchart

' + + assert html_to_markdown(html_text, base_url="https://example.org") == ( + "![Treatment flowchart](https://example.org/uploads/flowchart.png)" + )