From b6964d0391a702f2908fba06384a54dae031f23a Mon Sep 17 00:00:00 2001 From: Muhamed Kouate Date: Wed, 8 Jul 2026 18:36:00 +0200 Subject: [PATCH 1/2] add IDSA scraper --- datasets/amfv_datasets/scraping/__init__.py | 28 ++ datasets/amfv_datasets/scraping/cli.py | 46 ++- datasets/amfv_datasets/scraping/idsa.py | 399 ++++++++++++++++++++ datasets/test/test_scraping_cli.py | 132 +++++++ datasets/test/test_scraping_idsa.py | 320 ++++++++++++++++ 5 files changed, 921 insertions(+), 4 deletions(-) create mode 100644 datasets/amfv_datasets/scraping/idsa.py create mode 100644 datasets/test/test_scraping_idsa.py diff --git a/datasets/amfv_datasets/scraping/__init__.py b/datasets/amfv_datasets/scraping/__init__.py index cd7cdb0..d8a0aa0 100644 --- a/datasets/amfv_datasets/scraping/__init__.py +++ b/datasets/amfv_datasets/scraping/__init__.py @@ -17,6 +17,23 @@ first_matching_urls, html_to_markdown, ) +from amfv_datasets.scraping.idsa import ( + IDSA_DATASET_DISPLAY_NAME, + IDSA_DATASET_NAME, + LISTING_URL, + IDSAFetchError, + IDSAGuidelineListingPage, + IDSAGuidelineRef, + idsa_ref_from_url, + list_practice_guidelines, + scrape_idsa, +) +from amfv_datasets.scraping.idsa import ( + build_guideline_text as build_idsa_guideline_text, +) +from amfv_datasets.scraping.idsa import ( + scrape_guideline as scrape_idsa_guideline, +) from amfv_datasets.scraping.nice import ( GuidanceListingPage, GuidanceRef, @@ -31,7 +48,13 @@ __all__ = [ "GuidanceRef", "GuidanceListingPage", + "IDSAFetchError", + "IDSA_DATASET_DISPLAY_NAME", + "IDSA_DATASET_NAME", + "IDSAGuidelineListingPage", + "IDSAGuidelineRef", "LinkMode", + "LISTING_URL", "NiceFetchError", "OutputFormat", "ScrapeError", @@ -40,6 +63,7 @@ "ScraperSource", "USER_AGENT", "absolute_unique_urls", + "build_idsa_guideline_text", "build_guideline_text", "clean_text", "document_title", @@ -47,8 +71,12 @@ "first_matching_urls", "guidance_ref_from_url", "html_to_markdown", + "idsa_ref_from_url", + "list_practice_guidelines", "list_published_guidance", "scrape_guideline", + "scrape_idsa", + "scrape_idsa_guideline", "scrape_listing_documents", "scrape_nice", ] diff --git a/datasets/amfv_datasets/scraping/cli.py b/datasets/amfv_datasets/scraping/cli.py index e8aece3..7317045 100644 --- a/datasets/amfv_datasets/scraping/cli.py +++ b/datasets/amfv_datasets/scraping/cli.py @@ -8,6 +8,7 @@ from collections.abc import Iterable from dataclasses import asdict from enum import StrEnum +from itertools import chain from pathlib import Path from typing import Annotated, TextIO @@ -26,6 +27,7 @@ from amfv_datasets.scraping.base import ScrapedDocument, ScrapeRun from amfv_datasets.scraping.html import LinkMode +from amfv_datasets.scraping.idsa import scrape_idsa from amfv_datasets.scraping.nice import scrape_nice @@ -33,6 +35,7 @@ class ScraperSource(StrEnum): """Supported scraper sources.""" ALL = "all" + IDSA = "idsa" NICE = "nice" @@ -53,6 +56,8 @@ def scrape_documents( documents: int | None, link_mode: LinkMode, url: str | None = None, + include_archived: bool = False, + include_in_development: bool = False, ) -> ScrapeRun: """Configure a scrape for a source. @@ -64,17 +69,37 @@ def scrape_documents( link_mode: Whether links are kept as markdown links or stripped to their visible text. url: Source URL to scrape as a single document (default: None). + include_archived: Whether IDSA archived guidelines are included + (default: False). + include_in_development: Whether IDSA in-development guidelines are + included (default: False). """ if documents is not None and documents < 1: raise ValueError(f"documents must be at least 1; got {documents}") + scrape_runs: list[ScrapeRun] = [] for selected_source in _expand_source(source): match selected_source: + case ScraperSource.IDSA: + scrape_runs.append( + scrape_idsa( + documents=documents, + link_mode=link_mode, + url=url, + include_archived=include_archived, + include_in_development=include_in_development, + ) + ) case ScraperSource.NICE: - return scrape_nice(documents=documents, link_mode=link_mode, url=url) + scrape_runs.append(scrape_nice(documents=documents, link_mode=link_mode, url=url)) case ScraperSource.ALL: raise AssertionError("expanded source cannot be all") - raise AssertionError(f"unsupported source: {source}") + if not scrape_runs: + raise AssertionError(f"unsupported source: {source}") + if len(scrape_runs) == 1: + return scrape_runs[0] + total = None if any(run.total is None for run in scrape_runs) else sum(run.total or 0 for run in scrape_runs) + return ScrapeRun(documents=chain.from_iterable(run.documents for run in scrape_runs), total=total) def write_jsonl(documents: Iterable[ScrapedDocument], output: TextIO) -> int: @@ -125,7 +150,7 @@ def write_markdown_files(documents: Iterable[ScrapedDocument], output_path: Path def _expand_source(source: ScraperSource) -> tuple[ScraperSource, ...]: if source is ScraperSource.ALL: - return (ScraperSource.NICE,) + return (ScraperSource.NICE, ScraperSource.IDSA) return (source,) @@ -135,6 +160,8 @@ def run( url: Annotated[str | None, typer.Option("--url", help="Source URL to scrape as a single document.")] = None, documents: Annotated[str, typer.Option("--documents", help="Number of documents to scrape, or 'all'.")] = "1", link_mode: Annotated[LinkMode, typer.Option("--links", help="Whether to keep markdown links or strip links to text.")] = LinkMode.KEEP, # noqa: E501 + include_archived: Annotated[bool, typer.Option("--include-archived/--exclude-archived", help="Include archived guidelines for sources that expose archival status.")] = False, # noqa: E501 + include_in_development: Annotated[bool, typer.Option("--include-in-development/--exclude-in-development", help="Include in-development guidelines for sources that expose development status.")] = False, # noqa: E501 output_format: Annotated[OutputFormat, typer.Option("--format", "-f", help="Output format.")] = OutputFormat.JSONL, output_path: Annotated[Path | None, typer.Option("--output", "-o", help="Output JSONL file, markdown directory, or Hugging Face dataset directory. JSONL defaults to stdout.")] = None, # noqa: E501 progress: Annotated[bool, typer.Option("--progress/--no-progress", help="Show a Rich progress bar.")] = True, @@ -147,6 +174,10 @@ def run( documents: Number of documents to scrape, or "all" (default: "1"). link_mode: Whether links are kept as markdown links or stripped to their visible text (default: LinkMode.KEEP). + include_archived: Whether archived guidelines are included for sources + that expose archival status (default: False). + include_in_development: Whether in-development guidelines are included + for sources that expose development status (default: False). output_format: Output format to write (default: OutputFormat.JSONL). output_path: Output JSONL file, markdown directory, or Hugging Face dataset directory. When unset, JSONL is written to stdout (default: @@ -154,7 +185,14 @@ def run( progress: Whether to show a Rich progress bar (default: True). """ parsed_documents = _parse_documents(documents) - scrape_run = scrape_documents(source, documents=parsed_documents, link_mode=link_mode, url=url) + scrape_run = scrape_documents( + source, + documents=parsed_documents, + link_mode=link_mode, + url=url, + include_archived=include_archived, + include_in_development=include_in_development, + ) scraped_documents = scrape_run.documents if progress: scraped_documents = _progress_documents(scraped_documents, total=scrape_run.total) diff --git a/datasets/amfv_datasets/scraping/idsa.py b/datasets/amfv_datasets/scraping/idsa.py new file mode 100644 index 0000000..9941b3f --- /dev/null +++ b/datasets/amfv_datasets/scraping/idsa.py @@ -0,0 +1,399 @@ +"""Scrape IDSA practice guidelines into normalized markdown documents.""" + +from __future__ import annotations + +import re +from collections.abc import Iterable +from dataclasses import dataclass +from urllib.parse import urljoin, urlparse + +import httpx +from lxml import html as lxml_html + +from amfv_datasets.scraping.base import ( + ScrapedDocument, + ScrapeError, + ScrapeRun, + default_client, + scrape_listing_documents, +) +from amfv_datasets.scraping.html import LinkMode, absolute_unique_urls, clean_text, document_title, html_to_markdown + +BASE_URL = "https://www.idsociety.org" +LISTING_URL = f"{BASE_URL}/practice-guideline/all-practice-guidelines" +IDSA_DATASET_NAME = "idsa-webscrape" +IDSA_DATASET_DISPLAY_NAME = "IDSA Webscrape" +DOCUMENT_DELAY_SECONDS = 5.0 +SHORT_CONTENT_CHARS = 10_000 + +_IDSA_PATH_RE = re.compile(r"^/practice-guideline/(?P[^/]+)(?:/|$)", re.IGNORECASE) +_BACK_TO_TOP_RE = re.compile(r"^\s*back to top\s*$", re.IGNORECASE) +_TABLE_OF_CONTENTS_RE = re.compile(r"^\s*table\s+of\s+contents\s*$", re.IGNORECASE) +_PDF_TEXT_RE = re.compile(r"\b(download\s+)?pdf\b", re.IGNORECASE) +_WHITESPACE_RE = re.compile(r"\s+") + + +class IDSAFetchError(ScrapeError): + """Raised when a practice guideline cannot be sourced from IDSA.""" + + +@dataclass(frozen=True) +class IDSAGuidelineRef: + """An IDSA practice guideline reference from the A-Z listing.""" + + title: str + slug: str + page_url: str + year: int | None + statuses: tuple[str, ...] + + +@dataclass(frozen=True) +class IDSAGuidelineListingPage: + """An IDSA practice guideline listing page.""" + + refs: list[IDSAGuidelineRef] + total: int | None + + +def idsa_ref_from_url(url: str) -> IDSAGuidelineRef: + """Parse an IDSA practice guideline URL into a canonical guideline reference. + + Args: + url: IDSA practice guideline URL to parse. + """ + parsed = urlparse(url.strip()) + if parsed.scheme not in {"http", "https"} or parsed.netloc.lower() not in { + "www.idsociety.org", + "idsociety.org", + }: + raise IDSAFetchError(f"Enter an IDSA practice guideline URL from idsociety.org; got {url!r}") + + match = _IDSA_PATH_RE.match(parsed.path.rstrip("/") + "/") + if not match: + raise IDSAFetchError(f"Enter a URL like https://www.idsociety.org/practice-guideline/example/; got {url!r}") + + slug = match.group("slug").lower() + return IDSAGuidelineRef( + title=slug.replace("-", " ").title(), + slug=slug, + page_url=_page_url(slug), + year=None, + statuses=(), + ) + + +def list_practice_guidelines( + client: httpx.Client, + *, + include_archived: bool = False, + include_in_development: bool = False, +) -> IDSAGuidelineListingPage: + """Return IDSA practice guideline refs from the A-Z listing. + + Args: + client: HTTP client used to fetch the listing page. + include_archived: Whether archived guidelines are included (default: False). + include_in_development: Whether in-development guidelines are included (default: False). + """ + response = client.get(LISTING_URL) + response.raise_for_status() + refs = _parse_listing( + response.text, + include_archived=include_archived, + include_in_development=include_in_development, + ) + return IDSAGuidelineListingPage(refs=refs, total=len(refs)) + + +def build_guideline_text( + client: httpx.Client, + ref: IDSAGuidelineRef, + *, + link_mode: LinkMode = LinkMode.KEEP, +) -> tuple[str, int, str, dict[str, list[str]]]: + """Scrape a guideline page into markdown text, section count, title, and links. + + Args: + client: HTTP client used to fetch the guideline page. + ref: IDSA guideline reference to scrape. + link_mode: Whether links are kept as markdown links or stripped to their + visible text (default: LinkMode.KEEP). + """ + response = client.get(ref.page_url) + response.raise_for_status() + title = document_title(response.text, fallback=ref.title) + content_html = _guideline_content_html(response.text) + content = html_to_markdown(content_html, link_mode=link_mode, base_url=BASE_URL) + if not content: + raise IDSAFetchError(f"No readable content for IDSA guideline '{ref.slug}'") + return content, _section_count(content_html), title, _links_metadata(content_html) + + +def scrape_guideline( + client: httpx.Client, + ref: IDSAGuidelineRef, + *, + link_mode: LinkMode = LinkMode.KEEP, +) -> ScrapedDocument: + """Scrape an IDSA practice guideline into a normalized document. + + Args: + client: HTTP client used to fetch the guideline page. + ref: IDSA guideline reference to scrape. + link_mode: Whether links are kept as markdown links or stripped to their + visible text (default: LinkMode.KEEP). + """ + content, section_count, title, links_metadata = build_guideline_text(client, ref, link_mode=link_mode) + return ScrapedDocument( + source="idsa", + external_id=f"idsa-{ref.slug}", + title=title, + url=ref.page_url, + content=content, + section_count=section_count, + metadata={ + "year": ref.year, + "statuses": list(ref.statuses), + "slug": ref.slug, + "listing_url": LISTING_URL, + "content_length_chars": len(content), + "quality_flags": _quality_flags(content), + **links_metadata, + }, + ) + + +def scrape_idsa( + *, + documents: int | None, + link_mode: LinkMode = LinkMode.KEEP, + url: str | None = None, + include_archived: bool = False, + include_in_development: bool = False, +) -> ScrapeRun: + """Scrape IDSA practice guidelines from a URL or the A-Z listing. + + Args: + documents: Number of documents to scrape. Ignored when `url` is set. + When unset, every included IDSA listing item is scraped (default: + None). + link_mode: Whether links are kept as markdown links or stripped to their + visible text (default: LinkMode.KEEP). + url: IDSA source URL to scrape as a single document (default: None). + include_archived: Whether archived guidelines are included (default: False). + include_in_development: Whether in-development guidelines are included + (default: False). + """ + if url is not None: + + def scrape_url() -> Iterable[ScrapedDocument]: + with default_client() as client: + yield scrape_guideline(client, idsa_ref_from_url(url), link_mode=link_mode) + + return ScrapeRun(documents=scrape_url(), total=1) + + with default_client() as client: + listing = list_practice_guidelines( + client, + include_archived=include_archived, + include_in_development=include_in_development, + ) + total = listing.total if documents is None or listing.total is None else min(documents, listing.total) + return ScrapeRun( + total=total, + documents=scrape_listing_documents( + documents=documents, + client_factory=default_client, + first_page_items=listing.refs, + list_page=lambda _client, _page: (), + scrape_item=lambda client, ref: scrape_guideline(client, ref, link_mode=link_mode), + document_delay_seconds=DOCUMENT_DELAY_SECONDS, + ), + ) + + +def _parse_listing( + html_text: str, + *, + include_archived: bool, + include_in_development: bool, +) -> list[IDSAGuidelineRef]: + doc = lxml_html.fromstring(html_text) + items = doc.xpath( + "//div[contains(concat(' ', normalize-space(@class), ' '), ' alpha-listing ')]" + "//li[.//a[contains(concat(' ', normalize-space(@class), ' '), ' list-pages__link ')]]" + ) + refs: list[IDSAGuidelineRef] = [] + for item in items: + link = item.xpath(".//a[contains(concat(' ', normalize-space(@class), ' '), ' list-pages__link ')][1]") + if not link: + continue + statuses = tuple( + clean_text(status, drop_numeric_citations=False) + for status in item.xpath( + ".//*[contains(concat(' ', normalize-space(@class), ' '), ' category-dot ')]/text()" + ) + ) + if not _included_statuses( + statuses, + include_archived=include_archived, + include_in_development=include_in_development, + ): + continue + href = link[0].get("href") + if not href: + continue + page_url = urljoin(BASE_URL, href).split("#")[0].split("?")[0] + match = _IDSA_PATH_RE.match(urlparse(page_url).path.rstrip("/") + "/") + if not match: + continue + refs.append( + IDSAGuidelineRef( + title=clean_text(link[0].text_content(), drop_numeric_citations=False), + slug=match.group("slug").lower(), + page_url=page_url, + year=_parse_year(item), + statuses=statuses, + ) + ) + return refs + + +def _included_statuses( + statuses: tuple[str, ...], + *, + include_archived: bool, + include_in_development: bool, +) -> bool: + has_current = "Current" in statuses + has_archived = "Archived" in statuses + has_in_development = "In Development" in statuses + if has_archived and not include_archived: + return False + if has_in_development and not include_in_development: + return False + return has_current or (has_archived and include_archived) or (has_in_development and include_in_development) + + +def _parse_year(item: lxml_html.HtmlElement) -> int | None: + values = item.xpath(".//*[contains(concat(' ', normalize-space(@class), ' '), ' list-pages__year ')]/text()") + if not values: + return None + value = clean_text(values[0], drop_numeric_citations=False) + return int(value) if value.isdigit() else None + + +def _guideline_content_html(html_text: str) -> str: + doc = lxml_html.fromstring(html_text) + candidates = doc.xpath("//div[contains(concat(' ', normalize-space(@class), ' '), ' idsaPracticeGuidelinePage ')]") + if not candidates: + candidates = doc.xpath("//*[contains(concat(' ', normalize-space(@class), ' '), ' body-container ')]") + if not candidates: + candidates = doc.xpath("//div[contains(concat(' ', normalize-space(@class), ' '), ' standardpage-col-left ')]") + if not candidates: + raise IDSAFetchError("Page markup changed (no IDSA guideline content container)") + content = candidates[-1] + _remove_noise(content) + content_html = lxml_html.tostring(content, encoding="unicode") + if not clean_text(content.text_content(), drop_numeric_citations=False): + raise IDSAFetchError("Page markup changed (empty IDSA guideline content container)") + return content_html + + +def _remove_noise(content: lxml_html.HtmlElement) -> None: + noise_xpath = ( + ".//*[self::script or self::style or self::noscript or self::svg or self::button" + " or contains(concat(' ', normalize-space(@class), ' '), ' table-of-contents ')" + " or contains(concat(' ', normalize-space(@class), ' '), ' toc ')" + " or contains(concat(' ', normalize-space(@class), ' '), ' TableOfContents ')" + " or contains(concat(' ', normalize-space(@class), ' '), ' status-section ')" + " or contains(concat(' ', normalize-space(@class), ' '), ' view-all-guidance ')" + " or contains(concat(' ', normalize-space(@class), ' '), ' share ')" + " or contains(concat(' ', normalize-space(@class), ' '), ' addthis ')" + " or contains(concat(' ', normalize-space(@class), ' '), ' social ')]" + ) + for element in content.xpath(noise_xpath): + _drop_element(element) + _remove_table_of_contents(content) + for element in content.xpath(".//*[self::a or self::p or self::div or self::span]"): + if _BACK_TO_TOP_RE.match(element.text_content()): + _drop_element(element) + + +def _remove_table_of_contents(content: lxml_html.HtmlElement) -> None: + headings = content.xpath(".//*[self::h1 or self::h2 or self::h3 or self::h4 or self::h5 or self::h6]") + for heading in headings: + if not _TABLE_OF_CONTENTS_RE.match(heading.text_content()): + continue + for sibling in list(heading.itersiblings()): + if _heading_level(sibling) is not None: + break + _drop_element(sibling) + _drop_element(heading) + + +def _heading_level(element: lxml_html.HtmlElement) -> int | None: + tag = element.tag.lower() if isinstance(element.tag, str) else "" + if len(tag) == 2 and tag.startswith("h") and tag[1].isdigit(): + return int(tag[1]) + return None + + +def _drop_element(element: lxml_html.HtmlElement) -> None: + parent = element.getparent() + if parent is not None: + parent.remove(element) + + +def _links_metadata(content_html: str) -> dict[str, list[str]]: + doc = lxml_html.fromstring(content_html) + external_links: list[str] = [] + pdf_links: list[str] = [] + for link in doc.xpath(".//a[@href]"): + url = urljoin(BASE_URL, link.get("href")).split("#")[0].split("?")[0] + if urlparse(url).netloc.lower() in {"www.idsociety.org", "idsociety.org"}: + continue + external_links.append(url) + link_text = _WHITESPACE_RE.sub(" ", link.text_content()).strip() + if url.lower().endswith(".pdf") or _PDF_TEXT_RE.search(link_text): + pdf_links.append(url) + return { + "external_links": absolute_unique_urls(external_links, base_url=BASE_URL), + "pdf_links": absolute_unique_urls(pdf_links, base_url=BASE_URL), + } + + +def _section_count(content_html: str) -> int: + doc = lxml_html.fromstring(content_html) + headings = doc.xpath(".//*[self::h1 or self::h2 or self::h3]") + return max(1, len(headings)) + + +def _quality_flags(content: str) -> list[str]: + flags: list[str] = [] + if len(content) < SHORT_CONTENT_CHARS: + flags.append("short_content") + return flags + + +def _page_url(slug: str) -> str: + return f"{BASE_URL}/practice-guideline/{slug}/" + + +__all__ = [ + "BASE_URL", + "DOCUMENT_DELAY_SECONDS", + "IDSA_DATASET_DISPLAY_NAME", + "IDSA_DATASET_NAME", + "IDSAFetchError", + "IDSAGuidelineListingPage", + "IDSAGuidelineRef", + "LISTING_URL", + "SHORT_CONTENT_CHARS", + "build_guideline_text", + "idsa_ref_from_url", + "list_practice_guidelines", + "scrape_guideline", + "scrape_idsa", +] diff --git a/datasets/test/test_scraping_cli.py b/datasets/test/test_scraping_cli.py index 44f714f..bc4eccd 100644 --- a/datasets/test/test_scraping_cli.py +++ b/datasets/test/test_scraping_cli.py @@ -70,11 +70,15 @@ def fake_scrape_documents( documents: int | None, link_mode: LinkMode, url: str | None = None, + include_archived: bool = False, + include_in_development: bool = False, ) -> ScrapeRun: assert source is ScraperSource.NICE assert documents == 3 assert link_mode is LinkMode.STRIP assert url is None + assert include_archived is False + assert include_in_development is False return ScrapeRun([_document()], total=3) monkeypatch.setattr("amfv_datasets.scraping.cli.scrape_documents", fake_scrape_documents) @@ -105,6 +109,8 @@ def fake_scrape_documents( documents: int | None, link_mode: LinkMode, url: str | None = None, + include_archived: bool = False, + include_in_development: bool = False, ) -> ScrapeRun: return ScrapeRun([_document()], total=None) @@ -137,6 +143,8 @@ def fake_scrape_documents( documents: int | None, link_mode: LinkMode, url: str | None = None, + include_archived: bool = False, + include_in_development: bool = False, ) -> ScrapeRun: return ScrapeRun([_document()], total=7) @@ -204,11 +212,15 @@ def fake_scrape_documents( documents: int | None, link_mode: LinkMode, url: str | None = None, + include_archived: bool = False, + include_in_development: bool = False, ) -> ScrapeRun: assert source is ScraperSource.ALL assert documents is None assert link_mode is LinkMode.KEEP assert url is None + assert include_archived is False + assert include_in_development is False return ScrapeRun([_document()], total=12) monkeypatch.setattr("amfv_datasets.scraping.cli.scrape_documents", fake_scrape_documents) @@ -219,6 +231,115 @@ def fake_scrape_documents( assert json.loads(result.stdout.splitlines()[0])["external_id"] == "nice-ng1" +def test_cli_run_accepts_idsa_status_flags(monkeypatch: pytest.MonkeyPatch) -> None: + """The CLI forwards IDSA status inclusion flags.""" + runner = CliRunner() + + def fake_scrape_documents( + source: ScraperSource, + *, + documents: int | None, + link_mode: LinkMode, + url: str | None = None, + include_archived: bool = False, + include_in_development: bool = False, + ) -> ScrapeRun: + assert source is ScraperSource.IDSA + assert documents == 2 + assert link_mode is LinkMode.KEEP + assert url is None + assert include_archived is True + assert include_in_development is True + return ScrapeRun([_idsa_document()], total=2) + + monkeypatch.setattr("amfv_datasets.scraping.cli.scrape_documents", fake_scrape_documents) + + result = runner.invoke( + app, + ["--source", "idsa", "--documents", "2", "--include-archived", "--include-in-development"], + ) + + assert result.exit_code == 0 + assert json.loads(result.stdout.splitlines()[0])["external_id"] == "idsa-current-guideline" + assert "scraped 1 documents from idsa" in result.stderr + + +def test_scrape_documents_dispatches_idsa(monkeypatch: pytest.MonkeyPatch) -> None: + """The scrape dispatcher calls the IDSA scraper for the IDSA source.""" + + def fake_scrape_idsa( + *, + documents: int | None, + link_mode: LinkMode, + url: str | None, + include_archived: bool, + include_in_development: bool, + ) -> ScrapeRun: + assert documents == 1 + assert link_mode is LinkMode.STRIP + assert url == "https://www.idsociety.org/practice-guideline/current-guideline/" + assert include_archived is True + assert include_in_development is False + return ScrapeRun([_idsa_document()], total=1) + + monkeypatch.setattr("amfv_datasets.scraping.cli.scrape_idsa", fake_scrape_idsa) + + from amfv_datasets.scraping.cli import scrape_documents + + scrape_run = scrape_documents( + ScraperSource.IDSA, + documents=1, + link_mode=LinkMode.STRIP, + url="https://www.idsociety.org/practice-guideline/current-guideline/", + include_archived=True, + ) + + assert list(scrape_run.documents) == [_idsa_document()] + + +def test_scrape_documents_all_includes_idsa(monkeypatch: pytest.MonkeyPatch) -> None: + """The all source combines NICE and IDSA scrape runs.""" + + def fake_scrape_nice( + *, + documents: int | None, + link_mode: LinkMode, + url: str | None, + ) -> ScrapeRun: + assert documents == 1 + assert link_mode is LinkMode.KEEP + assert url is None + return ScrapeRun([_document()], total=1) + + def fake_scrape_idsa( + *, + documents: int | None, + link_mode: LinkMode, + url: str | None, + include_archived: bool, + include_in_development: bool, + ) -> ScrapeRun: + assert documents == 1 + assert link_mode is LinkMode.KEEP + assert url is None + assert include_archived is False + assert include_in_development is False + return ScrapeRun([_idsa_document()], total=1) + + monkeypatch.setattr("amfv_datasets.scraping.cli.scrape_nice", fake_scrape_nice) + monkeypatch.setattr("amfv_datasets.scraping.cli.scrape_idsa", fake_scrape_idsa) + + from amfv_datasets.scraping.cli import scrape_documents + + scrape_run = scrape_documents(ScraperSource.ALL, documents=1, link_mode=LinkMode.KEEP) + + assert scrape_run.total == 2 + assert [document.external_id for document in scrape_run.documents] == [ + "nice-ng1", + "idsa-current-guideline", + ] + + def _document() -> ScrapedDocument: return ScrapedDocument( source="nice", @@ -230,6 +351,17 @@ def _document() -> ScrapedDocument: ) +def _idsa_document() -> ScrapedDocument: + return ScrapedDocument( + source="idsa", + external_id="idsa-current-guideline", + title="Current Guideline", + url="https://www.idsociety.org/practice-guideline/current-guideline/", + content="content", + metadata={"slug": "current-guideline"}, + ) + + class _TextSink: def __init__(self) -> None: self.value = "" diff --git a/datasets/test/test_scraping_idsa.py b/datasets/test/test_scraping_idsa.py new file mode 100644 index 0000000..a83c4ae --- /dev/null +++ b/datasets/test/test_scraping_idsa.py @@ -0,0 +1,320 @@ +"""Tests for IDSA scraping helpers.""" + +import httpx +import pytest + +from amfv_datasets.scraping.html import LinkMode +from amfv_datasets.scraping.idsa import ( + BASE_URL, + LISTING_URL, + IDSAFetchError, + IDSAGuidelineRef, + build_guideline_text, + idsa_ref_from_url, + list_practice_guidelines, + scrape_guideline, + scrape_idsa, +) + + +def test_list_practice_guidelines_parses_and_filters_default_statuses() -> None: + """The IDSA listing includes only current non-development guidelines by default.""" + client = httpx.Client(transport=httpx.MockTransport(_listing_handler), base_url=BASE_URL) + + listing = list_practice_guidelines(client) + + assert listing.total == 2 + assert listing.refs == [ + IDSAGuidelineRef( + title="Current Guideline", + slug="current-guideline", + page_url="https://www.idsociety.org/practice-guideline/current-guideline/", + year=2024, + statuses=("Current",), + ), + IDSAGuidelineRef( + title="Current Endorsed Guideline", + slug="current-endorsed-guideline", + page_url="https://www.idsociety.org/practice-guideline/current-endorsed-guideline/", + year=2023, + statuses=("Current", "Endorsed"), + ), + ] + + +def test_list_practice_guidelines_can_include_archived_statuses() -> None: + """Archived guidelines are included only when requested.""" + client = httpx.Client(transport=httpx.MockTransport(_listing_handler), base_url=BASE_URL) + + listing = list_practice_guidelines(client, include_archived=True) + + assert [ref.slug for ref in listing.refs] == [ + "current-guideline", + "current-endorsed-guideline", + "archived-guideline", + ] + + +def test_list_practice_guidelines_can_include_in_development_statuses() -> None: + """In-development guidelines are included only when requested.""" + client = httpx.Client(transport=httpx.MockTransport(_listing_handler), base_url=BASE_URL) + + listing = list_practice_guidelines(client, include_in_development=True) + + assert [ref.slug for ref in listing.refs] == [ + "current-guideline", + "current-endorsed-guideline", + "development-guideline", + ] + + +def test_list_practice_guidelines_requires_both_flags_for_archived_development_statuses() -> None: + """Guidelines marked both archived and in development require both inclusion flags.""" + client = httpx.Client(transport=httpx.MockTransport(_listing_handler), base_url=BASE_URL) + + listing = list_practice_guidelines(client, include_archived=True, include_in_development=True) + + assert [ref.slug for ref in listing.refs] == [ + "current-guideline", + "current-endorsed-guideline", + "archived-guideline", + "development-guideline", + "archived-development-guideline", + ] + + +def test_idsa_ref_from_url_normalizes_practice_guideline_url() -> None: + """IDSA practice guideline URLs are normalized to canonical refs.""" + assert idsa_ref_from_url("https://www.idsociety.org/practice-guideline/Current-Guideline/?utm=1") == ( + IDSAGuidelineRef( + title="Current Guideline", + slug="current-guideline", + page_url="https://www.idsociety.org/practice-guideline/current-guideline/", + year=None, + statuses=(), + ) + ) + + +@pytest.mark.parametrize( + "url", + [ + "https://example.com/practice-guideline/current-guideline/", + "https://www.idsociety.org/news/current-guideline/", + ], + ids=["wrong-domain", "wrong-path"], +) +def test_idsa_ref_from_url_rejects_non_guideline_urls(url: str) -> None: + """Only IDSA practice guideline URLs are accepted.""" + with pytest.raises(IDSAFetchError): + idsa_ref_from_url(url) + + +def test_build_guideline_text_extracts_content_and_link_metadata() -> None: + """Guideline page content is converted to markdown and noisy UI is stripped.""" + client = httpx.Client(transport=httpx.MockTransport(_guideline_handler), base_url=BASE_URL) + + content, section_count, title, links_metadata = build_guideline_text( + client, + IDSAGuidelineRef( + title="Current Guideline", + slug="current-guideline", + page_url="https://www.idsociety.org/practice-guideline/current-guideline/", + year=2024, + statuses=("Current",), + ), + ) + + assert title == "Current Guideline" + assert section_count == 3 + assert "# Current Guideline" in content + assert "## Abstract" in content + assert "## Recommendations" in content + assert "Recommendation text with [evidence](https://doi.org/10.1093/cid/example)." in content + assert "Back to top" not in content + assert "Table of Contents" not in content + assert "https://www.idsociety.org#abstract" not in content + assert links_metadata == { + "external_links": [ + "https://academic.oup.com/example.pdf", + "https://doi.org/10.1093/cid/example", + ], + "pdf_links": ["https://academic.oup.com/example.pdf"], + } + + stripped_content, _section_count, _title, _links_metadata = build_guideline_text( + client, + IDSAGuidelineRef( + title="Current Guideline", + slug="current-guideline", + page_url="https://www.idsociety.org/practice-guideline/current-guideline/", + year=2024, + statuses=("Current",), + ), + link_mode=LinkMode.STRIP, + ) + assert "Recommendation text with evidence." in stripped_content + + +def test_scrape_idsa_returns_normalized_document() -> None: + """The IDSA scraper returns normalized scraped documents.""" + + def handler(request: httpx.Request) -> httpx.Response: + if str(request.url) == LISTING_URL: + return httpx.Response(200, text=_listing_html()) + if str(request.url) == "https://www.idsociety.org/practice-guideline/current-guideline/": + return httpx.Response(200, text=_guideline_html()) + raise AssertionError(f"Unexpected URL: {request.url}") + + original_client_factory = "amfv_datasets.scraping.idsa.default_client" + transport = httpx.MockTransport(handler) + + class ClientFactory: + def __call__(self) -> httpx.Client: + return httpx.Client(transport=transport) + + with pytest.MonkeyPatch.context() as monkeypatch: + monkeypatch.setattr(original_client_factory, ClientFactory()) + scrape_run = scrape_idsa(documents=1) + documents = list(scrape_run.documents) + + assert scrape_run.total == 1 + assert len(documents) == 1 + assert documents[0].source == "idsa" + assert documents[0].external_id == "idsa-current-guideline" + assert documents[0].metadata["statuses"] == ["Current"] + assert documents[0].metadata["year"] == 2024 + assert documents[0].metadata["pdf_links"] == ["https://academic.oup.com/example.pdf"] + assert documents[0].metadata["content_length_chars"] == len(documents[0].content) + assert documents[0].metadata["quality_flags"] == ["short_content"] + + +def test_scrape_guideline_does_not_flag_long_content() -> None: + """Long guideline content records length without short-content quality flags.""" + + def handler(request: httpx.Request) -> httpx.Response: + assert str(request.url) == "https://www.idsociety.org/practice-guideline/long-guideline/" + return httpx.Response(200, text=_long_guideline_html()) + + client = httpx.Client(transport=httpx.MockTransport(handler), base_url=BASE_URL) + + document = scrape_guideline( + client, + IDSAGuidelineRef( + title="Long Guideline", + slug="long-guideline", + page_url="https://www.idsociety.org/practice-guideline/long-guideline/", + year=2024, + statuses=("Current",), + ), + ) + + assert document.metadata["content_length_chars"] == len(document.content) + assert document.metadata["quality_flags"] == [] + + +def _listing_handler(request: httpx.Request) -> httpx.Response: + assert str(request.url) == LISTING_URL + return httpx.Response(200, text=_listing_html()) + + +def _guideline_handler(request: httpx.Request) -> httpx.Response: + assert str(request.url) == "https://www.idsociety.org/practice-guideline/current-guideline/" + return httpx.Response(200, text=_guideline_html()) + + +def _listing_html() -> str: + return """ + +
+ +
+ + """ + + +def _guideline_html() -> str: + return """ + + Current Guideline | IDSA +
+ +

Intro column noise.

+
+
+

Download PDF

+

Current Guideline

+

Table of Contents

+ + +

Published January 1, 2024

+

Abstract

+

Useful abstract.

+

Recommendations

+

Recommendation text with evidence.

+

Back to top

+
+ + """ + + +def _long_guideline_html() -> str: + long_text = "Recommendation text. " * 700 + return f""" + + Long Guideline | IDSA +
+

Long Guideline

+

Recommendations

+

{long_text}

+
+ + """ From 5a157e47eaa6c4dcf169404531bfc03bccd20d6f Mon Sep 17 00:00:00 2001 From: Muhamed Kouate Date: Wed, 8 Jul 2026 19:04:51 +0200 Subject: [PATCH 2/2] simplify IDSA scraper --- datasets/amfv_datasets/scraping/__init__.py | 12 - datasets/amfv_datasets/scraping/idsa.py | 22 +- datasets/test/test_scraping_idsa.py | 318 ++++++++------------ 3 files changed, 132 insertions(+), 220 deletions(-) diff --git a/datasets/amfv_datasets/scraping/__init__.py b/datasets/amfv_datasets/scraping/__init__.py index d8a0aa0..199e053 100644 --- a/datasets/amfv_datasets/scraping/__init__.py +++ b/datasets/amfv_datasets/scraping/__init__.py @@ -18,19 +18,12 @@ html_to_markdown, ) from amfv_datasets.scraping.idsa import ( - IDSA_DATASET_DISPLAY_NAME, - IDSA_DATASET_NAME, - LISTING_URL, IDSAFetchError, - IDSAGuidelineListingPage, IDSAGuidelineRef, idsa_ref_from_url, list_practice_guidelines, scrape_idsa, ) -from amfv_datasets.scraping.idsa import ( - build_guideline_text as build_idsa_guideline_text, -) from amfv_datasets.scraping.idsa import ( scrape_guideline as scrape_idsa_guideline, ) @@ -49,12 +42,8 @@ "GuidanceRef", "GuidanceListingPage", "IDSAFetchError", - "IDSA_DATASET_DISPLAY_NAME", - "IDSA_DATASET_NAME", - "IDSAGuidelineListingPage", "IDSAGuidelineRef", "LinkMode", - "LISTING_URL", "NiceFetchError", "OutputFormat", "ScrapeError", @@ -63,7 +52,6 @@ "ScraperSource", "USER_AGENT", "absolute_unique_urls", - "build_idsa_guideline_text", "build_guideline_text", "clean_text", "document_title", diff --git a/datasets/amfv_datasets/scraping/idsa.py b/datasets/amfv_datasets/scraping/idsa.py index 9941b3f..1feffcc 100644 --- a/datasets/amfv_datasets/scraping/idsa.py +++ b/datasets/amfv_datasets/scraping/idsa.py @@ -21,10 +21,8 @@ BASE_URL = "https://www.idsociety.org" LISTING_URL = f"{BASE_URL}/practice-guideline/all-practice-guidelines" -IDSA_DATASET_NAME = "idsa-webscrape" -IDSA_DATASET_DISPLAY_NAME = "IDSA Webscrape" DOCUMENT_DELAY_SECONDS = 5.0 -SHORT_CONTENT_CHARS = 10_000 +_SHORT_CONTENT_CHARS = 10_000 _IDSA_PATH_RE = re.compile(r"^/practice-guideline/(?P[^/]+)(?:/|$)", re.IGNORECASE) _BACK_TO_TOP_RE = re.compile(r"^\s*back to top\s*$", re.IGNORECASE) @@ -106,20 +104,12 @@ def list_practice_guidelines( return IDSAGuidelineListingPage(refs=refs, total=len(refs)) -def build_guideline_text( +def _build_guideline_text( client: httpx.Client, ref: IDSAGuidelineRef, *, link_mode: LinkMode = LinkMode.KEEP, ) -> tuple[str, int, str, dict[str, list[str]]]: - """Scrape a guideline page into markdown text, section count, title, and links. - - Args: - client: HTTP client used to fetch the guideline page. - ref: IDSA guideline reference to scrape. - link_mode: Whether links are kept as markdown links or stripped to their - visible text (default: LinkMode.KEEP). - """ response = client.get(ref.page_url) response.raise_for_status() title = document_title(response.text, fallback=ref.title) @@ -144,7 +134,7 @@ def scrape_guideline( link_mode: Whether links are kept as markdown links or stripped to their visible text (default: LinkMode.KEEP). """ - content, section_count, title, links_metadata = build_guideline_text(client, ref, link_mode=link_mode) + content, section_count, title, links_metadata = _build_guideline_text(client, ref, link_mode=link_mode) return ScrapedDocument( source="idsa", external_id=f"idsa-{ref.slug}", @@ -372,7 +362,7 @@ def _section_count(content_html: str) -> int: def _quality_flags(content: str) -> list[str]: flags: list[str] = [] - if len(content) < SHORT_CONTENT_CHARS: + if len(content) < _SHORT_CONTENT_CHARS: flags.append("short_content") return flags @@ -384,14 +374,10 @@ def _page_url(slug: str) -> str: __all__ = [ "BASE_URL", "DOCUMENT_DELAY_SECONDS", - "IDSA_DATASET_DISPLAY_NAME", - "IDSA_DATASET_NAME", "IDSAFetchError", "IDSAGuidelineListingPage", "IDSAGuidelineRef", "LISTING_URL", - "SHORT_CONTENT_CHARS", - "build_guideline_text", "idsa_ref_from_url", "list_practice_guidelines", "scrape_guideline", diff --git a/datasets/test/test_scraping_idsa.py b/datasets/test/test_scraping_idsa.py index a83c4ae..2c6073f 100644 --- a/datasets/test/test_scraping_idsa.py +++ b/datasets/test/test_scraping_idsa.py @@ -9,19 +9,25 @@ LISTING_URL, IDSAFetchError, IDSAGuidelineRef, - build_guideline_text, idsa_ref_from_url, list_practice_guidelines, scrape_guideline, scrape_idsa, ) +_LISTING_CASES = ( + ("current-guideline", "Current Guideline", 2024, ("Current",)), + ("current-endorsed-guideline", "Current Endorsed Guideline", 2023, ("Current", "Endorsed")), + ("endorsed-guideline", "Endorsed Guideline", 2022, ("Endorsed",)), + ("archived-guideline", "Archived Guideline", 2021, ("Archived",)), + ("development-guideline", "Development Guideline", 2020, ("In Development",)), + ("archived-development-guideline", "Archived Development Guideline", 2019, ("Archived", "In Development")), +) -def test_list_practice_guidelines_parses_and_filters_default_statuses() -> None: - """The IDSA listing includes only current non-development guidelines by default.""" - client = httpx.Client(transport=httpx.MockTransport(_listing_handler), base_url=BASE_URL) - listing = list_practice_guidelines(client) +def test_list_practice_guidelines_parses_listing_fields() -> None: + """The IDSA listing parser normalizes title, URL, year, and statuses.""" + listing = list_practice_guidelines(_listing_client()) assert listing.total == 2 assert listing.refs == [ @@ -42,45 +48,36 @@ def test_list_practice_guidelines_parses_and_filters_default_statuses() -> None: ] -def test_list_practice_guidelines_can_include_archived_statuses() -> None: - """Archived guidelines are included only when requested.""" - client = httpx.Client(transport=httpx.MockTransport(_listing_handler), base_url=BASE_URL) - - listing = list_practice_guidelines(client, include_archived=True) - - assert [ref.slug for ref in listing.refs] == [ - "current-guideline", - "current-endorsed-guideline", - "archived-guideline", - ] - - -def test_list_practice_guidelines_can_include_in_development_statuses() -> None: - """In-development guidelines are included only when requested.""" - client = httpx.Client(transport=httpx.MockTransport(_listing_handler), base_url=BASE_URL) - - listing = list_practice_guidelines(client, include_in_development=True) - - assert [ref.slug for ref in listing.refs] == [ - "current-guideline", - "current-endorsed-guideline", - "development-guideline", - ] - - -def test_list_practice_guidelines_requires_both_flags_for_archived_development_statuses() -> None: - """Guidelines marked both archived and in development require both inclusion flags.""" - client = httpx.Client(transport=httpx.MockTransport(_listing_handler), base_url=BASE_URL) +@pytest.mark.parametrize( + ("kwargs", "expected_slugs"), + [ + ({}, ["current-guideline", "current-endorsed-guideline"]), + ( + {"include_archived": True}, + ["current-guideline", "current-endorsed-guideline", "archived-guideline"], + ), + ( + {"include_in_development": True}, + ["current-guideline", "current-endorsed-guideline", "development-guideline"], + ), + ( + {"include_archived": True, "include_in_development": True}, + [ + "current-guideline", + "current-endorsed-guideline", + "archived-guideline", + "development-guideline", + "archived-development-guideline", + ], + ), + ], + ids=["default", "include-archived", "include-in-development", "include-both"], +) +def test_list_practice_guidelines_filters_statuses(kwargs: dict[str, bool], expected_slugs: list[str]) -> None: + """Status flags preserve the intended IDSA listing inclusion policy.""" + listing = list_practice_guidelines(_listing_client(), **kwargs) - listing = list_practice_guidelines(client, include_archived=True, include_in_development=True) - - assert [ref.slug for ref in listing.refs] == [ - "current-guideline", - "current-endorsed-guideline", - "archived-guideline", - "development-guideline", - "archived-development-guideline", - ] + assert [ref.slug for ref in listing.refs] == expected_slugs def test_idsa_ref_from_url_normalizes_practice_guideline_url() -> None: @@ -110,71 +107,47 @@ def test_idsa_ref_from_url_rejects_non_guideline_urls(url: str) -> None: idsa_ref_from_url(url) -def test_build_guideline_text_extracts_content_and_link_metadata() -> None: +def test_scrape_guideline_extracts_content_and_link_metadata() -> None: """Guideline page content is converted to markdown and noisy UI is stripped.""" - client = httpx.Client(transport=httpx.MockTransport(_guideline_handler), base_url=BASE_URL) + document = scrape_guideline(_guideline_client(_guideline_html()), _ref()) + + assert document.title == "Current Guideline" + assert document.section_count == 3 + assert "# Current Guideline" in document.content + assert "## Abstract" in document.content + assert "## Recommendations" in document.content + assert "[Download PDF](https://academic.oup.com/example.pdf)" in document.content + assert "Recommendation text with [evidence](https://doi.org/10.1093/cid/example)." in document.content + assert "Back to top" not in document.content + assert "Table of Contents" not in document.content + assert "https://www.idsociety.org#abstract" not in document.content + assert document.metadata["external_links"] == [ + "https://academic.oup.com/example.pdf", + "https://doi.org/10.1093/cid/example", + ] + assert document.metadata["pdf_links"] == ["https://academic.oup.com/example.pdf"] + assert document.metadata["content_length_chars"] == len(document.content) + assert document.metadata["quality_flags"] == ["short_content"] - content, section_count, title, links_metadata = build_guideline_text( - client, - IDSAGuidelineRef( - title="Current Guideline", - slug="current-guideline", - page_url="https://www.idsociety.org/practice-guideline/current-guideline/", - year=2024, - statuses=("Current",), - ), - ) - assert title == "Current Guideline" - assert section_count == 3 - assert "# Current Guideline" in content - assert "## Abstract" in content - assert "## Recommendations" in content - assert "Recommendation text with [evidence](https://doi.org/10.1093/cid/example)." in content - assert "Back to top" not in content - assert "Table of Contents" not in content - assert "https://www.idsociety.org#abstract" not in content - assert links_metadata == { - "external_links": [ - "https://academic.oup.com/example.pdf", - "https://doi.org/10.1093/cid/example", - ], - "pdf_links": ["https://academic.oup.com/example.pdf"], - } - - stripped_content, _section_count, _title, _links_metadata = build_guideline_text( - client, - IDSAGuidelineRef( - title="Current Guideline", - slug="current-guideline", - page_url="https://www.idsociety.org/practice-guideline/current-guideline/", - year=2024, - statuses=("Current",), - ), - link_mode=LinkMode.STRIP, - ) - assert "Recommendation text with evidence." in stripped_content +def test_scrape_guideline_strips_links_when_requested() -> None: + """The IDSA scraper honors the shared link-mode option.""" + document = scrape_guideline(_guideline_client(_guideline_html()), _ref(), link_mode=LinkMode.STRIP) + + assert "Recommendation text with evidence." in document.content + assert "[evidence]" not in document.content def test_scrape_idsa_returns_normalized_document() -> None: """The IDSA scraper returns normalized scraped documents.""" - - def handler(request: httpx.Request) -> httpx.Response: - if str(request.url) == LISTING_URL: - return httpx.Response(200, text=_listing_html()) - if str(request.url) == "https://www.idsociety.org/practice-guideline/current-guideline/": - return httpx.Response(200, text=_guideline_html()) - raise AssertionError(f"Unexpected URL: {request.url}") - - original_client_factory = "amfv_datasets.scraping.idsa.default_client" - transport = httpx.MockTransport(handler) + transport = httpx.MockTransport(_scrape_idsa_handler) class ClientFactory: def __call__(self) -> httpx.Client: return httpx.Client(transport=transport) with pytest.MonkeyPatch.context() as monkeypatch: - monkeypatch.setattr(original_client_factory, ClientFactory()) + monkeypatch.setattr("amfv_datasets.scraping.idsa.default_client", ClientFactory()) scrape_run = scrape_idsa(documents=1) documents = list(scrape_run.documents) @@ -184,111 +157,80 @@ def __call__(self) -> httpx.Client: assert documents[0].external_id == "idsa-current-guideline" assert documents[0].metadata["statuses"] == ["Current"] assert documents[0].metadata["year"] == 2024 - assert documents[0].metadata["pdf_links"] == ["https://academic.oup.com/example.pdf"] - assert documents[0].metadata["content_length_chars"] == len(documents[0].content) - assert documents[0].metadata["quality_flags"] == ["short_content"] + assert documents[0].metadata["slug"] == "current-guideline" + assert documents[0].metadata["listing_url"] == LISTING_URL -def test_scrape_guideline_does_not_flag_long_content() -> None: - """Long guideline content records length without short-content quality flags.""" +@pytest.mark.parametrize( + ("paragraph", "expected_flags"), + [ + ("Useful abstract.", ["short_content"]), + ("Recommendation text. " * 700, []), + ], + ids=["short", "long"], +) +def test_scrape_guideline_sets_content_quality_flags(paragraph: str, expected_flags: list[str]) -> None: + """Short content is flagged while long content is left unflagged.""" + document = scrape_guideline(_guideline_client(_guideline_html(paragraph=paragraph)), _ref()) - def handler(request: httpx.Request) -> httpx.Response: - assert str(request.url) == "https://www.idsociety.org/practice-guideline/long-guideline/" - return httpx.Response(200, text=_long_guideline_html()) + assert document.metadata["content_length_chars"] == len(document.content) + assert document.metadata["quality_flags"] == expected_flags - client = httpx.Client(transport=httpx.MockTransport(handler), base_url=BASE_URL) - document = scrape_guideline( - client, - IDSAGuidelineRef( - title="Long Guideline", - slug="long-guideline", - page_url="https://www.idsociety.org/practice-guideline/long-guideline/", - year=2024, - statuses=("Current",), - ), - ) +def _listing_client() -> httpx.Client: + def handler(request: httpx.Request) -> httpx.Response: + assert str(request.url) == LISTING_URL + return httpx.Response(200, text=_listing_html()) - assert document.metadata["content_length_chars"] == len(document.content) - assert document.metadata["quality_flags"] == [] + return httpx.Client(transport=httpx.MockTransport(handler), base_url=BASE_URL) -def _listing_handler(request: httpx.Request) -> httpx.Response: - assert str(request.url) == LISTING_URL - return httpx.Response(200, text=_listing_html()) +def _guideline_client(content_html: str) -> httpx.Client: + def handler(request: httpx.Request) -> httpx.Response: + assert str(request.url) == "https://www.idsociety.org/practice-guideline/current-guideline/" + return httpx.Response(200, text=content_html) + return httpx.Client(transport=httpx.MockTransport(handler), base_url=BASE_URL) -def _guideline_handler(request: httpx.Request) -> httpx.Response: - assert str(request.url) == "https://www.idsociety.org/practice-guideline/current-guideline/" - return httpx.Response(200, text=_guideline_html()) + +def _scrape_idsa_handler(request: httpx.Request) -> httpx.Response: + if str(request.url) == LISTING_URL: + return httpx.Response(200, text=_listing_html()) + if str(request.url) == "https://www.idsociety.org/practice-guideline/current-guideline/": + return httpx.Response(200, text=_guideline_html()) + raise AssertionError(f"Unexpected URL: {request.url}") def _listing_html() -> str: - return """ - -
- -
- + items = "\n".join( + _listing_item(slug=slug, title=title, year=year, statuses=statuses) + for slug, title, year, statuses in _LISTING_CASES + ) + return f'
    {items}
' + + +def _listing_item(*, slug: str, title: str, year: int, statuses: tuple[str, ...]) -> str: + categories = "".join(f'
  • {status}
    • ' for status in statuses) + return f""" +
  • +
      {categories}
    + {title} + {year} +
    • """ -def _guideline_html() -> str: - return """ +def _guideline_html(*, title: str = "Current Guideline", paragraph: str = "Useful abstract.") -> str: + return f""" - Current Guideline | IDSA + {title} | IDSA

    Intro column noise.

    Download PDF

    -

    Current Guideline

    +

    {title}

    Table of Contents

    • Abstract
      • @@ -297,7 +239,7 @@ def _guideline_html() -> str:

      Published January 1, 2024

      Abstract

      -

      Useful abstract.

      +

      {paragraph}

      Recommendations

      Recommendation text with evidence.

      Back to top

      @@ -306,15 +248,11 @@ def _guideline_html() -> str: """ -def _long_guideline_html() -> str: - long_text = "Recommendation text. " * 700 - return f""" - - Long Guideline | IDSA -
      -

      Long Guideline

      -

      Recommendations

      -

      {long_text}

      -
      - - """ +def _ref() -> IDSAGuidelineRef: + return IDSAGuidelineRef( + title="Current Guideline", + slug="current-guideline", + page_url="https://www.idsociety.org/practice-guideline/current-guideline/", + year=2024, + statuses=("Current",), + )