diff --git a/datasets/amfv_datasets/scraping/__init__.py b/datasets/amfv_datasets/scraping/__init__.py index cd7cdb0..45debf3 100644 --- a/datasets/amfv_datasets/scraping/__init__.py +++ b/datasets/amfv_datasets/scraping/__init__.py @@ -9,6 +9,16 @@ scrape_listing_documents, ) from amfv_datasets.scraping.cli import OutputFormat, ScraperSource +from amfv_datasets.scraping.drugs_com import ( + DrugsComFetchError, + DrugsComPageRef, + build_drugs_com_article_text, + drugscom_ref_from_url, + list_drug_refs, + list_two_letter_pages, + scrape_drugs_com, + scrape_drugs_com_page, +) from amfv_datasets.scraping.html import ( LinkMode, absolute_unique_urls, @@ -29,6 +39,9 @@ ) __all__ = [ + "DRUGSCOM_BASE_URL", + "DrugsComFetchError", + "DrugsComPageRef", "GuidanceRef", "GuidanceListingPage", "LinkMode", @@ -40,14 +53,20 @@ "ScraperSource", "USER_AGENT", "absolute_unique_urls", + "build_drugs_com_article_text", "build_guideline_text", "clean_text", - "document_title", "default_client", + "document_title", + "drugscom_ref_from_url", "first_matching_urls", "guidance_ref_from_url", "html_to_markdown", + "list_drug_refs", "list_published_guidance", + "list_two_letter_pages", + "scrape_drugs_com", + "scrape_drugs_com_page", "scrape_guideline", "scrape_listing_documents", "scrape_nice", diff --git a/datasets/amfv_datasets/scraping/base.py b/datasets/amfv_datasets/scraping/base.py index 0708ee8..08926f0 100644 --- a/datasets/amfv_datasets/scraping/base.py +++ b/datasets/amfv_datasets/scraping/base.py @@ -74,7 +74,10 @@ def scrape_listing_documents[ListingItemT]( documents: int | None, client_factory: Callable[[], AbstractContextManager[httpx.Client]], list_page: Callable[[httpx.Client, int], Iterable[ListingItemT]], - scrape_item: Callable[[httpx.Client, ListingItemT], ScrapedDocument], + scrape_item: Callable[ + [httpx.Client, ListingItemT], + ScrapedDocument | None, + ], document_delay_seconds: float = 5.0, first_page_items: Iterable[ListingItemT] | None = None, ) -> Iterable[ScrapedDocument]: @@ -99,6 +102,7 @@ def scrape_listing_documents[ListingItemT]( with client_factory() as client: page = 1 scraped = 0 + attempted = 0 page_items = list(first_page_items) if first_page_items is not None else None while documents is None or scraped < documents: if page_items is None: @@ -111,9 +115,16 @@ def scrape_listing_documents[ListingItemT]( for item in items: if documents is not None and scraped >= documents: break - if scraped and document_delay_seconds: + if attempted and document_delay_seconds: time.sleep(document_delay_seconds) - yield scrape_item(client, item) + + document = scrape_item(client, item) + attempted += 1 + + if document is None: + continue + + yield document scraped += 1 page += 1 diff --git a/datasets/amfv_datasets/scraping/cli.py b/datasets/amfv_datasets/scraping/cli.py index e8aece3..5dc33fb 100644 --- a/datasets/amfv_datasets/scraping/cli.py +++ b/datasets/amfv_datasets/scraping/cli.py @@ -8,6 +8,7 @@ from collections.abc import Iterable from dataclasses import asdict from enum import StrEnum +from itertools import chain from pathlib import Path from typing import Annotated, TextIO @@ -25,6 +26,7 @@ ) from amfv_datasets.scraping.base import ScrapedDocument, ScrapeRun +from amfv_datasets.scraping.drugs_com import scrape_drugs_com from amfv_datasets.scraping.html import LinkMode from amfv_datasets.scraping.nice import scrape_nice @@ -34,6 +36,7 @@ class ScraperSource(StrEnum): ALL = "all" NICE = "nice" + DRUGSCOM = "drugscom" class OutputFormat(StrEnum): @@ -61,20 +64,50 @@ def scrape_documents( implemented source. documents: Number of documents to scrape. When unset, each source runs until it is exhausted (default: None). - link_mode: Whether links are kept as markdown links or stripped to their - visible text. + link_mode: Whether links are kept as markdown links or stripped to + their visible text. url: Source URL to scrape as a single document (default: None). """ if documents is not None and documents < 1: raise ValueError(f"documents must be at least 1; got {documents}") + scrape_runs: list[ScrapeRun] = [] + for selected_source in _expand_source(source): match selected_source: case ScraperSource.NICE: - return scrape_nice(documents=documents, link_mode=link_mode, url=url) + scrape_runs.append( + scrape_nice( + documents=documents, + link_mode=link_mode, + url=url, + ) + ) + + case ScraperSource.DRUGSCOM: + scrape_runs.append( + scrape_drugs_com( + documents=documents, + link_mode=link_mode, + url=url, + ) + ) + case ScraperSource.ALL: raise AssertionError("expanded source cannot be all") - raise AssertionError(f"unsupported source: {source}") + + if not scrape_runs: + raise AssertionError(f"unsupported source: {source}") + + if len(scrape_runs) == 1: + return scrape_runs[0] + + total = None if any(run.total is None for run in scrape_runs) else sum(run.total or 0 for run in scrape_runs) + + return ScrapeRun( + documents=chain.from_iterable(run.documents for run in scrape_runs), + total=total, + ) def write_jsonl(documents: Iterable[ScrapedDocument], output: TextIO) -> int: @@ -125,7 +158,10 @@ def write_markdown_files(documents: Iterable[ScrapedDocument], output_path: Path def _expand_source(source: ScraperSource) -> tuple[ScraperSource, ...]: if source is ScraperSource.ALL: - return (ScraperSource.NICE,) + return ( + ScraperSource.NICE, + ScraperSource.DRUGSCOM, + ) return (source,) diff --git a/datasets/amfv_datasets/scraping/drugs_com.py b/datasets/amfv_datasets/scraping/drugs_com.py new file mode 100644 index 0000000..49c28aa --- /dev/null +++ b/datasets/amfv_datasets/scraping/drugs_com.py @@ -0,0 +1,539 @@ +from __future__ import annotations + +import logging +import re +from collections.abc import Iterable +from dataclasses import dataclass +from typing import Any +from urllib.parse import unquote, urlparse + +import httpx +from lxml import html as lxml_html + +from amfv_datasets.scraping.base import ( + ScrapedDocument, + ScrapeError, + ScrapeRun, + default_client, + scrape_listing_documents, +) +from amfv_datasets.scraping.html import LinkMode, html_to_markdown + +logger = logging.getLogger(__name__) + +BASE_URL = "https://www.drugs.com" +ALPHA_PATH = "/alpha" + +DRUGSCOM_DATASET_NAME = "drugscom-webscrape" +DRUGSCOM_DATASET_DISPLAY_NAME = "Drugs.com Webscrape" + +DOCUMENT_DELAY_SECONDS = 5.0 + +APPROXIMATE_ARTICLE_COUNT = 24_000 + +LETTERS: tuple[str, ...] = tuple("abcdefghijklmnopqrstuvwxyz") + ("0-9",) + +_DRUG_NAMESPACES = ( + "mtm", + "cons", + "pro", + "monograph", + "cdi", +) + +_DRUG_LINK_RE = re.compile(r"^/(?:(?P" + "|".join(_DRUG_NAMESPACES) + r")/)?(?P[a-z0-9][a-z0-9-]*)\.html$") + +_TWO_LETTER_LINK_RE = re.compile(r"^/alpha/(?P[a-z]{2})\.html$") + +_EXCLUDED_ROOT_SLUGS = frozenset( + { + "drug_information", + "pill_identification", + "drug_interactions", + "search_advanced", + "sitemap", + "news", + "professionals", + } +) + +_TRUNCATE_FROM_HEADING_RE = re.compile( + r"^#{1,3}\s*more about\b", + re.IGNORECASE | re.MULTILINE, +) + +_MIDPAGE_DROP_HEADING_RE = re.compile( + r"^#{2,3}\s*(?:related/similar drugs|does .+ interact with my other drugs?\??)\s*$", + re.IGNORECASE | re.MULTILINE, +) + +_ANY_HEADING_RE = re.compile( + r"^#{1,3}\s", + re.MULTILINE, +) + +_MARKDOWN_LINK_RE = re.compile(r"\[([^\]]+)\]\([^)]*\)") + +_REVIEWED_BYLINE_RE = re.compile( + r"Medically reviewed by (?P.+?)\.\s*" + r"Last updated on (?P[^.\n]+)\.?", + re.DOTALL, +) + +_COPYRIGHT_LINE_RE = re.compile( + r"Copyright \d{4}-\d{4} (?P[^.\n]+?)\.?" + r"(?:\s*Version:[^\n]*)?$", + re.MULTILINE, +) + +_PRONUNCIATION_LINE_RE = re.compile( + r"^Play pronunciation\.$", + re.MULTILINE, +) + +_BOILERPLATE_LINES = frozenset( + { + "Print page", + "My Meds", + } +) + + +def _plain_text(value: str) -> str: + """Collapse markdown links into visible text.""" + return _MARKDOWN_LINK_RE.sub(r"\1", value).strip() + + +class DrugsComFetchError(ScrapeError): + """Raised when a Drugs.com page cannot be fetched or parsed.""" + + +@dataclass(frozen=True) +class DrugsComPageRef: + """Reference to a Drugs.com detail page.""" + + slug: str + name: str + + @property + def path(self) -> str: + """Return the page's absolute path.""" + return f"/{self.slug}.html" + + @property + def page_url(self) -> str: + """Return the canonical page URL.""" + return f"{BASE_URL}{self.path}" + + @property + def external_id(self) -> str: + """Return the unique external identifier.""" + return f"drugscom-{self.slug.replace('/', '-')}" + + +def drugscom_ref_from_url(url: str) -> DrugsComPageRef: + """Convert a Drugs.com URL into a page reference.""" + parsed = urlparse(url.strip()) + + if parsed.scheme not in {"http", "https"}: + raise DrugsComFetchError(f"Invalid Drugs.com URL: {url!r}") + + if parsed.netloc.lower() not in { + "www.drugs.com", + "drugs.com", + }: + raise DrugsComFetchError(f"Not a Drugs.com URL: {url!r}") + + match = _DRUG_LINK_RE.match(unquote(parsed.path)) + + if not match: + raise DrugsComFetchError(f"Invalid drug page URL: {url!r}") + + if match.group("ns") is None and match.group("slug") in _EXCLUDED_ROOT_SLUGS: + raise DrugsComFetchError(f"Not a drug page: {url!r}") + + namespace = match.group("ns") + + slug = match.group("slug") if namespace is None else f"{namespace}/{match.group('slug')}" + + return DrugsComPageRef( + slug=slug, + name=match.group("slug").replace("-", " "), + ) + + +def _get( + client: httpx.Client, + path: str, +) -> lxml_html.HtmlElement: + """Fetch a Drugs.com path and parse HTML.""" + response = client.get(f"{BASE_URL}{path}") + + try: + response.raise_for_status() + except httpx.HTTPStatusError as error: + raise DrugsComFetchError(f"Drugs.com returned {response.status_code} for {path!r}") from error + + return lxml_html.fromstring(response.text) + + +def list_two_letter_pages( + client: httpx.Client, + letter: str, +) -> list[str]: + """Return two-letter alpha pages linked from a letter page.""" + root = _get( + client, + f"{ALPHA_PATH}/{letter}.html", + ) + + paths: list[str] = [] + seen: set[str] = set() + + for href in root.xpath("//a/@href"): + path = urlparse(href).path + + if _TWO_LETTER_LINK_RE.match(path) and path not in seen: + seen.add(path) + paths.append(path) + + return paths + + +def list_drug_refs( + client: httpx.Client, + listing_path: str, +) -> list[DrugsComPageRef]: + """Return drug page references from an alpha listing page.""" + root = _get( + client, + listing_path, + ) + + refs: list[DrugsComPageRef] = [] + seen: set[str] = set() + + for anchor in root.xpath("//a[@href]"): + href = anchor.get("href") + + if not href: + continue + + path = urlparse(href).path + + if href != path: + continue + + match = _DRUG_LINK_RE.match(path) + + if not match: + continue + + namespace = match.group("ns") + slug_name = match.group("slug") + + if namespace is None and slug_name in _EXCLUDED_ROOT_SLUGS: + continue + + slug = slug_name if namespace is None else f"{namespace}/{slug_name}" + + if slug in seen: + continue + + seen.add(slug) + + name = " ".join((anchor.text_content() or slug_name).split()) + + refs.append( + DrugsComPageRef( + slug=slug, + name=name, + ) + ) + + return refs + + +def _strip_chrome_html( + page_root: lxml_html.HtmlElement, +) -> lxml_html.HtmlElement: + """Remove site navigation and unrelated HTML.""" + content = page_root.xpath("//*[@id='content']") + + root = content[0] if content else page_root + + for tag in ( + "header", + "nav", + "footer", + "script", + "style", + "form", + "aside", + "noscript", + ): + for element in root.xpath(f".//{tag}"): + parent = element.getparent() + + if parent is not None: + parent.remove(element) + + return root + + +def _clean_markdown( + content: str, +) -> tuple[str, dict[str, str]]: + """Remove boilerplate and extract metadata.""" + metadata: dict[str, str] = {} + + reviewed = _REVIEWED_BYLINE_RE.search(content) + + if reviewed: + metadata["reviewed_by"] = _plain_text(reviewed.group("reviewer")) + metadata["last_updated"] = reviewed.group("updated").strip() + + content = _REVIEWED_BYLINE_RE.sub( + "", + content, + count=1, + ) + + copyright_match = _COPYRIGHT_LINE_RE.search(content) + + if copyright_match: + metadata["data_source"] = copyright_match.group("holder").strip() + + content = _COPYRIGHT_LINE_RE.sub( + "", + content, + count=1, + ) + + content = _PRONUNCIATION_LINE_RE.sub( + "", + content, + ) + + content = "\n".join(line for line in content.splitlines() if line.strip() not in _BOILERPLATE_LINES) + + truncate_match = _TRUNCATE_FROM_HEADING_RE.search(content) + + if truncate_match: + content = content[: truncate_match.start()] + + while True: + drop_match = _MIDPAGE_DROP_HEADING_RE.search(content) + + if not drop_match: + break + + next_heading = _ANY_HEADING_RE.search( + content, + drop_match.end(), + ) + + end = next_heading.start() if next_heading else len(content) + + content = content[: drop_match.start()] + content[end:] + + content = re.sub( + r"\n{3,}", + "\n\n", + content, + ).strip() + + return content, metadata + + +def build_drugs_com_article_text( + client: httpx.Client, + ref: DrugsComPageRef, + *, + link_mode: LinkMode = LinkMode.KEEP, +) -> tuple[str, str, dict[str, Any]]: + """Scrape one page into markdown.""" + page_root = _get( + client, + ref.path, + ) + + title_nodes = page_root.xpath("//h1") + + title = " ".join((title_nodes[0].text_content() or ref.name).split()) if title_nodes else ref.name + + article_root = _strip_chrome_html(page_root) + + raw_content = html_to_markdown( + lxml_html.tostring( + article_root, + encoding="unicode", + ), + link_mode=link_mode, + base_url=BASE_URL, + ) + + if not raw_content: + raise DrugsComFetchError(f"No readable content for {ref.slug!r}") + + content, extra_metadata = _clean_markdown(raw_content) + + if not content: + raise DrugsComFetchError(f"No content after cleanup for {ref.slug!r}") + + namespace = ref.slug.split("/", 1)[0] if "/" in ref.slug else "consumer" + + metadata: dict[str, Any] = { + "namespace": namespace, + "content_length_chars": len(content), + "license": "proprietary", + **extra_metadata, + } + + return content, title, metadata + + +def scrape_drugs_com_page( + client: httpx.Client, + ref: DrugsComPageRef, + *, + link_mode: LinkMode = LinkMode.KEEP, +) -> ScrapedDocument: + """Scrape one Drugs.com page into a normalized document.""" + content, title, metadata = build_drugs_com_article_text( + client, + ref, + link_mode=link_mode, + ) + + return ScrapedDocument( + source="drugscom", + external_id=ref.external_id, + title=title, + url=ref.page_url, + content=content, + section_count=max( + content.count("\n#"), + 1, + ), + metadata=metadata, + ) + + +def _scrape_or_skip( + client: httpx.Client, + ref: DrugsComPageRef, + *, + link_mode: LinkMode, +) -> ScrapedDocument | None: + """Scrape one page, skipping failures.""" + try: + return scrape_drugs_com_page( + client, + ref, + link_mode=link_mode, + ) + except DrugsComFetchError: + logger.warning( + "Skipping Drugs.com page %r", + ref.slug, + exc_info=True, + ) + return None + + +def scrape_drugs_com( + *, + documents: int | None, + link_mode: LinkMode = LinkMode.KEEP, + url: str | None = None, +) -> ScrapeRun: + """Scrape Drugs.com documents.""" + if url is not None: + + def scrape_url() -> Iterable[ScrapedDocument]: + with default_client() as client: + yield scrape_drugs_com_page( + client, + drugscom_ref_from_url(url), + link_mode=link_mode, + ) + + return ScrapeRun( + documents=scrape_url(), + total=1, + ) + + letter_queue: list[str] = list(LETTERS) + subpage_queue: list[str] = [] + exhausted = False + + def list_page( + client: httpx.Client, + _page: int, + ) -> list[DrugsComPageRef]: + nonlocal exhausted + + if exhausted: + return [] + + while True: + while not subpage_queue: + if not letter_queue: + exhausted = True + return [] + + letter = letter_queue.pop(0) + + two_letter_pages = list_two_letter_pages( + client, + letter, + ) + + subpage_queue.extend(two_letter_pages or [f"{ALPHA_PATH}/{letter}.html"]) + + listing_path = subpage_queue.pop(0) + + refs = list_drug_refs( + client, + listing_path, + ) + + if refs: + return refs + + return ScrapeRun( + total=(documents if documents is not None else APPROXIMATE_ARTICLE_COUNT), + documents=( + document + for document in scrape_listing_documents( + documents=documents, + client_factory=default_client, + list_page=list_page, + scrape_item=lambda client, ref: _scrape_or_skip( + client, + ref, + link_mode=link_mode, + ), + document_delay_seconds=DOCUMENT_DELAY_SECONDS, + ) + if document is not None + ), + ) + + +__all__ = [ + "APPROXIMATE_ARTICLE_COUNT", + "BASE_URL", + "DOCUMENT_DELAY_SECONDS", + "DRUGSCOM_DATASET_DISPLAY_NAME", + "DRUGSCOM_DATASET_NAME", + "DrugsComFetchError", + "DrugsComPageRef", + "build_drugs_com_article_text", + "drugscom_ref_from_url", + "list_drug_refs", + "list_two_letter_pages", + "scrape_drugs_com", + "scrape_drugs_com_page", +] diff --git a/datasets/amfv_datasets/scraping/html.py b/datasets/amfv_datasets/scraping/html.py index 2bd1c96..67effca 100644 --- a/datasets/amfv_datasets/scraping/html.py +++ b/datasets/amfv_datasets/scraping/html.py @@ -117,8 +117,13 @@ def html_to_markdown( def _absolutize_links(html_text: str, *, base_url: str) -> str: root = lxml_html.fragment_fromstring(html_text, create_parent="div") + for link in root.xpath(".//a[@href]"): link.set("href", urljoin(base_url, link.get("href"))) + + for image in root.xpath(".//img[@src]"): + image.set("src", urljoin(base_url, image.get("src"))) + return "".join(lxml_html.tostring(child, encoding="unicode") for child in root) diff --git a/datasets/test/test_drugs_com.py b/datasets/test/test_drugs_com.py new file mode 100644 index 0000000..882c200 --- /dev/null +++ b/datasets/test/test_drugs_com.py @@ -0,0 +1,311 @@ +"""Tests for Drugs.com scraping helpers.""" + +import httpx +import pytest + +from amfv_datasets.scraping import base +from amfv_datasets.scraping.drugs_com import ( + BASE_URL, + DrugsComFetchError, + DrugsComPageRef, + build_drugs_com_article_text, + drugscom_ref_from_url, + list_drug_refs, + list_two_letter_pages, + scrape_drugs_com, + scrape_drugs_com_page, +) +from amfv_datasets.scraping.html import LinkMode + +_NAV_CHROME = """ +
+""" + + +def _html_client(routes: dict[str, str]) -> httpx.Client: + def handler(request: httpx.Request) -> httpx.Response: + path = request.url.path + + if path not in routes: + return httpx.Response(404, text="not found") + + return httpx.Response(200, text=routes[path]) + + return httpx.Client( + transport=httpx.MockTransport(handler), + base_url=BASE_URL, + ) + + +def test_list_two_letter_pages_returns_only_linked_pairs() -> None: + """Letter pairs with no drugs render as unlinked text and are skipped.""" + page = ( + _NAV_CHROME + '

Drugs: Z

' + '
' + ) + + paths = list_two_letter_pages( + _html_client({"/alpha/z.html": page}), + "z", + ) + + assert paths == [ + "/alpha/za.html", + "/alpha/zc.html", + ] + + +def test_list_drug_refs_covers_every_namespace_dedupes_and_skips_pro_toggle() -> None: + """Every drug namespace is picked up; chrome and pro variants are not.""" + page = ( + _NAV_CHROME + '" + ) + + refs = list_drug_refs( + _html_client({"/alpha/za.html": page}), + "/alpha/za.html", + ) + + assert refs == [ + DrugsComPageRef(slug="zaltrap", name="Zaltrap"), + DrugsComPageRef(slug="mtm/zaditor", name="Zaditor"), + DrugsComPageRef(slug="cons/zagam", name="Zagam"), + DrugsComPageRef(slug="pro/zafemy", name="Zafemy"), + DrugsComPageRef( + slug="monograph/zanidatamab-hrii", + name="Zanidatamab-hrii", + ), + ] + + +def test_build_drugs_com_article_text_strips_chrome_and_boilerplate_sections() -> None: + """Nav/footer chrome, trailing resource blocks, and widgets are removed.""" + page = ( + _NAV_CHROME + '
' + "

Zoloft

" + "

Medically reviewed by Melisa Puckey, BPharm. Last updated on Aug 23, 2023.

" + "

What is Zoloft?

" + "

Zoloft is an antidepressant that belongs to a group of drugs called SSRIs.

" + "

Related/similar drugs

" + "

Vraylar is a once a day antipsychotic medication.

" + "

What other drugs will affect Zoloft?

" + "

Zoloft can cause a serious heart problem.

" + "

More about Zoloft (sertraline)

" + "
  • Check interactions
" + "

Copyright 1996-2026 Cerner Multum, Inc. Version: 29.01.

" + "
" + "
Drugs.com Mobile App download links
" + ) + + content, title, metadata = build_drugs_com_article_text( + _html_client({"/zoloft.html": page}), + DrugsComPageRef(slug="zoloft", name="Zoloft"), + link_mode=LinkMode.STRIP, + ) + + assert title == "Zoloft" + assert "Mobile App" not in content + assert "Vraylar" not in content + assert "Related/similar drugs" not in content + assert "Check interactions" not in content + assert "More about Zoloft" not in content + assert "Copyright" not in content + assert "belongs to a group of drugs called SSRIs" in content + assert "serious heart problem" in content + assert metadata["reviewed_by"] == "Melisa Puckey, BPharm" + assert metadata["last_updated"] == "Aug 23, 2023" + assert metadata["data_source"] == "Cerner Multum, Inc" + assert metadata["namespace"] == "consumer" + assert metadata["license"] == "proprietary" + + +def test_build_drugs_com_article_text_handles_a_linked_reviewer_byline() -> None: + """Linked reviewer names are extracted correctly into metadata.""" + page = ( + _NAV_CHROME + '
' + "

Zoloft

" + '

Medically reviewed by ' + "Melisa Puckey, BPharm. Last updated on Aug 23, 2023.

" + "

What is Zoloft?

" + "

Zoloft is an antidepressant.

" + "
" + ) + + content, _, metadata = build_drugs_com_article_text( + _html_client({"/zoloft.html": page}), + DrugsComPageRef(slug="zoloft", name="Zoloft"), + ) + + assert metadata["reviewed_by"] == "Melisa Puckey, BPharm" + assert metadata["last_updated"] == "Aug 23, 2023" + assert "Medically reviewed by" not in content + assert "bpharm.html" not in content + + +def test_scrape_drugs_com_page_normalizes_into_a_scraped_document() -> None: + """Scraped pages follow shared source and external ID conventions.""" + page = _NAV_CHROME + '

Zafirlukast

Zafirlukast treats asthma.

' + + document = scrape_drugs_com_page( + _html_client({"/mtm/zafirlukast.html": page}), + DrugsComPageRef( + slug="mtm/zafirlukast", + name="Zafirlukast", + ), + ) + + assert document.source == "drugscom" + assert document.external_id == "drugscom-mtm-zafirlukast" + assert document.url == "https://www.drugs.com/mtm/zafirlukast.html" + assert "Zafirlukast treats asthma" in document.content + assert document.metadata["namespace"] == "mtm" + + +def test_build_drugs_com_article_text_rejects_a_content_free_page() -> None: + """Pages with no remaining content raise an error.""" + page = _NAV_CHROME + "
only chrome here
" + + with pytest.raises(DrugsComFetchError): + build_drugs_com_article_text( + _html_client({"/nonexistent.html": page}), + DrugsComPageRef( + slug="nonexistent", + name="Nonexistent", + ), + ) + + +def test_drugscom_ref_from_url_parses_bare_and_namespaced_urls() -> None: + """Namespace prefixes fold into slug values.""" + assert drugscom_ref_from_url("https://www.drugs.com/zoloft.html") == DrugsComPageRef( + slug="zoloft", + name="zoloft", + ) + + assert drugscom_ref_from_url("https://www.drugs.com/pro/zafemy.html") == DrugsComPageRef( + slug="pro/zafemy", + name="zafemy", + ) + + +@pytest.mark.parametrize( + "url", + [ + "https://example.org/zoloft.html", + "https://www.drugs.com/support/about.html", + "https://www.drugs.com/drug_information.html", + "ftp://www.drugs.com/zoloft.html", + ], +) +def test_drugscom_ref_from_url_rejects_non_drug_urls(url: str) -> None: + """Non-Drugs.com hosts and unrelated pages are rejected.""" + with pytest.raises(DrugsComFetchError): + drugscom_ref_from_url(url) + + +def test_scrape_drugs_com_threads_two_level_pagination_with_a_no_subpage_fallback( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Pagination walks letter pages, subpages, and direct drug listings.""" + a_index = _NAV_CHROME + '

Drugs: A

' + + aa_listing = _NAV_CHROME + '

Drugs: Aa

' + + aaa_page = _NAV_CHROME + '

Aaa

Aaa is a fictional drug.

' + + digits_index = ( + _NAV_CHROME + '

Drugs: 0-9

' + ) + + five_htp_page = _NAV_CHROME + '

5-HTP

5-HTP is a supplement.

' + + routes = { + "/alpha/a.html": a_index, + "/alpha/aa.html": aa_listing, + "/aaa.html": aaa_page, + "/alpha/0-9.html": digits_index, + "/5-htp.html": five_htp_page, + } + + empty_index_template = _NAV_CHROME + '

Drugs: {letter}

' + + for letter in "bcdefghijklmnopqrstuvwxy": + routes[f"/alpha/{letter}.html"] = empty_index_template.format(letter=letter) + + routes["/alpha/z.html"] = empty_index_template.format(letter="z") + + monkeypatch.setattr( + "amfv_datasets.scraping.drugs_com.default_client", + lambda: _html_client(routes), + ) + + monkeypatch.setattr( + base.time, + "sleep", + lambda _seconds: None, + ) + + documents = list(scrape_drugs_com(documents=None).documents) + + assert [document.external_id for document in documents] == [ + "drugscom-aaa", + "drugscom-5-htp", + ] + assert "Aaa is a fictional drug" in documents[0].content + assert "5-HTP is a supplement" in documents[1].content + + +def test_scrape_drugs_com_skips_pages_it_cannot_serve( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A failed drug page does not stop the crawler.""" + a_index = _NAV_CHROME + '

Drugs: A

' + + aa_listing = ( + _NAV_CHROME + '

Drugs: Aa

' + '
' + ) + + abraxane_page = _NAV_CHROME + '

Abraxane

Abraxane treats cancer.

' + + routes = { + "/alpha/a.html": a_index, + "/alpha/aa.html": aa_listing, + "/abraxane.html": abraxane_page, + } + + empty_index_template = _NAV_CHROME + '

Drugs: {letter}

' + + for letter in list("bcdefghijklmnopqrstuvwxyz") + ["0-9"]: + routes[f"/alpha/{letter}.html"] = empty_index_template.format(letter=letter) + + monkeypatch.setattr( + "amfv_datasets.scraping.drugs_com.default_client", + lambda: _html_client(routes), + ) + + monkeypatch.setattr( + base.time, + "sleep", + lambda _seconds: None, + ) + + documents = list(scrape_drugs_com(documents=None).documents) + + assert [document.external_id for document in documents] == ["drugscom-abraxane"] diff --git a/datasets/test/test_scraping_base.py b/datasets/test/test_scraping_base.py index eed3c83..fc28a7a 100644 --- a/datasets/test/test_scraping_base.py +++ b/datasets/test/test_scraping_base.py @@ -149,3 +149,50 @@ def scrape_item(client: httpx.Client, item: str) -> ScrapedDocument: assert [document.external_id for document in documents] == ["item-1", "item-2", "item-3"] assert delays == [5.0, 5.0] + + +def test_scrape_listing_documents_skips_none_documents() -> None: + """Items returning None are skipped without counting as scraped.""" + + class _FakeClient: + def __enter__(self) -> "_FakeClient": + return self + + def __exit__(self, *args: object) -> None: + return None + + def client_factory(): + return _FakeClient() + + def list_page(client: httpx.Client, page: int) -> list[str]: + return ["keep-1", "skip", "keep-2"] if page == 1 else [] + + def scrape_item( + client: httpx.Client, + item: str, + ) -> ScrapedDocument | None: + if item == "skip": + return None + + return ScrapedDocument( + source="test", + external_id=item, + title=item, + url=f"https://example.org/{item}", + content="content", + ) + + documents = list( + scrape_listing_documents( + documents=2, + client_factory=client_factory, + list_page=list_page, + scrape_item=scrape_item, + document_delay_seconds=0, + ) + ) + + assert [doc.external_id for doc in documents] == [ + "keep-1", + "keep-2", + ] diff --git a/datasets/test/test_scraping_cli.py b/datasets/test/test_scraping_cli.py index 44f714f..cd9787d 100644 --- a/datasets/test/test_scraping_cli.py +++ b/datasets/test/test_scraping_cli.py @@ -60,7 +60,9 @@ def test_write_markdown_files_saves_documents_to_directory(tmp_path: Path) -> No ) -def test_cli_run_writes_jsonl_to_stdout(monkeypatch: pytest.MonkeyPatch) -> None: +def test_cli_run_writes_jsonl_to_stdout( + monkeypatch: pytest.MonkeyPatch, +) -> None: """The CLI run command emits scraped documents as JSONL.""" runner = CliRunner() @@ -77,7 +79,10 @@ def fake_scrape_documents( assert url is None return ScrapeRun([_document()], total=3) - monkeypatch.setattr("amfv_datasets.scraping.cli.scrape_documents", fake_scrape_documents) + monkeypatch.setattr( + "amfv_datasets.scraping.cli.scrape_documents", + fake_scrape_documents, + ) def fake_progress( documents: Iterator[ScrapedDocument], @@ -86,16 +91,25 @@ def fake_progress( ) -> Iterator[ScrapedDocument]: yield from documents - monkeypatch.setattr("amfv_datasets.scraping.cli._progress_documents", fake_progress) + monkeypatch.setattr( + "amfv_datasets.scraping.cli._progress_documents", + fake_progress, + ) - result = runner.invoke(app, ["--source", "nice", "--documents", "3", "--links", "strip"]) + result = runner.invoke( + app, + ["--source", "nice", "--documents", "3", "--links", "strip"], + ) assert result.exit_code == 0 assert json.loads(result.stdout.splitlines()[0])["external_id"] == "nice-ng1" assert "scraped 1 documents from nice" in result.stderr -def test_cli_run_can_disable_progress_for_file_output(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: +def test_cli_run_can_disable_progress_for_file_output( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: """Progress can be disabled explicitly for automation-friendly runs.""" runner = CliRunner() @@ -115,18 +129,33 @@ def fail_progress( ) -> Iterator[ScrapedDocument]: raise AssertionError("progress must be disabled") - monkeypatch.setattr("amfv_datasets.scraping.cli.scrape_documents", fake_scrape_documents) - monkeypatch.setattr("amfv_datasets.scraping.cli._progress_documents", fail_progress) + monkeypatch.setattr( + "amfv_datasets.scraping.cli.scrape_documents", + fake_scrape_documents, + ) + monkeypatch.setattr( + "amfv_datasets.scraping.cli._progress_documents", + fail_progress, + ) result = runner.invoke( app, - ["--source", "nice", "--output", str(tmp_path / "out.jsonl"), "--no-progress"], + [ + "--source", + "nice", + "--output", + str(tmp_path / "out.jsonl"), + "--no-progress", + ], ) assert result.exit_code == 0 -def test_cli_run_uses_progress_by_default(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: +def test_cli_run_uses_progress_by_default( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: """Progress is enabled by default.""" runner = CliRunner() progress_calls = 0 @@ -150,16 +179,32 @@ def fake_progress( assert total == 7 yield from documents - monkeypatch.setattr("amfv_datasets.scraping.cli.scrape_documents", fake_scrape_documents) - monkeypatch.setattr("amfv_datasets.scraping.cli._progress_documents", fake_progress) + monkeypatch.setattr( + "amfv_datasets.scraping.cli.scrape_documents", + fake_scrape_documents, + ) + monkeypatch.setattr( + "amfv_datasets.scraping.cli._progress_documents", + fake_progress, + ) - result = runner.invoke(app, ["--source", "nice", "--output", str(tmp_path / "out.jsonl")]) + result = runner.invoke( + app, + [ + "--source", + "nice", + "--output", + str(tmp_path / "out.jsonl"), + ], + ) assert result.exit_code == 0 assert progress_calls == 1 -def test_cli_run_accepts_source_url(monkeypatch: pytest.MonkeyPatch) -> None: +def test_cli_run_accepts_source_url( + monkeypatch: pytest.MonkeyPatch, +) -> None: """The CLI accepts a source URL as the scrape target.""" runner = CliRunner() @@ -182,41 +227,136 @@ def fake_progress( assert total == 1 yield from documents - monkeypatch.setattr("amfv_datasets.scraping.cli.scrape_nice", fake_scrape_nice) - monkeypatch.setattr("amfv_datasets.scraping.cli._progress_documents", fake_progress) + monkeypatch.setattr( + "amfv_datasets.scraping.cli.scrape_nice", + fake_scrape_nice, + ) + monkeypatch.setattr( + "amfv_datasets.scraping.cli._progress_documents", + fake_progress, + ) result = runner.invoke( app, - ["--source", "nice", "--url", "https://www.nice.org.uk/guidance/ta1138/chapter/4-Implementation"], + [ + "--source", + "nice", + "--url", + "https://www.nice.org.uk/guidance/ta1138/chapter/4-Implementation", + ], ) assert result.exit_code == 0 assert json.loads(result.stdout.splitlines()[0])["external_id"] == "nice-ng1" -def test_cli_run_accepts_all_documents(monkeypatch: pytest.MonkeyPatch) -> None: - """The CLI accepts --documents all.""" - runner = CliRunner() +def _document() -> ScrapedDocument: + return ScrapedDocument( + source="nice", + external_id="nice-ng1", + title="Guideline 1", + url="https://www.nice.org.uk/guidance/ng1", + content="content", + metadata={"ref": "NG1"}, + ) - def fake_scrape_documents( - source: ScraperSource, + +def _drugscom_document() -> ScrapedDocument: + return ScrapedDocument( + source="drugscom", + external_id="drugscom-asthma", + title="Asthma", + url="https://www.drugs.com/condition/asthma.html", + content="content", + metadata={"slug": "asthma"}, + ) + + +class _TextSink: + def __init__(self) -> None: + self.value = "" + + def write(self, text: str) -> int: + self.value += text + return len(text) + + +def test_scrape_documents_dispatches_drugscom( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The scrape dispatcher calls the Drugs.com scraper.""" + + def fake_scrape_drugs_com( *, documents: int | None, link_mode: LinkMode, - url: str | None = None, + url: str | None, ) -> ScrapeRun: - assert source is ScraperSource.ALL - assert documents is None - assert link_mode is LinkMode.KEEP - assert url is None - return ScrapeRun([_document()], total=12) + assert documents == 1 + assert link_mode is LinkMode.STRIP + assert url == "https://www.drugs.com/condition/asthma.html" + return ScrapeRun([_drugscom_document()], total=1) - monkeypatch.setattr("amfv_datasets.scraping.cli.scrape_documents", fake_scrape_documents) + monkeypatch.setattr( + "amfv_datasets.scraping.cli.scrape_drugs_com", + fake_scrape_drugs_com, + ) - result = runner.invoke(app, ["--source", "all", "--documents", "all"]) + from amfv_datasets.scraping.cli import scrape_documents - assert result.exit_code == 0 - assert json.loads(result.stdout.splitlines()[0])["external_id"] == "nice-ng1" + scrape_run = scrape_documents( + ScraperSource.DRUGSCOM, + documents=1, + link_mode=LinkMode.STRIP, + url="https://www.drugs.com/condition/asthma.html", + ) + + assert list(scrape_run.documents) == [_drugscom_document()] + + +def test_scrape_documents_all_includes_drugscom( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The all source combines NICE and Drugs.com scrape runs.""" + + def fake_scrape_nice( + *, + documents: int | None, + link_mode: LinkMode, + url: str | None, + ) -> ScrapeRun: + return ScrapeRun([_document()], total=1) + + def fake_scrape_drugs_com( + *, + documents: int | None, + link_mode: LinkMode, + url: str | None, + ) -> ScrapeRun: + return ScrapeRun([_drugscom_document()], total=1) + + monkeypatch.setattr( + "amfv_datasets.scraping.cli.scrape_nice", + fake_scrape_nice, + ) + monkeypatch.setattr( + "amfv_datasets.scraping.cli.scrape_drugs_com", + fake_scrape_drugs_com, + ) + + from amfv_datasets.scraping.cli import scrape_documents + + scrape_run = scrape_documents( + ScraperSource.ALL, + documents=1, + link_mode=LinkMode.KEEP, + ) + + assert scrape_run.total == 2 + assert [doc.external_id for doc in scrape_run.documents] == [ + "nice-ng1", + "drugscom-asthma", + ] def _document() -> ScrapedDocument: @@ -230,6 +370,17 @@ def _document() -> ScrapedDocument: ) +def _drugscom_document() -> ScrapedDocument: + return ScrapedDocument( + source="drugscom", + external_id="drugscom-asthma", + title="Asthma", + url="https://www.drugs.com/condition/asthma.html", + content="content", + metadata={"slug": "asthma"}, + ) + + class _TextSink: def __init__(self) -> None: self.value = "" diff --git a/datasets/test/test_scraping_html.py b/datasets/test/test_scraping_html.py index d34a8fa..dbdaa34 100644 --- a/datasets/test/test_scraping_html.py +++ b/datasets/test/test_scraping_html.py @@ -68,3 +68,13 @@ def test_html_to_markdown_can_strip_links() -> None: html_text = '

Offer treatment.

' assert html_to_markdown(html_text, link_mode=LinkMode.STRIP) == "Offer treatment." + + +def test_html_to_markdown_absolutizes_images() -> None: + """Relative image sources are preserved as absolute markdown image URLs.""" + html_text = 'Treatment flowchart' + + assert html_to_markdown( + html_text, + base_url="https://example.org/guideline/", + ) == ("![Treatment flowchart](https://example.org/images/flowchart.png)")