Add IDSA scraper

datasets/amfv_datasets/scraping/__init__.py

@@ -17,6 +17,16 @@
     first_matching_urls,
     html_to_markdown,
 )
+from amfv_datasets.scraping.idsa import (
+    IDSAFetchError,
+    IDSAGuidelineRef,
+    idsa_ref_from_url,
+    list_practice_guidelines,
+    scrape_idsa,
+)
+from amfv_datasets.scraping.idsa import (
+    scrape_guideline as scrape_idsa_guideline,
+)
 from amfv_datasets.scraping.nice import (
     GuidanceListingPage,
     GuidanceRef,
@@ -31,6 +41,8 @@
 __all__ = [
     "GuidanceRef",
     "GuidanceListingPage",
+    "IDSAFetchError",
+    "IDSAGuidelineRef",
     "LinkMode",
     "NiceFetchError",
     "OutputFormat",
@@ -47,8 +59,12 @@
     "first_matching_urls",
     "guidance_ref_from_url",
     "html_to_markdown",
+    "idsa_ref_from_url",
+    "list_practice_guidelines",
     "list_published_guidance",
     "scrape_guideline",
+    "scrape_idsa",
+    "scrape_idsa_guideline",
     "scrape_listing_documents",
     "scrape_nice",
 ]

datasets/amfv_datasets/scraping/cli.py

@@ -8,6 +8,7 @@
 from collections.abc import Iterable
 from dataclasses import asdict
 from enum import StrEnum
+from itertools import chain
 from pathlib import Path
 from typing import Annotated, TextIO
 
@@ -26,13 +27,15 @@
 
 from amfv_datasets.scraping.base import ScrapedDocument, ScrapeRun
 from amfv_datasets.scraping.html import LinkMode
+from amfv_datasets.scraping.idsa import scrape_idsa
 from amfv_datasets.scraping.nice import scrape_nice
 
 
 class ScraperSource(StrEnum):
     """Supported scraper sources."""
 
     ALL = "all"
+    IDSA = "idsa"
     NICE = "nice"
 
 
@@ -53,6 +56,8 @@ def scrape_documents(
     documents: int | None,
     link_mode: LinkMode,
     url: str | None = None,
+    include_archived: bool = False,
+    include_in_development: bool = False,
 ) -> ScrapeRun:
     """Configure a scrape for a source.
 
@@ -64,17 +69,37 @@ def scrape_documents(
         link_mode: Whether links are kept as markdown links or stripped to their
             visible text.
         url: Source URL to scrape as a single document (default: None).
+        include_archived: Whether IDSA archived guidelines are included
+            (default: False).
+        include_in_development: Whether IDSA in-development guidelines are
+            included (default: False).
     """
     if documents is not None and documents < 1:
         raise ValueError(f"documents must be at least 1; got {documents}")
 
+    scrape_runs: list[ScrapeRun] = []
     for selected_source in _expand_source(source):
         match selected_source:
+            case ScraperSource.IDSA:
+                scrape_runs.append(
+                    scrape_idsa(
+                        documents=documents,
+                        link_mode=link_mode,
+                        url=url,
+                        include_archived=include_archived,
+                        include_in_development=include_in_development,
+                    )
+                )
             case ScraperSource.NICE:
-                return scrape_nice(documents=documents, link_mode=link_mode, url=url)
+                scrape_runs.append(scrape_nice(documents=documents, link_mode=link_mode, url=url))
             case ScraperSource.ALL:
                 raise AssertionError("expanded source cannot be all")
-    raise AssertionError(f"unsupported source: {source}")
+    if not scrape_runs:
+        raise AssertionError(f"unsupported source: {source}")
+    if len(scrape_runs) == 1:
+        return scrape_runs[0]
+    total = None if any(run.total is None for run in scrape_runs) else sum(run.total or 0 for run in scrape_runs)
+    return ScrapeRun(documents=chain.from_iterable(run.documents for run in scrape_runs), total=total)
 
 
 def write_jsonl(documents: Iterable[ScrapedDocument], output: TextIO) -> int:
@@ -125,7 +150,7 @@ def write_markdown_files(documents: Iterable[ScrapedDocument], output_path: Path
 
 def _expand_source(source: ScraperSource) -> tuple[ScraperSource, ...]:
     if source is ScraperSource.ALL:
-        return (ScraperSource.NICE,)
+        return (ScraperSource.NICE, ScraperSource.IDSA)
     return (source,)
 
 
@@ -135,6 +160,8 @@ def run(
     url: Annotated[str | None, typer.Option("--url", help="Source URL to scrape as a single document.")] = None,
     documents: Annotated[str, typer.Option("--documents", help="Number of documents to scrape, or 'all'.")] = "1",
     link_mode: Annotated[LinkMode, typer.Option("--links", help="Whether to keep markdown links or strip links to text.")] = LinkMode.KEEP,  # noqa: E501
+    include_archived: Annotated[bool, typer.Option("--include-archived/--exclude-archived", help="Include archived guidelines for sources that expose archival status.")] = False,  # noqa: E501
+    include_in_development: Annotated[bool, typer.Option("--include-in-development/--exclude-in-development", help="Include in-development guidelines for sources that expose development status.")] = False,  # noqa: E501
     output_format: Annotated[OutputFormat, typer.Option("--format", "-f", help="Output format.")] = OutputFormat.JSONL,
     output_path: Annotated[Path | None, typer.Option("--output", "-o", help="Output JSONL file, markdown directory, or Hugging Face dataset directory. JSONL defaults to stdout.")] = None,  # noqa: E501
     progress: Annotated[bool, typer.Option("--progress/--no-progress", help="Show a Rich progress bar.")] = True,
@@ -147,14 +174,25 @@ def run(
         documents: Number of documents to scrape, or "all" (default: "1").
         link_mode: Whether links are kept as markdown links or stripped to their
             visible text (default: LinkMode.KEEP).
+        include_archived: Whether archived guidelines are included for sources
+            that expose archival status (default: False).
+        include_in_development: Whether in-development guidelines are included
+            for sources that expose development status (default: False).
         output_format: Output format to write (default: OutputFormat.JSONL).
         output_path: Output JSONL file, markdown directory, or Hugging Face
             dataset directory. When unset, JSONL is written to stdout (default:
             None).
         progress: Whether to show a Rich progress bar (default: True).
     """
     parsed_documents = _parse_documents(documents)
-    scrape_run = scrape_documents(source, documents=parsed_documents, link_mode=link_mode, url=url)
+    scrape_run = scrape_documents(
+        source,
+        documents=parsed_documents,
+        link_mode=link_mode,
+        url=url,
+        include_archived=include_archived,
+        include_in_development=include_in_development,
+    )
     scraped_documents = scrape_run.documents
     if progress:
         scraped_documents = _progress_documents(scraped_documents, total=scrape_run.total)