add who guideline scraper

datasets/amfv_datasets/scraping/LICENSE_NOTES.md

@@ -0,0 +1,30 @@
+# Source licensing notes for scraped corpora
+
+## WHO (World Health Organization)
+
+Publications on [who.int](https://www.who.int/publications) published since
+November 2016 are licensed under **Creative Commons Attribution-NonCommercial-
+ShareAlike 3.0 IGO** (CC BY-NC-SA 3.0 IGO).
+
+- **Non-commercial use and adaptation** are permitted.
+- **Attribution** to WHO is required.
+- **Share-alike**: derivatives must use the same or a similar licence.
+
+Pre-2017 publications were not reissued under this licence. Use
+`metadata.publication_date` to filter when building corpora.
+
+### Suggested attribution
+
+> © World Health Organization {year}. *{publication title}*.
+> Licensed under CC BY-NC-SA 3.0 IGO.
+> https://creativecommons.org/licenses/by-nc-sa/3.0/igo/
+
+Each scraped document also records `metadata.license` and
+`metadata.attribution`.
+
+### Milestone 1 content scope
+
+The WHO scraper collects the HTML **Overview** section from each publication
+landing page (`metadata.content_scope = "overview"`). Full guideline text is
+typically available only as a linked PDF (`metadata.download_url`). PDF
+extraction is intentionally deferred.

datasets/amfv_datasets/scraping/__init__.py

@@ -27,6 +27,16 @@
     scrape_guideline,
     scrape_nice,
 )
+from amfv_datasets.scraping.who import (
+    WhoFetchError,
+    WhoListingPage,
+    WhoPublicationRef,
+    build_publication_text,
+    list_publications,
+    publication_ref_from_url,
+    scrape_publication,
+    scrape_who,
+)
 
 __all__ = [
     "GuidanceRef",
@@ -39,16 +49,24 @@
     "ScrapedDocument",
     "ScraperSource",
     "USER_AGENT",
+    "WhoFetchError",
+    "WhoListingPage",
+    "WhoPublicationRef",
     "absolute_unique_urls",
     "build_guideline_text",
+    "build_publication_text",
     "clean_text",
     "document_title",
     "default_client",
     "first_matching_urls",
     "guidance_ref_from_url",
     "html_to_markdown",
     "list_published_guidance",
+    "list_publications",
+    "publication_ref_from_url",
     "scrape_guideline",
     "scrape_listing_documents",
     "scrape_nice",
+    "scrape_publication",
+    "scrape_who",
 ]

datasets/amfv_datasets/scraping/cli.py

@@ -27,13 +27,15 @@
 from amfv_datasets.scraping.base import ScrapedDocument, ScrapeRun
 from amfv_datasets.scraping.html import LinkMode
 from amfv_datasets.scraping.nice import scrape_nice
+from amfv_datasets.scraping.who import scrape_who
 
 
 class ScraperSource(StrEnum):
     """Supported scraper sources."""
 
     ALL = "all"
     NICE = "nice"
+    WHO = "who"
 
 
 class OutputFormat(StrEnum):
@@ -72,6 +74,8 @@ def scrape_documents(
         match selected_source:
             case ScraperSource.NICE:
                 return scrape_nice(documents=documents, link_mode=link_mode, url=url)
+            case ScraperSource.WHO:
+                return scrape_who(documents=documents, link_mode=link_mode, url=url)
             case ScraperSource.ALL:
                 raise AssertionError("expanded source cannot be all")
     raise AssertionError(f"unsupported source: {source}")
@@ -125,7 +129,7 @@ def write_markdown_files(documents: Iterable[ScrapedDocument], output_path: Path
 
 def _expand_source(source: ScraperSource) -> tuple[ScraperSource, ...]:
     if source is ScraperSource.ALL:
-        return (ScraperSource.NICE,)
+        return (ScraperSource.NICE, ScraperSource.WHO)
     return (source,)