diff --git a/prospector/extract.py b/prospector/extract.py
index 6c3faa3..cdb8342 100644
--- a/prospector/extract.py
+++ b/prospector/extract.py
@@ -78,10 +78,17 @@ def discover_extra_pages(homepage_html: str, base_url: str) -> list[tuple[str, s
href = (node.attributes.get("href") or "").strip()
if not href or href.startswith(("#", "mailto:", "tel:", "javascript:")):
continue
- absolute = urljoin(base_url.rstrip("/") + "/", href)
- if urlparse(absolute).hostname != base_host:
+ try:
+ absolute = urljoin(base_url.rstrip("/") + "/", href)
+ if urlparse(absolute).hostname != base_host:
+ continue
+ path = urlparse(absolute).path.lower()
+ except ValueError:
+ # A malformed href is the page author's mistake, not a reason to
+ # lose the company. An unfilled template placeholder such as
+ # "http://[BookingLink]" reads as an IPv6 literal to urlsplit
+ # (3.11.4+), which then raises from ipaddress. Skip the link.
continue
- path = urlparse(absolute).path.lower()
for kind, keywords in PAGE_KEYWORDS.items():
if kind not in found and any(k in path for k in keywords):
found[kind] = absolute
diff --git a/prospector/models.py b/prospector/models.py
index 5bc94ee..a482b26 100644
--- a/prospector/models.py
+++ b/prospector/models.py
@@ -61,6 +61,10 @@ class ResearchResult:
# pages rather than supplied in the input row.
email_evidence: "Evidence | None" = None
city: str | None = None
+ # How many of the company's own pages were actually read. Distinguishes
+ # "no site to read" from "read the site, nothing published" when explaining
+ # an unreachable company (008 FR-010).
+ pages_fetched: int = 0
sources_consulted: list[str] = field(default_factory=list)
failures: list[str] = field(default_factory=list)
diff --git a/prospector/pipeline.py b/prospector/pipeline.py
index cb7dd8a..cb38b98 100644
--- a/prospector/pipeline.py
+++ b/prospector/pipeline.py
@@ -128,7 +128,7 @@ def _process_company(
) -> tuple[Prospect, Draft | None]:
research = _research(company, settings, fetcher, verbose=verbose)
if not company.email:
- raise NoEmailFound(company.bucket_reason or "no email address found")
+ raise NoEmailFound(_no_email_reason(research))
prospect = _score(company, research, settings)
draft: Draft | None = None
if no_llm:
@@ -147,6 +147,21 @@ def _process_company(
return prospect, draft
+def _no_email_reason(research: ResearchResult) -> str:
+ """Why this company is unreachable, judged AFTER recovery ran (008 FR-010).
+
+ The ingest-time reason describes the input row ("blank email"), so reporting
+ it here reads as though recovery never happened. What an operator needs is
+ which stage ran out of road: a keyword returning listings with no websites
+ is a sourcing problem, while sites that publish no address is a vertical
+ that simply prefers contact forms."""
+ if not research.website:
+ return "no website could be resolved"
+ if not research.pages_fetched:
+ return f"no page could be fetched from {research.website}"
+ return "no published address on any fetched page"
+
+
def _research(company: Company, settings: Settings, fetcher: Fetcher, *, verbose: bool) -> ResearchResult:
research = ResearchResult(website=company.website)
info = resolve.resolve(company, settings, fetcher)
@@ -165,6 +180,8 @@ def _research(company: Company, settings: Settings, fetcher: Fetcher, *, verbose
if html is not None:
pages.append(extracting.PageContent(kind, url, html))
+ research.pages_fetched = len(pages)
+
# 008 FR-006: no supplied address -> look for one the company publishes on
# the pages we already fetched. No new request is made (FR-011).
if not company.email and pages:
diff --git a/tests/integration/test_email_recovery_batch.py b/tests/integration/test_email_recovery_batch.py
index 2f8f679..a5443a9 100644
--- a/tests/integration/test_email_recovery_batch.py
+++ b/tests/integration/test_email_recovery_batch.py
@@ -76,6 +76,12 @@ def test_unreachable_company_is_named(self, tmp_path, stubs):
summary, _ = run(tmp_path, stubs)
assert [name for name, _ in summary.skipped_companies] == ["Unreachable Ducts"]
+ def test_skip_reason_describes_the_outcome_not_the_input_row(self, tmp_path, stubs):
+ """FR-010: "blank email" restates the CSV. The operator needs to know
+ which stage ran out of road — the site was read and published nothing."""
+ summary, _ = run(tmp_path, stubs)
+ assert summary.skipped_companies[0][1] == "no published address on any fetched page"
+
def test_recovered_address_lands_in_the_note(self, tmp_path, stubs):
_, vault_dir = run(tmp_path, stubs)
text = (vault_dir / "recoverable-ducts.md").read_text(encoding="utf-8")
diff --git a/tests/unit/test_extract.py b/tests/unit/test_extract.py
index 0d3a58d..0adcfdc 100644
--- a/tests/unit/test_extract.py
+++ b/tests/unit/test_extract.py
@@ -25,6 +25,14 @@ def test_external_links_ignored(self):
html = 'About'
assert discover_extra_pages(html, "https://acmeduct.com") == []
+ def test_malformed_href_does_not_lose_the_page(self):
+ # An unfilled template placeholder reads as an IPv6 literal to urlsplit
+ # and raises from ipaddress. One bad link must not cost the company.
+ html = 'Bookc'
+ assert discover_extra_pages(html, "https://acmeduct.com") == [
+ ("contact", "https://acmeduct.com/contact")
+ ]
+
def test_capped_at_three(self):
html = (
'at'
diff --git a/tests/unit/test_pipeline_reasons.py b/tests/unit/test_pipeline_reasons.py
new file mode 100644
index 0000000..a158c99
--- /dev/null
+++ b/tests/unit/test_pipeline_reasons.py
@@ -0,0 +1,20 @@
+"""008 FR-010: an unreachable company is explained by what research found, not
+by what the input row looked like. The three reasons separate a sourcing problem
+(listings with no websites) from a vertical that publishes no addresses.
+"""
+
+from prospector.models import ResearchResult
+from prospector.pipeline import _no_email_reason
+
+
+class TestNoEmailReason:
+ def test_no_website_resolved(self):
+ assert _no_email_reason(ResearchResult()) == "no website could be resolved"
+
+ def test_website_resolved_but_unreadable(self):
+ research = ResearchResult(website="https://acmeduct.com", pages_fetched=0)
+ assert _no_email_reason(research) == "no page could be fetched from https://acmeduct.com"
+
+ def test_pages_read_but_nothing_published(self):
+ research = ResearchResult(website="https://acmeduct.com", pages_fetched=3)
+ assert _no_email_reason(research) == "no published address on any fetched page"