fix(pdf): rejoin every printed word break, deciding the hyphen from the document (#650)

.github/workflows/ci.yml

@@ -252,6 +252,7 @@ jobs:
           tests/test_pipeline_parity.py
           tests/test_pdf_xml_amount_recall.py
           tests/test_pdf_xml_prose_recall.py
+          tests/test_pdf_word_break_recall.py
           tests/test_front_matter_parity.py
           tests/test_xml_compare.py
           tests/test_toc_tree.py

scripts/regen_word_break_residuals.py

@@ -0,0 +1,88 @@
+"""Regenerate the word-break residual fixture read by tests/test_pdf_word_break_recall.py.
+
+A residual is a printed word break whose reconstruction disagrees with the bill's XML
+read IN CONTEXT: neither candidate form is attested in the document's own text or its
+sibling version's, so `pdf_text._shape_keeps_hyphen` decides it from letter case alone,
+and case cannot tell a lowercase-continuation compound (`government-` / `driven`) from
+a syllable break (`equip-` / `ment`).
+
+The file also records, per version, how many sites the aligned oracle cannot decide at
+all. Those are sites the gate does not cover, so the count is asserted rather than
+ignored.
+
+Run after an INTENTIONAL change to the break rule, then review the JSON diff:
+
+    uv run python scripts/regen_word_break_residuals.py
+
+The test asserts SET EQUALITY against this file, so both directions show up in review:
+a new entry is a site that stopped resolving, a removed entry is one that started.
+Never add an entry just to clear a red run -- establish which form the bill actually
+uses first, because a wrong entry silently blesses an invented word.
+
+Note this measures the SINGLE-DOCUMENT path (`extract_clean_pages`). The shipped
+comparison additionally lets each version borrow the other where its own text is silent
+(`compare/pdf.py`, own evidence first), which can only settle breaks this path leaves
+to the fallback, so this fixture is an upper bound on what a reader actually sees.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+_ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(_ROOT / "src"))
+sys.path.insert(0, str(_ROOT))
+sys.path.insert(0, str(_ROOT / "tests"))
+
+from pdf_corpus import cached_pages, dual_format_versions  # noqa: E402
+
+from tests import test_pdf_word_break_recall as gate  # noqa: E402
+
+
+def main() -> int:
+    rows: list[dict[str, str]] = []
+    undecided: dict[str, int] = {}
+    for bill, xml_path, pdf_path in dual_format_versions():
+        version = f"{bill}/{pdf_path.stem}"
+        pages = cached_pages(pdf_path)
+        oracle = gate.XmlOracle(xml_path)
+        seen: set[tuple[str, str]] = set()
+        n_undecided = 0
+        for join in gate._joins(gate._merge_groups(pages)):
+            keep = gate._canon(f"{join['left']}-{join['right']}")
+            drop = gate._canon(f"{join['left']}{join['right']}")
+            verdict = oracle.verdict(keep, drop, join["prev"], join["next"])
+            if verdict == "UNDECIDED":
+                n_undecided += 1
+                continue
+            if gate._canon(join["produced"]) == (keep if verdict == "KEEP" else drop):
+                continue
+            key = (join["left"], join["right"])
+            if key in seen:
+                continue
+            seen.add(key)
+            rows.append(
+                {
+                    "version": version,
+                    "left": join["left"],
+                    "right": join["right"],
+                    "produced": join["produced"],
+                    "expected": keep if verdict == "KEEP" else drop,
+                    "reason": "no in-document or sibling evidence for either form; decided by case shape",
+                }
+            )
+        undecided[version] = n_undecided
+        print(f"{version:52s} residuals={len(seen):4d} undecided={n_undecided:4d}", flush=True)
+
+    rows.sort(key=lambda r: (r["version"], r["left"], r["right"]))
+    out = gate._RESIDUALS_PATH
+    out.parent.mkdir(parents=True, exist_ok=True)
+    out.write_text(json.dumps({"residuals": rows, "undecided": undecided}, indent=2, sort_keys=True) + "\n")
+    print(f"\nwrote {len(rows)} residuals and {sum(undecided.values())} undecided sites to {out.relative_to(_ROOT)}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

src/deltatrack/compare/pdf.py

@@ -3,7 +3,8 @@
 This is the in-process wrap of the existing PDF pipeline, with the inputs coming
 from uploaded bytes instead of files on disk:
 
-    extract_clean_pages()  (parsers.pdf_text)
+    extract_print_pages()  (parsers.pdf_text)   — both sides, before either is merged
+    merge_print_pages()    (parsers.pdf_text)   — own evidence first, sibling as fallback
     diff_pdfs()            (diff_pdf)
     pdf_full_text()        (parsers.pdf_text)   — both paths (full text + offsets)
     pdf_diff_to_canonical()(formatters.canonical) — both paths (JSON out / embedded)
@@ -22,7 +23,13 @@
 from deltatrack.diff_pdf import PdfDiff, diff_pdfs
 from deltatrack.formatters.canonical import pdf_diff_to_canonical
 from deltatrack.formatters.diff_html import format_diff_html
-from deltatrack.parsers.pdf_text import Page, extract_clean_pages, pdf_full_text, pdf_full_text_print
+from deltatrack.parsers.pdf_text import (
+    Page,
+    extract_print_pages,
+    merge_print_pages,
+    pdf_full_text,
+    pdf_full_text_print,
+)
 
 
 class UnsupportedLayoutError(ValueError):
@@ -119,8 +126,23 @@ def _extract_and_diff(
         start_path.write_bytes(start_bytes)
         end_path.write_bytes(end_bytes)
 
-        old_pages = extract_clean_pages(start_path)
-        new_pages = extract_clean_pages(end_path)
+        # Read both documents before merging either, so each side can borrow the other's
+        # spellings for breaks its own text leaves open (#650). Two versions of one bill
+        # are near-identical, so a compound one version never happens to spell out
+        # unbroken is often spelled out in the other.
+        #
+        # Each side keeps its OWN evidence first and consults the sibling only where it
+        # is silent. Merging the two into one index and taking a majority would let the
+        # larger document overrule the smaller about its own text: if v1 writes
+        # `Non-Dedicated` and never `NonDedicated`, while v2 writes `NonDedicated` more
+        # often, a pooled majority renders both as `NonDedicated`. That corrupts v1,
+        # which was never ambiguous, and erases a real spelling change between the two
+        # versions, so the diff stops reporting a difference the documents have.
+        old_read = extract_print_pages(start_path)
+        new_read = extract_print_pages(end_path)
+        old_evidence, new_evidence = old_read.evidence(), new_read.evidence()
+        old_pages = merge_print_pages(old_read, old_evidence.then(new_evidence))
+        new_pages = merge_print_pages(new_read, new_evidence.then(old_evidence))
 
     if _is_unnumbered_layout(old_pages) or _is_unnumbered_layout(new_pages):
         raise UnsupportedLayoutError(_DECLINE_MESSAGE)

src/deltatrack/parsers/pdf_anchors.py

@@ -647,8 +647,17 @@ def _split_major_run(run, column_width):
 
 
 def _join_major_run(segment) -> str:
-    """Join one segment's lines into a major name: de-hyphenate a GPO soft wrap
-    (``INTEL-`` + ``LIGENCE`` → ``INTELLIGENCE``), else space-join."""
+    """Join one segment's lines into a major name, space-joining each line onto the last.
+
+    The de-hyphenating branch (``INTEL-`` + ``LIGENCE`` → ``INTELLIGENCE``) is DORMANT
+    since #650: `extract_clean_pages` now rejoins a wrapped heading before the anchor
+    parser ever sees it, so a segment line no longer ends mid-word -- measured at 0
+    de-hyphenations across the fixture corpus. It is kept for a `Page` built by hand,
+    and it is NOT the project's rule for whether a break hyphen survives. It drops the
+    hyphen unconditionally, which is right for the all-caps headings it was written for
+    and wrong in general (it would render a wrapped ``NON-`` / ``DEDICATED`` as
+    ``NONDEDICATED``). `pdf_text.BreakEvidence` decides this now.
+    """
     text = segment[0][1].text.strip()
     for _page, ln in segment[1:]:
         seg = ln.text.strip()

src/deltatrack/parsers/pdf_blocks.py

@@ -117,6 +117,16 @@ def _rejoin_cross_page_hyphens(lines: list[_IndexedLine]) -> list[_IndexedLine]:
     lowercase continuation) so real compounds like `Child-Rescue`, which
     continue uppercase, are preserved. Anchors never start lowercase, so a
     TITLE/SEC heading opening a page is never absorbed.
+
+    DORMANT since #650, and deliberately not the live rule. `extract_clean_pages`
+    now joins page-seam breaks itself, deciding the hyphen from evidence this
+    function does not have, so every `Page` the production path builds arrives with
+    nothing left here to do -- measured at 0 joins across the whole fixture corpus.
+    What remains is a fallback for a `Page` assembled by hand (tests, and the
+    anchor parser's single-page `parse_lines`), which cannot contain a page seam
+    anyway. Do not read the lowercase-continuation guard below as the project's
+    answer to whether a break hyphen survives: it is the pre-#650 answer, and
+    `pdf_text.BreakEvidence` is the current one.
     """
     merged: list[_IndexedLine] = []
     i = 0