Tie the abstract's error rates to the abstract

scripts/check_paper.py

@@ -27,7 +27,13 @@
     and the thin-donor support count — are pinned inside a
     whitespace-normalized slice of that section, so the same numeral
     occurring elsewhere in the paper cannot satisfy them and line
-    rewrapping cannot split a multi-word literal;
+    rewrapping cannot split a multi-word literal. The guard reads each
+    error rate the abstract quotes from the front matter's ``abstract: |``
+    block alone, as Quarto and pandoc read it, beside the rule it scores,
+    and every rate the abstract gives in that rule's wording must equal
+    the registered value. A copy in other metadata, in a comment, in the
+    body or inside the abstract cannot rescue a drifted figure, and a swap
+    between rules fails;
 (e) the README miscounts the in-kind receipts or omits a calculator
     commit that their provenance records.
 
@@ -72,6 +78,7 @@
     published_standard_errors,
     uncertainty_figures,
 )
+from paper_text import block_scalar_span, stated_values
 
 REPO = Path(__file__).resolve().parent.parent
 DATA = REPO / "data"
@@ -164,6 +171,19 @@ def section(start: str, end: str | None = None, text: str = QMD) -> str:
     return " ".join(text[first:last].split())
 
 
+def front_matter_block(key: str, text: str = QMD) -> str:
+    """Whitespace-normalized text of the front matter's one ``key: |``
+    literal block, read as Quarto and pandoc read it. Pins that read it see
+    no other metadata field, comment or body text. When ``block_scalar_span``
+    finds no such block, or finds another line that sets the key, the slice
+    is empty, so every pin inside it fails instead of reading other text."""
+    span = block_scalar_span(text, key)
+    if span is None:
+        failures.append(f"paper must set {key} only in one {key}: | block of its front matter")
+        return ""
+    return " ".join(text[span[0] : span[1]].split())
+
+
 COUNT_WORDS = (
     "zero",
     "one",
@@ -724,9 +744,37 @@ def mae(rule: str) -> float:
     has_number("2.63") and abs(realized_2025 - 0.0263) < 0.0005,
 )
 
+# Figures in the abstract also appear in the body or its tables, so their
+# abstract ties read only the front matter's `abstract: |` block, as Quarto
+# and pandoc read it. Other metadata fields, comments and body text fall
+# outside that block, so a copy there cannot rescue a drifted figure, and a
+# metadata block in the body that sets the abstract fails every tie. Each tie
+# collects every rate the abstract gives in its rule's wording and requires
+# them all to equal the registered value, so a swap between rules fails, and
+# so does a changed rate that leaves a correct copy elsewhere in the abstract.
+ABSTRACT = front_matter_block("abstract")
+
 # Current retrospective results in the abstract are a separate experiment.
-for rule, value in current_ce["projection_evaluation"]["summary_2020_2025"].items():
+ABSTRACT_RETROSPECTIVE_RULES = {
+    "cpi_u": "with the Consumer Price Index",
+    "fcsuti_cpi": "with a composite price index",
+    "replication_ratio": "with replicated consumption growth",
+    "blend": "with an equal blend",
+}
+retrospective = current_ce["projection_evaluation"]["summary_2020_2025"]
+check(
+    "abstract names the retrospective rules and their count",
+    set(retrospective) == set(ABSTRACT_RETROSPECTIVE_RULES)
+    and set(stated_values("compare {} projection rules", ABSTRACT, r"\S+"))
+    == {count_word(len(retrospective))},
+)
+for rule, value in retrospective.items():
     check(f"current projection prose {rule}", has_number(f"{value:.2f}"))
+    rule_phrase = ABSTRACT_RETROSPECTIVE_RULES.get(rule, "<unnamed rule>")
+    check(
+        f"abstract current projection {rule} {value:.2f}",
+        set(stated_values(f"{{}} percent {rule_phrase}", ABSTRACT)) == {f"{value:.2f}"},
+    )
 
 # (e) The September evaluation, re-derived from data/evaluation_2025.json.
 ev = json.loads((DATA / "evaluation_2025.json").read_text())
@@ -736,13 +784,36 @@ def mae(rule: str) -> float:
 repl = R["CE replication growth ratio alone"]
 comp = R["FCSUti-composite CPI aging alone"]
 cpiu = R["All-Items CPI-U aging (status quo)"]
-for label, value, literal in [
-    ("evaluation amended MAE 1.17", amended["mae"], "1.17"),
-    ("evaluation replication MAE 0.75", repl["mae"], "0.75"),
-    ("evaluation composite MAE 2.02", comp["mae"], "2.02"),
-    ("evaluation CPI-U MAE 2.58", cpiu["mae"], "2.58"),
+# The abstract quotes three of these errors, and the body restates each.
+for label, value, literal, abstract_phrase in [
+    (
+        "evaluation amended MAE 1.17",
+        amended["mae"],
+        "1.17",
+        "blend yields a mean absolute error of {} percent",
+    ),
+    (
+        "evaluation replication MAE 0.75",
+        repl["mae"],
+        "0.75",
+        "{} percent for consumption growth alone",
+    ),
+    ("evaluation composite MAE 2.02", comp["mae"], "2.02", None),
+    (
+        "evaluation CPI-U MAE 2.58",
+        cpiu["mae"],
+        "2.58",
+        "{} percent for CPI-U adjustment",
+    ),
 ]:
-    check(label, has_number(literal) and f"{value:.2%}".rstrip("%") == literal)
+    rounds_to_literal = f"{value:.2%}".rstrip("%") == literal
+    check(label, has_number(literal) and rounds_to_literal)
+    if abstract_phrase is not None:
+        check(
+            f"abstract {label}",
+            rounds_to_literal
+            and set(stated_values(abstract_phrase, ABSTRACT)) == {literal},
+        )
 check(
     "forecast record evaluation original MAE 0.98",
     has_number("0.98", text=record_text)
@@ -1100,7 +1171,6 @@ def block_end_years(entry: dict) -> list[int]:
 INKIND_TEXT = section(
     "# Imputing in-kind benefits {#sec-inkind}", "# Projecting thresholds {#sec-projection}"
 )
-ABSTRACT = " ".join(QMD.split("# Introduction", 1)[0].split())
 EVALUATION_TEXT = section(
     "## Evaluation against published thresholds {#sec-evaluation}",
     "## Validation of the price comparator",

scripts/paper_text.py

@@ -0,0 +1,123 @@
+"""Text helpers that the paper guard and its tests share.
+
+The guard reads the abstract through ``block_scalar_span``, and the mutation
+tests place their edits with the same function, so both work on the same text.
+Wherever pandoc or Quarto is installed, ``tests/test_paper_text.py`` checks
+that the parser reads the words that tool renders as the abstract, or fails
+closed.
+"""
+
+from __future__ import annotations
+
+import re
+
+FRONT_MATTER_ENDS = ("---", "...")
+SIGNED_NUMBER = r"[−-]?\d[\d,]*(?:\.\d+)?"
+LINE = re.compile(r"[^\n]*\n|[^\n]+\Z")
+
+
+def _bare(line: str) -> str:
+    """A line without its ending and without trailing spaces or tabs."""
+    return line.removesuffix("\n").removesuffix("\r").rstrip(" \t")
+
+
+def _key_matchers(key: str):
+    """Two matchers for a line that YAML could read as setting ``key``. The
+    first finds the key, plain or quoted, at the line's start before a colon,
+    or in an explicit ``? key`` line. The second also finds it in flow style,
+    after ``{`` or ``,``, as in ``{key: value}``."""
+    name = re.escape(key)
+    spelled = rf"""(?:{name}|"{name}"|'{name}')"""
+    sets = rf"(?:\?[ \t]+{spelled}[ \t]*(?::|\r?$)|{spelled}[ \t]*:)"
+    return re.compile(sets).match, re.compile(rf"(?:^|[{{,])[ \t]*{sets}").search
+
+
+def block_scalar_span(text: str, key: str) -> tuple[int, int] | None:
+    """Character span of the literal block under the front matter's one
+    top-level ``key: |`` line, read as Quarto and pandoc read it, or None.
+
+    Lines end at line feeds. The front matter opens with a ``---`` line and
+    text on the next line, and closes at the next ``---`` or ``...`` line.
+    The block takes its indentation from the spaces that lead its first line
+    with text and ends at the next line with fewer, such as another field or
+    a trailing comment. Lines of whitespace alone stay inside it. Indented
+    keys belong to other mappings and do not count.
+
+    It returns None, so every tie that reads the block fails, when
+
+    - the text lacks such front matter, or the front matter never closes;
+    - the front matter names the key other than once, or names it on a line
+      other than ``key: |``. A quoted ``"key"`` or an explicit ``? key`` also
+      names it;
+    - any line after the front matter names the key, at the line's start or
+      in flow style such as ``{key: value}``, since pandoc lets a later
+      metadata block, even inside a list item or block quote, replace the
+      front matter's value;
+    - a tab falls within the block's indentation on one of its lines, where
+      Quarto and pandoc read the line differently, or a carriage return
+      stands alone in the front matter, since pandoc drops it;
+    - the block holds no text.
+    """
+    lines = LINE.findall(text)
+    if len(lines) < 2 or _bare(lines[0]) != "---" or not _bare(lines[1]):
+        return None
+    header = []
+    offset = len(lines[0])
+    for end, line in enumerate(lines[1:], 1):
+        if _bare(line) in FRONT_MATTER_ENDS:
+            break
+        header.append((offset, line))
+        offset += len(line)
+    else:
+        return None
+    if any("\r" in line.removesuffix("\n").removesuffix("\r") for line in lines[: end + 1]):
+        return None
+    key_at_start, key_anywhere = _key_matchers(key)
+    starts = [index for index, (_, line) in enumerate(header) if key_at_start(line)]
+    if len(starts) != 1 or _bare(header[starts[0]][1]) != f"{key}: |":
+        return None
+    if any(key_anywhere(line.replace("\r", "").lstrip(" \t>")) for line in lines[end + 1 :]):
+        return None
+    first = last = starts[0] + 1
+    indent = None
+    while last < len(header):
+        line = header[last][1]
+        if line.strip():
+            spaces = len(line) - len(line.lstrip(" "))
+            indent = spaces if indent is None else indent
+            if "\t" in line[: max(indent, 1)]:
+                return None
+            if spaces < max(indent, 1):
+                break
+        last += 1
+    block = header[first:last]
+    if not "".join(line for _, line in block).strip():
+        return None
+    return block[0][0], block[-1][0] + len(block[-1][1])
+
+
+def _whitespace_tolerant(part: str) -> str:
+    """Regex for a template fragment in which any whitespace run matches any
+    whitespace run in the text."""
+    tokens = part.split()
+    if not tokens:
+        return r"\s+" if part else ""
+    lead = r"\s+" if part[0].isspace() else ""
+    trail = r"\s+" if part[-1].isspace() else ""
+    return lead + r"\s+".join(re.escape(token) for token in tokens) + trail
+
+
+def stated_values(template: str, text: str, value: str = SIGNED_NUMBER) -> list[str]:
+    """Every value that ``text`` gives in the ``{}`` slot of ``template``, in
+    order.
+
+    By default the slot takes a signed number at numeric-token boundaries, so
+    the text's "12.29" or "−2.29" never reads as "2.29". A template without
+    exactly one slot matches nothing.
+    """
+    parts = template.split("{}")
+    if len(parts) != 2:
+        return []
+    before, after = (_whitespace_tolerant(part) for part in parts)
+    slot = rf"(?<![\d.,])({value})(?![\d]|[.,]\d)"
+    return re.findall(before + slot + after, text)