From dd95210c07c706eb0b6a9a096fda0ab40e097b84 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 6 Nov 2026 19:12:59 -0500 Subject: [PATCH 1/5] Tie the abstract's error rates to the abstract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The guard checked the 2020–2025 retrospective errors (2.29, 1.65, 0.51 and 0.90 percent) and the 2025 evaluation errors (1.17, 2.58 and 0.75 percent) against the whole document. The body restates 1.17, 2.58 and 0.75, so the guard passed when one of them changed in the abstract alone. The guard now also reads each of these figures from the abstract alone, beside the rule it scores, so a swap between rules fails too. It also ties the abstract's count of four projection rules to the retrospective artifact. The abstract slice now comes from section(): when "abstract: |" or "# Introduction" goes missing or repeats, the slice comes back empty and every abstract tie fails instead of reading the whole paper. New tests change each figure in the abstract alone, swap two pairs, and rerun every case with the original wording restated in the body. Another test renames, repeats and restyles the slice's markers. No paper text or number changes. Co-Authored-By: Claude Opus 5.5 --- scripts/check_paper.py | 69 +++++++++++++++--- tests/test_paper_guard.py | 145 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 205 insertions(+), 9 deletions(-) diff --git a/scripts/check_paper.py b/scripts/check_paper.py index 28e3dfb..6e5e1ca 100644 --- a/scripts/check_paper.py +++ b/scripts/check_paper.py @@ -27,7 +27,10 @@ and the thin-donor support count — are pinned inside a whitespace-normalized slice of that section, so the same numeral occurring elsewhere in the paper cannot satisfy them and line - rewrapping cannot split a multi-word literal; + rewrapping cannot split a multi-word literal. The guard reads each + error rate the abstract quotes from the abstract alone, beside the + rule it scores, so a copy in the body cannot rescue a drifted + abstract figure and a swap between rules fails; (e) the README miscounts the in-kind receipts or omits a calculator commit that their provenance records. @@ -724,9 +727,35 @@ def mae(rule: str) -> float: has_number("2.63") and abs(realized_2025 - 0.0263) < 0.0005, ) +# Figures in the abstract also appear in the body or its tables, so their +# abstract ties read only the abstract, where a copy elsewhere cannot rescue a +# drifted figure. Each tie holds a figure beside the rule it scores, so a swap +# between rules fails too. Both markers must occur exactly once; otherwise the +# slice comes back empty and every abstract tie fails. +ABSTRACT = section("abstract: |", "# Introduction") + # Current retrospective results in the abstract are a separate experiment. -for rule, value in current_ce["projection_evaluation"]["summary_2020_2025"].items(): +ABSTRACT_RETROSPECTIVE_RULES = { + "cpi_u": "with the Consumer Price Index", + "fcsuti_cpi": "with a composite price index", + "replication_ratio": "with replicated consumption growth", + "blend": "with an equal blend", +} +retrospective = current_ce["projection_evaluation"]["summary_2020_2025"] +check( + "abstract names the retrospective rules and their count", + set(retrospective) == set(ABSTRACT_RETROSPECTIVE_RULES) + and has_number( + f"compare {count_word(len(retrospective))} projection rules", text=ABSTRACT + ), +) +for rule, value in retrospective.items(): check(f"current projection prose {rule}", has_number(f"{value:.2f}")) + rule_phrase = ABSTRACT_RETROSPECTIVE_RULES.get(rule, "") + check( + f"abstract current projection {rule} {value:.2f}", + has_number(f"{value:.2f} percent {rule_phrase}", text=ABSTRACT), + ) # (e) The September evaluation, re-derived from data/evaluation_2025.json. ev = json.loads((DATA / "evaluation_2025.json").read_text()) @@ -736,13 +765,36 @@ def mae(rule: str) -> float: repl = R["CE replication growth ratio alone"] comp = R["FCSUti-composite CPI aging alone"] cpiu = R["All-Items CPI-U aging (status quo)"] -for label, value, literal in [ - ("evaluation amended MAE 1.17", amended["mae"], "1.17"), - ("evaluation replication MAE 0.75", repl["mae"], "0.75"), - ("evaluation composite MAE 2.02", comp["mae"], "2.02"), - ("evaluation CPI-U MAE 2.58", cpiu["mae"], "2.58"), +# The abstract quotes three of these errors, and the body restates each. +for label, value, literal, abstract_phrase in [ + ( + "evaluation amended MAE 1.17", + amended["mae"], + "1.17", + "blend yields a mean absolute error of {} percent", + ), + ( + "evaluation replication MAE 0.75", + repl["mae"], + "0.75", + "{} percent for consumption growth alone", + ), + ("evaluation composite MAE 2.02", comp["mae"], "2.02", None), + ( + "evaluation CPI-U MAE 2.58", + cpiu["mae"], + "2.58", + "{} percent for CPI-U adjustment", + ), ]: - check(label, has_number(literal) and f"{value:.2%}".rstrip("%") == literal) + rounds_to_literal = f"{value:.2%}".rstrip("%") == literal + check(label, has_number(literal) and rounds_to_literal) + if abstract_phrase is not None: + check( + f"abstract {label}", + rounds_to_literal + and has_number(abstract_phrase.format(literal), text=ABSTRACT), + ) check( "forecast record evaluation original MAE 0.98", has_number("0.98", text=record_text) @@ -1100,7 +1152,6 @@ def block_end_years(entry: dict) -> list[int]: INKIND_TEXT = section( "# Imputing in-kind benefits {#sec-inkind}", "# Projecting thresholds {#sec-projection}" ) -ABSTRACT = " ".join(QMD.split("# Introduction", 1)[0].split()) EVALUATION_TEXT = section( "## Evaluation against published thresholds {#sec-evaluation}", "## Validation of the price comparator", diff --git a/tests/test_paper_guard.py b/tests/test_paper_guard.py index bdbd31f..0ed824b 100644 --- a/tests/test_paper_guard.py +++ b/tests/test_paper_guard.py @@ -446,6 +446,96 @@ ), ) +# Each case drifts figures in the abstract alone: (name, pattern, replacement, +# failing ties, decoy). The body or a generated table states every one of these +# figures too. The decoy restates the original wording under another heading, +# so a tie that read the whole document would still pass. +ABSTRACT_PROSE_MUTATIONS = ( + ( + "2020-2025 CPI-U error", + r"2\.29(\s+)percent", + r"2.30\1percent", + ("abstract current projection cpi_u 2.29",), + "We obtain 2.29 percent with the Consumer Price Index.", + ), + ( + "2020-2025 composite error", + r"1\.65(\s+)percent", + r"1.66\1percent", + ("abstract current projection fcsuti_cpi 1.65",), + "We obtain 1.65 percent with a composite price index.", + ), + ( + "2020-2025 replication error", + r"0\.51(\s+)percent", + r"0.52\1percent", + ("abstract current projection replication_ratio 0.51",), + "We obtain 0.51 percent with replicated consumption growth.", + ), + ( + "2020-2025 blend error", + r"0\.90(\s+)percent", + r"0.91\1percent", + ("abstract current projection blend 0.90",), + "We obtain 0.90 percent with an equal blend.", + ), + ( + "2025 blend error", + r"1\.17(\s+)percent", + r"1.18\1percent", + ("abstract evaluation amended MAE 1.17",), + "The blend yields a mean absolute error of 1.17 percent.", + ), + ( + "2025 CPI-U error", + r"2\.58(\s+)percent", + r"2.59\1percent", + ("abstract evaluation CPI-U MAE 2.58",), + "We obtain 2.58 percent for CPI-U adjustment.", + ), + ( + "2025 replication error", + r"0\.75(\s+)percent", + r"0.74\1percent", + ("abstract evaluation replication MAE 0.75",), + "We obtain 0.75 percent for consumption growth alone.", + ), + ( + "rule count", + r"compare(\s+)four(\s+)projection", + r"compare\1five\2projection", + ("abstract names the retrospective rules and their count",), + "We compare four projection rules.", + ), + ( + "swapped 2020-2025 errors", + r"2\.29(.*?)1\.65", + r"1.65\g<1>2.29", + ( + "abstract current projection cpi_u 2.29", + "abstract current projection fcsuti_cpi 1.65", + ), + "We obtain 2.29 percent with the Consumer Price Index and 1.65 percent " + "with a composite price index.", + ), + ( + "swapped 2025 errors", + r"1\.17(.*?)0\.75", + r"0.75\g<1>1.17", + ( + "abstract evaluation amended MAE 1.17", + "abstract evaluation replication MAE 0.75", + ), + "The blend yields a mean absolute error of 1.17 percent, and we obtain " + "0.75 percent for consumption growth alone.", + ), +) + + +def failure_lines(stdout): + """The labels the guard lists as failures, one per line.""" + return [line[4:] for line in stdout.splitlines() if line.startswith(" - ")] + def reflect_replicates(uncertainty, year): """Mirror each of one year's replicate thresholds around its full-sample @@ -1165,6 +1255,61 @@ def assert_article_mutation_fails( self.assertNotEqual(result.returncode, 0, label) self.assertIn(expected, result.stdout) + def run_abstract_mutation(self, pattern, replacement, decoy=None): + self.install_article() + self.mutate_article("abstract", pattern, replacement) + if decoy is not None: + self.edit_article((), DISCUSSION_HEADING, decoy) + return self.run_guard() + + def test_abstract_figures_drift_alone(self): + """The body restates 1.17, 2.58 and 0.75, so only a tie that reads the + abstract can catch a change made there alone.""" + for name, pattern, replacement, labels, _ in ABSTRACT_PROSE_MUTATIONS: + with self.subTest(name=name): + result = self.run_abstract_mutation(pattern, replacement) + self.assertNotEqual(result.returncode, 0) + for label in labels: + self.assertIn(label, failure_lines(result.stdout)) + self.install_article() + + def test_abstract_pins_reject_a_figure_restated_in_the_body(self): + """With the original wording restated in the body, only the abstract's + own ties fail.""" + for name, pattern, replacement, labels, decoy in ABSTRACT_PROSE_MUTATIONS: + with self.subTest(name=name): + result = self.run_abstract_mutation(pattern, replacement, decoy) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(sorted(failure_lines(result.stdout)), sorted(labels)) + self.install_article() + + def test_abstract_slice_fails_closed_without_unique_markers(self): + """A renamed, repeated or restyled marker empties the abstract slice, + so every abstract tie fails instead of reading the whole paper.""" + cases = ( + ("renamed introduction", (("# Introduction\n", "# Background\n"),)), + ( + "repeated introduction", + ((DISCUSSION_HEADING, "# Introduction\n\n" + DISCUSSION_HEADING),), + ), + ("folded abstract", (("abstract: |\n", "abstract: >\n"),)), + ) + for name, edits in cases: + with self.subTest(case=name): + self.install_article() + self.edit_article(edits) + result = self.run_guard() + self.assertNotEqual(result.returncode, 0) + failures = failure_lines(result.stdout) + for label in ( + "paper section heading not unique: 'abstract: |'", + "abstract evaluation amended MAE 1.17", + "abstract current projection cpi_u 2.29", + "in-kind prose: abstract in-kind lift range", + ): + self.assertIn(label, failures) + self.install_article() + def test_rolling_prose_figures_drift_from_the_sealed_forecast(self): for label, edits, _, _, expected in ROLLING_PROSE_MUTATIONS: with self.subTest(figure=label): From f95d8d70dd4f5b3c299cb8b06c2cb675ef7096d4 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 7 Nov 2026 12:07:44 -0500 Subject: [PATCH 2/5] Read the abstract from its front-matter block and require every stated rate to match The abstract ties read section("abstract: |", "# Introduction"), which also took in the fields after the abstract, the closing delimiter and any body text before the introduction, so a copy there could rescue a drifted figure. scripts/paper_text.py now finds the one top-level `abstract: |` key in the front matter and returns only its indented block. A missing, repeated or restyled key leaves the slice empty and fails every abstract tie. Each tie now collects every rate the abstract gives in its rule's wording and requires them all to equal the registered value, so a correct copy elsewhere in the abstract cannot hide a changed one, and a sign flip fails. The mutation tests place decoys in another metadata field, in body text before the introduction, inside the abstract and under the discussion heading, and check that renaming, demoting, repeating or moving the introduction heading changes nothing. tests/test_paper_text.py adds unit and seeded property tests for the parser and the slot reader. Co-Authored-By: Claude Opus 5.5 --- scripts/check_paper.py | 42 ++-- scripts/paper_text.py | 76 ++++++++ tests/test_paper_guard.py | 139 +++++++++++-- tests/test_paper_text.py | 398 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 622 insertions(+), 33 deletions(-) create mode 100644 scripts/paper_text.py create mode 100644 tests/test_paper_text.py diff --git a/scripts/check_paper.py b/scripts/check_paper.py index 6e5e1ca..75eba7a 100644 --- a/scripts/check_paper.py +++ b/scripts/check_paper.py @@ -28,9 +28,11 @@ whitespace-normalized slice of that section, so the same numeral occurring elsewhere in the paper cannot satisfy them and line rewrapping cannot split a multi-word literal. The guard reads each - error rate the abstract quotes from the abstract alone, beside the - rule it scores, so a copy in the body cannot rescue a drifted - abstract figure and a swap between rules fails; + error rate the abstract quotes from the front matter's ``abstract: |`` + block alone, beside the rule it scores, and every rate the abstract + gives in that rule's wording must equal the registered value. A copy + in other metadata, in the body or inside the abstract cannot rescue a + drifted figure, and a swap between rules fails; (e) the README miscounts the in-kind receipts or omits a calculator commit that their provenance records. @@ -75,6 +77,7 @@ published_standard_errors, uncertainty_figures, ) +from paper_text import block_scalar_span, stated_values REPO = Path(__file__).resolve().parent.parent DATA = REPO / "data" @@ -167,6 +170,18 @@ def section(start: str, end: str | None = None, text: str = QMD) -> str: return " ".join(text[first:last].split()) +def front_matter_block(key: str, text: str = QMD) -> str: + """Whitespace-normalized text of the front matter's one ``key: |`` + literal block. Pins read from it see neither other metadata fields nor + body text. A missing, repeated or restyled key yields an empty slice, so + every pin inside it fails instead of reading other text.""" + span = block_scalar_span(text, key) + if span is None: + failures.append(f"paper front matter lacks exactly one {key}: | block") + return "" + return " ".join(text[span[0] : span[1]].split()) + + COUNT_WORDS = ( "zero", "one", @@ -728,11 +743,13 @@ def mae(rule: str) -> float: ) # Figures in the abstract also appear in the body or its tables, so their -# abstract ties read only the abstract, where a copy elsewhere cannot rescue a -# drifted figure. Each tie holds a figure beside the rule it scores, so a swap -# between rules fails too. Both markers must occur exactly once; otherwise the -# slice comes back empty and every abstract tie fails. -ABSTRACT = section("abstract: |", "# Introduction") +# abstract ties read only the indented lines under the front matter's +# `abstract: |` key. Other metadata fields and body text fall outside that +# block, so a copy there cannot rescue a drifted figure. Each tie collects +# every rate the abstract gives in its rule's wording and requires them all to +# equal the registered value, so a swap between rules fails, and so does a +# changed rate that leaves a correct copy elsewhere in the abstract. +ABSTRACT = front_matter_block("abstract") # Current retrospective results in the abstract are a separate experiment. ABSTRACT_RETROSPECTIVE_RULES = { @@ -745,16 +762,15 @@ def mae(rule: str) -> float: check( "abstract names the retrospective rules and their count", set(retrospective) == set(ABSTRACT_RETROSPECTIVE_RULES) - and has_number( - f"compare {count_word(len(retrospective))} projection rules", text=ABSTRACT - ), + and set(stated_values("compare {} projection rules", ABSTRACT, r"\S+")) + == {count_word(len(retrospective))}, ) for rule, value in retrospective.items(): check(f"current projection prose {rule}", has_number(f"{value:.2f}")) rule_phrase = ABSTRACT_RETROSPECTIVE_RULES.get(rule, "") check( f"abstract current projection {rule} {value:.2f}", - has_number(f"{value:.2f} percent {rule_phrase}", text=ABSTRACT), + set(stated_values(f"{{}} percent {rule_phrase}", ABSTRACT)) == {f"{value:.2f}"}, ) # (e) The September evaluation, re-derived from data/evaluation_2025.json. @@ -793,7 +809,7 @@ def mae(rule: str) -> float: check( f"abstract {label}", rounds_to_literal - and has_number(abstract_phrase.format(literal), text=ABSTRACT), + and set(stated_values(abstract_phrase, ABSTRACT)) == {literal}, ) check( "forecast record evaluation original MAE 0.98", diff --git a/scripts/paper_text.py b/scripts/paper_text.py new file mode 100644 index 0000000..bcd3980 --- /dev/null +++ b/scripts/paper_text.py @@ -0,0 +1,76 @@ +"""Text helpers that the paper guard and its tests share. + +The guard reads the abstract through ``block_scalar_span``, and the mutation +tests place their edits with the same function, so both work on the same text. +""" + +from __future__ import annotations + +import re + +FRONT_MATTER_ENDS = ("---", "...") +SIGNED_NUMBER = r"[−-]?\d[\d,]*(?:\.\d+)?" + + +def block_scalar_span(text: str, key: str) -> tuple[int, int] | None: + """Character span of the literal block under the front matter's one + top-level ``key: |`` line: the indented and blank lines that follow it, up + to the next unindented line. + + Returns None when the text opens without front matter, the front matter + never closes, the key appears other than exactly once as ``key: |``, or + the block holds no text. Indented keys belong to other mappings and do not + count, and a ``key: |`` line in the body lies outside the front matter. + """ + lines = text.splitlines(keepends=True) + if not lines or lines[0].rstrip() != "---": + return None + header = [] + offset = len(lines[0]) + for line in lines[1:]: + if line.rstrip() in FRONT_MATTER_ENDS: + break + header.append((offset, line)) + offset += len(line) + else: + return None + key_line = re.compile(rf"{re.escape(key)}\s*:") + starts = [index for index, (_, line) in enumerate(header) if key_line.match(line)] + if len(starts) != 1 or header[starts[0]][1].rstrip() != f"{key}: |": + return None + first = last = starts[0] + 1 + while last < len(header) and ( + not header[last][1].strip() or header[last][1][0] in " \t" + ): + last += 1 + block = header[first:last] + if not "".join(line for _, line in block).strip(): + return None + return block[0][0], block[-1][0] + len(block[-1][1]) + + +def _whitespace_tolerant(part: str) -> str: + """Regex for a template fragment in which any whitespace run matches any + whitespace run in the text.""" + tokens = part.split() + if not tokens: + return r"\s+" if part else "" + lead = r"\s+" if part[0].isspace() else "" + trail = r"\s+" if part[-1].isspace() else "" + return lead + r"\s+".join(re.escape(token) for token in tokens) + trail + + +def stated_values(template: str, text: str, value: str = SIGNED_NUMBER) -> list[str]: + """Every value that ``text`` gives in the ``{}`` slot of ``template``, in + order. + + By default the slot takes a signed number at numeric-token boundaries, so + the text's "12.29" or "−2.29" never reads as "2.29". A template without + exactly one slot matches nothing. + """ + parts = template.split("{}") + if len(parts) != 2: + return [] + before, after = (_whitespace_tolerant(part) for part in parts) + slot = rf"(?\n"),)), + ( + "moved introduction after a decoy", + ( + ("# Introduction\n", "# Background\n"), + (DISCUSSION_HEADING, f"{decoy}\n\n## Introduction\n\n{DISCUSSION_HEADING}"), + ), + ), + ) + for name, edits in cases: + with self.subTest(case=name): + self.install_article() + self.edit_article(edits) + unchanged = self.run_guard() + self.assertEqual(unchanged.returncode, 0, unchanged.stdout) + self.mutate_article("abstract", r"1\.17(\s+)percent", r"1.18\1percent") + drifted = self.run_guard() + self.assertEqual( + failure_lines(drifted.stdout), ["abstract evaluation amended MAE 1.17"] + ) + self.install_article() + + def test_abstract_block_fails_closed_without_one_literal_key(self): + """A folded, repeated or relocated `abstract: |` key empties the + abstract slice, so every abstract tie fails instead of reading other + text.""" + prose = (self.repo / "paper/index.qmd").read_text() + span = PAPER_TEXT.block_scalar_span(prose, "abstract") + self.assertIsNotNone(span, "fixture has no abstract block") + block = prose[span[0] : span[1]] + key = "abstract: |\n" + cases = ( + ("folded abstract", ((key, "abstract: >\n"),)), + ("repeated abstract key", ((block, block + key + block),)), + ( + "abstract key only in the body", + ((key, "summary: |\n"), ("\n# Introduction\n", f"\n{key}{block}\n# Introduction\n")), + ), ) for name, edits in cases: with self.subTest(case=name): @@ -1302,9 +1402,8 @@ def test_abstract_slice_fails_closed_without_unique_markers(self): self.assertNotEqual(result.returncode, 0) failures = failure_lines(result.stdout) for label in ( - "paper section heading not unique: 'abstract: |'", - "abstract evaluation amended MAE 1.17", - "abstract current projection cpi_u 2.29", + "paper front matter lacks exactly one abstract: | block", + *ABSTRACT_TIE_LABELS, "in-kind prose: abstract in-kind lift range", ): self.assertIn(label, failures) diff --git a/tests/test_paper_text.py b/tests/test_paper_text.py new file mode 100644 index 0000000..ead9522 --- /dev/null +++ b/tests/test_paper_text.py @@ -0,0 +1,398 @@ +"""Tests for scripts/paper_text.py, the helpers through which the paper guard +reads the front matter's abstract and the rates the abstract states. + +The unit cases pin each rule of the block parser and the slot reader. The +seeded property tests build random front matter and rate statements, then check +that the abstract's words round-trip, that text outside the block never changes +what the guard reads, and that each value reads as its whole numeric token. +""" + +import importlib.util +import random +import textwrap +import unittest +from pathlib import Path + +REPO = Path(__file__).resolve().parents[1] + + +def load_paper_text(): + """The helpers the guard imports, loaded from this checkout.""" + spec = importlib.util.spec_from_file_location( + "paper_text_under_test", REPO / "scripts/paper_text.py" + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +PAPER_TEXT = load_paper_text() +block_scalar_span = PAPER_TEXT.block_scalar_span +stated_values = PAPER_TEXT.stated_values + +MINUS = "\u2212" +CPI_U = "{} percent with the Consumer Price Index" +ABSTRACT_LINES = " First line of the abstract,\n then its second line.\n" +PAPER = ( + "---\n" + 'title: "Thresholds"\n' + "format:\n" + " pdf:\n" + " include-in-header:\n" + " text: |\n" + " \\usepackage{needspace}\n" + "abstract: |\n" + f"{ABSTRACT_LINES}" + "---\n" + "# Introduction\n" + "\n" + "Body text.\n" +) + + +def block_lines(text, key="abstract"): + """The raw lines of the block, or None when the parser finds none.""" + span = block_scalar_span(text, key) + return None if span is None else text[span[0] : span[1]] + + +def block_text(text, key="abstract"): + """The block's words joined by single spaces, as the guard reads them.""" + lines = block_lines(text, key) + return None if lines is None else " ".join(lines.split()) + + +class BlockScalarSpanTests(unittest.TestCase): + def test_span_covers_exactly_the_indented_lines(self): + self.assertEqual(block_lines(PAPER), ABSTRACT_LINES) + + def test_trailing_spaces_after_the_indicator_are_allowed(self): + text = PAPER.replace("abstract: |\n", "abstract: | \n") + self.assertEqual(block_lines(text), ABSTRACT_LINES) + + def test_restyled_key_lines_fail_closed(self): + for line in ( + "abstract: >", + "abstract: >-", + "abstract: |-", + "abstract: |+", + "abstract: |2", + "abstract:", + "abstract:|", + "abstract : |", + "abstract: |", + ): + with self.subTest(line=line): + text = PAPER.replace("abstract: |\n", f"{line}\n") + self.assertIsNone(block_scalar_span(text, "abstract")) + + def test_a_repeated_top_level_key_fails_closed(self): + for repeat in ( + f"abstract: |\n{ABSTRACT_LINES}", + "abstract: A second abstract\n", + "abstract:\n - a list\n", + ): + before = PAPER.replace('title: "Thresholds"\n', f'title: "Thresholds"\n{repeat}') + after = PAPER.replace("---\n# Introduction", f"{repeat}---\n# Introduction") + for where, text in (("before", before), ("after", after)): + with self.subTest(repeat=repeat, where=where): + self.assertIsNone(block_scalar_span(text, "abstract")) + + def test_keys_that_share_the_prefix_do_not_count(self): + extra = 'abstract-title: "Summary"\nabstracts: |\n Another block.\n' + text = PAPER.replace('title: "Thresholds"\n', f'title: "Thresholds"\n{extra}') + self.assertEqual(block_lines(text), ABSTRACT_LINES) + + def test_indented_keys_belong_to_other_mappings(self): + nested = PAPER.replace( + " text: |\n", " abstract: |\n Nested copy.\n text: |\n" + ) + self.assertEqual(block_lines(nested), ABSTRACT_LINES) + self.assertIsNone(block_scalar_span(PAPER, "text")) + only_nested = nested.replace(f"abstract: |\n{ABSTRACT_LINES}", "") + self.assertIsNone(block_scalar_span(only_nested, "abstract")) + + def test_a_key_in_the_body_lies_outside_the_front_matter(self): + text = PAPER.replace("abstract: |\n", "summary: |\n").replace( + "Body text.\n", f"Body text.\n\nabstract: |\n{ABSTRACT_LINES}" + ) + self.assertIsNone(block_scalar_span(text, "abstract")) + + def test_the_block_ends_at_the_next_top_level_field(self): + text = PAPER.replace( + "---\n# Introduction", 'note: "2.29 percent"\n---\n# Introduction' + ) + self.assertEqual(block_lines(text), ABSTRACT_LINES) + + def test_blank_lines_stay_inside_the_block(self): + text = PAPER.replace(ABSTRACT_LINES, f"{ABSTRACT_LINES}\n A second paragraph.\n") + self.assertEqual( + block_text(text), + "First line of the abstract, then its second line. A second paragraph.", + ) + + def test_dots_also_close_the_front_matter(self): + text = PAPER.replace("---\n# Introduction", "...\n# Introduction") + self.assertEqual(block_lines(text), ABSTRACT_LINES) + + def test_missing_or_unterminated_front_matter_fails_closed(self): + cases = { + "unterminated": PAPER.replace("---\n# Introduction", "# Introduction"), + "no opening delimiter": PAPER.removeprefix("---\n"), + "text before the delimiter": "Preface.\n" + PAPER, + "empty": "", + } + for name, text in cases.items(): + with self.subTest(case=name): + self.assertIsNone(block_scalar_span(text, "abstract")) + + def test_an_empty_block_fails_closed(self): + for filler in ("", "\n", " \n\n"): + with self.subTest(filler=filler): + text = PAPER.replace(ABSTRACT_LINES, filler) + self.assertIsNone(block_scalar_span(text, "abstract")) + + def test_windows_line_endings_keep_the_span_exact(self): + text = PAPER.replace("\n", "\r\n") + self.assertEqual(block_lines(text), ABSTRACT_LINES.replace("\n", "\r\n")) + + def test_the_key_matches_literally(self): + text = PAPER.replace("abstract: |\n", "abXtract: |\n") + self.assertIsNone(block_scalar_span(text, "ab.tract")) + dotted = PAPER.replace("abstract: |\n", "ab.tract: |\n") + self.assertEqual(block_lines(dotted, "ab.tract"), ABSTRACT_LINES) + lookalike = dotted.replace( + 'title: "Thresholds"\n', 'title: "Thresholds"\nabXtract: Other\n' + ) + self.assertEqual(block_lines(lookalike, "ab.tract"), ABSTRACT_LINES) + + +class StatedValuesTests(unittest.TestCase): + def test_reads_the_value_in_the_slot(self): + text = "we obtain 2.29 percent with the Consumer Price Index for All Urban" + self.assertEqual(stated_values(CPI_U, text), ["2.29"]) + + def test_keeps_a_sign(self): + for sign in (MINUS, "-"): + with self.subTest(sign=sign): + text = f"of {sign}2.29 percent with the Consumer Price Index" + self.assertEqual(stated_values(CPI_U, text), [f"{sign}2.29"]) + + def test_reads_whole_numeric_tokens(self): + cases = { + "12.29": ["12.29"], + "2.290": ["2.290"], + "1,234.5": ["1,234.5"], + "1.2.29": [], + "2.29.1": [], + } + for number, expected in cases.items(): + with self.subTest(number=number): + text = f"of {number} percent with the Consumer Price Index" + self.assertEqual(stated_values(CPI_U, text), expected) + + def test_lists_every_statement_in_order(self): + text = ( + "2.29 percent with the Consumer Price Index and, in a later " + "sentence, 2.30 percent with the Consumer Price Index" + ) + self.assertEqual(stated_values(CPI_U, text), ["2.29", "2.30"]) + + def test_any_whitespace_run_meets_a_template_space(self): + text = "2.29\n percent with the\tConsumer\n Price Index" + self.assertEqual(stated_values(CPI_U, text), ["2.29"]) + + def test_a_template_space_needs_whitespace(self): + text = "2.29percent with the Consumer Price Index" + self.assertEqual(stated_values(CPI_U, text), []) + + def test_the_slot_can_take_a_word(self): + text = "We compare four projection rules." + self.assertEqual( + stated_values("compare {} projection rules", text, r"\S+"), ["four"] + ) + + def test_a_template_needs_exactly_one_slot(self): + text = "2.29 and 1.65 percent with the Consumer Price Index" + for template in ("percent with the Consumer Price Index", "{} and {} percent"): + with self.subTest(template=template): + self.assertEqual(stated_values(template, text), []) + + def test_sentence_punctuation_ends_a_number(self): + text = "an error of 1.17. Later, an error of 0.75, and then" + self.assertEqual(stated_values("error of {}", text), ["1.17", "0.75"]) + + +VOCABULARY = ( + "thresholds", + "tenure", + "renters", + "spending", + "survey", + "window", + "basic", + "needs", + "records", + "growth", +) +DECOY_STATEMENT = "We obtain 9.99 percent with the Consumer Price Index." + + +def random_number(rng): + """A signed decimal, integer or thousands-separated number.""" + kind = rng.randrange(3) + if kind == 0: + digits = f"{rng.uniform(0, 100):.{rng.randint(1, 4)}f}" + elif kind == 1: + digits = str(rng.randint(0, 999)) + else: + digits = f"{rng.randint(1_000, 9_999_999):,}" + return rng.choice(("", "", MINUS, "-")) + digits + + +def random_space(rng): + return rng.choice((" ", " ", "\n", "\n ", "\t", " \n ")) + + +def random_document(rng, statement=""): + """Front matter holding one abstract literal block among other fields, + and a body, as lists the property tests can edit, plus the abstract's + words. `statement` lands at a random place in the abstract.""" + filler = VOCABULARY + ("2.29", "percent", f"{MINUS}0.51", "1,234") + paragraphs = [ + [rng.choice(filler) for _ in range(rng.randint(1, 30))] + for _ in range(rng.randint(1, 3)) + ] + if statement: + words = rng.choice(paragraphs) + at = rng.randint(0, len(words)) + words[at:at] = statement.split() + indent = " " * rng.choice((1, 2, 4)) + width = rng.randint(8, 72) + block = [] + for number, words in enumerate(paragraphs): + if number: + block.append("") + block.extend( + indent + line + for line in textwrap.wrap( + " ".join(words), width, break_long_words=False, break_on_hyphens=False + ) + ) + fields = [ + ['title: "Thresholds"'], + [ + "format:", + " pdf:", + " include-in-header:", + " text: |", + " abstract: |", + " Nested copy 9.99.", + ], + ['abstract-title: "Summary"'], + ["keywords:", " - poverty", " - thresholds"], + ] + rng.shuffle(fields) + fields.insert(rng.randint(0, len(fields)), ["abstract: |", *block]) + body = ["# Introduction", "", "abstract: |", " Body copy 9.99.", ""] + abstract = " ".join(word for words in paragraphs for word in words) + return fields, rng.choice(("---", "...")), body, abstract + + +def render(fields, terminator, body): + lines = ["---", *(line for field in fields for line in field), terminator, *body] + return "\n".join(lines) + "\n" + + +def add_decoys(rng, fields, body): + """Restate a rate as another top-level field and in the body.""" + decoy_fields = ( + [f'note: "{DECOY_STATEMENT}"'], + ["notes:", f" - {DECOY_STATEMENT}"], + ["abstract-note: |", f" {DECOY_STATEMENT}"], + ) + fields.insert(rng.randint(0, len(fields)), list(rng.choice(decoy_fields))) + at = rng.randint(0, len(body)) + body[at:at] = [DECOY_STATEMENT, "abstract: |", f" {DECOY_STATEMENT}"] + + +class PropertyTests(unittest.TestCase): + ROUNDS = 300 + + def test_the_abstract_round_trips(self): + rng = random.Random(20261107) + for _ in range(self.ROUNDS): + fields, terminator, body, abstract = random_document(rng) + text = render(fields, terminator, body) + self.assertEqual(block_text(text), abstract, text) + + def test_text_outside_the_block_never_changes_what_the_guard_reads(self): + rng = random.Random(20261108) + for _ in range(self.ROUNDS): + fields, terminator, body, abstract = random_document(rng) + add_decoys(rng, fields, body) + text = render(fields, terminator, body) + self.assertEqual(block_text(text), abstract, text) + + def test_a_second_or_restyled_key_fails_closed(self): + rng = random.Random(20261109) + for _ in range(self.ROUNDS): + fields, terminator, body, _ = random_document(rng) + if rng.random() < 0.5: + second = rng.choice((["abstract: |", " Copy."], ["abstract: Copy."])) + fields.insert(rng.randint(0, len(fields)), list(second)) + else: + index = next( + i for i, field in enumerate(fields) if field[0] == "abstract: |" + ) + restyled = rng.choice( + ("abstract: >", "abstract: |-", "abstract:", "abstract : |") + ) + fields[index] = [restyled, *fields[index][1:]] + text = render(fields, terminator, body) + self.assertIsNone(block_scalar_span(text, "abstract"), text) + + def test_a_rate_in_the_abstract_reaches_its_tie_however_it_wraps(self): + """A rate the abstract states reads back exactly, whatever the + wrapping and indentation, and a copy outside the block never adds to + what the tie sees.""" + rng = random.Random(20261110) + for _ in range(self.ROUNDS): + value = random_number(rng) + statement = f"we obtain {value} percent with the Consumer Price Index" + fields, terminator, body, _ = random_document(rng, statement) + add_decoys(rng, fields, body) + text = render(fields, terminator, body) + self.assertEqual(stated_values(CPI_U, block_text(text)), [value], text) + + def test_values_round_trip_through_any_template_spacing(self): + rng = random.Random(20261111) + for _ in range(self.ROUNDS): + words = rng.sample(VOCABULARY, rng.randint(1, 4)) + cut = rng.randint(0, len(words)) + template = " ".join([*words[:cut], "{}", *words[cut:]]) + values = [random_number(rng) for _ in range(rng.randint(1, 3))] + text = " ; ".join( + random_space(rng).join([*words[:cut], value, *words[cut:]]) + for value in values + ) + self.assertEqual(stated_values(template, text), values, text) + + def test_a_longer_number_never_reads_as_the_value_it_contains(self): + rng = random.Random(20261112) + for _ in range(self.ROUNDS): + value = f"{rng.uniform(0, 10):.2f}" + digit = str(rng.randrange(10)) + for glued, expected in ( + (digit + value, [digit + value]), + (value + digit, [value + digit]), + (f"{digit},{value}", [f"{digit},{value}"]), + (f"{value}.{digit}", []), + (f"{digit}.{value}", []), + ): + text = f"of {glued} percent with the Consumer Price Index" + self.assertEqual(stated_values(CPI_U, text), expected, glued) + + +if __name__ == "__main__": + unittest.main() From ad209c96699fe4b0d12f246ea06fa37bc5758a99 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 7 Nov 2026 17:03:37 -0500 Subject: [PATCH 3/5] Read the abstract as pandoc reads it block_scalar_span now follows pandoc's reading of the front matter: - a comment indented less than the block ends the block; - the block takes its indentation from its first line with text; - a quoted key or an explicit "? abstract" also names the key; - a line after the front matter that sets the key, at its start or in flow style, fails closed, since pandoc lets a later metadata block, even inside a list item or block quote, replace the abstract; - a tab in a block line's indentation fails closed, since Quarto rejects it, and so does a lone carriage return in the front matter, which pandoc drops; - lines end only at line feeds, and front matter needs text on the line after its opening dashes. The guard's label for a missing or ambiguous block becomes "paper must set abstract only in one abstract: | block of its front matter". A new guard test shows that a comment after the block cannot satisfy a tie, and the fails-closed test adds a quoted repeated key and a body metadata block. tests/test_paper_text.py grows to 40 tests, among them a differential test that compares the parser with pandoc wherever pandoc is installed. Co-Authored-By: Claude Opus 5.5 --- scripts/check_paper.py | 25 +-- scripts/paper_text.py | 78 ++++++-- tests/test_paper_guard.py | 49 ++++- tests/test_paper_text.py | 396 +++++++++++++++++++++++++++++++++++--- 4 files changed, 486 insertions(+), 62 deletions(-) diff --git a/scripts/check_paper.py b/scripts/check_paper.py index 75eba7a..c6062f4 100644 --- a/scripts/check_paper.py +++ b/scripts/check_paper.py @@ -29,10 +29,11 @@ occurring elsewhere in the paper cannot satisfy them and line rewrapping cannot split a multi-word literal. The guard reads each error rate the abstract quotes from the front matter's ``abstract: |`` - block alone, beside the rule it scores, and every rate the abstract - gives in that rule's wording must equal the registered value. A copy - in other metadata, in the body or inside the abstract cannot rescue a - drifted figure, and a swap between rules fails; + block alone, as pandoc reads it, beside the rule it scores, and every + rate the abstract gives in that rule's wording must equal the + registered value. A copy in other metadata, in a comment, in the body + or inside the abstract cannot rescue a drifted figure, and a swap + between rules fails; (e) the README miscounts the in-kind receipts or omits a calculator commit that their provenance records. @@ -172,12 +173,13 @@ def section(start: str, end: str | None = None, text: str = QMD) -> str: def front_matter_block(key: str, text: str = QMD) -> str: """Whitespace-normalized text of the front matter's one ``key: |`` - literal block. Pins read from it see neither other metadata fields nor - body text. A missing, repeated or restyled key yields an empty slice, so - every pin inside it fails instead of reading other text.""" + literal block, read as pandoc reads it. Pins that read it see no other + metadata field, comment or body text. When ``block_scalar_span`` finds + no such block, or finds another line that sets the key, the slice is + empty, so every pin inside it fails instead of reading other text.""" span = block_scalar_span(text, key) if span is None: - failures.append(f"paper front matter lacks exactly one {key}: | block") + failures.append(f"paper must set {key} only in one {key}: | block of its front matter") return "" return " ".join(text[span[0] : span[1]].split()) @@ -743,9 +745,10 @@ def mae(rule: str) -> float: ) # Figures in the abstract also appear in the body or its tables, so their -# abstract ties read only the indented lines under the front matter's -# `abstract: |` key. Other metadata fields and body text fall outside that -# block, so a copy there cannot rescue a drifted figure. Each tie collects +# abstract ties read only the front matter's `abstract: |` block, as pandoc +# reads it. Other metadata fields, comments and body text fall outside that +# block, so a copy there cannot rescue a drifted figure, and a metadata block +# in the body that sets the abstract fails every tie. Each tie collects # every rate the abstract gives in its rule's wording and requires them all to # equal the registered value, so a swap between rules fails, and so does a # changed rate that leaves a correct copy elsewhere in the abstract. diff --git a/scripts/paper_text.py b/scripts/paper_text.py index bcd3980..2c38c0a 100644 --- a/scripts/paper_text.py +++ b/scripts/paper_text.py @@ -2,6 +2,8 @@ The guard reads the abstract through ``block_scalar_span``, and the mutation tests place their edits with the same function, so both work on the same text. +Wherever pandoc is installed, ``tests/test_paper_text.py`` checks that the +parser reads the words pandoc renders as the abstract, or fails closed. """ from __future__ import annotations @@ -10,38 +12,82 @@ FRONT_MATTER_ENDS = ("---", "...") SIGNED_NUMBER = r"[−-]?\d[\d,]*(?:\.\d+)?" +LINE = re.compile(r"[^\n]*\n|[^\n]+\Z") + + +def _bare(line: str) -> str: + """A line without its ending and without trailing spaces or tabs.""" + return line.removesuffix("\n").removesuffix("\r").rstrip(" \t") + + +def _key_matchers(key: str): + """Two matchers for a line that YAML could read as setting ``key``. The + first finds the key, plain or quoted, at the line's start before a colon, + or in an explicit ``? key`` line. The second also finds it in flow style, + after ``{`` or ``,``, as in ``{key: value}``.""" + name = re.escape(key) + spelled = rf"""(?:{name}|"{name}"|'{name}')""" + sets = rf"(?:\?[ \t]+{spelled}[ \t]*(?::|\r?$)|{spelled}[ \t]*:)" + return re.compile(sets).match, re.compile(rf"(?:^|[{{,])[ \t]*{sets}").search def block_scalar_span(text: str, key: str) -> tuple[int, int] | None: """Character span of the literal block under the front matter's one - top-level ``key: |`` line: the indented and blank lines that follow it, up - to the next unindented line. + top-level ``key: |`` line, read as pandoc reads it, or None. - Returns None when the text opens without front matter, the front matter - never closes, the key appears other than exactly once as ``key: |``, or - the block holds no text. Indented keys belong to other mappings and do not - count, and a ``key: |`` line in the body lies outside the front matter. + Lines end at line feeds. The front matter opens with a ``---`` line and + text on the next line, and closes at the next ``---`` or ``...`` line. + The block takes its indentation from its first line with text and ends at + the next line with less, such as another field or a trailing comment. + Lines of whitespace alone stay inside it. Indented keys belong to other + mappings and do not count. + + It returns None, so every tie that reads the block fails, when + + - the text lacks such front matter, or the front matter never closes; + - the front matter names the key other than once, or names it on a line + other than ``key: |``. A quoted ``"key"`` or an explicit ``? key`` also + names it; + - any line after the front matter names the key, at the line's start or + in flow style such as ``{key: value}``, since pandoc lets a later + metadata block, even inside a list item or block quote, replace the + front matter's value; + - a tab falls in the indentation of a line in the block, since Quarto + rejects tab indentation that pandoc accepts, or a carriage return + stands alone in the front matter, since pandoc drops it; + - the block holds no text. """ - lines = text.splitlines(keepends=True) - if not lines or lines[0].rstrip() != "---": + lines = LINE.findall(text) + if len(lines) < 2 or _bare(lines[0]) != "---" or not _bare(lines[1]): return None header = [] offset = len(lines[0]) - for line in lines[1:]: - if line.rstrip() in FRONT_MATTER_ENDS: + for end, line in enumerate(lines[1:], 1): + if _bare(line) in FRONT_MATTER_ENDS: break header.append((offset, line)) offset += len(line) else: return None - key_line = re.compile(rf"{re.escape(key)}\s*:") - starts = [index for index, (_, line) in enumerate(header) if key_line.match(line)] - if len(starts) != 1 or header[starts[0]][1].rstrip() != f"{key}: |": + if any("\r" in line.removesuffix("\n").removesuffix("\r") for line in lines[: end + 1]): + return None + key_at_start, key_anywhere = _key_matchers(key) + starts = [index for index, (_, line) in enumerate(header) if key_at_start(line)] + if len(starts) != 1 or _bare(header[starts[0]][1]) != f"{key}: |": + return None + if any(key_anywhere(line.replace("\r", "").lstrip(" \t>")) for line in lines[end + 1 :]): return None first = last = starts[0] + 1 - while last < len(header) and ( - not header[last][1].strip() or header[last][1][0] in " \t" - ): + indent = None + while last < len(header): + line = header[last][1] + if line.strip(): + leading = line[: len(line) - len(line.lstrip(" \t"))] + if "\t" in leading: + return None + indent = len(leading) if indent is None else indent + if len(leading) < max(indent, 1): + break last += 1 block = header[first:last] if not "".join(line for _, line in block).strip(): diff --git a/tests/test_paper_guard.py b/tests/test_paper_guard.py index aa0e533..9aab2e4 100644 --- a/tests/test_paper_guard.py +++ b/tests/test_paper_guard.py @@ -1377,10 +1377,44 @@ def test_abstract_ties_ignore_the_introduction_heading(self): ) self.install_article() - def test_abstract_block_fails_closed_without_one_literal_key(self): - """A folded, repeated or relocated `abstract: |` key empties the - abstract slice, so every abstract tie fails instead of reading other - text.""" + def test_abstract_ties_skip_a_comment_after_the_block(self): + """YAML ends the abstract's block at a line indented less than the + block, a comment included, so a comment that keeps wording the + abstract dropped cannot satisfy that wording's tie.""" + cases = ( + ( + "CPI-U rule renamed", + r"with(\s+)the(\s+)Consumer(\s+)Price(\s+)Index", + r"with\1the\2CPI-U\3price\4index", + "2.29 percent with the Consumer Price Index", + ["abstract current projection cpi_u 2.29"], + ), + ( + "stage-change bound changed", + r"less(\s+)than(\s+)one(\s+)point", + r"less\1than\2half\3a point", + "growth by less than one point per tenure", + ["in-kind prose: abstract stage changes under one point per tenure"], + ), + ) + for name, pattern, replacement, comment, labels in cases: + with self.subTest(case=name): + self.install_article() + self.mutate_article("abstract", pattern, replacement) + path = self.repo / "paper/index.qmd" + prose = path.read_text() + span = PAPER_TEXT.block_scalar_span(prose, "abstract") + self.assertIsNotNone(span, "fixture has no abstract block") + path.write_text(f"{prose[: span[1]]} # {comment}\n{prose[span[1] :]}") + result = self.run_guard() + self.assertEqual(sorted(failure_lines(result.stdout)), sorted(labels)) + self.install_article() + + def test_abstract_block_fails_closed_unless_one_key_sets_the_abstract(self): + """A folded, repeated or relocated `abstract: |` key, or a metadata + block in the body that sets the abstract, empties the abstract slice, + so every abstract tie fails instead of reading text that the paper + does not render as its abstract.""" prose = (self.repo / "paper/index.qmd").read_text() span = PAPER_TEXT.block_scalar_span(prose, "abstract") self.assertIsNotNone(span, "fixture has no abstract block") @@ -1393,6 +1427,11 @@ def test_abstract_block_fails_closed_without_one_literal_key(self): "abstract key only in the body", ((key, "summary: |\n"), ("\n# Introduction\n", f"\n{key}{block}\n# Introduction\n")), ), + ("quoted abstract key repeated", ((block, f'{block}"abstract": |\n{block}'),)), + ( + "body metadata block sets the abstract", + ((DISCUSSION_HEADING, f"---\n{key}{block}---\n\n{DISCUSSION_HEADING}"),), + ), ) for name, edits in cases: with self.subTest(case=name): @@ -1402,7 +1441,7 @@ def test_abstract_block_fails_closed_without_one_literal_key(self): self.assertNotEqual(result.returncode, 0) failures = failure_lines(result.stdout) for label in ( - "paper front matter lacks exactly one abstract: | block", + "paper must set abstract only in one abstract: | block of its front matter", *ABSTRACT_TIE_LABELS, "in-kind prose: abstract in-kind lift range", ): diff --git a/tests/test_paper_text.py b/tests/test_paper_text.py index ead9522..42bf98d 100644 --- a/tests/test_paper_text.py +++ b/tests/test_paper_text.py @@ -4,11 +4,18 @@ The unit cases pin each rule of the block parser and the slot reader. The seeded property tests build random front matter and rate statements, then check that the abstract's words round-trip, that text outside the block never changes -what the guard reads, and that each value reads as its whole numeric token. +what the guard reads, that a second key or a key after the front matter makes +the parser fail closed, and that each value reads as its whole numeric token. +Wherever pandoc is installed, a differential test checks that the parser reads +the words pandoc renders as the abstract, or fails closed. CI installs no +pandoc, so it skips that test. """ import importlib.util import random +import shutil +import subprocess +import tempfile import textwrap import unittest from pathlib import Path @@ -33,6 +40,8 @@ def load_paper_text(): MINUS = "\u2212" CPI_U = "{} percent with the Consumer Price Index" ABSTRACT_LINES = " First line of the abstract,\n then its second line.\n" +FOUR_SPACE_LINES = " First line of the abstract,\n then its second line.\n" +CLOSE = "---\n# Introduction" PAPER = ( "---\n" 'title: "Thresholds"\n' @@ -62,6 +71,21 @@ def block_text(text, key="abstract"): return None if lines is None else " ".join(lines.split()) +def with_lines(lines): + """PAPER with `lines` in place of the abstract's lines.""" + return PAPER.replace(ABSTRACT_LINES, lines) + + +def with_body(insert): + """PAPER with `insert` after its body text.""" + return PAPER.replace("Body text.\n", f"Body text.\n\n{insert}\nMore text.\n") + + +NESTED_FLOW_KEY = PAPER.replace( + 'title: "Thresholds"\n', 'title: "Thresholds"\nlinks: {abstract: "Nested 9.99."}\n' +) + + class BlockScalarSpanTests(unittest.TestCase): def test_span_covers_exactly_the_indented_lines(self): self.assertEqual(block_lines(PAPER), ABSTRACT_LINES) @@ -77,10 +101,14 @@ def test_restyled_key_lines_fail_closed(self): "abstract: |-", "abstract: |+", "abstract: |2", + "abstract: | # a comment", "abstract:", "abstract:|", "abstract : |", "abstract: |", + "abstract:\t|", + '"abstract": |', + "'abstract': |", ): with self.subTest(line=line): text = PAPER.replace("abstract: |\n", f"{line}\n") @@ -91,55 +119,148 @@ def test_a_repeated_top_level_key_fails_closed(self): f"abstract: |\n{ABSTRACT_LINES}", "abstract: A second abstract\n", "abstract:\n - a list\n", + '"abstract": |\n A quoted copy.\n', + "'abstract': A quoted copy.\n", + "? abstract\n: |\n An explicit copy.\n", ): before = PAPER.replace('title: "Thresholds"\n', f'title: "Thresholds"\n{repeat}') - after = PAPER.replace("---\n# Introduction", f"{repeat}---\n# Introduction") + after = PAPER.replace(CLOSE, f"{repeat}{CLOSE}") for where, text in (("before", before), ("after", after)): with self.subTest(repeat=repeat, where=where): self.assertIsNone(block_scalar_span(text, "abstract")) def test_keys_that_share_the_prefix_do_not_count(self): - extra = 'abstract-title: "Summary"\nabstracts: |\n Another block.\n' + extra = ( + 'abstract-title: "Summary"\nabstracts: |\n Another block.\n' + '"abstract-note": Another field.\n' + ) text = PAPER.replace('title: "Thresholds"\n', f'title: "Thresholds"\n{extra}') self.assertEqual(block_lines(text), ABSTRACT_LINES) - def test_indented_keys_belong_to_other_mappings(self): + def test_nested_keys_belong_to_other_mappings(self): nested = PAPER.replace( " text: |\n", " abstract: |\n Nested copy.\n text: |\n" ) self.assertEqual(block_lines(nested), ABSTRACT_LINES) + self.assertEqual(block_lines(NESTED_FLOW_KEY), ABSTRACT_LINES) self.assertIsNone(block_scalar_span(PAPER, "text")) only_nested = nested.replace(f"abstract: |\n{ABSTRACT_LINES}", "") self.assertIsNone(block_scalar_span(only_nested, "abstract")) - def test_a_key_in_the_body_lies_outside_the_front_matter(self): - text = PAPER.replace("abstract: |\n", "summary: |\n").replace( + def test_a_key_after_the_front_matter_fails_closed(self): + """pandoc lets a metadata block anywhere in the body, even inside a + list item or block quote, replace the front matter's abstract, so any + line after the front matter that names the key, at its start or in + flow style, empties the slice.""" + for name, insert in { + "metadata block": "---\nabstract: |\n Body copy.\n---\n", + "metadata block closed by dots": "---\nabstract: |\n Body copy.\n...\n", + "list item": "- item\n\n ---\n abstract: |\n Body copy.\n ---\n", + "block quote": "> ---\n> abstract: |\n> Body copy.\n> ---\n", + "quoted key": '---\n"abstract": |\n Body copy.\n---\n', + "explicit key": "---\n? abstract\n: |\n Body copy.\n---\n", + "bare key line": "abstract: Body copy.\n", + "flow mapping": "---\n{abstract: Body copy.}\n---\n", + "quoted flow key after a comma": '---\n{title: X, "abstract": Body copy.}\n---\n', + "explicit flow key": "---\n{? abstract : Body copy.}\n---\n", + "flow key on a line led by a comma": "---\n{title: X\n, abstract: Body copy.}\n---\n", + "flow mapping in a block quote": "> ---\n> {abstract: Body copy.}\n> ---\n", + }.items(): + with self.subTest(case=name): + self.assertIsNone(block_scalar_span(with_body(insert), "abstract")) + right_after = PAPER.replace(CLOSE, f"---\n---\nabstract: |\n Body copy.\n{CLOSE}") + self.assertIsNone(block_scalar_span(right_after, "abstract")) + moved = PAPER.replace("abstract: |\n", "summary: |\n").replace( "Body text.\n", f"Body text.\n\nabstract: |\n{ABSTRACT_LINES}" ) - self.assertIsNone(block_scalar_span(text, "abstract")) + self.assertIsNone(block_scalar_span(moved, "abstract")) + + def test_prose_that_mentions_the_abstract_does_not_name_the_key(self): + for line in ( + "The abstract: 9.99 percent.", + "Abstract: 9.99 percent.", + "abstracts: 9.99 percent", + "abstract-note: 9.99 percent", + "abstract", + "? abstracts", + "The title, abstract and keywords.", + "{abstracts: 9.99}", + ): + with self.subTest(line=line): + self.assertEqual(block_lines(with_body(line)), ABSTRACT_LINES) def test_the_block_ends_at_the_next_top_level_field(self): - text = PAPER.replace( - "---\n# Introduction", 'note: "2.29 percent"\n---\n# Introduction' - ) + text = PAPER.replace(CLOSE, f'note: "2.29 percent"\n{CLOSE}') self.assertEqual(block_lines(text), ABSTRACT_LINES) + def test_the_first_line_with_text_sets_the_indentation(self): + deeper = " First line,\n a deeper line,\n and a third.\n" + self.assertEqual(block_lines(with_lines(deeper)), deeper) + shallower = " Four spaces,\n then two.\n" + self.assertEqual(block_lines(with_lines(shallower)), " Four spaces,\n") + leading_blank = "\n" + ABSTRACT_LINES + self.assertEqual(block_lines(with_lines(leading_blank)), leading_blank) + + def test_a_trailing_comment_ends_the_block(self): + """YAML ends the block at a line indented less than its first line, + a comment included, so the comment's text stays out of the slice.""" + comment = "# We obtain 9.99 percent with the Consumer Price Index.\n" + for lines, indent in ( + (ABSTRACT_LINES, 0), + (ABSTRACT_LINES, 1), + (ABSTRACT_LINES + "\n", 1), + (ABSTRACT_LINES + "\t\n", 1), + (FOUR_SPACE_LINES, 3), + ): + with self.subTest(lines=lines, indent=indent): + text = with_lines(lines + " " * indent + comment) + self.assertEqual(block_lines(text), lines) + + def test_a_hash_line_indented_like_the_block_is_text(self): + """A literal block keeps a "#" line at its own indentation as text, + even after a line of whitespace.""" + for lines in ( + ABSTRACT_LINES + " # 9.99 percent\n", + ABSTRACT_LINES + "\t\n # 9.99 percent\n", + ): + with self.subTest(lines=lines): + self.assertEqual(block_lines(with_lines(lines)), lines) + def test_blank_lines_stay_inside_the_block(self): - text = PAPER.replace(ABSTRACT_LINES, f"{ABSTRACT_LINES}\n A second paragraph.\n") - self.assertEqual( - block_text(text), - "First line of the abstract, then its second line. A second paragraph.", - ) + for blank in ("\n", " \n", "\t\n", " \t \n"): + with self.subTest(blank=blank): + text = with_lines(f"{ABSTRACT_LINES}{blank} A second paragraph.\n") + self.assertEqual( + block_text(text), + "First line of the abstract, then its second line. A second paragraph.", + ) + + def test_a_tab_in_a_lines_indentation_fails_closed(self): + for lines in ( + ABSTRACT_LINES + "\tTabbed text.\n", + ABSTRACT_LINES + " \tTabbed text.\n", + ABSTRACT_LINES + " \tTabbed text.\n", + "\tTabbed first line.\n" + ABSTRACT_LINES, + ): + with self.subTest(lines=lines): + self.assertIsNone(block_scalar_span(with_lines(lines), "abstract")) + inner = " First line,\twith a tab inside it.\n" + self.assertEqual(block_lines(with_lines(inner)), inner) def test_dots_also_close_the_front_matter(self): - text = PAPER.replace("---\n# Introduction", "...\n# Introduction") + text = PAPER.replace(CLOSE, "...\n# Introduction") self.assertEqual(block_lines(text), ABSTRACT_LINES) def test_missing_or_unterminated_front_matter_fails_closed(self): cases = { - "unterminated": PAPER.replace("---\n# Introduction", "# Introduction"), + "unterminated": PAPER.replace(CLOSE, "# Introduction"), "no opening delimiter": PAPER.removeprefix("---\n"), "text before the delimiter": "Preface.\n" + PAPER, + "blank line after the delimiter": PAPER.replace("---\n", "---\n\n", 1), + "spaces after the delimiter": PAPER.replace("---\n", "---\n \n", 1), + "separator after the closing dashes": PAPER.replace( + CLOSE, "---\u2028\n# Introduction" + ), "empty": "", } for name, text in cases.items(): @@ -156,6 +277,27 @@ def test_windows_line_endings_keep_the_span_exact(self): text = PAPER.replace("\n", "\r\n") self.assertEqual(block_lines(text), ABSTRACT_LINES.replace("\n", "\r\n")) + def test_a_lone_carriage_return_in_the_front_matter_fails_closed(self): + """pandoc drops a carriage return that no line feed follows, so the + parser fails closed rather than guess how the line reads.""" + for name, text in { + "every line": PAPER.replace("\n", "\r"), + "inside the abstract": with_lines(" First line\rof the abstract.\n"), + "inside the title": PAPER.replace('"Thresholds"', '"Thresh\rolds"'), + }.items(): + with self.subTest(case=name): + self.assertIsNone(block_scalar_span(text, "abstract")) + body = PAPER.replace("Body text.\n", "Body\rtext.\n") + self.assertEqual(block_lines(body), ABSTRACT_LINES) + + def test_only_line_feeds_end_lines(self): + """A form feed or Unicode line separator stays inside its line, so it + cannot end the block early.""" + for separator in ("\u2028", "\u2029", "\x85", "\x0c", "\x0b"): + with self.subTest(separator=repr(separator)): + lines = f" First line,{separator}then more.\n Second line.\n" + self.assertEqual(block_lines(with_lines(lines)), lines) + def test_the_key_matches_literally(self): text = PAPER.replace("abstract: |\n", "abXtract: |\n") self.assertIsNone(block_scalar_span(text, "ab.tract")) @@ -257,7 +399,9 @@ def random_space(rng): def random_document(rng, statement=""): """Front matter holding one abstract literal block among other fields, and a body, as lists the property tests can edit, plus the abstract's - words. `statement` lands at a random place in the abstract.""" + words. `statement` lands at a random place in the abstract. Lines after + the block's first may sit deeper, and blank lines between paragraphs may + hold spaces or a tab.""" filler = VOCABULARY + ("2.29", "percent", f"{MINUS}0.51", "1,234") paragraphs = [ [rng.choice(filler) for _ in range(rng.randint(1, 30))] @@ -272,13 +416,12 @@ def random_document(rng, statement=""): block = [] for number, words in enumerate(paragraphs): if number: - block.append("") - block.extend( - indent + line - for line in textwrap.wrap( - " ".join(words), width, break_long_words=False, break_on_hyphens=False - ) - ) + block.append(rng.choice(("", " ", indent, "\t"))) + for line in textwrap.wrap( + " ".join(words), width, break_long_words=False, break_on_hyphens=False + ): + deeper = rng.choice(("", "", "", " ")) if block else "" + block.append(indent + deeper + line) fields = [ ['title: "Thresholds"'], [ @@ -294,7 +437,7 @@ def random_document(rng, statement=""): ] rng.shuffle(fields) fields.insert(rng.randint(0, len(fields)), ["abstract: |", *block]) - body = ["# Introduction", "", "abstract: |", " Body copy 9.99.", ""] + body = ["# Introduction", "", "Body copy 9.99.", ""] abstract = " ".join(word for words in paragraphs for word in words) return fields, rng.choice(("---", "...")), body, abstract @@ -305,15 +448,20 @@ def render(fields, terminator, body): def add_decoys(rng, fields, body): - """Restate a rate as another top-level field and in the body.""" + """Restate a rate in another top-level field, in a comment after the + abstract indented less than its block, and in the body's prose.""" decoy_fields = ( [f'note: "{DECOY_STATEMENT}"'], ["notes:", f" - {DECOY_STATEMENT}"], ["abstract-note: |", f" {DECOY_STATEMENT}"], ) + abstract = next(field for field in fields if field[0] == "abstract: |") + indent = len(abstract[1]) - len(abstract[1].lstrip(" ")) + blank = rng.choice(([], [""], ["\t"])) + abstract.extend([*blank, " " * rng.randrange(indent) + f"# {DECOY_STATEMENT}"]) fields.insert(rng.randint(0, len(fields)), list(rng.choice(decoy_fields))) at = rng.randint(0, len(body)) - body[at:at] = [DECOY_STATEMENT, "abstract: |", f" {DECOY_STATEMENT}"] + body[at:at] = [DECOY_STATEMENT, "", f"The abstract: {DECOY_STATEMENT}"] class PropertyTests(unittest.TestCase): @@ -339,19 +487,61 @@ def test_a_second_or_restyled_key_fails_closed(self): for _ in range(self.ROUNDS): fields, terminator, body, _ = random_document(rng) if rng.random() < 0.5: - second = rng.choice((["abstract: |", " Copy."], ["abstract: Copy."])) + second = rng.choice( + ( + ["abstract: |", " Copy."], + ["abstract: Copy."], + ['"abstract": |', " Copy."], + ["'abstract': Copy."], + ["? abstract", ": Copy."], + ) + ) fields.insert(rng.randint(0, len(fields)), list(second)) else: index = next( i for i, field in enumerate(fields) if field[0] == "abstract: |" ) restyled = rng.choice( - ("abstract: >", "abstract: |-", "abstract:", "abstract : |") + ( + "abstract: >", + "abstract: |-", + "abstract:", + "abstract : |", + '"abstract": |', + "abstract: | # note", + ) ) fields[index] = [restyled, *fields[index][1:]] text = render(fields, terminator, body) self.assertIsNone(block_scalar_span(text, "abstract"), text) + def test_a_key_after_the_front_matter_fails_closed(self): + """However a line after the front matter names the key, and whatever + container or metadata block holds it, the parser fails closed.""" + rng = random.Random(20261113) + spellings = ( + "abstract: |", + "abstract: Copy.", + "abstract : |", + '"abstract": |', + "'abstract': |", + "? abstract", + "{abstract: Copy.}", + '{title: X, "abstract": Copy.}', + "{? abstract : Copy.}", + ) + containers = ("", " ", " ", "> ", "> ", "> > ") + for _ in range(self.ROUNDS): + fields, terminator, body, _ = random_document(rng) + container = rng.choice(containers) + lines = [container + rng.choice(spellings), container + " Body copy 9.99."] + if rng.random() < 0.5: + lines = [container + "---", *lines, container + rng.choice(("---", "..."))] + at = rng.randint(0, len(body)) + body[at:at] = lines + text = render(fields, terminator, body) + self.assertIsNone(block_scalar_span(text, "abstract"), text) + def test_a_rate_in_the_abstract_reaches_its_tie_however_it_wraps(self): """A rate the abstract states reads back exactly, whatever the wrapping and indentation, and a copy outside the block never adds to @@ -394,5 +584,151 @@ def test_a_longer_number_never_reads_as_the_value_it_contains(self): self.assertEqual(stated_values(CPI_U, text), expected, glued) +PANDOC = shutil.which("pandoc") + + +def words(text): + """The words of `text`, less any bare "#" marks, which markdown reads as + a heading's level.""" + return " ".join(token for token in text.split() if token.strip("#")) + + +# Documents on which the parser and pandoc have disagreed, or could: comments +# and tabs near the block, odd line endings, repeated keys, flow-style keys +# and metadata blocks in the body. +EDGE_CASES = { + "fixture": PAPER, + "comment at indent 0 after the block": with_lines(ABSTRACT_LINES + "# 9.99 percent\n"), + "comment at indent 1 after the block": with_lines(ABSTRACT_LINES + " # 9.99 percent\n"), + "comment at indent 3 after a 4-space block": with_lines( + FOUR_SPACE_LINES + " # 9.99 percent\n" + ), + "blank line, then a comment at indent 1": with_lines( + ABSTRACT_LINES + "\n # 9.99 percent\n" + ), + "tab line, then a comment at indent 1": with_lines( + ABSTRACT_LINES + "\t\n # 9.99 percent\n" + ), + "hash line at the block's indentation": with_lines( + ABSTRACT_LINES + " # 9.99 percent\n" + ), + "tab line, then a hash line at the block's indentation": with_lines( + ABSTRACT_LINES + "\t\n # 9.99 percent\n" + ), + "tab-indented line after the block": with_lines(ABSTRACT_LINES + "\tTabbed 9.99.\n"), + "space and tab before a line": with_lines(ABSTRACT_LINES + " \tTabbed 9.99.\n"), + "tab after the block's indentation": with_lines( + " First,\n \tafter a tab 9.99,\n third.\n" + ), + "tab-indented first line": with_lines("\tFirst 9.99.\n second.\n"), + "tab line inside the block": with_lines(" First,\n\t\n second 9.99.\n"), + "line indented less than the first": with_lines(" Four 1.11,\n two 9.99.\n"), + "line indented more than the first": with_lines(" First,\n deeper 9.99,\n third.\n"), + "leading blank line deeper than the text": with_lines(" \n" + ABSTRACT_LINES), + "U+2028 inside a line": with_lines(" First line\u2028then 9.99.\n Second.\n"), + "form feed inside a line": with_lines(" First line\x0cthen 9.99.\n Second.\n"), + "carriage returns alone": PAPER.replace("\n", "\r"), + "carriage return inside a line": with_lines(" First\rsecond 9.99.\n third.\n"), + "Windows line endings": PAPER.replace("\n", "\r\n"), + "blank line after the opening dashes": PAPER.replace("---\n", "---\n\n", 1), + "closing dashes, then U+2028": PAPER.replace(CLOSE, "---\u2028\n# Introduction"), + "closing dashes with trailing spaces": PAPER.replace(CLOSE, "--- \n# Introduction"), + "closing dots": PAPER.replace(CLOSE, "...\n# Introduction"), + "quoted key repeated after": PAPER.replace(CLOSE, f'"abstract": |\n Decoy 9.99.\n{CLOSE}'), + "single-quoted key repeated after": PAPER.replace( + CLOSE, f"'abstract': |\n Decoy 9.99.\n{CLOSE}" + ), + "explicit key repeated after": PAPER.replace( + CLOSE, f"? abstract\n: |\n Decoy 9.99.\n{CLOSE}" + ), + "plain key repeated after": PAPER.replace(CLOSE, f"abstract: |\n Decoy 9.99.\n{CLOSE}"), + "key line with a comment": PAPER.replace("abstract: |\n", "abstract: | # note\n"), + "folded key": PAPER.replace("abstract: |\n", "abstract: >\n"), + "metadata block in the body": with_body("---\nabstract: |\n Body copy 9.99.\n---\n"), + "metadata block in a list item": with_body( + "- item\n\n ---\n abstract: |\n Body copy 9.99.\n ---\n" + ), + "metadata block in a block quote": with_body( + "> ---\n> abstract: |\n> Body copy 9.99.\n> ---\n" + ), + "metadata block right after the front matter": PAPER.replace( + CLOSE, f"---\n---\nabstract: |\n Body copy 9.99.\n{CLOSE}" + ), + "metadata block that sets another key": with_body("---\nnote: |\n Body copy 9.99.\n---\n"), + "flow mapping in the body": with_body("---\n{abstract: Body copy 9.99.}\n---\n"), + "flow key after a comma in the body": with_body( + '---\n{title: X, "abstract": Body copy 9.99.}\n---\n' + ), + "nested flow key in the body": with_body("---\n{meta: {abstract: Body copy 9.99.}}\n---\n"), + "nested flow key in the front matter": NESTED_FLOW_KEY, + "key line outside any metadata block": with_body("abstract: |\n Body copy 9.99.\n"), + "metadata block in a code fence": with_body( + "```yaml\n---\nabstract: |\n Body copy 9.99.\n---\n```\n" + ), + "byte order mark": "\ufeff" + PAPER, +} + + +@unittest.skipUnless(PANDOC, "pandoc is not installed") +class PandocAgreementTests(unittest.TestCase): + """Wherever the parser reads an abstract, pandoc renders the same words, + or rejects the document, which then cannot render at all.""" + + ROUNDS = 100 + + @classmethod + def setUpClass(cls): + cls.directory = tempfile.TemporaryDirectory() + cls.template = Path(cls.directory.name) / "abstract.txt" + cls.template.write_text("$abstract$\n") + + @classmethod + def tearDownClass(cls): + cls.directory.cleanup() + + def rendered(self, text): + """The words pandoc renders as the abstract, or None when pandoc + rejects the document.""" + result = subprocess.run( + [PANDOC, "--from=markdown", "--to=plain", f"--template={self.template}"], + input=text.encode(), + capture_output=True, + ) + return None if result.returncode else words(result.stdout.decode()) + + def agrees(self, text): + """Whether the parser read an abstract that pandoc renders, after + asserting that pandoc renders the parser's words.""" + span = block_scalar_span(text, "abstract") + if span is None: + return False + rendered = self.rendered(text) + if rendered is None: + return False + self.assertEqual(words(text[span[0] : span[1]]), rendered, text) + return True + + def test_pandoc_renders_the_fixture(self): + self.assertEqual(self.rendered(PAPER), words(ABSTRACT_LINES)) + + def test_edge_cases(self): + for name, text in EDGE_CASES.items(): + with self.subTest(case=name): + self.agrees(text) + + def test_random_documents(self): + """Every random document with decoys outside its abstract parses, + renders and agrees.""" + rng = random.Random(20261114) + agreed = 0 + for _ in range(self.ROUNDS): + value = random_number(rng) + statement = f"we obtain {value} percent with the Consumer Price Index" + fields, terminator, body, _ = random_document(rng, statement) + add_decoys(rng, fields, body) + agreed += self.agrees(render(fields, terminator, body)) + self.assertEqual(agreed, self.ROUNDS) + + if __name__ == "__main__": unittest.main() From 3ea9a9970c72d42a2f284106b1752026fa591cd6 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 8 Nov 2026 00:43:39 -0500 Subject: [PATCH 4/5] Fail closed only on a tab within the abstract's indentation Quarto 1.9.36 renders a tab that follows the block's indentation as text, as pandoc does, so the parser now reads such a line instead of failing closed. A tab within the indentation still fails closed: there Quarto rejects the line, or ends the block, where pandoc reads text. The parser counts only leading spaces as the block's indentation. The parser tests now separate the tabs that fail closed from the tabs that read as text, and the edge cases add both kinds. A comparison with Quarto's rendered abstract over 333 documents (47 edge cases, 86 shared cases, 40 random documents and 160 random documents edited so Quarto renders them) found 207 agreements, 41 documents where the parser fails closed, 36 that both refuse and 48 that Quarto cannot render. One document diverges: an aliased key in a body metadata block, which the pull request lists as out of scope. Co-Authored-By: Claude Opus 5.5 --- scripts/paper_text.py | 20 ++++++++++---------- tests/test_paper_text.py | 23 +++++++++++++++++++---- 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/scripts/paper_text.py b/scripts/paper_text.py index 2c38c0a..14dd88c 100644 --- a/scripts/paper_text.py +++ b/scripts/paper_text.py @@ -37,10 +37,10 @@ def block_scalar_span(text: str, key: str) -> tuple[int, int] | None: Lines end at line feeds. The front matter opens with a ``---`` line and text on the next line, and closes at the next ``---`` or ``...`` line. - The block takes its indentation from its first line with text and ends at - the next line with less, such as another field or a trailing comment. - Lines of whitespace alone stay inside it. Indented keys belong to other - mappings and do not count. + The block takes its indentation from the spaces that lead its first line + with text and ends at the next line with fewer, such as another field or + a trailing comment. Lines of whitespace alone stay inside it. Indented + keys belong to other mappings and do not count. It returns None, so every tie that reads the block fails, when @@ -52,8 +52,8 @@ def block_scalar_span(text: str, key: str) -> tuple[int, int] | None: in flow style such as ``{key: value}``, since pandoc lets a later metadata block, even inside a list item or block quote, replace the front matter's value; - - a tab falls in the indentation of a line in the block, since Quarto - rejects tab indentation that pandoc accepts, or a carriage return + - a tab falls within the block's indentation on one of its lines, where + Quarto and pandoc read the line differently, or a carriage return stands alone in the front matter, since pandoc drops it; - the block holds no text. """ @@ -82,11 +82,11 @@ def block_scalar_span(text: str, key: str) -> tuple[int, int] | None: while last < len(header): line = header[last][1] if line.strip(): - leading = line[: len(line) - len(line.lstrip(" \t"))] - if "\t" in leading: + spaces = len(line) - len(line.lstrip(" ")) + indent = spaces if indent is None else indent + if "\t" in line[: max(indent, 1)]: return None - indent = len(leading) if indent is None else indent - if len(leading) < max(indent, 1): + if spaces < max(indent, 1): break last += 1 block = header[first:last] diff --git a/tests/test_paper_text.py b/tests/test_paper_text.py index 42bf98d..1265cf2 100644 --- a/tests/test_paper_text.py +++ b/tests/test_paper_text.py @@ -235,17 +235,28 @@ def test_blank_lines_stay_inside_the_block(self): "First line of the abstract, then its second line. A second paragraph.", ) - def test_a_tab_in_a_lines_indentation_fails_closed(self): + def test_a_tab_within_the_indentation_fails_closed(self): + """Quarto rejects a tab within the block's indentation, or ends the + block there, where pandoc reads the line as text.""" for lines in ( ABSTRACT_LINES + "\tTabbed text.\n", ABSTRACT_LINES + " \tTabbed text.\n", - ABSTRACT_LINES + " \tTabbed text.\n", + ABSTRACT_LINES + "\t# A comment.\n", + FOUR_SPACE_LINES + " \tTabbed text.\n", "\tTabbed first line.\n" + ABSTRACT_LINES, ): with self.subTest(lines=lines): self.assertIsNone(block_scalar_span(with_lines(lines), "abstract")) - inner = " First line,\twith a tab inside it.\n" - self.assertEqual(block_lines(with_lines(inner)), inner) + + def test_a_tab_after_the_indentation_is_text(self): + for lines in ( + " First line,\twith a tab inside it.\n", + ABSTRACT_LINES + " \tTabbed text.\n", + " \tTabbed first line,\n then its second line.\n", + " First line,\n \ta deeper tabbed line.\n", + ): + with self.subTest(lines=lines): + self.assertEqual(block_lines(with_lines(lines)), lines) def test_dots_also_close_the_front_matter(self): text = PAPER.replace(CLOSE, "...\n# Introduction") @@ -621,6 +632,10 @@ def words(text): " First,\n \tafter a tab 9.99,\n third.\n" ), "tab-indented first line": with_lines("\tFirst 9.99.\n second.\n"), + "tab after the first line's indentation": with_lines(" \tFirst 9.99,\n second.\n"), + "tab after a deeper line's indentation": with_lines(" First,\n \tdeeper 9.99,\n third.\n"), + "tab within a 4-space block's indentation": with_lines(FOUR_SPACE_LINES + " \tTabbed 9.99.\n"), + "tab, then a hash, after the block": with_lines(ABSTRACT_LINES + "\t# 9.99 percent\n"), "tab line inside the block": with_lines(" First,\n\t\n second 9.99.\n"), "line indented less than the first": with_lines(" Four 1.11,\n two 9.99.\n"), "line indented more than the first": with_lines(" First,\n deeper 9.99,\n third.\n"), From 39b5dcbed762ea6054c048bceaf3b92874d5ace7 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 8 Nov 2026 00:50:48 -0500 Subject: [PATCH 5/5] Check the parser against the abstract Quarto renders Quarto renders the paper, and it reads the front matter with its own YAML parser before pandoc runs, so the two can disagree. Quarto rejects a `...` terminator, a line holding only a tab and a repeated key, and it reads a tab-led `#` line after the block as a comment where pandoc reads text. Wherever Quarto is installed, three new tests render the fixture, the 47 edge cases and 12 random documents with decoys, and require Quarto's abstract to show the parser's words, or Quarto to reject the document. The random documents close with `---`, empty their tab-only lines and rename the `notes` list, all of which Quarto would reject. CI installs no Quarto, so it skips them, as it skips the pandoc tests. The guard's and parser's docstrings now say they read the abstract as Quarto and pandoc read it. Co-Authored-By: Claude Opus 5.5 --- scripts/check_paper.py | 30 ++++----- scripts/paper_text.py | 7 ++- tests/test_paper_text.py | 131 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 147 insertions(+), 21 deletions(-) diff --git a/scripts/check_paper.py b/scripts/check_paper.py index c6062f4..0a9b2a5 100644 --- a/scripts/check_paper.py +++ b/scripts/check_paper.py @@ -29,10 +29,10 @@ occurring elsewhere in the paper cannot satisfy them and line rewrapping cannot split a multi-word literal. The guard reads each error rate the abstract quotes from the front matter's ``abstract: |`` - block alone, as pandoc reads it, beside the rule it scores, and every - rate the abstract gives in that rule's wording must equal the - registered value. A copy in other metadata, in a comment, in the body - or inside the abstract cannot rescue a drifted figure, and a swap + block alone, as Quarto and pandoc read it, beside the rule it scores, + and every rate the abstract gives in that rule's wording must equal + the registered value. A copy in other metadata, in a comment, in the + body or inside the abstract cannot rescue a drifted figure, and a swap between rules fails; (e) the README miscounts the in-kind receipts or omits a calculator commit that their provenance records. @@ -173,10 +173,10 @@ def section(start: str, end: str | None = None, text: str = QMD) -> str: def front_matter_block(key: str, text: str = QMD) -> str: """Whitespace-normalized text of the front matter's one ``key: |`` - literal block, read as pandoc reads it. Pins that read it see no other - metadata field, comment or body text. When ``block_scalar_span`` finds - no such block, or finds another line that sets the key, the slice is - empty, so every pin inside it fails instead of reading other text.""" + literal block, read as Quarto and pandoc read it. Pins that read it see + no other metadata field, comment or body text. When ``block_scalar_span`` + finds no such block, or finds another line that sets the key, the slice + is empty, so every pin inside it fails instead of reading other text.""" span = block_scalar_span(text, key) if span is None: failures.append(f"paper must set {key} only in one {key}: | block of its front matter") @@ -745,13 +745,13 @@ def mae(rule: str) -> float: ) # Figures in the abstract also appear in the body or its tables, so their -# abstract ties read only the front matter's `abstract: |` block, as pandoc -# reads it. Other metadata fields, comments and body text fall outside that -# block, so a copy there cannot rescue a drifted figure, and a metadata block -# in the body that sets the abstract fails every tie. Each tie collects -# every rate the abstract gives in its rule's wording and requires them all to -# equal the registered value, so a swap between rules fails, and so does a -# changed rate that leaves a correct copy elsewhere in the abstract. +# abstract ties read only the front matter's `abstract: |` block, as Quarto +# and pandoc read it. Other metadata fields, comments and body text fall +# outside that block, so a copy there cannot rescue a drifted figure, and a +# metadata block in the body that sets the abstract fails every tie. Each tie +# collects every rate the abstract gives in its rule's wording and requires +# them all to equal the registered value, so a swap between rules fails, and +# so does a changed rate that leaves a correct copy elsewhere in the abstract. ABSTRACT = front_matter_block("abstract") # Current retrospective results in the abstract are a separate experiment. diff --git a/scripts/paper_text.py b/scripts/paper_text.py index 14dd88c..03e1121 100644 --- a/scripts/paper_text.py +++ b/scripts/paper_text.py @@ -2,8 +2,9 @@ The guard reads the abstract through ``block_scalar_span``, and the mutation tests place their edits with the same function, so both work on the same text. -Wherever pandoc is installed, ``tests/test_paper_text.py`` checks that the -parser reads the words pandoc renders as the abstract, or fails closed. +Wherever pandoc or Quarto is installed, ``tests/test_paper_text.py`` checks +that the parser reads the words that tool renders as the abstract, or fails +closed. """ from __future__ import annotations @@ -33,7 +34,7 @@ def _key_matchers(key: str): def block_scalar_span(text: str, key: str) -> tuple[int, int] | None: """Character span of the literal block under the front matter's one - top-level ``key: |`` line, read as pandoc reads it, or None. + top-level ``key: |`` line, read as Quarto and pandoc read it, or None. Lines end at line feeds. The front matter opens with a ``---`` line and text on the next line, and closes at the next ``---`` or ``...`` line. diff --git a/tests/test_paper_text.py b/tests/test_paper_text.py index 1265cf2..bd04901 100644 --- a/tests/test_paper_text.py +++ b/tests/test_paper_text.py @@ -6,11 +6,14 @@ that the abstract's words round-trip, that text outside the block never changes what the guard reads, that a second key or a key after the front matter makes the parser fail closed, and that each value reads as its whole numeric token. -Wherever pandoc is installed, a differential test checks that the parser reads -the words pandoc renders as the abstract, or fails closed. CI installs no -pandoc, so it skips that test. +Wherever pandoc or Quarto is installed, differential tests check that the +parser reads the words that tool renders as the abstract, or fails closed. +Quarto renders the paper and reads the front matter itself before pandoc does, +so the two can disagree. CI installs neither, so it skips those tests. """ +import concurrent.futures +import html.parser import importlib.util import random import shutil @@ -708,6 +711,7 @@ def rendered(self, text): [PANDOC, "--from=markdown", "--to=plain", f"--template={self.template}"], input=text.encode(), capture_output=True, + check=False, ) return None if result.returncode else words(result.stdout.decode()) @@ -745,5 +749,126 @@ def test_random_documents(self): self.assertEqual(agreed, self.ROUNDS) +QUARTO = shutil.which("quarto") + + +class QuartoAbstract(html.parser.HTMLParser): + """Collects the text of the first ``
`` on a page + that Quarto renders, less the block title Quarto sets above it.""" + + VOID = frozenset( + ("area", "br", "col", "embed", "hr", "img", "input", "link", "meta", "source", "wbr") + ) + + def __init__(self): + super().__init__() + self.found = False + self.depth = 0 + self.title_depth = None + self.parts = [] + + def handle_starttag(self, tag, attrs): + if tag in self.VOID or (self.found and not self.depth): + return + classes = (dict(attrs).get("class") or "").split() + if self.depth: + self.depth += 1 + if self.title_depth is None and "block-title" in classes: + self.title_depth = self.depth + elif tag == "div" and "abstract" in classes: + self.found = True + self.depth = 1 + + def handle_endtag(self, tag): + if tag in self.VOID or not self.depth: + return + if self.depth == self.title_depth: + self.title_depth = None + self.depth -= 1 + + def handle_data(self, data): + if self.depth and self.title_depth is None: + self.parts.append(data) + + +def renderable(fields): + """Edit a random document's fields into ones Quarto accepts: it rejects a + line holding only a tab and a list under ``notes``.""" + for field in fields: + field[:] = ["" if line == "\t" else line for line in field] + if field[0] == "notes:": + field[0] = "remarks:" + + +@unittest.skipUnless(QUARTO, "quarto is not installed") +class QuartoAgreementTests(unittest.TestCase): + """Wherever the parser reads an abstract, the page Quarto renders shows + the same words as its abstract, or Quarto rejects the document, which + then cannot render at all.""" + + ROUNDS = 12 + JOBS = 4 + + def rendered(self, text): + """The words of the abstract on the page Quarto renders, "" when the + page has none, or None when Quarto rejects the document.""" + with tempfile.TemporaryDirectory() as directory: + folder = Path(directory) + (folder / "doc.qmd").write_bytes(text.encode()) + result = subprocess.run( + [QUARTO, "render", "doc.qmd", "--to", "html"], + cwd=folder, + capture_output=True, + check=False, + timeout=600, + ) + page = folder / "doc.html" + if result.returncode or not page.exists(): + return None + reader = QuartoAbstract() + reader.feed(page.read_text()) + return words(" ".join(reader.parts)) + + def compare(self, documents): + """The parser's words and Quarto's for each document the parser + reads an abstract from, rendering JOBS documents at a time.""" + parsed = {} + for name, text in documents.items(): + span = block_scalar_span(text, "abstract") + if span is not None: + parsed[name] = words(text[span[0] : span[1]]) + with concurrent.futures.ThreadPoolExecutor(self.JOBS) as pool: + rendered = pool.map(lambda name: self.rendered(documents[name]), parsed) + return {name: (ours, theirs) for (name, ours), theirs in zip(parsed.items(), rendered)} + + def test_quarto_renders_the_fixture(self): + self.assertEqual(self.rendered(PAPER), words(ABSTRACT_LINES)) + + def test_edge_cases(self): + for name, (ours, theirs) in self.compare(EDGE_CASES).items(): + with self.subTest(case=name): + if theirs is not None: + self.assertEqual(ours, theirs) + + def test_random_documents(self): + """Random documents with decoys outside the abstract, edited into + ones Quarto accepts and closed with ``---``, all parse, render and + agree.""" + rng = random.Random(20261116) + documents = {} + for round_number in range(self.ROUNDS): + value = random_number(rng) + statement = f"we obtain {value} percent with the Consumer Price Index" + fields, _, body, _ = random_document(rng, statement) + add_decoys(rng, fields, body) + renderable(fields) + documents[round_number] = render(fields, "---", body) + compared = self.compare(documents) + self.assertEqual(len(compared), self.ROUNDS) + for round_number, (ours, theirs) in compared.items(): + with self.subTest(round=round_number): + self.assertEqual(ours, theirs) + + if __name__ == "__main__": unittest.main()