Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@ papers/<slug>/
lean/ the authors' Lean project, unchanged
links.json which passage goes with which declaration, plain titles, notes and comparisons
checks/ optional Lean files of ours that test a statement against the authors' Lean
comparator/ optional files of a Comparator run: challenge, solution, configuration, receipt
generated/ recorded declaration data and its provenance, plus any compiled checks
```

Expand Down Expand Up @@ -176,6 +177,17 @@ reports for it. `check` stops if a check file has changed since `extract` compil
says that the links, notes and comparisons have not been reviewed by the authors, until an author
has reviewed them.

A paper can show a [Comparator](https://github.com/leanprover/comparator) run. Name the folder
holding the run's files under `[comparator] evidence` in `paper.toml`; pages that show the same
Lean name one shared folder, such as `"../pcp-full-argument/comparator"`. The folder holds
`Challenge.lean`, `Solution.lean`, the configuration, the rejected control and a `receipt.json`
recording the run. The build reads each configured theorem's statement from `Challenge.lean`,
with its lines, and links the files at the commit it builds. It stops if a file's SHA-256 differs
from the receipt, if the challenge states no theorem of a configured name, or if the receipt does
not record an accepted run and a rejected control. Each checked declaration then has a closed
"Comparator evidence" fold beneath its axiom report. Comparator tests a proof against the
challenge statement; it does not test whether that statement means what the paper says.

## Repository map

Authored paper files and checked-in declaration records live under `papers/`.
Expand Down
7 changes: 7 additions & 0 deletions docs/paper.toml.example
Original file line number Diff line number Diff line change
Expand Up @@ -51,3 +51,10 @@ page_label = "{page}" # how the paper numbers its pages, such as "191:{p
# example = "isoR-clam"
# or
# example = ["isoR-clam", "closed-world", "indexIsoL"]

# A Comparator run on some of the Lean's theorems. Name the folder of the run's files:
# Challenge.lean, Solution.lean, the configuration, the rejected control and receipt.json.
# Papers that show the same Lean can name one shared folder. The build stops if a file
# does not match the receipt's SHA-256 or the challenge states no configured theorem.
# [comparator]
# evidence = "comparator"
3 changes: 3 additions & 0 deletions papers/pcp-exposition/paper.toml
Original file line number Diff line number Diff line change
Expand Up @@ -24,3 +24,6 @@ language = "lean"
[reading.companions]
"../long/reasoning-record.md" = "pcp-full-argument"
"../short/paper.md" = "pcp-specialist"

[comparator]
evidence = "../pcp-full-argument/comparator"
3 changes: 3 additions & 0 deletions papers/pcp-full-argument/paper.toml
Original file line number Diff line number Diff line change
Expand Up @@ -24,3 +24,6 @@ language = "lean"
[reading.companions]
"../short/paper.md" = "pcp-specialist"
"../expository/main.md" = "pcp-exposition"

[comparator]
evidence = "comparator"
3 changes: 3 additions & 0 deletions papers/pcp-reading-guide/paper.toml
Original file line number Diff line number Diff line change
Expand Up @@ -24,3 +24,6 @@ scope = "This page links Sections 3 and 4 of the guide to the five Lean files be

[reading]
language = "lean"

[comparator]
evidence = "../pcp-full-argument/comparator"
3 changes: 3 additions & 0 deletions papers/pcp-specialist/paper.toml
Original file line number Diff line number Diff line change
Expand Up @@ -24,3 +24,6 @@ language = "lean"
[reading.companions]
"../long/reasoning-record.md" = "pcp-full-argument"
"../expository/main.md" = "pcp-exposition"

[comparator]
evidence = "../pcp-full-argument/comparator"
7 changes: 7 additions & 0 deletions skills/add-paper/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,13 @@ local branch and put `"fidelity_held": "<reason>"` in `links.json` in place of `
`check` then skips the comparison step for that paper, and the page says nothing about
comparisons.

If Comparator has judged some of the authors' theorems against a challenge file, put the
run's files in one folder (`Challenge.lean`, `Solution.lean`, the configuration, the rejected
control and `receipt.json`) and name it under `[comparator] evidence` in `paper.toml`; see the
[README](../../README.md#what-paperpy-check-establishes). The build checks the files against
the receipt's hashes and shows the evidence under each checked declaration. Comparator does not test
whether the challenge matches the paper, so its result never replaces a comparison.

## 6. Run the comparison

```sh
Expand Down
104 changes: 104 additions & 0 deletions tests/test_paper.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
"""The parts of the pipeline that decide whether a link is real."""

import hashlib
import json
import sys
import unittest
from pathlib import Path
Expand Down Expand Up @@ -513,3 +515,105 @@ def test_each_example_is_the_authors_lines(self):
for line in example["code"]:
self.assertEqual(line["text"], lines[line["n"] - 1])
self.assertEqual(example["code"][-1]["n"], min(example["last"], example["decl_last"]))


class ComparatorEvidence(unittest.TestCase):
"""Comparator's run files become data for the page only when they match its receipt."""

challenge = ("import X\n\nnamespace PCP\n\ntheorem first (n : Nat) :\n n = n := by\n sorry\n\nend PCP\n\n"
"namespace PCP.Inner\n\ntheorem second : True := by\n sorry\n\nend PCP.Inner\n")

def setUp(self):
import tempfile
self.scratch = tempfile.TemporaryDirectory()
self.addCleanup(self.scratch.cleanup)
root = Path(self.scratch.name)
self.saved = paper.ROOT, paper.PAPERS
paper.ROOT, paper.PAPERS = root, root / "papers"
self.addCleanup(lambda: setattr(paper, "ROOT", self.saved[0]) or setattr(paper, "PAPERS", self.saved[1]))
self.folder = root / "papers" / "one"
self.runs = self.folder / "comparator"
(self.runs / "negative").mkdir(parents=True)
names = ["PCP.first", "PCP.Inner.second"]
files = {"Challenge.lean": self.challenge, "Solution.lean": "import Y\n",
"comparator.json": json.dumps({"theorem_names": names, "permitted_axioms": ["propext"]}),
"negative/Challenge.lean": self.challenge.replace("n = n", "n = n + 0"),
"negative/comparator.json": "{}"}
for name, text in files.items():
(self.runs / name).write_text(text, encoding="utf-8")
self.receipt = {
"run": "https://github.com/o/r/actions/runs/1", "date": "2026-11-16", "toolchain": "leanprover/lean4:v4.35.0",
"sandbox": "bubblewrap 0.9.0", "mode": "standard; sandboxed; --paranoid was not completed",
"permitted_axioms": ["propext"],
"accepted": {"config": "comparator.json", "theorem_names": names, "exit": 0,
"output": ["Lean default kernel accepts the solution", "Your solution is okay!"]},
"rejected_control": {"config": "negative/comparator.json", "change": "n + 0 in place of n", "exit": 1,
"output": ["error: Challenge and solution theorem statement do not match: 'PCP.first'"]},
"sha256": {name: hashlib.sha256((self.runs / name).read_bytes()).hexdigest()
for name in ("Challenge.lean", "Solution.lean", "comparator.json", "negative/Challenge.lean")},
"limits": "Comparator does not compare the statements with the paper.",
}
self.write_receipt()
self.config = {"comparator": {"evidence": "comparator"}}

def write_receipt(self):
(self.runs / "receipt.json").write_text(json.dumps(self.receipt), encoding="utf-8")

def evidence(self, config=None):
import os
old = os.environ.get("PROOF_READER_COMMIT")
os.environ["PROOF_READER_COMMIT"] = "a" * 40
try:
return paper.comparator_evidence(self.folder, config or self.config)
finally:
if old is None:
del os.environ["PROOF_READER_COMMIT"]
else:
os.environ["PROOF_READER_COMMIT"] = old

def test_reads_each_statement_under_its_full_name(self):
found = paper.challenge_statements(self.challenge)
self.assertEqual(found, {"PCP.first": [5, 6], "PCP.Inner.second": [13, 13]})

def test_builds_the_statement_and_fixed_links(self):
built = self.evidence()
first = built["theorems"]["PCP.first"]
self.assertEqual(first["statement"], "theorem first (n : Nat) :\n n = n := by")
self.assertTrue(first["url"].endswith(f"/blob/{'a' * 40}/papers/one/comparator/Challenge.lean#L5-L6"))
self.assertTrue(built["links"]["receipt"].endswith("/papers/one/comparator/receipt.json"))
self.assertTrue(built["links"]["control"].endswith("/papers/one/comparator/negative/Challenge.lean"))
self.assertEqual((built["kernel"], built["paranoid"], built["control"]["theorem"]),
("Lean default kernel", "not completed", "PCP.first"))

def test_another_paper_shares_the_folder(self):
other = paper.PAPERS / "two"
other.mkdir()
built = paper.comparator_evidence(other, {"comparator": {"evidence": "../one/comparator"}})
self.assertEqual(built["folder"], "papers/one/comparator")

def test_no_key_means_no_evidence(self):
self.assertIsNone(paper.comparator_evidence(self.folder, {}))

def test_a_file_that_does_not_match_its_hash_stops_the_build(self):
(self.runs / "Challenge.lean").write_text(self.challenge.replace("n = n", "n = 0"), encoding="utf-8")
with self.assertRaisesRegex(paper.PaperError, "Challenge.lean does not match the sha256"):
self.evidence()

def test_a_theorem_the_challenge_does_not_state_stops_the_build(self):
names = ["PCP.first", "PCP.third"]
(self.runs / "comparator.json").write_text(json.dumps({"theorem_names": names, "permitted_axioms": ["propext"]}))
self.receipt["accepted"]["theorem_names"] = names
self.receipt["sha256"]["comparator.json"] = hashlib.sha256((self.runs / "comparator.json").read_bytes()).hexdigest()
self.write_receipt()
with self.assertRaisesRegex(paper.PaperError, "states no theorem called PCP.third"):
self.evidence()

def test_a_run_that_did_not_pass_stops_the_build(self):
self.receipt["accepted"]["exit"] = 1
self.write_receipt()
with self.assertRaisesRegex(paper.PaperError, "accepted run"):
self.evidence()

def test_a_folder_outside_papers_is_refused(self):
with self.assertRaisesRegex(paper.PaperError, "folder under papers/"):
self.evidence({"comparator": {"evidence": "../../.."}})
126 changes: 126 additions & 0 deletions tools/paper.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import hashlib
import html
import json
import os
import re
import shutil
import subprocess
Expand Down Expand Up @@ -60,6 +61,8 @@
CHECK_KEYWORDS = ("theorem", "lemma", "def")
AXIOMS_PRINTED = re.compile(
r"^'(.+?)' (?:depends on axioms: \[([^\]]*)\]|does not depend on any axioms)", re.M)
# Where this repository's files can be read at one commit, for links to Comparator's files.
REPOSITORY = "https://github.com/wcook04/proof-reader"


class PaperError(Exception):
Expand Down Expand Up @@ -689,6 +692,122 @@ def reachable(goals: list[str], uses: dict) -> set[str]:
return seen


def challenge_statements(text: str) -> dict[str, list[int]]:
"""Each theorem a Comparator challenge states, by its full name: the lines from its
keyword to the line that opens its proof."""
lines = text.splitlines()
spaces: list[str] = []
found: dict[str, list[int]] = {}
for number, line in enumerate(lines, 1):
opened = re.match(r"namespace\s+(\S+)\s*$", line)
closed = re.match(r"end\s+(\S+)\s*$", line)
if opened:
spaces.append(opened[1])
elif closed and spaces and spaces[-1] == closed[1]:
spaces.pop()
match = WRITTEN.match(line)
if match and not line[:1].isspace() and match.group(1).split()[0] in ("theorem", "lemma"):
name = ".".join([*spaces, match.group(2)])
if name in found:
raise PaperError(f"the challenge states {name} twice")
found[name] = [number, statement_end(lines, number, len(lines), "theorem")]
return found


def built_commit() -> str | None:
"""The commit this build reads, so links to the repository's files stay fixed."""
for key in ("PROOF_READER_COMMIT", "GITHUB_SHA"):
if os.environ.get(key, "").strip():
return os.environ[key].strip()
if (ROOT / ".git").exists():
run = subprocess.run(["git", "rev-parse", "HEAD"], cwd=ROOT, capture_output=True, text=True)
if run.returncode == 0 and re.fullmatch(r"[0-9a-f]{40}", run.stdout.strip()):
return run.stdout.strip()
return None


def comparator_evidence(folder: Path, config: dict) -> dict | None:
"""What Comparator checked for this paper, read from its run's files.

`[comparator] evidence` in paper.toml names a folder holding the challenge, the
solution, the configuration, the run's receipt and the rejected control; papers that
show the same Lean name one shared folder. A file that does not match the receipt's
hash, or a theorem name the challenge does not state, stops the build."""
wanted = config.get("comparator", {}).get("evidence")
if not wanted:
return None
where = (folder / wanted).resolve()
if not where.is_relative_to(PAPERS.resolve()) or not where.is_dir():
raise PaperError(f"[comparator] evidence must be a folder under papers/: {wanted}")
try:
receipt = json.loads((where / "receipt.json").read_text(encoding="utf-8"))
except FileNotFoundError:
raise PaperError(f"[comparator] {wanted} has no receipt.json") from None
accepted, control = receipt.get("accepted", {}), receipt.get("rejected_control", {})
hashes = receipt.get("sha256", {})
files = {"challenge": "Challenge.lean", "solution": "Solution.lean", "config": accepted.get("config", "")}
for name in (*files.values(), control.get("config", "")):
if not name or not (where / name).is_file():
raise PaperError(f"[comparator] {wanted}: {name or 'a configuration'} named by receipt.json is missing")
for name in files.values():
if name not in hashes:
raise PaperError(f"[comparator] {wanted}: receipt.json records no sha256 for {name}")
for name, digest in hashes.items():
path = where / name
if not path.is_file() or hashlib.sha256(path.read_bytes()).hexdigest() != digest:
raise PaperError(f"[comparator] {wanted}: {name} does not match the sha256 in receipt.json")
setup = json.loads((where / files["config"]).read_text(encoding="utf-8"))
names = setup.get("theorem_names", [])
if not names or names != accepted.get("theorem_names"):
raise PaperError(f"[comparator] {wanted}: receipt.json and {files['config']} name different theorems")
if setup.get("permitted_axioms") != receipt.get("permitted_axioms"):
raise PaperError(f"[comparator] {wanted}: receipt.json and {files['config']} permit different axioms")
output = accepted.get("output", [])
if accepted.get("exit") != 0 or "Your solution is okay!" not in output:
raise PaperError(f"[comparator] {wanted}: receipt.json does not record an accepted run")
kernel = next((line[: -len(" accepts the solution")] for line in output if line.endswith(" accepts the solution")), None)
if not kernel:
raise PaperError(f"[comparator] {wanted}: receipt.json does not record which kernel accepted the solution")
mode = receipt.get("mode", "")
# Only the one reading the page can state is accepted; a receipt saying anything else
# about --paranoid needs the page's wording extended first.
if "--paranoid was not completed" not in mode:
raise PaperError(f"[comparator] {wanted}: receipt.json mode must record that --paranoid was not completed")
rejected = next((m for line in control.get("output", [])
for m in [re.search(r"statement do not match: '([^']+)'", line)] if m), None)
if not control.get("exit") or not rejected or not control.get("change"):
raise PaperError(f"[comparator] {wanted}: receipt.json does not record a rejected control")
control_challenge = (Path(control["config"]).parent / "Challenge.lean").as_posix()
challenge = (where / "Challenge.lean").read_text(encoding="utf-8")
stated = challenge_statements(challenge)
missing = [n for n in names if n not in stated]
if missing:
raise PaperError(f"[comparator] {wanted}: Challenge.lean states no theorem called {', '.join(missing)}")
commit = built_commit()
if not commit:
print(" no commit known (set PROOF_READER_COMMIT): Comparator links point at main", file=sys.stderr)
base = f"{REPOSITORY}/blob/{commit or 'main'}/{where.relative_to(ROOT.resolve()).as_posix()}"
lines = challenge.splitlines()
theorems = {}
for name in names:
first, last = stated[name]
theorems[name] = {"first": first, "last": last, "statement": "\n".join(lines[first - 1:last]),
"url": f"{base}/Challenge.lean#L{first}-L{last}"}
return {
"folder": where.relative_to(ROOT.resolve()).as_posix(), "commit": commit,
"run": receipt["run"], "date": receipt["date"], "run_commit": receipt.get("commit"),
"toolchain": receipt["toolchain"], "sandbox": receipt["sandbox"], "kernel": kernel,
"paranoid": "not completed",
"permitted_axioms": receipt["permitted_axioms"],
"links": {"challenge": f"{base}/Challenge.lean", "solution": f"{base}/{files['solution']}",
"config": f"{base}/{files['config']}", "receipt": f"{base}/receipt.json",
"control": f"{base}/{control_challenge}"},
"control": {"change": control["change"], "theorem": rejected[1]},
"limits": receipt.get("limits", ""),
"theorems": theorems,
}


def edition(slug: str) -> dict:
data = resolve(slug)
config, shard_info, links = data["config"], data["shards"], data["links"]
Expand Down Expand Up @@ -790,6 +909,7 @@ def split(names: list[str], statement: bool, kind_of) -> list[str]:
"mentions": find_mentions(data["words"], names),
"declarations": declarations,
"checks": data["checks"],
"comparator": comparator_evidence(data["folder"], config),
"shelf": {"example": examples[0] if examples else None, "examples": examples},
"sheets": statement_sheets(links.get("claims", []), declarations, data["passages"], shard_info),
"coverage": {"sections": tied_sections, "ties": len(data["passages"])},
Expand Down Expand Up @@ -924,6 +1044,12 @@ def check(slug: str) -> None:
print(f" compared as {verdict}: {len(names)}" + (f" ({', '.join(names)})" if names else ""))
for item in built["checks"]:
print(f" check {item['name']} rests on: {', '.join(item['axioms']) or 'no axioms'}")
evidence = built["comparator"]
if evidence:
shown = {d["id"] for d in built["declarations"]}
on_page = [name for name in evidence["theorems"] if name in shown]
print(f" Comparator evidence from {evidence['folder']}: {len(on_page)} of {len(evidence['theorems'])} "
f"checked theorems on this page" + (f" ({', '.join(on_page)})" if on_page else ""))


def build(slugs: list[str]) -> None:
Expand Down
Loading