From 721f81aae9a23f3d9badad2a64ec7cfd8f942c50 Mon Sep 17 00:00:00 2001 From: Dang Nguyen Date: Wed, 1 Jul 2026 01:23:28 -0500 Subject: [PATCH] Add Anchored Review Format standard for AI review systems An open output standard so conformant review systems plug into the benchmark with no per-system adapter. A review is a list of comments, each with a verbatim quote and an explanation (the only fields scoring depends on), with everything else optional. Includes two integration profiles (CLI for systems the benchmark runs locally, hosted API for closed systems), a stdlib-only validator, a stdlib-only reference client for the API profile, and minimal/full/invalid examples. Adds only standard/; no changes to the reviewer package, benchmark, or adapters. Co-Authored-By: Claude Opus 4.8 (1M context) --- standard/README.md | 114 +++++++++++++ standard/examples/full.json | 30 ++++ standard/examples/invalid-missing-quote.json | 9 ++ standard/examples/minimal.json | 13 ++ standard/profile-api.md | 93 +++++++++++ standard/profile-cli.md | 48 ++++++ standard/reference/review_client.py | 147 +++++++++++++++++ standard/validate.py | 160 +++++++++++++++++++ 8 files changed, 614 insertions(+) create mode 100644 standard/README.md create mode 100644 standard/examples/full.json create mode 100644 standard/examples/invalid-missing-quote.json create mode 100644 standard/examples/minimal.json create mode 100644 standard/profile-api.md create mode 100644 standard/profile-cli.md create mode 100755 standard/reference/review_client.py create mode 100755 standard/validate.py diff --git a/standard/README.md b/standard/README.md new file mode 100644 index 0000000..270b04d --- /dev/null +++ b/standard/README.md @@ -0,0 +1,114 @@ +# Anchored Review Format + +This is a small, open standard for the output of an AI paper-review system. If a system adopts this standard, the benchmark can run it directly, with no custom integration code needed. + +A review in this standard is a list of comments, where every comment is anchored to a **verbatim quote** from the paper and paired with an **explanation** of the issue. A system connects in one of two ways: + +- **Command line** (open systems the benchmark can run): the system provides a command that takes a paper and writes a review. See `[profile-cli.md](profile-cli.md)`. +- **Hosted API** (closed systems the benchmark calls over the network): the system exposes an endpoint the benchmark submits papers to and polls for results. See `[profile-api.md](profile-api.md)`. + +Both return the same review payload, described below. + +## Payload format + +A review payload looks like this: + +```json +{ + "standard_version": "1.0", + "comments": [ + { + "quote": "we achieve a 51% improvement over the baseline", + "explanation": "The 51% figure is not supported by Table 3, which reports a 5.1% relative gain. This looks like a misplaced decimal." + } + ] +} +``` + +Two top-level fields are required: `standard_version` and `comments`. Each comment requires `quote` and `explanation`. This is everything the standard requires. + +Validate a payload with the bundled checker, which needs only Python: + +```bash +python validate.py your_review.json +``` + +A passing payload prints `✓ VALID` and exits 0. + +## Why these two fields + +The benchmark seeds known errors into clean papers and measures how many a system catches. To decide whether a comment caught a seeded error, the benchmark does two things: + +1. Match the comment's `quote` against the region of the paper that was changed. This match is fuzzy, but it works best when the quote is **exactly from the paper text**. +2. Read the comment's `explanation` and judge whether it identifies the same error that was seeded. + +So `quote` should be extracted verbatim from the paper, and `explanation` should say what is wrong and why. Those are the only fields the score depends on. + +## Full specification + +One payload is one system's review of one paper. + +### Top-level fields + + +| Field | Required | Type | Meaning | +| ------------------ | ----------- | ------ | ---------------------------------------------------------------------------------------------------------------------------------------- | +| `standard_version` | yes | string | Version of this standard, currently `"1.0"`. | +| `comments` | yes | array | The list of comments. See below. | +| `standard` | recommended | string | The constant `"anchored-review"`, so the payload is self-identifying. | +| `paper_id` | recommended | string | An id for the paper, such as an arXiv id or a slug. | +| `system` | recommended | string | The name of the review system. | +| `model` | optional | string | The underlying model id, if there is one. | +| `overall_feedback` | optional | string | A high-level assessment of the paper. | +| `paragraphs` | optional | array | The paper split into paragraphs. Include this only if the comments use `paragraph_index`. Each item is `{"index": int, "text": string}`. | + + +### Comment fields + + +| Field | Required | Type | Meaning | +| ----------------- | ----------- | --------------- | ------------------------------------------------------------- | +| `quote` | yes | string | A span copied from the paper that the comment is about. | +| `explanation` | yes | string | What is wrong with the quoted span and why. | +| `title` | recommended | string | A short label for the comment. | +| `severity` | optional | string | A free-text tier such as `major`, `moderate`, or `minor`. | +| `paragraph_index` | optional | integer or null | A 0-based index into `paragraphs`, if that array is provided. | +| `id` | optional | string | A stable id for the comment. | + + +Extra fields are preserved and ignored by the benchmark, so a system can carry its own metadata without breaking anything. + +## Connecting a system + +Each profile matches a different way a system runs. Both deliver the same payload above. + +- **Open source, or otherwise runnable by the benchmark:** the [command-line profile](profile-cli.md). The benchmark runs the system locally, so there is no hosting or inference cost for its authors, and this is the simplest to adopt. +- **Closed source:** the [hosted API profile](profile-api.md). The system runs behind an endpoint the benchmark calls. This is the option when the benchmark cannot run the system's code, such as for proprietary systems. + +## Examples + +- `[examples/minimal.json](examples/minimal.json)`: the smallest valid payload, with only the required fields. +- `[examples/full.json](examples/full.json)`: every field populated, including paragraphs and per-comment metadata. +- `[examples/invalid-missing-quote.json](examples/invalid-missing-quote.json)`: a payload that fails, showing what the validator reports. + +## Validating + +```bash +python validate.py examples/full.json +``` + +The checker reports two kinds of findings. **Errors** mean the payload does not conform and will not score. **Warnings** flag recommended fields that were left out, which are fine to ignore. To enforce the recommended fields too, treat warnings as failures: + +```bash +python validate.py your_review.json --strict +``` + +The checker uses only the Python standard library, so any Python 3 install can run it as is. + +## Versioning + +The `standard_version` field tracks the standard. Version `1.0` is the current release. Additions that keep older payloads valid will bump the minor version (`1.1`, `1.2`). Changes that can break older payloads will bump the major version (`2.0`). A system keeps emitting the version it built against, and its payloads stay readable. + +## Questions + +This standard grew out of the OpenAIReview project. If a field does not map cleanly onto what a system produces, or a field should be added, open an issue. The standard is meant to stay small, so the bar for new required fields is high, but new optional fields are welcome. \ No newline at end of file diff --git a/standard/examples/full.json b/standard/examples/full.json new file mode 100644 index 0000000..5be8907 --- /dev/null +++ b/standard/examples/full.json @@ -0,0 +1,30 @@ +{ + "standard": "anchored-review", + "standard_version": "1.0", + "paper_id": "2504.06303", + "system": "example-reviewer", + "model": "example-model-v1", + "overall_feedback": "The paper presents a clear method and strong empirical results. The main concerns are an unsupported headline number in the abstract and an inconsistent definition of the loss function between Section 2 and Appendix A.", + "paragraphs": [ + { "index": 0, "text": "Abstract. We present a method that achieves a 51% improvement over the baseline ..." }, + { "index": 1, "text": "Section 2. We define the training objective as L = -sum p log p ..." } + ], + "comments": [ + { + "id": "c0", + "title": "Unsupported headline improvement", + "quote": "we achieve a 51% improvement over the baseline", + "explanation": "The 51% figure is not supported by Table 3, which reports a 5.1% relative gain. This looks like a misplaced decimal or a transcription error.", + "severity": "major", + "paragraph_index": 0 + }, + { + "id": "c1", + "title": "Loss definition conflates two distributions", + "quote": "the loss is defined as L = -sum p log p", + "explanation": "The cross-entropy loss should sum over the true distribution times the log of the predicted distribution. As written the two distributions are conflated, which makes this the entropy of p rather than a training loss.", + "severity": "moderate", + "paragraph_index": 1 + } + ] +} diff --git a/standard/examples/invalid-missing-quote.json b/standard/examples/invalid-missing-quote.json new file mode 100644 index 0000000..d89b8ce --- /dev/null +++ b/standard/examples/invalid-missing-quote.json @@ -0,0 +1,9 @@ +{ + "standard_version": "1.0", + "comments": [ + { + "title": "This comment is missing its quote", + "explanation": "A comment must anchor to a verbatim span from the paper, but this one has no quote field, so it cannot be matched to anything and the validator rejects it." + } + ] +} diff --git a/standard/examples/minimal.json b/standard/examples/minimal.json new file mode 100644 index 0000000..7b39f2a --- /dev/null +++ b/standard/examples/minimal.json @@ -0,0 +1,13 @@ +{ + "standard_version": "1.0", + "comments": [ + { + "quote": "we achieve a 51% improvement over the baseline", + "explanation": "The 51% figure is not supported by Table 3, which reports a 5.1% relative gain. This looks like a misplaced decimal or a transcription error." + }, + { + "quote": "the loss is defined as L = -sum p log p", + "explanation": "The cross-entropy loss should sum over the true distribution times the log of the predicted distribution. As written the two distributions are conflated, which makes this the entropy of p rather than a training loss." + } + ] +} diff --git a/standard/profile-api.md b/standard/profile-api.md new file mode 100644 index 0000000..bb40387 --- /dev/null +++ b/standard/profile-api.md @@ -0,0 +1,93 @@ +# Hosted API profile + +This profile is for closed systems the benchmark cannot run itself. The system exposes an HTTP endpoint. The benchmark submits each paper, polls until the review is ready, and reads the result. + +A review takes minutes, so this is a job-style API. Submit returns immediately with an id, and the result is fetched by polling. + +## Endpoints + +### Submit a review + +``` +POST /v1/reviews +Authorization: Bearer +Content-Type: application/json + +{ + "paper": { "text": "" }, + "options": { } +} +``` + +Response: + +```json +{ "session_id": "abc123", "status": "queued" } +``` + +- `paper` carries the paper. Supporting `{"text": "..."}` is the minimum. A system may also accept `{"url": "..."}` or a multipart file upload, and the benchmark sends whichever the system declares. +- `options` is an open object for any parameters the system exposes (review mode, depth). It can be omitted when there are none. +- `status` is the job state (see below). + +### Fetch a review + +``` +GET /v1/reviews/{session_id} +Authorization: Bearer +``` + +Response while running: + +```json +{ "session_id": "abc123", "status": "running" } +``` + +Response when done: + +```json +{ + "session_id": "abc123", + "status": "completed", + "result": { + "standard_version": "1.0", + "comments": [ + { "quote": "...", "explanation": "..." } + ] + } +} +``` + +The `result` object is a payload in the [Anchored Review Format](README.md). Returning the standard field names (`quote`, `explanation`) is what lets the benchmark call the API with no per-system adapter. + +## Status values + + +| Status | Meaning | +| ----------- | ----------------------------------------- | +| `queued` | Accepted, not started. | +| `running` | In progress. Keep polling. | +| `completed` | Done. `result` holds the payload. | +| `failed` | Terminal error. Include an `error` field. | + + +Any other terminal-sounding value is treated as done or failed on a best-effort basis, but the four above keep the interaction unambiguous. + +## Authentication + +The example uses a bearer token. When an API uses a different header (for example `x-api-key`), the system declares the header name and the benchmark sets it. One scheme per system is sufficient. + +## Reference client + +`[reference/review_client.py](reference/review_client.py)` is a standalone client for this profile. It submits a paper, polls until the job is terminal, and returns the validated payload. It uses only the Python standard library, so the system's creators can run it against a staging endpoint to check that their output is compatible with the benchmark: + +```bash +python reference/review_client.py --base-url https://your-api.example.com --api-key paper.txt +``` + +It prints the returned payload and the validator's verdict. + +## Notes + +- Reviews are long-running. The client polls with a default timeout that can be raised for slow systems. +- The benchmark may submit several papers at once. Each `session_id` is independent. + diff --git a/standard/profile-cli.md b/standard/profile-cli.md new file mode 100644 index 0000000..4a94317 --- /dev/null +++ b/standard/profile-cli.md @@ -0,0 +1,48 @@ +# Command-line profile + +This profile is for systems the benchmark can run itself, such as an open-source repository or a locally installable package. The system provides a command that reviews one paper. The benchmark runs it on each paper and reads the review it writes. + +This is the simplest way to connect a system. There is no endpoint to host and no inference cost for the system's authors, since the benchmark supplies the compute. + +## The contract + +The system provides a single command that: + +1. Takes the path to a paper file as input. +2. Writes one review payload, conforming to the [Anchored Review Format](README.md), to a specified path. +3. Exits `0` on success and non-zero on failure. + +The benchmark invokes the command once per paper: + +```bash +your-review-command --out +``` + +- `` is a file the benchmark provides. The system declares which formats it accepts (for example PDF, Markdown, or LaTeX source), and the benchmark sends that format. +- `--out ` is where the command writes the payload. + +Any configuration the system needs (model choice, API keys for its own backend, review depth) can come from command-line flags or environment variables. + +## Format validation + +The system's creators can run this check to see if their output format is compatible with the benchmark. + +```bash +python validate.py +``` + +## Worked example + +The reference system in this repository, `openaireview`, satisfies this profile with: + +```bash +openaireview review paper.pdf --out review.json +``` + +The benchmark runs that command per paper and scores the `review.json` it produces. Another system's command takes its place, with whatever name and flags fit the tool. + +## Notes + +- The benchmark may run several papers in parallel, so each invocation should be independent and write only to its own `--out` path. +- A nondeterministic run is fine. The benchmark reports the run it observes. If the command exposes a seed, please document it to help us with reproducibility. + diff --git a/standard/reference/review_client.py b/standard/reference/review_client.py new file mode 100755 index 0000000..baf15ac --- /dev/null +++ b/standard/reference/review_client.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +"""Reference client for the Anchored Review Format hosted API profile. + +This script checks if your system is compatible with the benchmark. +Submits a paper to a conformant endpoint, polls until the review job is +terminal, and returns the review payload. It then runs the payload through the +bundled validator and prints the verdict, so a system's creators can point this +at a staging endpoint and confirm their API is compatible with the benchmark. + +This client follows the standard endpoints described in profile-api.md: + + POST /v1/reviews -> {session_id, status} + GET /v1/reviews/{id} -> {status, result?, error?} + +Usage: + python review_client.py --base-url https://api.example.com --api-key KEY paper.txt +""" + +import argparse +import json +import sys +import time +import urllib.error +import urllib.request +from pathlib import Path + +# Reuse the validator that ships next to this client. +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +import validate as _validate # noqa: E402 + +TERMINAL_OK = {"completed", "complete", "done", "ready", "succeeded", "success"} +TERMINAL_FAIL = {"failed", "error", "errored", "rejected", "cancelled", "canceled"} + + +class ReviewClient: + def __init__(self, base_url, api_key, *, auth_header="Authorization", + auth_scheme="Bearer", request_timeout=60.0): + self.base_url = base_url.rstrip("/") + self.api_key = api_key + self.auth_header = auth_header + self.auth_scheme = auth_scheme + self.request_timeout = request_timeout + + def _headers(self): + value = f"{self.auth_scheme} {self.api_key}".strip() if self.auth_scheme else self.api_key + return {self.auth_header: value, "Content-Type": "application/json"} + + def _request(self, method, url, body=None): + data = json.dumps(body).encode("utf-8") if body is not None else None + req = urllib.request.Request(url, data=data, headers=self._headers(), method=method) + try: + with urllib.request.urlopen(req, timeout=self.request_timeout) as resp: + return json.loads(resp.read().decode("utf-8")) + except urllib.error.HTTPError as e: + detail = e.read().decode("utf-8", "replace")[:500] + raise RuntimeError(f"{method} {url} failed (HTTP {e.code}): {detail}") from e + except urllib.error.URLError as e: + raise RuntimeError(f"{method} {url} failed: {e.reason}") from e + + def submit(self, paper_text, options=None): + """POST /v1/reviews. Returns the session id.""" + body = {"paper": {"text": paper_text}, "options": options or {}} + resp = self._request("POST", f"{self.base_url}/v1/reviews", body) + session_id = resp.get("session_id") or resp.get("id") + if not session_id: + raise RuntimeError(f"submit response missing session_id: {resp}") + return session_id + + def fetch(self, session_id): + """GET /v1/reviews/{id}. Returns the raw job object.""" + return self._request("GET", f"{self.base_url}/v1/reviews/{session_id}") + + def poll(self, session_id, *, interval=5.0, timeout=1200.0): + """Poll until the job is terminal. Returns the review payload.""" + deadline = time.monotonic() + timeout + last_status = None + while True: + job = self.fetch(session_id) + status = str(job.get("status", "")).lower() + if status != last_status: + print(f" status={status or ''}", file=sys.stderr, flush=True) + last_status = status + if status in TERMINAL_OK: + result = job.get("result") + if result is None: + raise RuntimeError(f"job {session_id} is {status} but has no 'result'") + return result + if status in TERMINAL_FAIL: + raise RuntimeError(f"job {session_id} {status}: {job.get('error', '(no error field)')}") + if time.monotonic() >= deadline: + raise TimeoutError(f"job {session_id} not done after {timeout:.0f}s (last status={status!r})") + time.sleep(interval) + + def review(self, paper_text, *, options=None, interval=5.0, timeout=1200.0): + """Submit a paper and return its review payload once ready.""" + session_id = self.submit(paper_text, options) + print(f" submitted, session_id={session_id}", file=sys.stderr, flush=True) + return self.poll(session_id, interval=interval, timeout=timeout) + + +def main(): + parser = argparse.ArgumentParser(description="Reference client for the hosted API profile.") + parser.add_argument("paper", help="Path to a paper text file to submit.") + parser.add_argument("--base-url", required=True, help="Base URL of the review API.") + parser.add_argument("--api-key", required=True, help="API key / token.") + parser.add_argument("--auth-header", default="Authorization", help="Auth header name (default: Authorization).") + parser.add_argument("--auth-scheme", default="Bearer", + help="Auth scheme prefix (default: Bearer). Pass '' for a bare key, e.g. x-api-key.") + parser.add_argument("--poll-interval", type=float, default=5.0, help="Seconds between polls.") + parser.add_argument("--timeout", type=float, default=1200.0, help="Max seconds to wait for a review.") + parser.add_argument("--out", help="Optional path to write the returned payload.") + args = parser.parse_args() + + paper_text = Path(args.paper).read_text(encoding="utf-8") + client = ReviewClient( + args.base_url, args.api_key, + auth_header=args.auth_header, auth_scheme=args.auth_scheme, + ) + + try: + payload = client.review(paper_text, interval=args.poll_interval, timeout=args.timeout) + except (RuntimeError, TimeoutError) as e: + print(f"✗ {e}", file=sys.stderr) + return 1 + + text = json.dumps(payload, indent=2, ensure_ascii=False) + if args.out: + Path(args.out).write_text(text, encoding="utf-8") + print(f"Payload written to {args.out}") + else: + print(text) + + report = _validate.Report() + _validate.validate(payload, report) + for w in report.warnings: + print(f" warning: {w}", file=sys.stderr) + for err in report.errors: + print(f" error: {err}", file=sys.stderr) + if report.errors: + print(f"\n✗ Returned payload is INVALID ({len(report.errors)} error(s)). See profile-api.md.", file=sys.stderr) + return 1 + print(f"\n✓ Returned payload is VALID ({len(payload.get('comments', []))} comment(s)). API is benchmark-ready.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/standard/validate.py b/standard/validate.py new file mode 100755 index 0000000..d781777 --- /dev/null +++ b/standard/validate.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +"""Validate a file against the Anchored Review Format. + +This checker uses only the Python standard library, so you can run it without +installing anything: + + python validate.py my_review.json + +It prints clear, line-by-line diagnostics and exits 0 if the file is valid, +1 if it is not. Warnings point out recommended fields you left out; they do +not fail the file unless you pass --strict. + +See README.md for the full format description. +""" + +import argparse +import json +import sys + +REQUIRED_TOP = ("standard_version", "comments") +RECOMMENDED_TOP = ("standard", "paper_id", "system") +REQUIRED_COMMENT = ("quote", "explanation") +RECOMMENDED_COMMENT = ("title",) + + +class Report: + def __init__(self): + self.errors = [] + self.warnings = [] + + def error(self, msg): + self.errors.append(msg) + + def warn(self, msg): + self.warnings.append(msg) + + +def _is_nonempty_str(value): + return isinstance(value, str) and value.strip() != "" + + +def validate(data, report): + """Run all checks on the parsed JSON, recording errors and warnings.""" + if not isinstance(data, dict): + report.error( + f"Top level must be a JSON object, got {type(data).__name__}. " + "Wrap your comments like {\"standard_version\": \"1.0\", \"comments\": [...]}." + ) + return + + for key in REQUIRED_TOP: + if key not in data: + report.error(f"Missing required top-level field: \"{key}\".") + + if "standard_version" in data and not _is_nonempty_str(data["standard_version"]): + report.error("\"standard_version\" must be a non-empty string, e.g. \"1.0\".") + + for key in RECOMMENDED_TOP: + if key not in data: + report.warn(f"Recommended top-level field missing: \"{key}\".") + + paragraphs = data.get("paragraphs") + n_paragraphs = None + if paragraphs is not None: + if not isinstance(paragraphs, list): + report.error("\"paragraphs\" must be an array if present.") + else: + n_paragraphs = len(paragraphs) + + comments = data.get("comments") + if comments is None: + return # already reported as a missing required field + if not isinstance(comments, list): + report.error("\"comments\" must be an array.") + return + if not comments: + report.warn("\"comments\" is empty. A review with no comments scores zero detections.") + + for i, comment in enumerate(comments): + _validate_comment(comment, i, n_paragraphs, report) + + +def _validate_comment(comment, i, n_paragraphs, report): + where = f"comments[{i}]" + if not isinstance(comment, dict): + report.error(f"{where} must be an object, got {type(comment).__name__}.") + return + + for key in REQUIRED_COMMENT: + if key not in comment: + report.error(f"{where} is missing required field \"{key}\".") + elif not _is_nonempty_str(comment[key]): + report.error(f"{where}.{key} must be a non-empty string.") + + for key in RECOMMENDED_COMMENT: + if key not in comment: + report.warn(f"{where} is missing recommended field \"{key}\".") + + pidx = comment.get("paragraph_index") + if pidx is not None: + if not isinstance(pidx, int) or isinstance(pidx, bool): + report.error(f"{where}.paragraph_index must be an integer or null.") + elif pidx < 0: + report.error(f"{where}.paragraph_index must be >= 0.") + elif n_paragraphs is not None and pidx >= n_paragraphs: + report.error( + f"{where}.paragraph_index is {pidx} but only {n_paragraphs} " + "paragraphs were provided." + ) + + +def main(): + parser = argparse.ArgumentParser( + description="Validate a file against the Anchored Review Format." + ) + parser.add_argument("file", help="Path to the JSON file to validate.") + parser.add_argument( + "--strict", + action="store_true", + help="Treat warnings as failures.", + ) + args = parser.parse_args() + + try: + with open(args.file, encoding="utf-8") as f: + raw = f.read() + except OSError as e: + print(f"✗ Could not read {args.file}: {e}", file=sys.stderr) + return 1 + + try: + data = json.loads(raw) + except json.JSONDecodeError as e: + print(f"✗ {args.file} is not valid JSON: {e}", file=sys.stderr) + return 1 + + report = Report() + validate(data, report) + + for w in report.warnings: + print(f" warning: {w}") + for err in report.errors: + print(f" error: {err}") + + n_comments = len(data["comments"]) if isinstance(data, dict) and isinstance(data.get("comments"), list) else 0 + failed = bool(report.errors) or (args.strict and report.warnings) + + print() + if failed: + why = "errors" if report.errors else "warnings (--strict)" + print(f"✗ INVALID — {len(report.errors)} error(s), {len(report.warnings)} warning(s). Fix the {why} above.") + return 1 + + suffix = f", {len(report.warnings)} warning(s)" if report.warnings else "" + print(f"✓ VALID — {n_comments} comment(s){suffix}. Ready to submit.") + return 0 + + +if __name__ == "__main__": + sys.exit(main())