diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml
new file mode 100644
index 0000000..0086358
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/config.yml
@@ -0,0 +1 @@
+blank_issues_enabled: true
diff --git a/.github/ISSUE_TEMPLATE/paper.yml b/.github/ISSUE_TEMPLATE/paper.yml
new file mode 100644
index 0000000..6a9a7cf
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/paper.yml
@@ -0,0 +1,61 @@
+name: Suggest a paper
+description: Suggest a paper that evaluates, analyzes, or improves a production coding agent.
+title: "[Paper]: "
+labels: [paper-candidate]
+body:
+ - type: input
+ id: title
+ attributes:
+ label: Paper title
+ validations:
+ required: true
+ - type: input
+ id: paper_url
+ attributes:
+ label: Primary-source URL
+ description: Official proceedings, DOI, OpenReview, or arXiv URL.
+ validations:
+ required: true
+ - type: dropdown
+ id: products
+ attributes:
+ label: Products studied
+ multiple: true
+ options:
+ - Claude Code
+ - Codex CLI
+ - Historical OpenAI Codex model
+ validations:
+ required: true
+ - type: dropdown
+ id: classification
+ attributes:
+ label: Suggested evidence class
+ options:
+ - Direct comparison
+ - Related method
+ - Evaluation only
+ - Historical model
+ validations:
+ required: true
+ - type: textarea
+ id: evidence
+ attributes:
+ label: Baseline and result evidence
+ description: Quote table or section numbers and paraphrase the result. Include model and budget details when available.
+ validations:
+ required: true
+ - type: input
+ id: artifact_url
+ attributes:
+ label: Artifact URL
+ description: Optional official code, data, or reproduction repository.
+ - type: checkboxes
+ id: checks
+ attributes:
+ label: Submission checks
+ options:
+ - label: I linked a primary research source rather than a blog summary.
+ required: true
+ - label: The paper studies the product agent, not only a model from the same vendor.
+ required: true
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
new file mode 100644
index 0000000..8a60058
--- /dev/null
+++ b/.github/pull_request_template.md
@@ -0,0 +1,15 @@
+## Summary
+
+Describe the paper additions or catalog corrections.
+
+## Evidence
+
+Link primary sources and identify the tables, sections, or appendices supporting quantitative claims.
+
+## Checklist
+
+- [ ] Inclusion criteria are satisfied.
+- [ ] Product, model, version, and budget fields are complete or explicitly unknown.
+- [ ] Comparability caveats are recorded.
+- [ ] `make check` passes.
+- [ ] No paper PDF has been committed.
diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml
new file mode 100644
index 0000000..f4a4184
--- /dev/null
+++ b/.github/workflows/validate.yml
@@ -0,0 +1,24 @@
+name: Validate catalog
+
+on:
+ pull_request:
+ push:
+ branches: [main]
+ workflow_dispatch:
+
+permissions:
+ contents: read
+
+jobs:
+ validate:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-python@v5
+ with:
+ python-version: "3.12"
+ cache: pip
+ - name: Install dependencies
+ run: python -m pip install -r requirements-dev.txt
+ - name: Validate catalog and generated files
+ run: make check
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..bdaace1
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,6 @@
+.venv/
+__pycache__/
+.pytest_cache/
+.ruff_cache/
+*.pyc
+.DS_Store
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
new file mode 100644
index 0000000..7f9b9fc
--- /dev/null
+++ b/CONTRIBUTING.md
@@ -0,0 +1,63 @@
+# Contributing
+
+Thank you for helping maintain an accurate, evidence-first catalog of research on production coding agents.
+
+## Inclusion criteria
+
+A paper belongs in the catalog when at least one of the following is true:
+
+1. It directly executes Claude Code or Codex CLI in an experiment.
+2. It proposes a method that wraps, configures, improves, or competes with one of those products.
+3. It introduces a benchmark or empirical study with product-level Claude Code or Codex CLI results.
+4. It studies the historical OpenAI Codex model and is clearly labeled `historical`.
+
+A paper does not belong in the primary catalog merely because it uses a Claude or GPT-family model. The product agent or harness must be part of the study.
+
+## Evidence standard
+
+Every entry must link to a primary source: the official proceedings page, DOI landing page, OpenReview record, arXiv record, or authors' artifact repository.
+
+Reported results must be paraphrased accurately and accompanied by comparison caveats. Do not infer that a comparison is same-model or same-budget unless the paper explicitly establishes it.
+
+Use these values when the paper is unclear:
+
+```yaml
+same_model: unknown
+same_budget: unknown
+```
+
+## Adding a paper
+
+1. Add one entry to `data/papers.yaml`.
+2. Use a stable identifier such as `short-title-year`.
+3. Choose the narrowest correct evidence class.
+4. Run `make build` to regenerate README tables.
+5. Run `make check` before opening a pull request.
+
+Do not manually edit content between `CATALOG:*:START` and `CATALOG:*:END` markers in `README.md`; those sections are generated.
+
+## Evidence classes
+
+- `direct`: a numeric head-to-head comparison with Claude Code or Codex CLI.
+- `related`: relevant product-level evidence, but not a clean numeric head-to-head.
+- `evaluation`: product evaluation without a proposed improvement method.
+- `historical`: pre-CLI OpenAI Codex model research.
+
+## Product naming
+
+Use canonical identifiers:
+
+- `claude-code`
+- `codex-cli`
+- `openai-codex-model` for historical model-only work
+
+Future products must be added to both `data/schema.json` and `docs/taxonomy.md` before use.
+
+## Pull request checklist
+
+- [ ] The paper satisfies the inclusion criteria.
+- [ ] All quantitative claims match the cited primary source.
+- [ ] Model, version, budget, and tool details are recorded or marked unknown.
+- [ ] Comparison caveats are explicit.
+- [ ] `make check` passes.
+- [ ] No copyrighted PDF has been committed.
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..6d22878
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 MicroMilo
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..da9f295
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,21 @@
+.PHONY: install build validate test lint check
+
+install:
+ python -m pip install -r requirements-dev.txt
+
+build:
+ python scripts/build_readme.py
+
+validate:
+ python scripts/validate.py
+
+test:
+ python -m pytest
+
+lint:
+ python -m ruff check scripts tests
+
+check: validate
+ python scripts/build_readme.py --check
+ python -m pytest
+ python -m ruff check scripts tests
diff --git a/README.md b/README.md
index 231661b..c1ed3b7 100644
--- a/README.md
+++ b/README.md
@@ -1,2 +1,137 @@
-# awesome-claude-code-codex-papers
-Evidence-first research papers on Claude Code, Codex CLI, and industrial coding agents
+# Awesome Claude Code & Codex Papers
+
+> Research papers that evaluate, analyze, or improve **Claude Code** and **Codex CLI**.
+
+This is an evidence-first catalog of research on production coding agents. It records not only which papers mention an agent, but also the exact baseline role, task, method, reported result, experimental controls, and reproducibility artifacts.
+
+The initial scope is intentionally narrow: **Claude Code** and **Codex CLI**. The schema is product-agnostic so future releases can add Kimi Code, Zed, Cursor, Gemini CLI, and other industrial coding agents without changing the data model.
+
+## Why this catalog exists
+
+Broad agent-paper lists answer “what has been published?” This catalog focuses on a more operational question:
+
+> When researchers evaluate or outperform a production coding agent, what did they add, what exactly improved, and was the comparison fair?
+
+Each entry is reviewed for:
+
+- the product and model used as the baseline;
+- the task, benchmark, and resource budget;
+- the method added around or instead of the baseline;
+- the reported metric and result;
+- same-model and same-budget comparability;
+- paper, code, data, and reproduction links.
+
+## Scope
+
+Included:
+
+- papers that directly run Claude Code or Codex CLI;
+- papers that improve, wrap, configure, or compare their agent harnesses;
+- benchmarks and empirical studies that report product-level results;
+- older OpenAI Codex model papers, clearly separated from Codex CLI.
+
+Not included in the primary catalog:
+
+- papers that only use a Claude or GPT model without the corresponding product agent;
+- blog posts or product comparisons without a research artifact;
+- claims that cannot be traced to a paper or official artifact.
+
+## Evidence classes
+
+- **Direct comparison** — the proposed method is numerically compared with Claude Code or Codex CLI.
+- **Related method** — the product is discussed or partially used, but the key numeric comparison is not a clean product-level head-to-head.
+- **Evaluation only** — the paper evaluates the product without proposing a method intended to beat it.
+- **Historical model** — the paper studies the pre-CLI OpenAI Codex model.
+
+## Direct comparisons
+
+
+| Paper | Venue | Product baseline | Task / benchmark | Method | Reported result | Controls |
+|---|---|---|---|---|---|---|
+| [Agentic Harness Engineering: Observability-Driven Automatic Evolution of Coding-Agent Harnesses](https://arxiv.org/abs/2604.25850) | arXiv 2026
preprint | Codex CLI | Automatic evolution of coding-agent harnesses
Terminal-Bench 2 and SWE-bench Verified | Closed-loop harness evolution using component, trajectory, and decision observability with evidence-backed edits and rollback
`harness-evolution` `observability` `memory` `verifier-loop` | Terminal-Bench 2 pass@1 improves from 69.7 to 77.0, above a human-designed Codex-CLI harness at 71.9; transfer uses 12% fewer tokens on SWE-bench Verified. | model: unknown
budget: unknown
evidence: medium |
+| [Comparing AI Agents to Cybersecurity Professionals in Real-World Penetration Testing](https://arxiv.org/abs/2512.09882)
[artifact](https://github.com/Stanford-Trinity/ARTEMIS) | ICLR 2026
main | Claude Code (Claude Sonnet 4)
Codex CLI (GPT-5) | Long-horizon penetration testing in a live enterprise network
Approximately 8,000 hosts across 12 subnets | Supervisor with dynamic expert agents, recursive task decomposition, parallel exploration, context management, triage, and reporting
`multi-agent` `dynamic-specialization` `parallelism` `memory` `verifier-loop` | ARTEMIS found 9 valid vulnerabilities with an 82% valid-submission rate, ranked second overall, and outperformed 9 of 10 human participants. | model: unknown
budget: different
evidence: medium |
+| [FormAct: Agentic Source Editing for Rich-Format Document Generation](https://openreview.net/forum?id=n7Ta0YEcgw) | ICML 2026
main | Codex CLI | Rich-format document generation and editing
RichDocBench and RichDocFuzz | HTML source editor, template retrieval, rendered-page review agent, iterative refinement, and edit-triggered context compression
`retrieval` `context-reduction` `verifier-loop` `visual-review` | Render correctness 4.81 versus 4.39 for multi-pass Codex; human rank-1 rate 0.760 versus 0.140. | model: unknown
budget: unknown
evidence: high |
+| [On the Impact of AGENTS.md Files on the Efficiency of AI Coding Agents](https://conf.researchr.org/details/icse-2026/jaws-2026-papers/31/On-the-Impact-of-AGENTS-md-Files-on-the-Efficiency-of-AI-Coding-Agents) | ICSE JAWs 2026
workshop | Claude Code
Codex CLI | Real pull-request tasks with and without repository instructions
10 repositories and 124 pull requests | Persistent repository-level build, test, style, and directory guidance in AGENTS.md
`repository-instructions` `structured-state` `context-reduction` | Median runtime falls 28.64% and output tokens fall 16.58% with comparable completion behavior. | model: same
budget: same
evidence: high |
+| [Prefactory: Automated Discovery and Application of Library-Adoption Refactorings](https://arxiv.org/abs/2607.17211) | arXiv 2026
preprint | Codex CLI | Discovery and application of library-adoption refactorings
PrefactoryBench | LLM-synthesized lexical and structural search heuristics, deterministic candidate ranking, targeted refactoring, and differential tests
`deterministic-search` `static-analysis` `test-feedback` `verifier-loop` | File-level detection 75 versus Codex 35; function-level detection 56 versus 32; 40 of 56 candidates yield test-validated refactorings. | model: unknown
budget: unknown
evidence: medium |
+| [QLCoder: A Query Synthesizer For Static Analysis of Security Vulnerabilities](https://proceedings.iclr.cc/paper_files/paper/2026/hash/2adf01ab15adde8820622f7f24bd516b-Abstract-Conference.html) | ICLR 2026
main | Claude Code | CodeQL query synthesis from CVE metadata
176 CVEs across 111 Java projects | CVE-grounded retrieval, AST guidance, CodeQL LSP tools, and iterative execution feedback
`retrieval` `verifier-loop` `static-analysis` | Correct-query rate 53.4% versus 10% for Claude Code-only; F1 0.70 versus 0.048 for IRIS and 0.073 for CodeQL suites. | model: unknown
budget: unknown
evidence: high |
+| [RepoOMP: Repository-Aware Hotspot OpenMP Parallelization via Dependency-Aware Context Reduction](https://arxiv.org/abs/2608.05855)
[artifact](https://github.com/Qlalq/RepoOMP_Simplified) | arXiv 2026
preprint | Claude Code | Repository-aware OpenMP hotspot parallelization
951 hotspots including 330 accepted real-world hotspots | Multi-granularity dependency graph, rule-or-LLM hotspot routing, reduced transformation context, and compile/workload/speedup validation
`repository-graph` `context-reduction` `deterministic-search` `performance-feedback` `verifier-loop` | Across nine detailed kernels, speedup improves 18% to 28% and token cost falls 47% to 68% relative to unstructured Claude Code. | model: unknown
budget: unknown
evidence: medium |
+| [RPG: A Repository Planning Graph for Unified and Scalable Codebase Generation](https://proceedings.iclr.cc/paper_files/paper/2026/file/9482f45fdd89aba9130bb04c44f788a9-Paper-Conference.pdf)
[artifact](https://github.com/microsoft/RPG-ZeroRepo) | ICLR 2026
main | Claude Code (Claude 4 Sonnet)
Codex CLI (o3-pro) | Repository-level codebase generation
RepoCraft | Persistent graph over features, functions, files, interfaces, and data flow with graph-guided generation and validation
`structured-state` `repository-graph` `dependency-aware-planning` `test-feedback` | ZeroRepo coverage 81.5% versus Claude Code 54.2% and Codex 28.4%; pass rate 69.7% versus Claude Code 33.9%. | model: different
budget: unknown
evidence: high |
+| [RustPrint: Documentation-Guided Agentic Codebase Migration from C to Rust](https://arxiv.org/abs/2605.14634) | arXiv 2026
preprint | Claude Code | Whole-codebase migration from C to Rust
Eight real C repositories from 11K to 84K lines of code | Architecture documentation blueprint, module and data-flow planning, compile feedback, documentation mismatch repair, and source-test translation
`structured-state` `dependency-aware-planning` `verifier-loop` `test-feedback` | In the reported Kimi-K2-Instruct setting, feature preservation is 93.26% versus 52.52% for agentic Claude Code; cross-evaluation test pass is 95.17% versus 79.85%. | model: different
budget: unknown
evidence: medium |
+| [Unlocking Model Potentials Through Adaptive Multi-Agent Scaffolding for Efficient Issue Resolution](https://arxiv.org/abs/2606.25514) | arXiv 2026
preprint | Claude Code | Repository issue resolution
SWE-bench Verified and SWE-bench Pro | Event-based agent communication, issue-quality routing, parallel patching and validation, and exploratory fallback
`multi-agent` `dynamic-specialization` `parallelism` `test-feedback` | SWE-bench Verified improves 3.6 to 8.4 percentage points and SWE-bench Pro 6.3 to 18.5 points; average cost is $1.18 lower than multi-agent Claude Code. | model: unknown
budget: unknown
evidence: medium |
+| [WebDesignIter: Co-Evolving Design Knowledge for Repository-Level Front-End Code Generation](https://arxiv.org/abs/2607.10621) | arXiv 2026
preprint | Claude Code
Codex CLI | Repository-level front-end code generation
Web-Bench | Persistent architecture and design knowledge graph, design-informed planning, targeted patches, and sandbox repair
`structured-state` `repository-graph` `retrieval` `test-feedback` | Average Pass@2 improves by 9.55 percentage points over existing baselines; removing design knowledge reduces Pass@1 by 11.4 points. | model: unknown
budget: unknown
evidence: medium |
+| [When Parallelism Pays Off: Cohesion-Aware Task Partitioning for Multi-Agent Coding](https://arxiv.org/abs/2606.00953) | arXiv 2026
preprint | Claude Code (Agent Teams) | Parallel multi-agent coding
28 DevEval and CodeProjectEval tasks | Static dependency graph partitioning, structural hub isolation, community detection, and dependency-aware scheduling
`repository-graph` `dependency-aware-planning` `multi-agent` `parallelism` | Pass rate improves 14.0%, wall-clock speedup reaches 2.10x, and API cost falls 35% relative to Claude Code Agent Teams and simpler schedules. | model: unknown
budget: unknown
evidence: medium |
+| [MLZero: A Multi-Agent System for End-to-end Machine Learning Automation](https://papers.neurips.cc/paper_files/paper/2025/hash/63ed15a46a143ff57484b38cd6b85d91-Abstract-Conference.html)
[artifact](https://github.com/autogluon/autogluon-assistant) | NeurIPS 2025
main | Codex CLI (GPT-4.1 and o4-mini) | End-to-end machine learning automation
MLE-Bench Lite and Multimodal AutoML Agent Benchmark | Hierarchical agents with multimodal perception, semantic and episodic memory, and a planning-coding-evaluation loop
`multi-agent` `memory` `verifier-loop` `test-feedback` | MLZero led MLE-Bench Lite with six gold medals and achieved 0.92 success on its multimodal benchmark; one appendix setting reports 6.5% errors versus 26.9% for Codex CLI. | model: unknown
budget: unknown
evidence: medium |
+
+
+## Related methods
+
+
+| Paper | Venue | Product baseline | Task / benchmark | Method | Reported result | Controls |
+|---|---|---|---|---|---|---|
+| [CodeGrep: An RL-Trained Retrieval Agent for LLM Coding Agents](https://arxiv.org/abs/2608.05886) | arXiv 2026
preprint | Claude Code (Claude-Code-like agent) | Repository retrieval for issue resolution
SWE-bench Verified | GRPO-trained 14B retrieval agent that issues multi-turn parallel grep, glob, and read operations for a frozen downstream coding agent
`retrieval` `context-reduction` `parallelism` | Resolve rate 27.0% versus 25.8% without retrieval; resolved tasks use 15% fewer rounds and 19% fewer tokens. | model: same
budget: unknown
evidence: contextual |
+| [SCATE: Learning to Supervise Coding Agents for Cost-Effective Test Generation](https://arxiv.org/abs/2607.08983) | arXiv 2026
preprint | Claude Code | Automated test generation
Repository-level test-generation benchmark | Contextual-bandit supervisor that selects testing actions from current coverage and testability state
`test-feedback` `verifier-loop` | Line coverage improves 32.3% and branch coverage 30.9% over agent-only Gemini CLI; adaptation is also evaluated with Claude Code. | model: unknown
budget: unknown
evidence: contextual |
+
+
+## Evaluation-only papers
+
+
+| Paper | Venue | Product baseline | Task / benchmark | Method | Reported result | Controls |
+|---|---|---|---|---|---|---|
+| [Engineering Pitfalls in AI Coding Tools: An Empirical Study of Bugs in Claude Code, Codex, and Gemini CLI](https://conf.researchr.org/details/fse-2026/fse-2026-industry-papers/12/Engineering-Pitfalls-in-AI-Coding-Tools-An-Empirical-Study-of-Bugs-in-Claude-Code-C) | FSE Industry Track 2026
conference | Claude Code
Codex CLI | Empirical analysis of product bug reports
More than 3,800 reported bugs | Evaluation-only manual taxonomy of functionality, integration, invocation, and command-execution failures
`observability` | More than 67% of studied bugs concern functionality; API, integration, and configuration account for 36.9% of root causes. | model: unknown
budget: unknown
evidence: high |
+| [Terminal-Bench 2.0: Advancing Agentic Terminal Intelligence through Better Evaluation and Verification](https://proceedings.iclr.cc/paper_files/paper/2026/file/444a3737adaee10d86ad2ef5f74468e6-Paper-Conference.pdf) | ICLR 2026
main | Claude Code (Claude Opus 4.5)
Codex CLI (GPT-5.2) | General terminal-agent evaluation
Terminal-Bench 2.0 | Evaluation-only benchmark with stronger task verification and a neutral terminal scaffold
`verifier-loop` | GPT-5.2 with Codex CLI scores 62.9; Claude Opus 4.5 scores 57.8 with Terminus 2 and 52.1 with Claude Code. | model: unknown
budget: unknown
evidence: high |
+| [BountyBench: Dollar Impact of AI Agent Attackers and Defenders on Real-World Cybersecurity Systems](https://proceedings.neurips.cc/paper_files/paper/2025/hash/faed4276b52ef762879db4142655c699-Abstract-Datasets_and_Benchmarks_Track.html) | NeurIPS 2025
dataset benchmark | Claude Code
Codex CLI (o3-high and o4-mini) | Vulnerability detection, exploitation, and patching
BountyBench | Evaluation-only benchmark of offensive and defensive cybersecurity agents
`verifier-loop` | Codex o3-high detects 12.5% and patches 90%; Claude Code patches 87.5% and exploits 57.5% in the reported settings. | model: unknown
budget: unknown
evidence: high |
+
+
+## Historical OpenAI Codex model papers
+
+
+| Paper | Venue | Product baseline | Task / benchmark | Method | Reported result | Controls |
+|---|---|---|---|---|---|---|
+| [Interactive Code Generation via Test-Driven User-Intent Formalization](https://arxiv.org/abs/2208.05950)
[artifact](https://github.com/microsoft/TiCoder) | AST at ICSE 2022
workshop | OpenAI Codex model (OpenAI Codex) | Interactive function-level code generation
MBPP and HumanEval | Generate tests to formalize user intent, collect lightweight feedback, and rank or prune candidate programs
`test-feedback` `verifier-loop` | With one to five simulated user queries, Codex pass@1 gains 22.49 to 37.71 points on MBPP and 24.79 to 53.98 points on HumanEval. | model: same
budget: different
evidence: high |
+
+
+## Method taxonomy
+
+The catalog currently tracks these recurring ways of improving coding agents:
+
+- persistent structured state and repository graphs;
+- retrieval and context reduction;
+- executable verification and feedback loops;
+- dependency-aware planning and parallelism;
+- multi-agent coordination and specialization;
+- harness observability and self-improvement;
+- repository-level instructions and configuration;
+- deterministic search or analysis before LLM invocation.
+
+See [docs/taxonomy.md](docs/taxonomy.md) for definitions.
+
+## Data and validation
+
+`data/papers.yaml` is the source of truth. The JSON Schema and project checks validate required metadata, unique identifiers, supported products, evidence classes, and comparison fields.
+
+```bash
+python -m pip install -r requirements-dev.txt
+make check
+```
+
+To regenerate the tables after editing the catalog:
+
+```bash
+make build
+```
+
+## Contributing
+
+Paper suggestions and corrections are welcome. Read [CONTRIBUTING.md](CONTRIBUTING.md), then open the paper-submission issue template or submit a pull request that updates `data/papers.yaml`.
+
+Please link to official paper and artifact pages. Do not upload or redistribute paper PDFs in this repository.
+
+## Related collections
+
+- [PurCL/ASE](https://github.com/PurCL/ASE) — a broad Agentic Software Engineering literature database.
+- [EuniAI/awesome-code-agents](https://github.com/EuniAI/awesome-code-agents) — a research map of code agents.
+- [Awesome Agent Harnesses](https://github.com/NeuraLiying/Awesome-Agent-Harnesses) — papers and engineering resources about agent harnesses.
+
+## License
+
+Code and catalog metadata are available under the [MIT License](LICENSE). Copyright remains with the original paper authors and publishers.
diff --git a/data/papers.yaml b/data/papers.yaml
new file mode 100644
index 0000000..46f6149
--- /dev/null
+++ b/data/papers.yaml
@@ -0,0 +1,510 @@
+catalog_version: 1
+reviewed_at: "2026-08-21"
+papers:
+ - id: qlcoder-2026
+ title: "QLCoder: A Query Synthesizer For Static Analysis of Security Vulnerabilities"
+ year: 2026
+ venue: ICLR
+ publication_status: main
+ classification: direct
+ paper_url: https://proceedings.iclr.cc/paper_files/paper/2026/hash/2adf01ab15adde8820622f7f24bd516b-Abstract-Conference.html
+ products:
+ - product: claude-code
+ role: baseline
+ model: not-reported
+ version: not-reported
+ task:
+ summary: CodeQL query synthesis from CVE metadata
+ benchmark: 176 CVEs across 111 Java projects
+ method:
+ summary: CVE-grounded retrieval, AST guidance, CodeQL LSP tools, and iterative execution feedback
+ tags: [retrieval, verifier-loop, static-analysis]
+ evidence:
+ result: "Correct-query rate 53.4% versus 10% for Claude Code-only; F1 0.70 versus 0.048 for IRIS and 0.073 for CodeQL suites."
+ same_model: unknown
+ same_budget: unknown
+ strength: high
+ caveats: Security-query synthesis is narrower than general software engineering.
+
+ - id: rpg-zerorepo-2026
+ title: "RPG: A Repository Planning Graph for Unified and Scalable Codebase Generation"
+ year: 2026
+ venue: ICLR
+ publication_status: main
+ classification: direct
+ paper_url: https://proceedings.iclr.cc/paper_files/paper/2026/file/9482f45fdd89aba9130bb04c44f788a9-Paper-Conference.pdf
+ artifact_url: https://github.com/microsoft/RPG-ZeroRepo
+ products:
+ - product: claude-code
+ role: baseline
+ model: Claude 4 Sonnet
+ version: not-reported
+ - product: codex-cli
+ role: baseline
+ model: o3-pro
+ version: not-reported
+ task:
+ summary: Repository-level codebase generation
+ benchmark: RepoCraft
+ method:
+ summary: Persistent graph over features, functions, files, interfaces, and data flow with graph-guided generation and validation
+ tags: [structured-state, repository-graph, dependency-aware-planning, test-feedback]
+ evidence:
+ result: "ZeroRepo coverage 81.5% versus Claude Code 54.2% and Codex 28.4%; pass rate 69.7% versus Claude Code 33.9%."
+ same_model: "no"
+ same_budget: unknown
+ strength: high
+ caveats: ZeroRepo uses o3-mini while the Claude Code baseline uses Claude 4 Sonnet, so model and harness effects are mixed.
+
+ - id: artemis-2026
+ title: "Comparing AI Agents to Cybersecurity Professionals in Real-World Penetration Testing"
+ year: 2026
+ venue: ICLR
+ publication_status: main
+ classification: direct
+ paper_url: https://arxiv.org/abs/2512.09882
+ artifact_url: https://github.com/Stanford-Trinity/ARTEMIS
+ products:
+ - product: claude-code
+ role: baseline
+ model: Claude Sonnet 4
+ version: not-reported
+ - product: codex-cli
+ role: baseline
+ model: GPT-5
+ version: not-reported
+ task:
+ summary: Long-horizon penetration testing in a live enterprise network
+ benchmark: Approximately 8,000 hosts across 12 subnets
+ method:
+ summary: Supervisor with dynamic expert agents, recursive task decomposition, parallel exploration, context management, triage, and reporting
+ tags: [multi-agent, dynamic-specialization, parallelism, memory, verifier-loop]
+ evidence:
+ result: "ARTEMIS found 9 valid vulnerabilities with an 82% valid-submission rate, ranked second overall, and outperformed 9 of 10 human participants."
+ same_model: unknown
+ same_budget: "no"
+ strength: medium
+ caveats: Runtime limits, model combinations, safety refusals, and product policies differ across systems.
+
+ - id: mlzero-2025
+ title: "MLZero: A Multi-Agent System for End-to-end Machine Learning Automation"
+ year: 2025
+ venue: NeurIPS
+ publication_status: main
+ classification: direct
+ paper_url: https://papers.neurips.cc/paper_files/paper/2025/hash/63ed15a46a143ff57484b38cd6b85d91-Abstract-Conference.html
+ artifact_url: https://github.com/autogluon/autogluon-assistant
+ products:
+ - product: codex-cli
+ role: baseline
+ model: GPT-4.1 and o4-mini
+ version: not-reported
+ task:
+ summary: End-to-end machine learning automation
+ benchmark: MLE-Bench Lite and Multimodal AutoML Agent Benchmark
+ method:
+ summary: Hierarchical agents with multimodal perception, semantic and episodic memory, and a planning-coding-evaluation loop
+ tags: [multi-agent, memory, verifier-loop, test-feedback]
+ evidence:
+ result: "MLZero led MLE-Bench Lite with six gold medals and achieved 0.92 success on its multimodal benchmark; one appendix setting reports 6.5% errors versus 26.9% for Codex CLI."
+ same_model: unknown
+ same_budget: unknown
+ strength: medium
+ caveats: Results concern machine-learning automation and should not be generalized to all software-engineering tasks.
+
+ - id: formact-2026
+ title: "FormAct: Agentic Source Editing for Rich-Format Document Generation"
+ year: 2026
+ venue: ICML
+ publication_status: main
+ classification: direct
+ paper_url: https://openreview.net/forum?id=n7Ta0YEcgw
+ products:
+ - product: codex-cli
+ role: baseline
+ model: not-reported
+ version: not-reported
+ task:
+ summary: Rich-format document generation and editing
+ benchmark: RichDocBench and RichDocFuzz
+ method:
+ summary: HTML source editor, template retrieval, rendered-page review agent, iterative refinement, and edit-triggered context compression
+ tags: [retrieval, context-reduction, verifier-loop, visual-review]
+ evidence:
+ result: "Render correctness 4.81 versus 4.39 for multi-pass Codex; human rank-1 rate 0.760 versus 0.140."
+ same_model: unknown
+ same_budget: unknown
+ strength: high
+ caveats: Codex has slightly higher content alignment, and the task is document formatting rather than source-code repair.
+
+ - id: agentic-harness-engineering-2026
+ title: "Agentic Harness Engineering: Observability-Driven Automatic Evolution of Coding-Agent Harnesses"
+ year: 2026
+ venue: arXiv
+ publication_status: preprint
+ classification: direct
+ paper_url: https://arxiv.org/abs/2604.25850
+ products:
+ - product: codex-cli
+ role: baseline
+ model: not-reported
+ version: not-reported
+ task:
+ summary: Automatic evolution of coding-agent harnesses
+ benchmark: Terminal-Bench 2 and SWE-bench Verified
+ method:
+ summary: Closed-loop harness evolution using component, trajectory, and decision observability with evidence-backed edits and rollback
+ tags: [harness-evolution, observability, memory, verifier-loop]
+ evidence:
+ result: "Terminal-Bench 2 pass@1 improves from 69.7 to 77.0, above a human-designed Codex-CLI harness at 71.9; transfer uses 12% fewer tokens on SWE-bench Verified."
+ same_model: unknown
+ same_budget: unknown
+ strength: medium
+ caveats: Preprint; model, harness, and evaluation-budget effects require careful separation.
+
+ - id: webdesigniter-2026
+ title: "WebDesignIter: Co-Evolving Design Knowledge for Repository-Level Front-End Code Generation"
+ year: 2026
+ venue: arXiv
+ publication_status: preprint
+ classification: direct
+ paper_url: https://arxiv.org/abs/2607.10621
+ products:
+ - product: claude-code
+ role: baseline
+ model: not-reported
+ version: not-reported
+ - product: codex-cli
+ role: baseline
+ model: not-reported
+ version: not-reported
+ task:
+ summary: Repository-level front-end code generation
+ benchmark: Web-Bench
+ method:
+ summary: Persistent architecture and design knowledge graph, design-informed planning, targeted patches, and sandbox repair
+ tags: [structured-state, repository-graph, retrieval, test-feedback]
+ evidence:
+ result: "Average Pass@2 improves by 9.55 percentage points over existing baselines; removing design knowledge reduces Pass@1 by 11.4 points."
+ same_model: unknown
+ same_budget: unknown
+ strength: medium
+ caveats: Front-end-specific benchmark and preprint status.
+
+ - id: prefactory-2026
+ title: "Prefactory: Automated Discovery and Application of Library-Adoption Refactorings"
+ year: 2026
+ venue: arXiv
+ publication_status: preprint
+ classification: direct
+ paper_url: https://arxiv.org/abs/2607.17211
+ products:
+ - product: codex-cli
+ role: baseline
+ model: not-reported
+ version: not-reported
+ task:
+ summary: Discovery and application of library-adoption refactorings
+ benchmark: PrefactoryBench
+ method:
+ summary: LLM-synthesized lexical and structural search heuristics, deterministic candidate ranking, targeted refactoring, and differential tests
+ tags: [deterministic-search, static-analysis, test-feedback, verifier-loop]
+ evidence:
+ result: "File-level detection 75 versus Codex 35; function-level detection 56 versus 32; 40 of 56 candidates yield test-validated refactorings."
+ same_model: unknown
+ same_budget: unknown
+ strength: medium
+ caveats: The strongest comparison concerns candidate detection, not universal end-to-end task success.
+
+ - id: rustprint-2026
+ title: "RustPrint: Documentation-Guided Agentic Codebase Migration from C to Rust"
+ year: 2026
+ venue: arXiv
+ publication_status: preprint
+ classification: direct
+ paper_url: https://arxiv.org/abs/2605.14634
+ products:
+ - product: claude-code
+ role: baseline
+ model: not-reported
+ version: not-reported
+ task:
+ summary: Whole-codebase migration from C to Rust
+ benchmark: Eight real C repositories from 11K to 84K lines of code
+ method:
+ summary: Architecture documentation blueprint, module and data-flow planning, compile feedback, documentation mismatch repair, and source-test translation
+ tags: [structured-state, dependency-aware-planning, verifier-loop, test-feedback]
+ evidence:
+ result: "In the reported Kimi-K2-Instruct setting, feature preservation is 93.26% versus 52.52% for agentic Claude Code; cross-evaluation test pass is 95.17% versus 79.85%."
+ same_model: "no"
+ same_budget: unknown
+ strength: medium
+ caveats: The comparison mixes model and scaffold choices and is specific to C-to-Rust migration.
+
+ - id: repoomp-2026
+ title: "RepoOMP: Repository-Aware Hotspot OpenMP Parallelization via Dependency-Aware Context Reduction"
+ year: 2026
+ venue: arXiv
+ publication_status: preprint
+ classification: direct
+ paper_url: https://arxiv.org/abs/2608.05855
+ artifact_url: https://github.com/Qlalq/RepoOMP_Simplified
+ products:
+ - product: claude-code
+ role: baseline
+ model: not-reported
+ version: not-reported
+ task:
+ summary: Repository-aware OpenMP hotspot parallelization
+ benchmark: 951 hotspots including 330 accepted real-world hotspots
+ method:
+ summary: Multi-granularity dependency graph, rule-or-LLM hotspot routing, reduced transformation context, and compile/workload/speedup validation
+ tags: [repository-graph, context-reduction, deterministic-search, performance-feedback, verifier-loop]
+ evidence:
+ result: "Across nine detailed kernels, speedup improves 18% to 28% and token cost falls 47% to 68% relative to unstructured Claude Code."
+ same_model: unknown
+ same_budget: unknown
+ strength: medium
+ caveats: Domain-specific high-performance-computing task and preprint status.
+
+ - id: icat-agent-2026
+ title: "Unlocking Model Potentials Through Adaptive Multi-Agent Scaffolding for Efficient Issue Resolution"
+ year: 2026
+ venue: arXiv
+ publication_status: preprint
+ classification: direct
+ paper_url: https://arxiv.org/abs/2606.25514
+ products:
+ - product: claude-code
+ role: baseline
+ model: not-reported
+ version: not-reported
+ task:
+ summary: Repository issue resolution
+ benchmark: SWE-bench Verified and SWE-bench Pro
+ method:
+ summary: Event-based agent communication, issue-quality routing, parallel patching and validation, and exploratory fallback
+ tags: [multi-agent, dynamic-specialization, parallelism, test-feedback]
+ evidence:
+ result: "SWE-bench Verified improves 3.6 to 8.4 percentage points and SWE-bench Pro 6.3 to 18.5 points; average cost is $1.18 lower than multi-agent Claude Code."
+ same_model: unknown
+ same_budget: unknown
+ strength: medium
+ caveats: Exact model parity varies by reported configuration; preprint.
+
+ - id: co-coder-2026
+ title: "When Parallelism Pays Off: Cohesion-Aware Task Partitioning for Multi-Agent Coding"
+ year: 2026
+ venue: arXiv
+ publication_status: preprint
+ classification: direct
+ paper_url: https://arxiv.org/abs/2606.00953
+ products:
+ - product: claude-code
+ role: baseline
+ model: not-reported
+ version: Agent Teams
+ task:
+ summary: Parallel multi-agent coding
+ benchmark: 28 DevEval and CodeProjectEval tasks
+ method:
+ summary: Static dependency graph partitioning, structural hub isolation, community detection, and dependency-aware scheduling
+ tags: [repository-graph, dependency-aware-planning, multi-agent, parallelism]
+ evidence:
+ result: "Pass rate improves 14.0%, wall-clock speedup reaches 2.10x, and API cost falls 35% relative to Claude Code Agent Teams and simpler schedules."
+ same_model: unknown
+ same_budget: unknown
+ strength: medium
+ caveats: Small 28-task evaluation and preprint status.
+
+ - id: agents-md-impact-2026
+ title: "On the Impact of AGENTS.md Files on the Efficiency of AI Coding Agents"
+ year: 2026
+ venue: ICSE JAWs
+ publication_status: workshop
+ classification: direct
+ paper_url: https://conf.researchr.org/details/icse-2026/jaws-2026-papers/31/On-the-Impact-of-AGENTS-md-Files-on-the-Efficiency-of-AI-Coding-Agents
+ products:
+ - product: claude-code
+ role: baseline
+ model: not-reported
+ version: not-reported
+ - product: codex-cli
+ role: baseline
+ model: not-reported
+ version: not-reported
+ task:
+ summary: Real pull-request tasks with and without repository instructions
+ benchmark: 10 repositories and 124 pull requests
+ method:
+ summary: Persistent repository-level build, test, style, and directory guidance in AGENTS.md
+ tags: [repository-instructions, structured-state, context-reduction]
+ evidence:
+ result: "Median runtime falls 28.64% and output tokens fall 16.58% with comparable completion behavior."
+ same_model: "yes"
+ same_budget: "yes"
+ strength: high
+ caveats: Workshop study and configuration intervention rather than a standalone agent.
+
+ - id: codegrep-2026
+ title: "CodeGrep: An RL-Trained Retrieval Agent for LLM Coding Agents"
+ year: 2026
+ venue: arXiv
+ publication_status: preprint
+ classification: related
+ paper_url: https://arxiv.org/abs/2608.05886
+ products:
+ - product: claude-code
+ role: evaluated
+ model: not-reported
+ version: Claude-Code-like agent
+ task:
+ summary: Repository retrieval for issue resolution
+ benchmark: SWE-bench Verified
+ method:
+ summary: GRPO-trained 14B retrieval agent that issues multi-turn parallel grep, glob, and read operations for a frozen downstream coding agent
+ tags: [retrieval, context-reduction, parallelism]
+ evidence:
+ result: "Resolve rate 27.0% versus 25.8% without retrieval; resolved tasks use 15% fewer rounds and 19% fewer tokens."
+ same_model: "yes"
+ same_budget: unknown
+ strength: contextual
+ caveats: The clean numeric comparison is against a frozen no-retrieval downstream agent, not necessarily the released Claude Code CLI.
+
+ - id: scate-2026
+ title: "SCATE: Learning to Supervise Coding Agents for Cost-Effective Test Generation"
+ year: 2026
+ venue: arXiv
+ publication_status: preprint
+ classification: related
+ paper_url: https://arxiv.org/abs/2607.08983
+ products:
+ - product: claude-code
+ role: evaluated
+ model: not-reported
+ version: not-reported
+ task:
+ summary: Automated test generation
+ benchmark: Repository-level test-generation benchmark
+ method:
+ summary: Contextual-bandit supervisor that selects testing actions from current coverage and testability state
+ tags: [test-feedback, verifier-loop]
+ evidence:
+ result: "Line coverage improves 32.3% and branch coverage 30.9% over agent-only Gemini CLI; adaptation is also evaluated with Claude Code."
+ same_model: unknown
+ same_budget: unknown
+ strength: contextual
+ caveats: The abstract does not provide an equally complete numeric head-to-head against Claude Code.
+
+ - id: terminal-bench-2-2026
+ title: "Terminal-Bench 2.0: Advancing Agentic Terminal Intelligence through Better Evaluation and Verification"
+ year: 2026
+ venue: ICLR
+ publication_status: main
+ classification: evaluation
+ paper_url: https://proceedings.iclr.cc/paper_files/paper/2026/file/444a3737adaee10d86ad2ef5f74468e6-Paper-Conference.pdf
+ products:
+ - product: claude-code
+ role: evaluated
+ model: Claude Opus 4.5
+ version: not-reported
+ - product: codex-cli
+ role: evaluated
+ model: GPT-5.2
+ version: not-reported
+ task:
+ summary: General terminal-agent evaluation
+ benchmark: Terminal-Bench 2.0
+ method:
+ summary: Evaluation-only benchmark with stronger task verification and a neutral terminal scaffold
+ tags: [verifier-loop]
+ evidence:
+ result: "GPT-5.2 with Codex CLI scores 62.9; Claude Opus 4.5 scores 57.8 with Terminus 2 and 52.1 with Claude Code."
+ same_model: unknown
+ same_budget: unknown
+ strength: high
+ caveats: Benchmark result; differences can reflect both model and harness, and no new method is proposed to beat the products.
+
+ - id: bountybench-2025
+ title: "BountyBench: Dollar Impact of AI Agent Attackers and Defenders on Real-World Cybersecurity Systems"
+ year: 2025
+ venue: NeurIPS
+ publication_status: dataset-benchmark
+ classification: evaluation
+ paper_url: https://proceedings.neurips.cc/paper_files/paper/2025/hash/faed4276b52ef762879db4142655c699-Abstract-Datasets_and_Benchmarks_Track.html
+ products:
+ - product: claude-code
+ role: evaluated
+ model: not-reported
+ version: not-reported
+ - product: codex-cli
+ role: evaluated
+ model: o3-high and o4-mini
+ version: not-reported
+ task:
+ summary: Vulnerability detection, exploitation, and patching
+ benchmark: BountyBench
+ method:
+ summary: Evaluation-only benchmark of offensive and defensive cybersecurity agents
+ tags: [verifier-loop]
+ evidence:
+ result: "Codex o3-high detects 12.5% and patches 90%; Claude Code patches 87.5% and exploits 57.5% in the reported settings."
+ same_model: unknown
+ same_budget: unknown
+ strength: high
+ caveats: Benchmark-only evidence across different tasks and model configurations.
+
+ - id: engineering-pitfalls-2026
+ title: "Engineering Pitfalls in AI Coding Tools: An Empirical Study of Bugs in Claude Code, Codex, and Gemini CLI"
+ year: 2026
+ venue: FSE Industry Track
+ publication_status: conference
+ classification: evaluation
+ paper_url: https://conf.researchr.org/details/fse-2026/fse-2026-industry-papers/12/Engineering-Pitfalls-in-AI-Coding-Tools-An-Empirical-Study-of-Bugs-in-Claude-Code-C
+ products:
+ - product: claude-code
+ role: evaluated
+ model: not-applicable
+ version: multiple
+ - product: codex-cli
+ role: evaluated
+ model: not-applicable
+ version: multiple
+ task:
+ summary: Empirical analysis of product bug reports
+ benchmark: More than 3,800 reported bugs
+ method:
+ summary: Evaluation-only manual taxonomy of functionality, integration, invocation, and command-execution failures
+ tags: [observability]
+ evidence:
+ result: "More than 67% of studied bugs concern functionality; API, integration, and configuration account for 36.9% of root causes."
+ same_model: unknown
+ same_budget: unknown
+ strength: high
+ caveats: Failure-taxonomy study; it does not propose an agent that outperforms the products.
+
+ - id: ticoder-2022
+ title: "Interactive Code Generation via Test-Driven User-Intent Formalization"
+ year: 2022
+ venue: AST at ICSE
+ publication_status: workshop
+ classification: historical
+ paper_url: https://arxiv.org/abs/2208.05950
+ artifact_url: https://github.com/microsoft/TiCoder
+ products:
+ - product: openai-codex-model
+ role: historical-baseline
+ model: OpenAI Codex
+ version: not-reported
+ task:
+ summary: Interactive function-level code generation
+ benchmark: MBPP and HumanEval
+ method:
+ summary: Generate tests to formalize user intent, collect lightweight feedback, and rank or prune candidate programs
+ tags: [test-feedback, verifier-loop]
+ evidence:
+ result: "With one to five simulated user queries, Codex pass@1 gains 22.49 to 37.71 points on MBPP and 24.79 to 53.98 points on HumanEval."
+ same_model: "yes"
+ same_budget: "no"
+ strength: high
+ caveats: This is the historical OpenAI Codex model, not the modern Codex CLI product.
diff --git a/data/schema.json b/data/schema.json
new file mode 100644
index 0000000..1e1cda7
--- /dev/null
+++ b/data/schema.json
@@ -0,0 +1,193 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "https://github.com/MicroMilo/awesome-claude-code-codex-papers/data/schema.json",
+ "title": "Claude Code and Codex Paper Catalog",
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["catalog_version", "reviewed_at", "papers"],
+ "properties": {
+ "catalog_version": {
+ "const": 1
+ },
+ "reviewed_at": {
+ "type": "string",
+ "format": "date"
+ },
+ "papers": {
+ "type": "array",
+ "minItems": 1,
+ "items": {
+ "$ref": "#/$defs/paper"
+ }
+ }
+ },
+ "$defs": {
+ "product": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["product", "role", "model", "version"],
+ "properties": {
+ "product": {
+ "enum": ["claude-code", "codex-cli", "openai-codex-model"]
+ },
+ "role": {
+ "enum": ["baseline", "evaluated", "host", "historical-baseline"]
+ },
+ "model": {
+ "type": "string",
+ "minLength": 1
+ },
+ "version": {
+ "type": "string",
+ "minLength": 1
+ }
+ }
+ },
+ "paper": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": [
+ "id",
+ "title",
+ "year",
+ "venue",
+ "publication_status",
+ "classification",
+ "paper_url",
+ "products",
+ "task",
+ "method",
+ "evidence"
+ ],
+ "properties": {
+ "id": {
+ "type": "string",
+ "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$"
+ },
+ "title": {
+ "type": "string",
+ "minLength": 5
+ },
+ "year": {
+ "type": "integer",
+ "minimum": 2020,
+ "maximum": 2100
+ },
+ "venue": {
+ "type": "string",
+ "minLength": 2
+ },
+ "publication_status": {
+ "enum": [
+ "main",
+ "conference",
+ "workshop",
+ "tool-demo",
+ "dataset-benchmark",
+ "journal",
+ "preprint"
+ ]
+ },
+ "classification": {
+ "enum": ["direct", "related", "evaluation", "historical"]
+ },
+ "paper_url": {
+ "type": "string",
+ "format": "uri",
+ "pattern": "^https://"
+ },
+ "artifact_url": {
+ "type": "string",
+ "format": "uri",
+ "pattern": "^https://"
+ },
+ "products": {
+ "type": "array",
+ "minItems": 1,
+ "items": {
+ "$ref": "#/$defs/product"
+ }
+ },
+ "task": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["summary", "benchmark"],
+ "properties": {
+ "summary": {
+ "type": "string",
+ "minLength": 3
+ },
+ "benchmark": {
+ "type": "string",
+ "minLength": 1
+ }
+ }
+ },
+ "method": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["summary", "tags"],
+ "properties": {
+ "summary": {
+ "type": "string",
+ "minLength": 3
+ },
+ "tags": {
+ "type": "array",
+ "uniqueItems": true,
+ "items": {
+ "enum": [
+ "structured-state",
+ "repository-graph",
+ "retrieval",
+ "context-reduction",
+ "verifier-loop",
+ "static-analysis",
+ "test-feedback",
+ "performance-feedback",
+ "multi-agent",
+ "dynamic-specialization",
+ "dependency-aware-planning",
+ "parallelism",
+ "memory",
+ "harness-evolution",
+ "observability",
+ "repository-instructions",
+ "deterministic-search",
+ "visual-review"
+ ]
+ }
+ }
+ }
+ },
+ "evidence": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["result", "same_model", "same_budget", "strength", "caveats"],
+ "properties": {
+ "result": {
+ "type": "string",
+ "minLength": 3
+ },
+ "same_model": {
+ "enum": ["yes", "no", "unknown"]
+ },
+ "same_budget": {
+ "enum": ["yes", "no", "unknown"]
+ },
+ "strength": {
+ "enum": ["high", "medium", "contextual"]
+ },
+ "caveats": {
+ "type": "string",
+ "minLength": 3
+ }
+ }
+ },
+ "notes": {
+ "type": "string"
+ }
+ }
+ }
+ }
+}
diff --git a/docs/evaluation-protocol.md b/docs/evaluation-protocol.md
new file mode 100644
index 0000000..54fb1fe
--- /dev/null
+++ b/docs/evaluation-protocol.md
@@ -0,0 +1,45 @@
+# Evaluation Protocol Checklist
+
+Use this checklist when reviewing a paper or designing a reproduction.
+
+## Product identity
+
+- Product name and CLI/runtime version
+- Backbone model and model snapshot
+- Reasoning effort or inference mode
+- Date of execution for drifting hosted products
+
+## Environment
+
+- Repository commit and task specification
+- Container or VM image
+- Network access
+- Tool and permission policy
+- Context files such as AGENTS.md or CLAUDE.md
+
+## Budget
+
+- Wall-clock timeout
+- Maximum turns or iterations
+- Input and output token limits
+- Maximum parallel agents
+- Monetary budget
+
+## Outcome
+
+- Task success or pass rate
+- Quality or domain-specific metric
+- Wall-clock latency
+- Token and monetary cost
+- Tool calls and repair iterations
+- Failure category
+
+## Minimum comparison matrix
+
+When practical, report three conditions:
+
+1. The production product configuration, such as Claude Code or Codex CLI.
+2. A neutral harness using the same model and budget.
+3. The proposed method using the same model and budget.
+
+Any variable that cannot be held fixed should be documented as a caveat rather than silently ignored.
diff --git a/docs/taxonomy.md b/docs/taxonomy.md
new file mode 100644
index 0000000..0db3402
--- /dev/null
+++ b/docs/taxonomy.md
@@ -0,0 +1,58 @@
+# Catalog Taxonomy
+
+## Products
+
+- `claude-code`: Anthropic's Claude Code product or CLI used as an end-to-end agent harness.
+- `codex-cli`: OpenAI Codex CLI used as an end-to-end agent harness.
+- `openai-codex-model`: the historical OpenAI Codex model, without the modern Codex CLI harness.
+
+The product field describes the evaluated harness, not merely the model provider. A paper using a Claude API model inside a custom scaffold is not automatically a Claude Code paper.
+
+## Evidence classes
+
+### Direct comparison
+
+The paper reports at least one numeric comparison between its proposed method and Claude Code or Codex CLI. Direct does not imply fair: `same_model`, `same_budget`, and caveats record comparability.
+
+### Related method
+
+The product is integrated, discussed, or used in a partial experiment, but the central result is not a clean product-level comparison.
+
+### Evaluation only
+
+The paper introduces a benchmark, dataset, measurement study, or failure analysis and reports product-level results without a method intended to outperform the product.
+
+### Historical model
+
+The paper studies the pre-CLI OpenAI Codex model. These entries are retained for research continuity but never mixed with modern Codex CLI results.
+
+## Method tags
+
+- `structured-state`: externalized plans, repository graphs, architecture descriptions, or durable state.
+- `repository-graph`: graph representation of files, symbols, features, dependencies, or data flow.
+- `retrieval`: learned or engineered selection of relevant code and evidence.
+- `context-reduction`: removal or compression of irrelevant context.
+- `verifier-loop`: an external executable judge provides iterative repair signals.
+- `static-analysis`: static-analysis queries, ASTs, LSP services, or program-analysis tools.
+- `test-feedback`: tests or coverage guide the next action.
+- `performance-feedback`: measured runtime or speedup guides acceptance and repair.
+- `multi-agent`: more than one active reasoning agent.
+- `dynamic-specialization`: roles or experts are created based on task state.
+- `dependency-aware-planning`: ordering or partitioning follows program dependencies.
+- `parallelism`: concurrent agent execution is a core mechanism.
+- `memory`: durable semantic, episodic, or task memory.
+- `harness-evolution`: automatic modification of tools, prompts, memory, middleware, or runtime policy.
+- `observability`: structured trajectory, component, or decision-level evidence.
+- `repository-instructions`: repository-scoped instruction files such as AGENTS.md.
+- `deterministic-search`: non-LLM rules or analyses narrow candidates before generation.
+- `visual-review`: rendered output is inspected and used as feedback.
+
+## Comparison controls
+
+`same_model` and `same_budget` use `yes`, `no`, or `unknown`.
+
+- `yes`: the paper explicitly holds the variable fixed.
+- `no`: the paper explicitly uses different models or budgets.
+- `unknown`: the source does not establish parity clearly enough.
+
+These fields describe experimental controls, not paper quality.
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 0000000..d448a3f
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,9 @@
+[tool.ruff]
+target-version = "py310"
+line-length = 100
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "UP"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
diff --git a/requirements-dev.txt b/requirements-dev.txt
new file mode 100644
index 0000000..ce63a7e
--- /dev/null
+++ b/requirements-dev.txt
@@ -0,0 +1,4 @@
+PyYAML==6.0.2
+jsonschema==4.25.0
+pytest==8.4.1
+ruff==0.12.9
diff --git a/scripts/build_readme.py b/scripts/build_readme.py
new file mode 100644
index 0000000..15349af
--- /dev/null
+++ b/scripts/build_readme.py
@@ -0,0 +1,170 @@
+#!/usr/bin/env python3
+"""Generate catalog tables in README.md from data/papers.yaml."""
+
+from __future__ import annotations
+
+import argparse
+import re
+import sys
+from pathlib import Path
+
+import yaml
+
+ROOT = Path(__file__).resolve().parents[1]
+CATALOG_PATH = ROOT / "data" / "papers.yaml"
+README_PATH = ROOT / "README.md"
+
+SECTIONS = {
+ "direct": ("", ""),
+ "related": ("", ""),
+ "evaluation": (
+ "",
+ "",
+ ),
+ "historical": (
+ "",
+ "",
+ ),
+}
+
+PRODUCT_LABELS = {
+ "claude-code": "Claude Code",
+ "codex-cli": "Codex CLI",
+ "openai-codex-model": "OpenAI Codex model",
+}
+
+CONTROL_LABELS = {
+ "yes": "same",
+ "no": "different",
+ "unknown": "unknown",
+}
+
+
+def load_catalog() -> dict:
+ with CATALOG_PATH.open(encoding="utf-8") as handle:
+ return yaml.safe_load(handle)
+
+
+def escape_cell(value: object) -> str:
+ return str(value).replace("|", "\\|").replace("\n", " ").strip()
+
+
+def format_paper(paper: dict) -> str:
+ title = escape_cell(paper["title"])
+ paper_link = f"[{title}]({paper['paper_url']})"
+ artifact_url = paper.get("artifact_url")
+ if artifact_url:
+ paper_link += f"
[artifact]({artifact_url})"
+ return paper_link
+
+
+def format_products(paper: dict) -> str:
+ rendered = []
+ for item in paper["products"]:
+ label = PRODUCT_LABELS[item["product"]]
+ details = []
+ if item["model"] not in {"not-reported", "not-applicable"}:
+ details.append(item["model"])
+ if item["version"] not in {"not-reported", "multiple"}:
+ details.append(item["version"])
+ if details:
+ label += f" ({', '.join(details)})"
+ rendered.append(escape_cell(label))
+ return "
".join(rendered)
+
+
+def format_venue(paper: dict) -> str:
+ status = paper["publication_status"].replace("-", " ")
+ return f"{escape_cell(paper['venue'])} {paper['year']}
{status}"
+
+
+def format_method(paper: dict) -> str:
+ method = escape_cell(paper["method"]["summary"])
+ tags = paper["method"]["tags"]
+ if tags:
+ method += "
" + " ".join(f"`{tag}`" for tag in tags)
+ return method
+
+
+def format_controls(paper: dict) -> str:
+ evidence = paper["evidence"]
+ model = CONTROL_LABELS[evidence["same_model"]]
+ budget = CONTROL_LABELS[evidence["same_budget"]]
+ strength = evidence["strength"]
+ return f"model: {model}
budget: {budget}
evidence: {strength}"
+
+
+def generate_table(papers: list[dict]) -> str:
+ if not papers:
+ return "_No reviewed papers in this class yet._"
+
+ header = (
+ "| Paper | Venue | Product baseline | Task / benchmark | "
+ "Method | Reported result | Controls |\n"
+ "|---|---|---|---|---|---|---|"
+ )
+ rows = []
+ for paper in sorted(papers, key=lambda item: (-item["year"], item["title"].lower())):
+ task = (
+ f"{escape_cell(paper['task']['summary'])}"
+ f"
{escape_cell(paper['task']['benchmark'])}"
+ )
+ row = "| " + " | ".join(
+ [
+ format_paper(paper),
+ format_venue(paper),
+ format_products(paper),
+ task,
+ format_method(paper),
+ escape_cell(paper["evidence"]["result"]),
+ format_controls(paper),
+ ]
+ ) + " |"
+ rows.append(row)
+ return header + "\n" + "\n".join(rows)
+
+
+def replace_section(readme: str, start: str, end: str, content: str) -> str:
+ pattern = re.compile(re.escape(start) + r".*?" + re.escape(end), re.DOTALL)
+ replacement = f"{start}\n{content}\n{end}"
+ updated, count = pattern.subn(lambda _: replacement, readme)
+ if count != 1:
+ raise RuntimeError(f"Expected one README section for {start}, found {count}")
+ return updated
+
+
+def render_readme() -> str:
+ catalog = load_catalog()
+ readme = README_PATH.read_text(encoding="utf-8")
+ papers = catalog["papers"]
+ for classification, (start, end) in SECTIONS.items():
+ selected = [paper for paper in papers if paper["classification"] == classification]
+ readme = replace_section(readme, start, end, generate_table(selected))
+ return readme
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "--check",
+ action="store_true",
+ help="Fail when README.md does not match the generated catalog tables.",
+ )
+ args = parser.parse_args()
+
+ rendered = render_readme()
+ current = README_PATH.read_text(encoding="utf-8")
+ if args.check:
+ if rendered != current:
+ print("README.md is stale; run `python scripts/build_readme.py`.", file=sys.stderr)
+ return 1
+ print("README.md catalog tables are current.")
+ return 0
+
+ README_PATH.write_text(rendered, encoding="utf-8")
+ print(f"Updated {README_PATH.relative_to(ROOT)}")
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/scripts/validate.py b/scripts/validate.py
new file mode 100644
index 0000000..0fa63ee
--- /dev/null
+++ b/scripts/validate.py
@@ -0,0 +1,98 @@
+#!/usr/bin/env python3
+"""Validate the paper catalog against its schema and project invariants."""
+
+from __future__ import annotations
+
+import json
+import sys
+from collections import Counter
+from pathlib import Path
+
+import yaml
+from jsonschema import Draft202012Validator, FormatChecker
+
+ROOT = Path(__file__).resolve().parents[1]
+CATALOG_PATH = ROOT / "data" / "papers.yaml"
+SCHEMA_PATH = ROOT / "data" / "schema.json"
+
+
+def load_files() -> tuple[dict, dict]:
+ with CATALOG_PATH.open(encoding="utf-8") as handle:
+ catalog = yaml.safe_load(handle)
+ with SCHEMA_PATH.open(encoding="utf-8") as handle:
+ schema = json.load(handle)
+ return catalog, schema
+
+
+def format_path(path: object) -> str:
+ parts = list(path)
+ return ".".join(str(part) for part in parts) or ""
+
+
+def schema_errors(catalog: dict, schema: dict) -> list[str]:
+ validator = Draft202012Validator(schema, format_checker=FormatChecker())
+ errors = sorted(validator.iter_errors(catalog), key=lambda error: list(error.path))
+ return [f"{format_path(error.path)}: {error.message}" for error in errors]
+
+
+def duplicate_values(values: list[str]) -> list[str]:
+ counts = Counter(values)
+ return sorted(value for value, count in counts.items() if count > 1)
+
+
+def invariant_errors(catalog: dict) -> list[str]:
+ errors = []
+ papers = catalog.get("papers", [])
+
+ for duplicate in duplicate_values([paper.get("id", "") for paper in papers]):
+ errors.append(f"duplicate paper id: {duplicate}")
+ for duplicate in duplicate_values([paper.get("title", "") for paper in papers]):
+ errors.append(f"duplicate paper title: {duplicate}")
+
+ for paper in papers:
+ paper_id = paper.get("id", "")
+ classification = paper.get("classification")
+ products = paper.get("products", [])
+ product_ids = [item.get("product") for item in products]
+
+ for duplicate in duplicate_values(product_ids):
+ errors.append(f"{paper_id}: duplicate product entry: {duplicate}")
+
+ if classification == "direct":
+ direct_products = {"claude-code", "codex-cli"}
+ has_baseline = any(
+ item.get("role") == "baseline" and item.get("product") in direct_products
+ for item in products
+ )
+ if not has_baseline:
+ errors.append(f"{paper_id}: direct entries require a product baseline")
+
+ historical_products = {"openai-codex-model"}
+ has_historical_product = any(
+ item.get("product") in historical_products for item in products
+ )
+ if classification == "historical" and not has_historical_product:
+ errors.append(f"{paper_id}: historical entries require a historical product")
+ if classification != "historical" and has_historical_product:
+ errors.append(f"{paper_id}: historical products must use historical classification")
+
+ return errors
+
+
+def main() -> int:
+ catalog, schema = load_files()
+ errors = schema_errors(catalog, schema) + invariant_errors(catalog)
+ if errors:
+ print("Catalog validation failed:", file=sys.stderr)
+ for error in errors:
+ print(f"- {error}", file=sys.stderr)
+ return 1
+
+ classes = Counter(paper["classification"] for paper in catalog["papers"])
+ summary = ", ".join(f"{name}={count}" for name, count in sorted(classes.items()))
+ print(f"Validated {len(catalog['papers'])} papers ({summary}).")
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/tests/test_catalog.py b/tests/test_catalog.py
new file mode 100644
index 0000000..ee73d38
--- /dev/null
+++ b/tests/test_catalog.py
@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+import subprocess
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def run_script(script: str, *args: str) -> subprocess.CompletedProcess[str]:
+ return subprocess.run(
+ [sys.executable, str(ROOT / "scripts" / script), *args],
+ cwd=ROOT,
+ check=False,
+ capture_output=True,
+ text=True,
+ )
+
+
+def test_catalog_is_valid() -> None:
+ result = run_script("validate.py")
+ assert result.returncode == 0, result.stderr
+
+
+def test_generated_readme_is_current() -> None:
+ result = run_script("build_readme.py", "--check")
+ assert result.returncode == 0, result.stderr